From 24a4244f19d29b9d416d0f0a7d566ffa4ad32122 Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Thu, 19 Jan 2023 16:59:14 +0100 Subject: [PATCH 01/12] add more information to existing errors logs and panics - '$ROOT/$FILE' for better localization in the code - add text where none was given - fix unnecessary sprintf nesting in influxv2 and prometheus metricrepo logging --- cmd/cc-backend/main.go | 28 +++---- internal/api/rest.go | 110 ++++++++++++------------- internal/auth/auth.go | 18 ++-- internal/auth/jwt.go | 20 ++--- internal/auth/ldap.go | 12 +-- internal/auth/local.go | 2 +- internal/auth/users.go | 10 +-- internal/config/config.go | 8 +- internal/graph/schema.resolvers.go | 4 +- internal/graph/stats.go | 8 +- internal/metricdata/cc-metric-store.go | 20 ++--- internal/metricdata/influxdb-v2.go | 42 +++++----- internal/metricdata/metricdata.go | 14 ++-- internal/metricdata/prometheus.go | 40 +++++---- internal/repository/dbConnection.go | 6 +- internal/repository/init.go | 22 ++--- internal/repository/job.go | 22 ++--- internal/repository/query.go | 6 +- internal/repository/user.go | 4 +- internal/routerConfig/routes.go | 8 +- internal/runtimeEnv/setup.go | 6 +- pkg/archive/archive.go | 2 +- pkg/archive/clusterConfig.go | 14 ++-- pkg/archive/fsBackend.go | 26 +++--- pkg/archive/nodelist.go | 16 ++-- pkg/lrucache/cache.go | 16 ++-- pkg/schema/job.go | 4 +- pkg/schema/metrics.go | 6 +- pkg/schema/validate.go | 6 +- pkg/units/units.go | 2 +- web/web.go | 5 +- 31 files changed, 254 insertions(+), 253 deletions(-) diff --git a/cmd/cc-backend/main.go b/cmd/cc-backend/main.go index 7c41d73..1113159 100644 --- a/cmd/cc-backend/main.go +++ b/cmd/cc-backend/main.go @@ -118,7 +118,7 @@ func main() { "ldap": config.Keys.LdapConfig, "jwt": config.Keys.JwtConfig, }); err != nil { - log.Fatal(err) + log.Fatalf("auth initialization failed: %v", err) } if d, err := time.ParseDuration(config.Keys.SessionMaxAge); err != nil { @@ -134,12 +134,12 @@ func main() { if err := authentication.AddUser(&auth.User{ Username: parts[0], Password: parts[2], Roles: strings.Split(parts[1], ","), }); err != nil { - log.Fatal(err) + log.Fatalf("adding '%s' user authentication failed: %v", parts[0], err) } } if flagDelUser != "" { if err := authentication.DelUser(flagDelUser); err != nil { - log.Fatal(err) + log.Fatalf("deleting user failed: %v", err) } } @@ -149,7 +149,7 @@ func main() { } if err := authentication.LdapAuth.Sync(); err != nil { - log.Fatal(err) + log.Fatalf("LDAP sync failed: %v", err) } log.Info("LDAP sync successfull") } @@ -157,16 +157,16 @@ func main() { if flagGenJWT != "" { user, err := authentication.GetUser(flagGenJWT) if err != nil { - log.Fatal(err) + log.Fatalf("could not get user from JWT: %v", err) } if !user.HasRole(auth.RoleApi) { - log.Warn("that user does not have the API role") + log.Warnf("user '%s' does not have the API role", user.Username) } jwt, err := authentication.JwtAuth.ProvideJWT(user) if err != nil { - log.Fatal(err) + log.Fatalf("failed to provide JWT to user '%s': %v", user.Username, err) } fmt.Printf("JWT for '%s': %s\n", user.Username, jwt) @@ -176,16 +176,16 @@ func main() { } if err := archive.Init(config.Keys.Archive, config.Keys.DisableArchive); err != nil { - log.Fatal(err) + log.Fatalf("failed to initialize archive: %s", err.Error()) } if err := metricdata.Init(config.Keys.DisableArchive); err != nil { - log.Fatal(err) + log.Fatalf("failed to initialize metricdata repository: %s", err.Error()) } if flagReinitDB { if err := repository.InitDB(); err != nil { - log.Fatal(err) + log.Fatal("failed to re-initialize repository DB: %s", err.Error()) } } @@ -361,7 +361,7 @@ func main() { // Start http or https server listener, err := net.Listen("tcp", config.Keys.Addr) if err != nil { - log.Fatal(err) + log.Fatalf("starting http listener failed: %v", err) } if !strings.HasSuffix(config.Keys.Addr, ":80") && config.Keys.RedirectHttpTo != "" { @@ -373,7 +373,7 @@ func main() { if config.Keys.HttpsCertFile != "" && config.Keys.HttpsKeyFile != "" { cert, err := tls.LoadX509KeyPair(config.Keys.HttpsCertFile, config.Keys.HttpsKeyFile) if err != nil { - log.Fatal(err) + log.Fatalf("loading X509 keypair failed: %v", err) } listener = tls.NewListener(listener, &tls.Config{ Certificates: []tls.Certificate{cert}, @@ -400,7 +400,7 @@ func main() { go func() { defer wg.Done() if err := server.Serve(listener); err != nil && err != http.ErrServerClosed { - log.Fatal(err) + log.Fatalf("starting server failed: %v", err) } }() @@ -424,7 +424,7 @@ func main() { for range time.Tick(30 * time.Minute) { err := jobRepo.StopJobsExceedingWalltimeBy(config.Keys.StopJobsExceedingWalltime) if err != nil { - log.Errorf("error while looking for jobs exceeding theire walltime: %s", err.Error()) + log.Errorf("error while looking for jobs exceeding their walltime: %s", err.Error()) } runtime.GC() } diff --git a/internal/api/rest.go b/internal/api/rest.go index dbc778c..004d183 100644 --- a/internal/api/rest.go +++ b/internal/api/rest.go @@ -134,7 +134,7 @@ type ApiTag struct { type TagJobApiRequest []*ApiTag func handleError(err error, statusCode int, rw http.ResponseWriter) { - log.Warnf("REST API: %s", err.Error()) + log.Warnf("API/REST > ERROR : %s", err.Error()) rw.Header().Add("Content-Type", "application/json") rw.WriteHeader(statusCode) json.NewEncoder(rw).Encode(ErrorResponse{ @@ -169,7 +169,7 @@ func decode(r io.Reader, val interface{}) error { // @router /jobs/ [get] func (api *RestApi) getJobs(rw http.ResponseWriter, r *http.Request) { if user := auth.GetUser(r.Context()); user != nil && !user.HasRole(auth.RoleApi) { - handleError(fmt.Errorf("missing role: %#v", auth.RoleApi), http.StatusForbidden, rw) + handleError(fmt.Errorf("API/REST > missing role: %#v", auth.RoleApi), http.StatusForbidden, rw) return } @@ -271,7 +271,7 @@ func (api *RestApi) getJobs(rw http.ResponseWriter, r *http.Request) { results = append(results, res) } - log.Debugf("/api/jobs: %d jobs returned", len(results)) + log.Debugf("API/REST > /api/jobs: %d jobs returned", len(results)) bw := bufio.NewWriter(rw) defer bw.Flush() if err := json.NewEncoder(bw).Encode(map[string]interface{}{ @@ -300,7 +300,7 @@ func (api *RestApi) getJobs(rw http.ResponseWriter, r *http.Request) { // @router /jobs/tag_job/{id} [post] func (api *RestApi) tagJob(rw http.ResponseWriter, r *http.Request) { if user := auth.GetUser(r.Context()); user != nil && !user.HasRole(auth.RoleApi) { - handleError(fmt.Errorf("missing role: %#v", auth.RoleApi), http.StatusForbidden, rw) + handleError(fmt.Errorf("API/REST > missing role: %#v", auth.RoleApi), http.StatusForbidden, rw) return } @@ -365,13 +365,13 @@ func (api *RestApi) tagJob(rw http.ResponseWriter, r *http.Request) { // @router /jobs/start_job/ [post] func (api *RestApi) startJob(rw http.ResponseWriter, r *http.Request) { if user := auth.GetUser(r.Context()); user != nil && !user.HasRole(auth.RoleApi) { - handleError(fmt.Errorf("missing role: %#v", auth.RoleApi), http.StatusForbidden, rw) + handleError(fmt.Errorf("API/REST > missing role: %#v", auth.RoleApi), http.StatusForbidden, rw) return } req := schema.JobMeta{BaseJob: schema.JobDefaults} if err := decode(r.Body, &req); err != nil { - handleError(fmt.Errorf("parsing request body failed: %w", err), http.StatusBadRequest, rw) + handleError(fmt.Errorf("API/REST > parsing request body failed: %w", err), http.StatusBadRequest, rw) return } @@ -391,12 +391,12 @@ func (api *RestApi) startJob(rw http.ResponseWriter, r *http.Request) { // Check if combination of (job_id, cluster_id, start_time) already exists: jobs, err := api.JobRepository.FindAll(&req.JobID, &req.Cluster, nil) if err != nil && err != sql.ErrNoRows { - handleError(fmt.Errorf("checking for duplicate failed: %w", err), http.StatusInternalServerError, rw) + handleError(fmt.Errorf("API/REST > checking for duplicate failed: %w", err), http.StatusInternalServerError, rw) return } else if err == nil { for _, job := range jobs { if (req.StartTime - job.StartTimeUnix) < 86400 { - handleError(fmt.Errorf("a job with that jobId, cluster and startTime already exists: dbid: %d", job.ID), http.StatusUnprocessableEntity, rw) + handleError(fmt.Errorf("API/REST > a job with that jobId, cluster and startTime already exists: dbid: %d", job.ID), http.StatusUnprocessableEntity, rw) return } } @@ -404,7 +404,7 @@ func (api *RestApi) startJob(rw http.ResponseWriter, r *http.Request) { id, err := api.JobRepository.Start(&req) if err != nil { - handleError(fmt.Errorf("insert into database failed: %w", err), http.StatusInternalServerError, rw) + handleError(fmt.Errorf("API/REST > insert into database failed: %w", err), http.StatusInternalServerError, rw) return } // unlock here, adding Tags can be async @@ -413,12 +413,12 @@ func (api *RestApi) startJob(rw http.ResponseWriter, r *http.Request) { for _, tag := range req.Tags { if _, err := api.JobRepository.AddTagOrCreate(id, tag.Type, tag.Name); err != nil { http.Error(rw, err.Error(), http.StatusInternalServerError) - handleError(fmt.Errorf("adding tag to new job %d failed: %w", id, err), http.StatusInternalServerError, rw) + handleError(fmt.Errorf("API/REST > adding tag to new job %d failed: %w", id, err), http.StatusInternalServerError, rw) return } } - log.Printf("new job (id: %d): cluster=%s, jobId=%d, user=%s, startTime=%d", id, req.Cluster, req.JobID, req.User, req.StartTime) + log.Printf("API/REST > new job (id: %d): cluster=%s, jobId=%d, user=%s, startTime=%d", id, req.Cluster, req.JobID, req.User, req.StartTime) rw.Header().Add("Content-Type", "application/json") rw.WriteHeader(http.StatusCreated) json.NewEncoder(rw).Encode(StartJobApiResponse{ @@ -446,14 +446,14 @@ func (api *RestApi) startJob(rw http.ResponseWriter, r *http.Request) { // @router /jobs/stop_job/{id} [post] func (api *RestApi) stopJobById(rw http.ResponseWriter, r *http.Request) { if user := auth.GetUser(r.Context()); user != nil && !user.HasRole(auth.RoleApi) { - handleError(fmt.Errorf("missing role: %#v", auth.RoleApi), http.StatusForbidden, rw) + handleError(fmt.Errorf("API/REST > missing role: %#v", auth.RoleApi), http.StatusForbidden, rw) return } // Parse request body: Only StopTime and State req := StopJobApiRequest{} if err := decode(r.Body, &req); err != nil { - handleError(fmt.Errorf("parsing request body failed: %w", err), http.StatusBadRequest, rw) + handleError(fmt.Errorf("API/REST > parsing request body failed: %w", err), http.StatusBadRequest, rw) return } @@ -464,17 +464,17 @@ func (api *RestApi) stopJobById(rw http.ResponseWriter, r *http.Request) { if ok { id, e := strconv.ParseInt(id, 10, 64) if e != nil { - handleError(fmt.Errorf("integer expected in path for id: %w", e), http.StatusBadRequest, rw) + handleError(fmt.Errorf("API/REST > integer expected in path for id: %w", e), http.StatusBadRequest, rw) return } job, err = api.JobRepository.FindById(id) } else { - handleError(errors.New("the parameter 'id' is required"), http.StatusBadRequest, rw) + handleError(errors.New("API/REST > the parameter 'id' is required"), http.StatusBadRequest, rw) return } if err != nil { - handleError(fmt.Errorf("finding job failed: %w", err), http.StatusUnprocessableEntity, rw) + handleError(fmt.Errorf("API/REST > finding job failed: %w", err), http.StatusUnprocessableEntity, rw) return } @@ -499,14 +499,14 @@ func (api *RestApi) stopJobById(rw http.ResponseWriter, r *http.Request) { // @router /jobs/stop_job/ [post] func (api *RestApi) stopJobByRequest(rw http.ResponseWriter, r *http.Request) { if user := auth.GetUser(r.Context()); user != nil && !user.HasRole(auth.RoleApi) { - handleError(fmt.Errorf("missing role: %#v", auth.RoleApi), http.StatusForbidden, rw) + handleError(fmt.Errorf("API/REST > missing role: %#v", auth.RoleApi), http.StatusForbidden, rw) return } // Parse request body req := StopJobApiRequest{} if err := decode(r.Body, &req); err != nil { - handleError(fmt.Errorf("parsing request body failed: %w", err), http.StatusBadRequest, rw) + handleError(fmt.Errorf("API/REST > parsing request body failed: %w", err), http.StatusBadRequest, rw) return } @@ -514,14 +514,14 @@ func (api *RestApi) stopJobByRequest(rw http.ResponseWriter, r *http.Request) { var job *schema.Job var err error if req.JobId == nil { - handleError(errors.New("the field 'jobId' is required"), http.StatusBadRequest, rw) + handleError(errors.New("API/REST > the field 'jobId' is required"), http.StatusBadRequest, rw) return } job, err = api.JobRepository.Find(req.JobId, req.Cluster, req.StartTime) if err != nil { - handleError(fmt.Errorf("finding job failed: %w", err), http.StatusUnprocessableEntity, rw) + handleError(fmt.Errorf("API/REST > finding job failed: %w", err), http.StatusUnprocessableEntity, rw) return } @@ -545,7 +545,7 @@ func (api *RestApi) stopJobByRequest(rw http.ResponseWriter, r *http.Request) { // @router /jobs/delete_job/{id} [delete] func (api *RestApi) deleteJobById(rw http.ResponseWriter, r *http.Request) { if user := auth.GetUser(r.Context()); user != nil && !user.HasRole(auth.RoleApi) { - handleError(fmt.Errorf("missing role: %#v", auth.RoleApi), http.StatusForbidden, rw) + handleError(fmt.Errorf("API/REST > missing role: %#v", auth.RoleApi), http.StatusForbidden, rw) return } @@ -555,23 +555,23 @@ func (api *RestApi) deleteJobById(rw http.ResponseWriter, r *http.Request) { if ok { id, e := strconv.ParseInt(id, 10, 64) if e != nil { - handleError(fmt.Errorf("integer expected in path for id: %w", e), http.StatusBadRequest, rw) + handleError(fmt.Errorf("API/REST > integer expected in path for id: %w", e), http.StatusBadRequest, rw) return } err = api.JobRepository.DeleteJobById(id) } else { - handleError(errors.New("the parameter 'id' is required"), http.StatusBadRequest, rw) + handleError(errors.New("API/REST > the parameter 'id' is required"), http.StatusBadRequest, rw) return } if err != nil { - handleError(fmt.Errorf("deleting job failed: %w", err), http.StatusUnprocessableEntity, rw) + handleError(fmt.Errorf("API/REST > deleting job failed: %w", err), http.StatusUnprocessableEntity, rw) return } rw.Header().Add("Content-Type", "application/json") rw.WriteHeader(http.StatusOK) json.NewEncoder(rw).Encode(DeleteJobApiResponse{ - Message: fmt.Sprintf("Successfully deleted job %s", id), + Message: fmt.Sprintf("API/REST > Successfully deleted job %s", id), }) } @@ -593,14 +593,14 @@ func (api *RestApi) deleteJobById(rw http.ResponseWriter, r *http.Request) { // @router /jobs/delete_job/ [delete] func (api *RestApi) deleteJobByRequest(rw http.ResponseWriter, r *http.Request) { if user := auth.GetUser(r.Context()); user != nil && !user.HasRole(auth.RoleApi) { - handleError(fmt.Errorf("missing role: %#v", auth.RoleApi), http.StatusForbidden, rw) + handleError(fmt.Errorf("API/REST > missing role: %#v", auth.RoleApi), http.StatusForbidden, rw) return } // Parse request body req := DeleteJobApiRequest{} if err := decode(r.Body, &req); err != nil { - handleError(fmt.Errorf("parsing request body failed: %w", err), http.StatusBadRequest, rw) + handleError(fmt.Errorf("API/REST > parsing request body failed: %w", err), http.StatusBadRequest, rw) return } @@ -608,27 +608,27 @@ func (api *RestApi) deleteJobByRequest(rw http.ResponseWriter, r *http.Request) var job *schema.Job var err error if req.JobId == nil { - handleError(errors.New("the field 'jobId' is required"), http.StatusBadRequest, rw) + handleError(errors.New("API/REST > the field 'jobId' is required"), http.StatusBadRequest, rw) return } job, err = api.JobRepository.Find(req.JobId, req.Cluster, req.StartTime) if err != nil { - handleError(fmt.Errorf("finding job failed: %w", err), http.StatusUnprocessableEntity, rw) + handleError(fmt.Errorf("API/REST > finding job failed: %w", err), http.StatusUnprocessableEntity, rw) return } err = api.JobRepository.DeleteJobById(job.ID) if err != nil { - handleError(fmt.Errorf("deleting job failed: %w", err), http.StatusUnprocessableEntity, rw) + handleError(fmt.Errorf("API/REST > deleting job failed: %w", err), http.StatusUnprocessableEntity, rw) return } rw.Header().Add("Content-Type", "application/json") rw.WriteHeader(http.StatusOK) json.NewEncoder(rw).Encode(DeleteJobApiResponse{ - Message: fmt.Sprintf("Successfully deleted job %d", job.ID), + Message: fmt.Sprintf("API/REST > Successfully deleted job %d", job.ID), }) } @@ -649,7 +649,7 @@ func (api *RestApi) deleteJobByRequest(rw http.ResponseWriter, r *http.Request) // @router /jobs/delete_job_before/{ts} [delete] func (api *RestApi) deleteJobBefore(rw http.ResponseWriter, r *http.Request) { if user := auth.GetUser(r.Context()); user != nil && !user.HasRole(auth.RoleApi) { - handleError(fmt.Errorf("missing role: %#v", auth.RoleApi), http.StatusForbidden, rw) + handleError(fmt.Errorf("API/REST > missing role: %#v", auth.RoleApi), http.StatusForbidden, rw) return } @@ -660,24 +660,24 @@ func (api *RestApi) deleteJobBefore(rw http.ResponseWriter, r *http.Request) { if ok { ts, e := strconv.ParseInt(id, 10, 64) if e != nil { - handleError(fmt.Errorf("integer expected in path for ts: %w", e), http.StatusBadRequest, rw) + handleError(fmt.Errorf("API/REST > integer expected in path for ts: %w", e), http.StatusBadRequest, rw) return } cnt, err = api.JobRepository.DeleteJobsBefore(ts) } else { - handleError(errors.New("the parameter 'ts' is required"), http.StatusBadRequest, rw) + handleError(errors.New("API/REST > the parameter 'ts' is required"), http.StatusBadRequest, rw) return } if err != nil { - handleError(fmt.Errorf("deleting jobs failed: %w", err), http.StatusUnprocessableEntity, rw) + handleError(fmt.Errorf("API/REST > deleting jobs failed: %w", err), http.StatusUnprocessableEntity, rw) return } rw.Header().Add("Content-Type", "application/json") rw.WriteHeader(http.StatusOK) json.NewEncoder(rw).Encode(DeleteJobApiResponse{ - Message: fmt.Sprintf("Successfully deleted %d jobs", cnt), + Message: fmt.Sprintf("API/REST > Successfully deleted %d jobs", cnt), }) } @@ -685,12 +685,12 @@ func (api *RestApi) checkAndHandleStopJob(rw http.ResponseWriter, job *schema.Jo // Sanity checks if job == nil || job.StartTime.Unix() >= req.StopTime || job.State != schema.JobStateRunning { - handleError(errors.New("stopTime must be larger than startTime and only running jobs can be stopped"), http.StatusBadRequest, rw) + handleError(errors.New("API/REST > stopTime must be larger than startTime and only running jobs can be stopped"), http.StatusBadRequest, rw) return } if req.State != "" && !req.State.Valid() { - handleError(fmt.Errorf("invalid job state: %#v", req.State), http.StatusBadRequest, rw) + handleError(fmt.Errorf("API/REST > invalid job state: %#v", req.State), http.StatusBadRequest, rw) return } else if req.State == "" { req.State = schema.JobStateCompleted @@ -700,11 +700,11 @@ func (api *RestApi) checkAndHandleStopJob(rw http.ResponseWriter, job *schema.Jo job.Duration = int32(req.StopTime - job.StartTime.Unix()) job.State = req.State if err := api.JobRepository.Stop(job.ID, job.Duration, job.State, job.MonitoringStatus); err != nil { - handleError(fmt.Errorf("marking job as stopped failed: %w", err), http.StatusInternalServerError, rw) + handleError(fmt.Errorf("API/REST > marking job as stopped failed: %w", err), http.StatusInternalServerError, rw) return } - log.Printf("archiving job... (dbid: %d): cluster=%s, jobId=%d, user=%s, startTime=%s", job.ID, job.Cluster, job.JobID, job.User, job.StartTime) + log.Printf("API/REST > archiving job... (dbid: %d): cluster=%s, jobId=%d, user=%s, startTime=%s", job.ID, job.Cluster, job.JobID, job.User, job.StartTime) // Send a response (with status OK). This means that erros that happen from here on forward // can *NOT* be communicated to the client. If reading from a MetricDataRepository or @@ -724,7 +724,7 @@ func (api *RestApi) checkAndHandleStopJob(rw http.ResponseWriter, job *schema.Jo // func (api *RestApi) importJob(rw http.ResponseWriter, r *http.Request) { // if user := auth.GetUser(r.Context()); user != nil && !user.HasRole(auth.RoleApi) { -// handleError(fmt.Errorf("missing role: %#v", auth.RoleApi), http.StatusForbidden, rw) +// handleError(fmt.Errorf("API/REST > missing role: %#v", auth.RoleApi), http.StatusForbidden, rw) // return // } @@ -733,12 +733,12 @@ func (api *RestApi) checkAndHandleStopJob(rw http.ResponseWriter, job *schema.Jo // Data *schema.JobData `json:"data"` // } // if err := decode(r.Body, &body); err != nil { -// handleError(fmt.Errorf("import failed: %s", err.Error()), http.StatusBadRequest, rw) +// handleError(fmt.Errorf("API/REST > import failed: %s", err.Error()), http.StatusBadRequest, rw) // return // } // if err := api.JobRepository.ImportJob(body.Meta, body.Data); err != nil { -// handleError(fmt.Errorf("import failed: %s", err.Error()), http.StatusUnprocessableEntity, rw) +// handleError(fmt.Errorf("API/REST > import failed: %s", err.Error()), http.StatusUnprocessableEntity, rw) // return // } @@ -793,7 +793,7 @@ func (api *RestApi) getJWT(rw http.ResponseWriter, r *http.Request) { me := auth.GetUser(r.Context()) if !me.HasRole(auth.RoleAdmin) { if username != me.Username { - http.Error(rw, "only admins are allowed to sign JWTs not for themselves", http.StatusForbidden) + http.Error(rw, "API/REST > only admins are allowed to sign JWTs not for themselves", http.StatusForbidden) return } } @@ -818,13 +818,13 @@ func (api *RestApi) createUser(rw http.ResponseWriter, r *http.Request) { rw.Header().Set("Content-Type", "text/plain") me := auth.GetUser(r.Context()) if !me.HasRole(auth.RoleAdmin) { - http.Error(rw, "only admins are allowed to create new users", http.StatusForbidden) + http.Error(rw, "API/REST > only admins are allowed to create new users", http.StatusForbidden) return } username, password, role, name, email := r.FormValue("username"), r.FormValue("password"), r.FormValue("role"), r.FormValue("name"), r.FormValue("email") if len(password) == 0 && role != auth.RoleApi { - http.Error(rw, "only API users are allowed to have a blank password (login will be impossible)", http.StatusBadRequest) + http.Error(rw, "API/REST > only API users are allowed to have a blank password (login will be impossible)", http.StatusBadRequest) return } @@ -838,12 +838,12 @@ func (api *RestApi) createUser(rw http.ResponseWriter, r *http.Request) { return } - rw.Write([]byte(fmt.Sprintf("User %#v successfully created!\n", username))) + rw.Write([]byte(fmt.Sprintf("API/REST > User %#v successfully created!\n", username))) } func (api *RestApi) deleteUser(rw http.ResponseWriter, r *http.Request) { if user := auth.GetUser(r.Context()); !user.HasRole(auth.RoleAdmin) { - http.Error(rw, "only admins are allowed to delete a user", http.StatusForbidden) + http.Error(rw, "API/REST > only admins are allowed to delete a user", http.StatusForbidden) return } @@ -858,7 +858,7 @@ func (api *RestApi) deleteUser(rw http.ResponseWriter, r *http.Request) { func (api *RestApi) getUsers(rw http.ResponseWriter, r *http.Request) { if user := auth.GetUser(r.Context()); !user.HasRole(auth.RoleAdmin) { - http.Error(rw, "only admins are allowed to fetch a list of users", http.StatusForbidden) + http.Error(rw, "API/REST > only admins are allowed to fetch a list of users", http.StatusForbidden) return } @@ -873,7 +873,7 @@ func (api *RestApi) getUsers(rw http.ResponseWriter, r *http.Request) { func (api *RestApi) updateUser(rw http.ResponseWriter, r *http.Request) { if user := auth.GetUser(r.Context()); !user.HasRole(auth.RoleAdmin) { - http.Error(rw, "only admins are allowed to update a user", http.StatusForbidden) + http.Error(rw, "API/REST > only admins are allowed to update a user", http.StatusForbidden) return } @@ -903,7 +903,7 @@ func (api *RestApi) updateConfiguration(rw http.ResponseWriter, r *http.Request) rw.Header().Set("Content-Type", "text/plain") key, value := r.FormValue("key"), r.FormValue("value") - fmt.Printf("KEY: %#v\nVALUE: %#v\n", key, value) + fmt.Printf("API/REST > KEY: %#v\nVALUE: %#v\n", key, value) if err := repository.GetUserCfgRepo().UpdateConfig(key, value, auth.GetUser(r.Context())); err != nil { http.Error(rw, err.Error(), http.StatusUnprocessableEntity) @@ -915,7 +915,7 @@ func (api *RestApi) updateConfiguration(rw http.ResponseWriter, r *http.Request) func (api *RestApi) putMachineState(rw http.ResponseWriter, r *http.Request) { if api.MachineStateDir == "" { - http.Error(rw, "not enabled", http.StatusNotFound) + http.Error(rw, "API/REST > machine state not enabled", http.StatusNotFound) return } @@ -928,7 +928,7 @@ func (api *RestApi) putMachineState(rw http.ResponseWriter, r *http.Request) { return } - filename := filepath.Join(dir, fmt.Sprintf("%s.json", host)) + filename := filepath.Join(dir, fmt.Sprintf("API/REST > %s.json", host)) f, err := os.Create(filename) if err != nil { http.Error(rw, err.Error(), http.StatusInternalServerError) @@ -946,12 +946,12 @@ func (api *RestApi) putMachineState(rw http.ResponseWriter, r *http.Request) { func (api *RestApi) getMachineState(rw http.ResponseWriter, r *http.Request) { if api.MachineStateDir == "" { - http.Error(rw, "not enabled", http.StatusNotFound) + http.Error(rw, "API/REST > machine state not enabled", http.StatusNotFound) return } vars := mux.Vars(r) - filename := filepath.Join(api.MachineStateDir, vars["cluster"], fmt.Sprintf("%s.json", vars["host"])) + filename := filepath.Join(api.MachineStateDir, vars["cluster"], fmt.Sprintf("API/REST > %s.json", vars["host"])) // Sets the content-type and 'Last-Modified' Header and so on automatically http.ServeFile(rw, r, filename) diff --git a/internal/auth/auth.go b/internal/auth/auth.go index 22fa0db..c119417 100644 --- a/internal/auth/auth.go +++ b/internal/auth/auth.go @@ -99,7 +99,7 @@ func Init(db *sqlx.DB, sessKey := os.Getenv("SESSION_KEY") if sessKey == "" { - log.Warn("environment variable 'SESSION_KEY' not set (will use non-persistent random key)") + log.Warn("AUTH/AUTH > environment variable 'SESSION_KEY' not set (will use non-persistent random key)") bytes := make([]byte, 32) if _, err := rand.Read(bytes); err != nil { return nil, err @@ -169,7 +169,7 @@ func (auth *Authentication) Login( user := (*User)(nil) if username != "" { if user, _ = auth.GetUser(username); err != nil { - // log.Warnf("login of unkown user %#v", username) + // log.Warnf("AUTH/AUTH > login of unkown user %#v", username) _ = err } } @@ -181,14 +181,14 @@ func (auth *Authentication) Login( user, err = authenticator.Login(user, rw, r) if err != nil { - log.Warnf("login failed: %s", err.Error()) + log.Warnf("AUTH/AUTH > user '%s' login failed: %s", user.Username, err.Error()) onfailure(rw, r, err) return } session, err := auth.sessionStore.New(r, "session") if err != nil { - log.Errorf("session creation failed: %s", err.Error()) + log.Errorf("AUTH/AUTH > session creation failed: %s", err.Error()) http.Error(rw, err.Error(), http.StatusInternalServerError) return } @@ -199,18 +199,18 @@ func (auth *Authentication) Login( session.Values["username"] = user.Username session.Values["roles"] = user.Roles if err := auth.sessionStore.Save(r, rw, session); err != nil { - log.Errorf("session save failed: %s", err.Error()) + log.Errorf("AUTH/AUTH > session save failed: %s", err.Error()) http.Error(rw, err.Error(), http.StatusInternalServerError) return } - log.Infof("login successfull: user: %#v (roles: %v)", user.Username, user.Roles) + log.Infof("AUTH/AUTH > login successfull: user: %#v (roles: %v)", user.Username, user.Roles) ctx := context.WithValue(r.Context(), ContextUserKey, user) onsuccess.ServeHTTP(rw, r.WithContext(ctx)) return } - log.Warn("login failed: no authenticator applied") + log.Warn("AUTH/AUTH > login failed: no authenticator applied") onfailure(rw, r, err) }) } @@ -226,7 +226,7 @@ func (auth *Authentication) Auth( for _, authenticator := range auth.authenticators { user, err := authenticator.Auth(rw, r) if err != nil { - log.Warnf("authentication failed: %s", err.Error()) + log.Warnf("AUTH/AUTH > authentication failed: %s", err.Error()) http.Error(rw, err.Error(), http.StatusUnauthorized) return } @@ -239,7 +239,7 @@ func (auth *Authentication) Auth( return } - log.Warnf("authentication failed: %s", "no authenticator applied") + log.Warnf("AUTH/AUTH > authentication failed: %s", "no authenticator applied") // http.Error(rw, http.StatusText(http.StatusUnauthorized), http.StatusUnauthorized) onfailure(rw, r, errors.New("unauthorized (login first or use a token)")) }) diff --git a/internal/auth/jwt.go b/internal/auth/jwt.go index ccc8240..ed8e413 100644 --- a/internal/auth/jwt.go +++ b/internal/auth/jwt.go @@ -41,7 +41,7 @@ func (ja *JWTAuthenticator) Init(auth *Authentication, conf interface{}) error { pubKey, privKey := os.Getenv("JWT_PUBLIC_KEY"), os.Getenv("JWT_PRIVATE_KEY") if pubKey == "" || privKey == "" { - log.Warn("environment variables 'JWT_PUBLIC_KEY' or 'JWT_PRIVATE_KEY' not set (token based authentication will not work)") + log.Warn("AUTH/JWT > environment variables 'JWT_PUBLIC_KEY' or 'JWT_PRIVATE_KEY' not set (token based authentication will not work)") } else { bytes, err := base64.StdEncoding.DecodeString(pubKey) if err != nil { @@ -75,20 +75,20 @@ func (ja *JWTAuthenticator) Init(auth *Authentication, conf interface{}) error { // Warn if other necessary settings are not configured if ja.config != nil { if ja.config.CookieName == "" { - log.Warn("cookieName for JWTs not configured (cross login via JWT cookie will fail)") + log.Warn("AUTH/JWT > cookieName for JWTs not configured (cross login via JWT cookie will fail)") } if !ja.config.ForceJWTValidationViaDatabase { - log.Warn("forceJWTValidationViaDatabase not set to true: CC will accept users and roles defined in JWTs regardless of its own database!") + log.Warn("AUTH/JWT > forceJWTValidationViaDatabase not set to true: CC will accept users and roles defined in JWTs regardless of its own database!") } if ja.config.TrustedExternalIssuer == "" { - log.Warn("trustedExternalIssuer for JWTs not configured (cross login via JWT cookie will fail)") + log.Warn("AUTH/JWT > trustedExternalIssuer for JWTs not configured (cross login via JWT cookie will fail)") } } else { - log.Warn("cookieName and trustedExternalIssuer for JWTs not configured (cross login via JWT cookie will fail)") + log.Warn("AUTH/JWT > cookieName and trustedExternalIssuer for JWTs not configured (cross login via JWT cookie will fail)") } } else { ja.publicKeyCrossLogin = nil - log.Warn("environment variable 'CROSS_LOGIN_JWT_PUBLIC_KEY' not set (cross login token based authentication will not work)") + log.Warn("AUTH/JWT > environment variable 'CROSS_LOGIN_JWT_PUBLIC_KEY' not set (cross login token based authentication will not work)") } return nil @@ -123,7 +123,7 @@ func (ja *JWTAuthenticator) Login( if t.Method == jwt.SigningMethodHS256 || t.Method == jwt.SigningMethodHS512 { return ja.loginTokenKey, nil } - return nil, fmt.Errorf("unkown signing method for login token: %s (known: HS256, HS512, EdDSA)", t.Method.Alg()) + return nil, fmt.Errorf("AUTH/JWT > unkown signing method for login token: %s (known: HS256, HS512, EdDSA)", t.Method.Alg()) }) if err != nil { return nil, err @@ -243,7 +243,7 @@ func (ja *JWTAuthenticator) Auth( // Deny any logins for unknown usernames if err != nil { - log.Warn("Could not find user from JWT in internal database.") + log.Warn("AUTH/JWT > Could not find user from JWT in internal database.") return nil, errors.New("unknown user") } @@ -264,7 +264,7 @@ func (ja *JWTAuthenticator) Auth( // Create a session so that we no longer need the JTW Cookie session, err := ja.auth.sessionStore.New(r, "session") if err != nil { - log.Errorf("session creation failed: %s", err.Error()) + log.Errorf("AUTH/JWT > session creation failed: %s", err.Error()) http.Error(rw, err.Error(), http.StatusInternalServerError) return nil, err } @@ -276,7 +276,7 @@ func (ja *JWTAuthenticator) Auth( session.Values["roles"] = roles if err := ja.auth.sessionStore.Save(r, rw, session); err != nil { - log.Errorf("session save failed: %s", err.Error()) + log.Errorf("AUTH/JWT > session save failed: %s", err.Error()) http.Error(rw, err.Error(), http.StatusInternalServerError) return nil, err } diff --git a/internal/auth/ldap.go b/internal/auth/ldap.go index d15bfea..6f974b5 100644 --- a/internal/auth/ldap.go +++ b/internal/auth/ldap.go @@ -33,7 +33,7 @@ func (la *LdapAuthenticator) Init( la.syncPassword = os.Getenv("LDAP_ADMIN_PASSWORD") if la.syncPassword == "" { - log.Warn("environment variable 'LDAP_ADMIN_PASSWORD' not set (ldap sync will not work)") + log.Warn("AUTH/LDAP > environment variable 'LDAP_ADMIN_PASSWORD' not set (ldap sync will not work)") } if la.config != nil && la.config.SyncInterval != "" { @@ -49,11 +49,11 @@ func (la *LdapAuthenticator) Init( go func() { ticker := time.NewTicker(interval) for t := range ticker.C { - log.Printf("LDAP sync started at %s", t.Format(time.RFC3339)) + log.Printf("AUTH/LDAP > sync started at %s", t.Format(time.RFC3339)) if err := la.Sync(); err != nil { - log.Errorf("LDAP sync failed: %s", err.Error()) + log.Errorf("AUTH/LDAP > sync failed: %s", err.Error()) } - log.Print("LDAP sync done") + log.Print("AUTH/LDAP > sync done") } }() } @@ -147,13 +147,13 @@ func (la *LdapAuthenticator) Sync() error { for username, where := range users { if where == IN_DB && la.config.SyncDelOldUsers { - log.Debugf("ldap-sync: remove %#v (does not show up in LDAP anymore)", username) + log.Debugf("AUTH/LDAP > sync: remove %#v (does not show up in LDAP anymore)", username) if _, err := la.auth.db.Exec(`DELETE FROM user WHERE user.username = ?`, username); err != nil { return err } } else if where == IN_LDAP { name := newnames[username] - log.Debugf("ldap-sync: add %#v (name: %#v, roles: [user], ldap: true)", username, name) + log.Debugf("AUTH/LDAP > sync: add %#v (name: %#v, roles: [user], ldap: true)", username, name) if _, err := la.auth.db.Exec(`INSERT INTO user (username, ldap, name, roles) VALUES (?, ?, ?, ?)`, username, 1, name, "[\""+RoleUser+"\"]"); err != nil { return err diff --git a/internal/auth/local.go b/internal/auth/local.go index a8468c2..0813e23 100644 --- a/internal/auth/local.go +++ b/internal/auth/local.go @@ -39,7 +39,7 @@ func (la *LocalAuthenticator) Login( r *http.Request) (*User, error) { if e := bcrypt.CompareHashAndPassword([]byte(user.Password), []byte(r.FormValue("password"))); e != nil { - return nil, fmt.Errorf("user '%s' provided the wrong password (%w)", user.Username, e) + return nil, fmt.Errorf("AUTH/LOCAL > user '%s' provided the wrong password (%w)", user.Username, e) } return user, nil diff --git a/internal/auth/users.go b/internal/auth/users.go index aea5eb3..009fe35 100644 --- a/internal/auth/users.go +++ b/internal/auth/users.go @@ -67,7 +67,7 @@ func (auth *Authentication) AddUser(user *User) error { return err } - log.Infof("new user %#v created (roles: %s, auth-source: %d)", user.Username, rolesJson, user.AuthSource) + log.Infof("AUTH/USERS > new user %#v created (roles: %s, auth-source: %d)", user.Username, rolesJson, user.AuthSource) return nil } @@ -121,12 +121,12 @@ func (auth *Authentication) AddRole( } if role != RoleAdmin && role != RoleApi && role != RoleUser && role != RoleSupport { - return fmt.Errorf("invalid user role: %#v", role) + return fmt.Errorf("AUTH/USERS > invalid user role: %#v", role) } for _, r := range user.Roles { if r == role { - return fmt.Errorf("user %#v already has role %#v", username, role) + return fmt.Errorf("AUTH/USERS > user %#v already has role %#v", username, role) } } @@ -144,7 +144,7 @@ func (auth *Authentication) RemoveRole(ctx context.Context, username string, rol } if role != RoleAdmin && role != RoleApi && role != RoleUser && role != RoleSupport { - return fmt.Errorf("invalid user role: %#v", role) + return fmt.Errorf("AUTH/USERS > invalid user role: %#v", role) } var exists bool @@ -164,7 +164,7 @@ func (auth *Authentication) RemoveRole(ctx context.Context, username string, rol } return nil } else { - return fmt.Errorf("user %#v already does not have role %#v", username, role) + return fmt.Errorf("AUTH/USERS > user %#v already does not have role %#v", username, role) } } diff --git a/internal/config/config.go b/internal/config/config.go index ca78495..700413a 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -49,20 +49,20 @@ func Init(flagConfigFile string) { raw, err := os.ReadFile(flagConfigFile) if err != nil { if !os.IsNotExist(err) { - log.Fatal(err) + log.Fatalf("CONFIG/CONFIG > ERROR: %v", err) } } else { if err := schema.Validate(schema.Config, bytes.NewReader(raw)); err != nil { - log.Fatalf("Validate config: %v\n", err) + log.Fatalf("CONFIG/CONFIG > Validate config: %v\n", err) } dec := json.NewDecoder(bytes.NewReader(raw)) dec.DisallowUnknownFields() if err := dec.Decode(&Keys); err != nil { - log.Fatal(err) + log.Fatalf("CONFIG/CONFIG > could not decode: %v", err) } if Keys.Clusters == nil || len(Keys.Clusters) < 1 { - log.Fatal("At least one cluster required in config!") + log.Fatal("CONFIG/CONFIG > At least one cluster required in config!") } } } diff --git a/internal/graph/schema.resolvers.go b/internal/graph/schema.resolvers.go index 1aa8a04..12d316c 100644 --- a/internal/graph/schema.resolvers.go +++ b/internal/graph/schema.resolvers.go @@ -51,7 +51,7 @@ func (r *mutationResolver) CreateTag(ctx context.Context, typeArg string, name s // DeleteTag is the resolver for the deleteTag field. func (r *mutationResolver) DeleteTag(ctx context.Context, id string) (string, error) { - panic(fmt.Errorf("not implemented: DeleteTag - deleteTag")) + panic(fmt.Errorf("GRAPH/RESOLVERS > not implemented: DeleteTag - deleteTag")) } // AddTagsToJob is the resolver for the addTagsToJob field. @@ -175,7 +175,7 @@ func (r *queryResolver) JobMetrics(ctx context.Context, id string, metrics []str for name, md := range data { for scope, metric := range md { if metric.Scope != schema.MetricScope(scope) { - panic("WTF?") + panic("GRAPH/RESOLVERS > metric.Scope != schema.MetricScope(scope) : Should not happen!") } res = append(res, &model.JobMetricWithName{ diff --git a/internal/graph/stats.go b/internal/graph/stats.go index ed74592..5d8c4a3 100644 --- a/internal/graph/stats.go +++ b/internal/graph/stats.go @@ -211,7 +211,7 @@ func (r *queryResolver) rooflineHeatmap( return nil, err } if len(jobs) > MAX_JOBS_FOR_ANALYSIS { - return nil, fmt.Errorf("too many jobs matched (max: %d)", MAX_JOBS_FOR_ANALYSIS) + return nil, fmt.Errorf("GRAPH/STATS > too many jobs matched (max: %d)", MAX_JOBS_FOR_ANALYSIS) } fcols, frows := float64(cols), float64(rows) @@ -233,14 +233,14 @@ func (r *queryResolver) rooflineHeatmap( flops_, membw_ := jobdata["flops_any"], jobdata["mem_bw"] if flops_ == nil && membw_ == nil { - return nil, fmt.Errorf("'flops_any' or 'mem_bw' missing for job %d", job.ID) + return nil, fmt.Errorf("GRAPH/STATS > 'flops_any' or 'mem_bw' missing for job %d", job.ID) } flops, ok1 := flops_["node"] membw, ok2 := membw_["node"] if !ok1 || !ok2 { // TODO/FIXME: - return nil, errors.New("todo: rooflineHeatmap() query not implemented for where flops_any or mem_bw not available at 'node' level") + return nil, errors.New("GRAPH/STATS > todo: rooflineHeatmap() query not implemented for where flops_any or mem_bw not available at 'node' level") } for n := 0; n < len(flops.Series); n++ { @@ -275,7 +275,7 @@ func (r *queryResolver) jobsFootprints(ctx context.Context, filter []*model.JobF return nil, err } if len(jobs) > MAX_JOBS_FOR_ANALYSIS { - return nil, fmt.Errorf("too many jobs matched (max: %d)", MAX_JOBS_FOR_ANALYSIS) + return nil, fmt.Errorf("GRAPH/STATS > too many jobs matched (max: %d)", MAX_JOBS_FOR_ANALYSIS) } avgs := make([][]schema.Float, len(metrics)) diff --git a/internal/metricdata/cc-metric-store.go b/internal/metricdata/cc-metric-store.go index c03717d..027c0e0 100644 --- a/internal/metricdata/cc-metric-store.go +++ b/internal/metricdata/cc-metric-store.go @@ -202,7 +202,7 @@ func (ccms *CCMetricStore) LoadData( for _, res := range row { if res.Error != nil { - errors = append(errors, fmt.Sprintf("failed to fetch '%s' from host '%s': %s", query.Metric, query.Hostname, *res.Error)) + errors = append(errors, fmt.Sprintf("METRICDATA/CCMS > failed to fetch '%s' from host '%s': %s", query.Metric, query.Hostname, *res.Error)) continue } @@ -245,7 +245,7 @@ func (ccms *CCMetricStore) LoadData( } if len(errors) != 0 { - return jobData, fmt.Errorf("cc-metric-store: %s", strings.Join(errors, ", ")) + return jobData, fmt.Errorf("METRICDATA/CCMS > Errors: %s", strings.Join(errors, ", ")) } return jobData, nil @@ -272,8 +272,8 @@ func (ccms *CCMetricStore) buildQueries( remoteName := ccms.toRemoteName(metric) mc := archive.GetMetricConfig(job.Cluster, metric) if mc == nil { - // return nil, fmt.Errorf("metric '%s' is not specified for cluster '%s'", metric, job.Cluster) - // log.Printf("metric '%s' is not specified for cluster '%s'", metric, job.Cluster) + // return nil, fmt.Errorf("METRICDATA/CCMS > metric '%s' is not specified for cluster '%s'", metric, job.Cluster) + // log.Printf("METRICDATA/CCMS > metric '%s' is not specified for cluster '%s'", metric, job.Cluster) continue } @@ -483,7 +483,7 @@ func (ccms *CCMetricStore) buildQueries( continue } - return nil, nil, fmt.Errorf("TODO: unhandled case: native-scope=%s, requested-scope=%s", nativeScope, requestedScope) + return nil, nil, fmt.Errorf("METRICDATA/CCMS > TODO: unhandled case: native-scope=%s, requested-scope=%s", nativeScope, requestedScope) } } } @@ -521,7 +521,7 @@ func (ccms *CCMetricStore) LoadStats( metric := ccms.toLocalName(query.Metric) data := res[0] if data.Error != nil { - return nil, fmt.Errorf("fetching %s for node %s failed: %s", metric, query.Hostname, *data.Error) + return nil, fmt.Errorf("METRICDATA/CCMS > fetching %s for node %s failed: %s", metric, query.Hostname, *data.Error) } metricdata, ok := stats[metric] @@ -531,7 +531,7 @@ func (ccms *CCMetricStore) LoadStats( } if data.Avg.IsNaN() || data.Min.IsNaN() || data.Max.IsNaN() { - return nil, fmt.Errorf("fetching %s for node %s failed: %s", metric, query.Hostname, "avg/min/max is NaN") + return nil, fmt.Errorf("METRICDATA/CCMS > fetching %s for node %s failed: %s", metric, query.Hostname, "avg/min/max is NaN") } metricdata[query.Hostname] = schema.MetricStatistics{ @@ -593,11 +593,11 @@ func (ccms *CCMetricStore) LoadNodeData( metric := ccms.toLocalName(query.Metric) qdata := res[0] if qdata.Error != nil { - errors = append(errors, fmt.Sprintf("fetching %s for node %s failed: %s", metric, query.Hostname, *qdata.Error)) + errors = append(errors, fmt.Sprintf("METRICDATA/CCMS > fetching %s for node %s failed: %s", metric, query.Hostname, *qdata.Error)) } if qdata.Avg.IsNaN() || qdata.Min.IsNaN() || qdata.Max.IsNaN() { - // return nil, fmt.Errorf("fetching %s for node %s failed: %s", metric, query.Hostname, "avg/min/max is NaN") + // return nil, fmt.Errorf("METRICDATA/CCMS > fetching %s for node %s failed: %s", metric, query.Hostname, "avg/min/max is NaN") qdata.Avg, qdata.Min, qdata.Max = 0., 0., 0. } @@ -627,7 +627,7 @@ func (ccms *CCMetricStore) LoadNodeData( } if len(errors) != 0 { - return data, fmt.Errorf("cc-metric-store: %s", strings.Join(errors, ", ")) + return data, fmt.Errorf("METRICDATA/CCMS > errors: %s", strings.Join(errors, ", ")) } return data, nil diff --git a/internal/metricdata/influxdb-v2.go b/internal/metricdata/influxdb-v2.go index ca1d566..51a7dbb 100644 --- a/internal/metricdata/influxdb-v2.go +++ b/internal/metricdata/influxdb-v2.go @@ -10,12 +10,12 @@ import ( "encoding/json" "errors" "fmt" - "log" "strings" "time" "github.com/ClusterCockpit/cc-backend/pkg/archive" "github.com/ClusterCockpit/cc-backend/pkg/schema" + "github.com/ClusterCockpit/cc-backend/pkg/log" influxdb2 "github.com/influxdata/influxdb-client-go/v2" influxdb2Api "github.com/influxdata/influxdb-client-go/v2/api" ) @@ -71,7 +71,7 @@ func (idb *InfluxDBv2DataRepository) LoadData( for _, h := range job.Resources { if h.HWThreads != nil || h.Accelerators != nil { // TODO - return nil, errors.New("the InfluxDB metric data repository does not yet support HWThreads or Accelerators") + return nil, errors.New("METRICDATA/INFLUXV2 > the InfluxDB metric data repository does not yet support HWThreads or Accelerators") } hostsConds = append(hostsConds, fmt.Sprintf(`r["hostname"] == "%s"`, h.Hostname)) } @@ -84,7 +84,7 @@ func (idb *InfluxDBv2DataRepository) LoadData( switch scope { case "node": // Get Finest Granularity, Groupy By Measurement and Hostname (== Metric / Node), Calculate Mean for 60s windows - // log.Println("Note: Scope 'node' requested. ") + // log.Info("METRICDATA/INFLUXV2 > Scope 'node' requested. ") query = fmt.Sprintf(` from(bucket: "%s") |> range(start: %s, stop: %s) @@ -97,10 +97,10 @@ func (idb *InfluxDBv2DataRepository) LoadData( idb.formatTime(job.StartTime), idb.formatTime(idb.epochToTime(job.StartTimeUnix+int64(job.Duration)+int64(1))), measurementsCond, hostsCond) case "socket": - log.Println("Note: Scope 'socket' requested, but not yet supported: Will return 'node' scope only. ") + log.Info("METRICDATA/INFLUXV2 > Scope 'socket' requested, but not yet supported: Will return 'node' scope only. ") continue case "core": - log.Println("Note: Scope 'core' requested, but not yet supported: Will return 'node' scope only. ") + log.Info("METRICDATA/INFLUXV2 > Scope 'core' requested, but not yet supported: Will return 'node' scope only. ") continue // Get Finest Granularity only, Set NULL to 0.0 // query = fmt.Sprintf(` @@ -114,9 +114,9 @@ func (idb *InfluxDBv2DataRepository) LoadData( // idb.formatTime(job.StartTime), idb.formatTime(idb.epochToTime(job.StartTimeUnix + int64(job.Duration) + int64(1) )), // measurementsCond, hostsCond) default: - log.Println("Note: Unknown Scope requested: Will return 'node' scope. ") + log.Info("METRICDATA/INFLUXV2 > Unknown Scope requested: Will return 'node' scope. ") continue - // return nil, errors.New("the InfluxDB metric data repository does not yet support other scopes than 'node'") + // return nil, errors.New("METRICDATA/INFLUXV2 > the InfluxDB metric data repository does not yet support other scopes than 'node'") } rows, err := idb.queryClient.Query(ctx, query) @@ -208,15 +208,15 @@ func (idb *InfluxDBv2DataRepository) LoadData( for _, scope := range scopes { if scope == "node" { // No 'socket/core' support yet for metric, nodes := range stats { - // log.Println(fmt.Sprintf("<< Add Stats for : Field %s >>", metric)) + // log.Debugf("<< Add Stats for : Field %s >>", metric) for node, stats := range nodes { - // log.Println(fmt.Sprintf("<< Add Stats for : Host %s : Min %.2f, Max %.2f, Avg %.2f >>", node, stats.Min, stats.Max, stats.Avg )) + // log.Debugf("<< Add Stats for : Host %s : Min %.2f, Max %.2f, Avg %.2f >>", node, stats.Min, stats.Max, stats.Avg ) for index, _ := range jobData[metric][scope].Series { - // log.Println(fmt.Sprintf("<< Try to add Stats to Series in Position %d >>", index)) + // log.Debugf("<< Try to add Stats to Series in Position %d >>", index) if jobData[metric][scope].Series[index].Hostname == node { - // log.Println(fmt.Sprintf("<< Match for Series in Position %d : Host %s >>", index, jobData[metric][scope].Series[index].Hostname)) + // log.Debugf("<< Match for Series in Position %d : Host %s >>", index, jobData[metric][scope].Series[index].Hostname) jobData[metric][scope].Series[index].Statistics = &schema.MetricStatistics{Avg: stats.Avg, Min: stats.Min, Max: stats.Max} - // log.Println(fmt.Sprintf("<< Result Inner: Min %.2f, Max %.2f, Avg %.2f >>", jobData[metric][scope].Series[index].Statistics.Min, jobData[metric][scope].Series[index].Statistics.Max, jobData[metric][scope].Series[index].Statistics.Avg)) + // log.Debugf("<< Result Inner: Min %.2f, Max %.2f, Avg %.2f >>", jobData[metric][scope].Series[index].Statistics.Min, jobData[metric][scope].Series[index].Statistics.Max, jobData[metric][scope].Series[index].Statistics.Avg) } } } @@ -228,9 +228,9 @@ func (idb *InfluxDBv2DataRepository) LoadData( // for _, scope := range scopes { // for _, met := range metrics { // for _, series := range jobData[met][scope].Series { - // log.Println(fmt.Sprintf("<< Result: %d data points for metric %s on %s with scope %s, Stats: Min %.2f, Max %.2f, Avg %.2f >>", + // log.Debugf("<< Result: %d data points for metric %s on %s with scope %s, Stats: Min %.2f, Max %.2f, Avg %.2f >>", // len(series.Data), met, series.Hostname, scope, - // series.Statistics.Min, series.Statistics.Max, series.Statistics.Avg)) + // series.Statistics.Min, series.Statistics.Max, series.Statistics.Avg) // } // } // } @@ -249,7 +249,7 @@ func (idb *InfluxDBv2DataRepository) LoadStats( for _, h := range job.Resources { if h.HWThreads != nil || h.Accelerators != nil { // TODO - return nil, errors.New("the InfluxDB metric data repository does not yet support HWThreads or Accelerators") + return nil, errors.New("METRICDATA/INFLUXV2 > the InfluxDB metric data repository does not yet support HWThreads or Accelerators") } hostsConds = append(hostsConds, fmt.Sprintf(`r["hostname"] == "%s"`, h.Hostname)) } @@ -258,7 +258,7 @@ func (idb *InfluxDBv2DataRepository) LoadStats( // lenMet := len(metrics) for _, metric := range metrics { - // log.Println(fmt.Sprintf("<< You are here: %s (Index %d of %d metrics)", metric, index, lenMet)) + // log.Debugf("<< You are here: %s (Index %d of %d metrics)", metric, index, lenMet) query := fmt.Sprintf(` data = from(bucket: "%s") @@ -285,17 +285,17 @@ func (idb *InfluxDBv2DataRepository) LoadStats( avg, avgok := row.ValueByKey("avg").(float64) if !avgok { - // log.Println(fmt.Sprintf(">> Assertion error for metric %s, statistic AVG. Expected 'float64', got %v", metric, avg)) + // log.Debugf(">> Assertion error for metric %s, statistic AVG. Expected 'float64', got %v", metric, avg) avg = 0.0 } min, minok := row.ValueByKey("min").(float64) if !minok { - // log.Println(fmt.Sprintf(">> Assertion error for metric %s, statistic MIN. Expected 'float64', got %v", metric, min)) + // log.Debugf(">> Assertion error for metric %s, statistic MIN. Expected 'float64', got %v", metric, min) min = 0.0 } max, maxok := row.ValueByKey("max").(float64) if !maxok { - // log.Println(fmt.Sprintf(">> Assertion error for metric %s, statistic MAX. Expected 'float64', got %v", metric, max)) + // log.Debugf(">> Assertion error for metric %s, statistic MAX. Expected 'float64', got %v", metric, max) max = 0.0 } @@ -319,7 +319,7 @@ func (idb *InfluxDBv2DataRepository) LoadNodeData( ctx context.Context) (map[string]map[string][]*schema.JobMetric, error) { // TODO : Implement to be used in Analysis- und System/Node-View - log.Println(fmt.Sprintf("LoadNodeData unimplemented for InfluxDBv2DataRepository, Args: cluster %s, metrics %v, nodes %v, scopes %v", cluster, metrics, nodes, scopes)) + log.Infof("METRICDATA/INFLUXV2 > LoadNodeData unimplemented for InfluxDBv2DataRepository, Args: cluster %s, metrics %v, nodes %v, scopes %v", cluster, metrics, nodes, scopes) - return nil, errors.New("unimplemented for InfluxDBv2DataRepository") + return nil, errors.New("METRICDATA/INFLUXV2 > unimplemented for InfluxDBv2DataRepository") } diff --git a/internal/metricdata/metricdata.go b/internal/metricdata/metricdata.go index 61ca4cc..2b68f55 100644 --- a/internal/metricdata/metricdata.go +++ b/internal/metricdata/metricdata.go @@ -60,7 +60,7 @@ func Init(disableArchive bool) error { case "test": mdr = &TestMetricDataRepository{} default: - return fmt.Errorf("unkown metric data repository '%s' for cluster '%s'", kind.Kind, cluster.Name) + return fmt.Errorf("METRICDATA/METRICDATA > unkown metric data repository '%s' for cluster '%s'", kind.Kind, cluster.Name) } if err := mdr.Init(cluster.MetricDataRepository); err != nil { @@ -90,7 +90,7 @@ func LoadData(job *schema.Job, repo, ok := metricDataRepos[job.Cluster] if !ok { - return fmt.Errorf("no metric data repository configured for '%s'", job.Cluster), 0, 0 + return fmt.Errorf("METRICDATA/METRICDATA > no metric data repository configured for '%s'", job.Cluster), 0, 0 } if scopes == nil { @@ -107,7 +107,7 @@ func LoadData(job *schema.Job, jd, err = repo.LoadData(job, metrics, scopes, ctx) if err != nil { if len(jd) != 0 { - log.Errorf("partial error: %s", err.Error()) + log.Errorf("METRICDATA/METRICDATA > partial error: %s", err.Error()) } else { return err, 0, 0 } @@ -182,7 +182,7 @@ func LoadAverages( repo, ok := metricDataRepos[job.Cluster] if !ok { - return fmt.Errorf("no metric data repository configured for '%s'", job.Cluster) + return fmt.Errorf("METRICDATA/METRICDATA > no metric data repository configured for '%s'", job.Cluster) } stats, err := repo.LoadStats(job, metrics, ctx) @@ -217,7 +217,7 @@ func LoadNodeData( repo, ok := metricDataRepos[cluster] if !ok { - return nil, fmt.Errorf("no metric data repository configured for '%s'", cluster) + return nil, fmt.Errorf("METRICDATA/METRICDATA > no metric data repository configured for '%s'", cluster) } if metrics == nil { @@ -229,14 +229,14 @@ func LoadNodeData( data, err := repo.LoadNodeData(cluster, metrics, nodes, scopes, from, to, ctx) if err != nil { if len(data) != 0 { - log.Errorf("partial error: %s", err.Error()) + log.Errorf("METRICDATA/METRICDATA > partial error: %s", err.Error()) } else { return nil, err } } if data == nil { - return nil, fmt.Errorf("the metric data repository for '%s' does not support this query", cluster) + return nil, fmt.Errorf("METRICDATA/METRICDATA > the metric data repository for '%s' does not support this query", cluster) } return data, nil diff --git a/internal/metricdata/prometheus.go b/internal/metricdata/prometheus.go index a6d00f9..db2bc6f 100644 --- a/internal/metricdata/prometheus.go +++ b/internal/metricdata/prometheus.go @@ -163,7 +163,7 @@ func (pdb *PrometheusDataRepository) Init(rawConfig json.RawMessage) error { rt = promcfg.NewBasicAuthRoundTripper(config.Username, prom_pw, "", promapi.DefaultRoundTripper) } else { if config.Username != "" { - return errors.New("Prometheus username provided, but PROMETHEUS_PASSWORD not set.") + return errors.New("METRICDATA/PROMETHEUS > Prometheus username provided, but PROMETHEUS_PASSWORD not set.") } } // init client @@ -184,9 +184,9 @@ func (pdb *PrometheusDataRepository) Init(rawConfig json.RawMessage) error { for metric, templ := range config.Templates { pdb.templates[metric], err = template.New(metric).Parse(templ) if err == nil { - log.Debugf("Added PromQL template for %s: %s", metric, templ) + log.Debugf("METRICDATA/PROMETHEUS > Added PromQL template for %s: %s", metric, templ) } else { - log.Errorf("Failed to parse PromQL template %s for metric %s", templ, metric) + log.Errorf("METRICDATA/PROMETHEUS > Failed to parse PromQL template %s for metric %s", templ, metric) } } return nil @@ -213,14 +213,14 @@ func (pdb *PrometheusDataRepository) FormatQuery( if templ, ok := pdb.templates[metric]; ok { err := templ.Execute(buf, args) if err != nil { - return "", errors.New(fmt.Sprintf("Error compiling template %v", templ)) + return "", errors.New(fmt.Sprintf("METRICDATA/PROMETHEUS > Error compiling template %v", templ)) } else { query := buf.String() - log.Debugf(fmt.Sprintf("PromQL: %s", query)) + log.Debugf("METRICDATA/PROMETHEUS > PromQL: %s", query) return query, nil } } else { - return "", errors.New(fmt.Sprintf("No PromQL for metric %s configured.", metric)) + return "", errors.New(fmt.Sprintf("METRICDATA/PROMETHEUS > No PromQL for metric %s configured.", metric)) } } @@ -283,16 +283,15 @@ func (pdb *PrometheusDataRepository) LoadData( for _, scope := range scopes { if scope != schema.MetricScopeNode { - logOnce.Do(func(){log.Infof(fmt.Sprintf("Note: Scope '%s' requested, but not yet supported: Will return 'node' scope only.", scope))}) + logOnce.Do(func(){log.Infof("METRICDATA/PROMETHEUS > Scope '%s' requested, but not yet supported: Will return 'node' scope only.", scope)}) continue } for _, metric := range metrics { metricConfig := archive.GetMetricConfig(job.Cluster, metric) if metricConfig == nil { - log.Errorf(fmt.Sprintf("Error in LoadData: Metric %s for cluster %s not configured", - metric, job.Cluster)) - return nil, errors.New("Prometheus querry error") + log.Errorf("METRICDATA/PROMETHEUS > Error in LoadData: Metric %s for cluster %s not configured", metric, job.Cluster) + return nil, errors.New("METRICDATA/PROMETHEUS > Prometheus query error") } query, err := pdb.FormatQuery(metric, scope, nodes, job.Cluster) if err != nil { @@ -308,11 +307,11 @@ func (pdb *PrometheusDataRepository) LoadData( result, warnings, err := pdb.queryClient.QueryRange(ctx, query, r) if err != nil { - log.Errorf(fmt.Sprintf("Prometheus query error in LoadData: %v\nQuery: %s", err, query)) - return nil, errors.New("Prometheus querry error") + log.Errorf("METRICDATA/PROMETHEUS > Prometheus query error in LoadData: %v\nQuery: %s", err, query) + return nil, errors.New("METRICDATA/PROMETHEUS > Prometheus query error") } if len(warnings) > 0 { - log.Warnf(fmt.Sprintf("Warnings: %v\n", warnings)) + log.Warnf("Warnings: %v\n", warnings) } // init data structures @@ -390,15 +389,14 @@ func (pdb *PrometheusDataRepository) LoadNodeData( } for _, scope := range scopes { if scope != schema.MetricScopeNode { - logOnce.Do(func(){log.Infof(fmt.Sprintf("Note: Scope '%s' requested, but not yet supported: Will return 'node' scope only.", scope))}) + logOnce.Do(func(){log.Infof("METRICDATA/PROMETHEUS > Note: Scope '%s' requested, but not yet supported: Will return 'node' scope only.", scope)}) continue } for _, metric := range metrics { metricConfig := archive.GetMetricConfig(cluster, metric) if metricConfig == nil { - log.Errorf(fmt.Sprintf("Error in LoadNodeData: Metric %s for cluster %s not configured", - metric, cluster)) - return nil, errors.New("Prometheus querry error") + log.Errorf("METRICDATA/PROMETHEUS > Error in LoadNodeData: Metric %s for cluster %s not configured", metric, cluster) + return nil, errors.New("METRICDATA/PROMETHEUS > Prometheus querry error") } query, err := pdb.FormatQuery(metric, scope, nodes, cluster) if err != nil { @@ -414,11 +412,11 @@ func (pdb *PrometheusDataRepository) LoadNodeData( result, warnings, err := pdb.queryClient.QueryRange(ctx, query, r) if err != nil { - log.Errorf(fmt.Sprintf("Prometheus query error in LoadNodeData: %v\n", err)) - return nil, errors.New("Prometheus querry error") + log.Errorf("METRICDATA/PROMETHEUS > Prometheus query error in LoadNodeData: %v\n", err) + return nil, errors.New("METRICDATA/PROMETHEUS > Prometheus querry error") } if len(warnings) > 0 { - log.Warnf(fmt.Sprintf("Warnings: %v\n", warnings)) + log.Warnf("METRICDATA/PROMETHEUS > Warnings: %v\n", warnings) } step := int64(metricConfig.Timestep) @@ -444,6 +442,6 @@ func (pdb *PrometheusDataRepository) LoadNodeData( } } t1 := time.Since(t0) - log.Debugf(fmt.Sprintf("LoadNodeData of %v nodes took %s", len(data), t1)) + log.Debugf("METRICDATA/PROMETHEUS > LoadNodeData of %v nodes took %s", len(data), t1) return data, nil } diff --git a/internal/repository/dbConnection.go b/internal/repository/dbConnection.go index d1759f0..987bf20 100644 --- a/internal/repository/dbConnection.go +++ b/internal/repository/dbConnection.go @@ -39,14 +39,14 @@ func Connect(driver string, db string) { } else if driver == "mysql" { dbHandle, err = sqlx.Open("mysql", fmt.Sprintf("%s?multiStatements=true", db)) if err != nil { - log.Fatal(err) + log.Fatalf("REPOSITORY/DBCONNECTION > sqlx.Open() error: %v", err) } dbHandle.SetConnMaxLifetime(time.Minute * 3) dbHandle.SetMaxOpenConns(10) dbHandle.SetMaxIdleConns(10) } else { - log.Fatalf("unsupported database driver: %s", driver) + log.Fatalf("REPOSITORY/DBCONNECTION > unsupported database driver: %s", driver) } dbConnInstance = &DBConnection{DB: dbHandle} @@ -55,7 +55,7 @@ func Connect(driver string, db string) { func GetConnection() *DBConnection { if dbConnInstance == nil { - log.Fatalf("Database connection not initialized!") + log.Fatalf("REPOSITORY/DBCONNECTION > Database connection not initialized!") } return dbConnInstance diff --git a/internal/repository/init.go b/internal/repository/init.go index 76973eb..1358100 100644 --- a/internal/repository/init.go +++ b/internal/repository/init.go @@ -95,7 +95,7 @@ func HandleImportFlag(flag string) error { for _, pair := range strings.Split(flag, ",") { files := strings.Split(pair, ":") if len(files) != 2 { - return fmt.Errorf("invalid import flag format") + return fmt.Errorf("REPOSITORY/INIT > invalid import flag format") } raw, err := os.ReadFile(files[0]) @@ -105,7 +105,7 @@ func HandleImportFlag(flag string) error { if config.Keys.Validate { if err := schema.Validate(schema.Meta, bytes.NewReader(raw)); err != nil { - return fmt.Errorf("validate job meta: %v", err) + return fmt.Errorf("REPOSITORY/INIT > validate job meta: %v", err) } } dec := json.NewDecoder(bytes.NewReader(raw)) @@ -122,7 +122,7 @@ func HandleImportFlag(flag string) error { if config.Keys.Validate { if err := schema.Validate(schema.Data, bytes.NewReader(raw)); err != nil { - return fmt.Errorf("validate job data: %v", err) + return fmt.Errorf("REPOSITORY/INIT > validate job data: %v", err) } } dec = json.NewDecoder(bytes.NewReader(raw)) @@ -139,7 +139,7 @@ func HandleImportFlag(flag string) error { return err } - return fmt.Errorf("a job with that jobId, cluster and startTime does already exist (dbid: %d)", job.ID) + return fmt.Errorf("REPOSITORY/INIT > a job with that jobId, cluster and startTime does already exist (dbid: %d)", job.ID) } job := schema.Job{ @@ -186,7 +186,7 @@ func HandleImportFlag(flag string) error { } } - log.Infof("Successfully imported a new job (jobId: %d, cluster: %s, dbid: %d)", job.JobID, job.Cluster, id) + log.Infof("REPOSITORY/INIT > successfully imported a new job (jobId: %d, cluster: %s, dbid: %d)", job.JobID, job.Cluster, id) } return nil } @@ -260,34 +260,34 @@ func InitDB() error { job.RawResources, err = json.Marshal(job.Resources) if err != nil { - log.Errorf("repository initDB()- %v", err) + log.Errorf("REPOSITORY/INIT > repository initDB(): %v", err) errorOccured++ continue } job.RawMetaData, err = json.Marshal(job.MetaData) if err != nil { - log.Errorf("repository initDB()- %v", err) + log.Errorf("REPOSITORY/INIT > repository initDB(): %v", err) errorOccured++ continue } if err := SanityChecks(&job.BaseJob); err != nil { - log.Errorf("repository initDB()- %v", err) + log.Errorf("REPOSITORY/INIT > repository initDB(): %v", err) errorOccured++ continue } res, err := stmt.Exec(job) if err != nil { - log.Errorf("repository initDB()- %v", err) + log.Errorf("REPOSITORY/INIT > repository initDB(): %v", err) errorOccured++ continue } id, err := res.LastInsertId() if err != nil { - log.Errorf("repository initDB()- %v", err) + log.Errorf("REPOSITORY/INIT > repository initDB(): %v", err) errorOccured++ continue } @@ -318,7 +318,7 @@ func InitDB() error { } if errorOccured > 0 { - log.Errorf("Error in import of %d jobs!", errorOccured) + log.Errorf("REPOSITORY/INIT > Error in import of %d jobs!", errorOccured) } if err := tx.Commit(); err != nil { diff --git a/internal/repository/job.go b/internal/repository/job.go index f305cd0..ca7c639 100644 --- a/internal/repository/job.go +++ b/internal/repository/job.go @@ -214,12 +214,12 @@ func (r *JobRepository) FindById(jobId int64) (*schema.Job, error) { func (r *JobRepository) Start(job *schema.JobMeta) (id int64, err error) { job.RawResources, err = json.Marshal(job.Resources) if err != nil { - return -1, fmt.Errorf("encoding resources field failed: %w", err) + return -1, fmt.Errorf("REPOSITORY/JOB > encoding resources field failed: %w", err) } job.RawMetaData, err = json.Marshal(job.MetaData) if err != nil { - return -1, fmt.Errorf("encoding metaData field failed: %w", err) + return -1, fmt.Errorf("REPOSITORY/JOB > encoding metaData field failed: %w", err) } res, err := r.DB.NamedExec(`INSERT INTO job ( @@ -259,9 +259,9 @@ func (r *JobRepository) DeleteJobsBefore(startTime int64) (int, error) { err := r.DB.Get(&cnt, qs) //ignore error as it will also occur in delete statement _, err = r.DB.Exec(`DELETE FROM job WHERE job.start_time < ?`, startTime) if err != nil { - log.Warnf(" DeleteJobsBefore(%d): error %v", startTime, err) + log.Warnf("REPOSITORY/JOB > DeleteJobsBefore(%d): error %v", startTime, err) } else { - log.Infof("DeleteJobsBefore(%d): Deleted %d jobs", startTime, cnt) + log.Infof("REPOSITORY/JOB > DeleteJobsBefore(%d): Deleted %d jobs", startTime, cnt) } return cnt, err } @@ -269,9 +269,9 @@ func (r *JobRepository) DeleteJobsBefore(startTime int64) (int, error) { func (r *JobRepository) DeleteJobById(id int64) error { _, err := r.DB.Exec(`DELETE FROM job WHERE job.id = ?`, id) if err != nil { - log.Warnf("DeleteJobById(%d): error %v", id, err) + log.Warnf("REPOSITORY/JOB > DeleteJobById(%d): error %v", id, err) } else { - log.Infof("DeleteJobById(%d): Success", id) + log.Infof("REPOSITORY/JOB > DeleteJobById(%d): Success", id) } return err } @@ -376,7 +376,7 @@ func (r *JobRepository) archivingWorker(){ // not using meta data, called to load JobMeta into Cache? // will fail if job meta not in repository if _, err := r.FetchMetadata(job); err != nil { - log.Errorf("archiving job (dbid: %d) failed: %s", job.ID, err.Error()) + log.Errorf("REPOSITORY/JOB > archiving job (dbid: %d) failed: %s", job.ID, err.Error()) r.UpdateMonitoringStatus(job.ID, schema.MonitoringStatusArchivingFailed) continue } @@ -385,18 +385,18 @@ func (r *JobRepository) archivingWorker(){ // TODO: Maybe use context with cancel/timeout here jobMeta, err := metricdata.ArchiveJob(job, context.Background()) if err != nil { - log.Errorf("archiving job (dbid: %d) failed: %s", job.ID, err.Error()) + log.Errorf("REPOSITORY/JOB > archiving job (dbid: %d) failed: %s", job.ID, err.Error()) r.UpdateMonitoringStatus(job.ID, schema.MonitoringStatusArchivingFailed) continue } // Update the jobs database entry one last time: if err := r.MarkArchived(job.ID, schema.MonitoringStatusArchivingSuccessful, jobMeta.Statistics); err != nil { - log.Errorf("archiving job (dbid: %d) failed: %s", job.ID, err.Error()) + log.Errorf("REPOSITORY/JOB > archiving job (dbid: %d) failed: %s", job.ID, err.Error()) continue } - log.Printf("archiving job (dbid: %d) successful", job.ID) + log.Printf("REPOSITORY/JOB > archiving job (dbid: %d) successful", job.ID) r.archivePending.Done() } } @@ -523,7 +523,7 @@ func (r *JobRepository) StopJobsExceedingWalltimeBy(seconds int) error { } if rowsAffected > 0 { - log.Warnf("%d jobs have been marked as failed due to running too long", rowsAffected) + log.Warnf("REPOSITORY/JOB > %d jobs have been marked as failed due to running too long", rowsAffected) } return nil } diff --git a/internal/repository/query.go b/internal/repository/query.go index fad6091..bc2d108 100644 --- a/internal/repository/query.go +++ b/internal/repository/query.go @@ -36,7 +36,7 @@ func (r *JobRepository) QueryJobs( } else if order.Order == model.SortDirectionEnumDesc { query = query.OrderBy(fmt.Sprintf("job.%s DESC", field)) } else { - return nil, errors.New("invalid sorting order") + return nil, errors.New("REPOSITORY/QUERY > invalid sorting order") } } @@ -54,7 +54,7 @@ func (r *JobRepository) QueryJobs( return nil, err } - log.Debugf("SQL query: `%s`, args: %#v", sql, args) + log.Debugf("REPOSITORY/QUERY > SQL query: `%s`, args: %#v", sql, args) rows, err := query.RunWith(r.stmtCache).Query() if err != nil { return nil, err @@ -209,7 +209,7 @@ var matchAllCap = regexp.MustCompile("([a-z0-9])([A-Z])") func toSnakeCase(str string) string { for _, c := range str { if c == '\'' || c == '\\' { - panic("A hacker (probably not)!!!") + panic("REPOSITORY/QUERY > toSnakeCase() attack vector!") } } diff --git a/internal/repository/user.go b/internal/repository/user.go index 2e036d9..80adfc8 100644 --- a/internal/repository/user.go +++ b/internal/repository/user.go @@ -42,12 +42,12 @@ func GetUserCfgRepo() *UserCfgRepo { FOREIGN KEY (username) REFERENCES user (username) ON DELETE CASCADE ON UPDATE NO ACTION);`) if err != nil { - log.Fatal(err) + log.Fatalf("REPOSITORY/USER > db.DB.exec() error: %v", err) } lookupConfigStmt, err := db.DB.Preparex(`SELECT confkey, value FROM configuration WHERE configuration.username = ?`) if err != nil { - log.Fatal(err) + log.Fatalf("REPOSITORY/USER > db.DB.Preparex() error: %v", err) } userCfgRepoInstance = &UserCfgRepo{ diff --git a/internal/routerConfig/routes.go b/internal/routerConfig/routes.go index 9424df7..fa03b26 100644 --- a/internal/routerConfig/routes.go +++ b/internal/routerConfig/routes.go @@ -61,12 +61,12 @@ func setupHomeRoute(i InfoType, r *http.Request) InfoType { State: []schema.JobState{schema.JobStateRunning}, }}, nil, nil) if err != nil { - log.Errorf("failed to count jobs: %s", err.Error()) + log.Errorf("ROUTERCONFIG/ROUTES > failed to count jobs: %s", err.Error()) runningJobs = map[string]int{} } totalJobs, err := jobRepo.CountGroupedJobs(r.Context(), model.AggregateCluster, nil, nil, nil) if err != nil { - log.Errorf("failed to count jobs: %s", err.Error()) + log.Errorf("ROUTERCONFIG/ROUTES > failed to count jobs: %s", err.Error()) totalJobs = map[string]int{} } from := time.Now().Add(-24 * time.Hour) @@ -75,7 +75,7 @@ func setupHomeRoute(i InfoType, r *http.Request) InfoType { Duration: &schema.IntRange{From: 0, To: graph.ShortJobDuration}, }}, nil, nil) if err != nil { - log.Errorf("failed to count jobs: %s", err.Error()) + log.Errorf("ROUTERCONFIG/ROUTES > failed to count jobs: %s", err.Error()) recentShortJobs = map[string]int{} } @@ -150,7 +150,7 @@ func setupTaglistRoute(i InfoType, r *http.Request) InfoType { tags, counts, err := jobRepo.CountTags(username) tagMap := make(map[string][]map[string]interface{}) if err != nil { - log.Errorf("GetTags failed: %s", err.Error()) + log.Errorf("ROUTERCONFIG/ROUTES > GetTags failed: %s", err.Error()) i["tagmap"] = tagMap return i } diff --git a/internal/runtimeEnv/setup.go b/internal/runtimeEnv/setup.go index e0e5543..7ac7138 100644 --- a/internal/runtimeEnv/setup.go +++ b/internal/runtimeEnv/setup.go @@ -40,14 +40,14 @@ func LoadEnv(file string) error { line = strings.TrimPrefix(line, "export ") parts := strings.SplitN(line, "=", 2) if len(parts) != 2 { - return fmt.Errorf("unsupported line: %#v", line) + return fmt.Errorf("RUNTIME/SETUP > unsupported line: %#v", line) } key := strings.TrimSpace(parts[0]) val := strings.TrimSpace(parts[1]) if strings.HasPrefix(val, "\"") { if !strings.HasSuffix(val, "\"") { - return fmt.Errorf("unsupported line: %#v", line) + return fmt.Errorf("RUNTIME/SETUP > unsupported line: %#v", line) } runes := []rune(val[1 : len(val)-1]) @@ -65,7 +65,7 @@ func LoadEnv(file string) error { case '"': sb.WriteRune('"') default: - return fmt.Errorf("unsupprorted escape sequence in quoted string: backslash %#v", runes[i]) + return fmt.Errorf("RUNTIME/SETUP > unsupprorted escape sequence in quoted string: backslash %#v", runes[i]) } continue } diff --git a/pkg/archive/archive.go b/pkg/archive/archive.go index 5bec30e..f0a7027 100644 --- a/pkg/archive/archive.go +++ b/pkg/archive/archive.go @@ -49,7 +49,7 @@ func Init(rawConfig json.RawMessage, disableArchive bool) error { // case "s3": // ar = &S3Archive{} default: - return fmt.Errorf("unkown archive backend '%s''", kind.Kind) + return fmt.Errorf("ARCHIVE/ARCHIVE > unkown archive backend '%s''", kind.Kind) } if err := ar.Init(rawConfig); err != nil { diff --git a/pkg/archive/clusterConfig.go b/pkg/archive/clusterConfig.go index 925c630..317dfb7 100644 --- a/pkg/archive/clusterConfig.go +++ b/pkg/archive/clusterConfig.go @@ -59,7 +59,7 @@ func initClusterConfig() error { nl, err := ParseNodeList(sc.Nodes) if err != nil { - return fmt.Errorf("in %s/cluster.json: %w", cluster.Name, err) + return fmt.Errorf("ARCHIVE/CLUSTERCONFIG > in %s/cluster.json: %w", cluster.Name, err) } nodeLists[cluster.Name][sc.Name] = nl } @@ -112,7 +112,7 @@ func AssignSubCluster(job *schema.BaseJob) error { cluster := GetCluster(job.Cluster) if cluster == nil { - return fmt.Errorf("unkown cluster: %#v", job.Cluster) + return fmt.Errorf("ARCHIVE/CLUSTERCONFIG > unkown cluster: %#v", job.Cluster) } if job.SubCluster != "" { @@ -121,11 +121,11 @@ func AssignSubCluster(job *schema.BaseJob) error { return nil } } - return fmt.Errorf("already assigned subcluster %#v unkown (cluster: %#v)", job.SubCluster, job.Cluster) + return fmt.Errorf("ARCHIVE/CLUSTERCONFIG > already assigned subcluster %#v unkown (cluster: %#v)", job.SubCluster, job.Cluster) } if len(job.Resources) == 0 { - return fmt.Errorf("job without any resources/hosts") + return fmt.Errorf("ARCHIVE/CLUSTERCONFIG > job without any resources/hosts") } host0 := job.Resources[0].Hostname @@ -141,7 +141,7 @@ func AssignSubCluster(job *schema.BaseJob) error { return nil } - return fmt.Errorf("no subcluster found for cluster %#v and host %#v", job.Cluster, host0) + return fmt.Errorf("ARCHIVE/CLUSTERCONFIG > no subcluster found for cluster %#v and host %#v", job.Cluster, host0) } func GetSubClusterByNode(cluster, hostname string) (string, error) { @@ -154,12 +154,12 @@ func GetSubClusterByNode(cluster, hostname string) (string, error) { c := GetCluster(cluster) if c == nil { - return "", fmt.Errorf("unkown cluster: %#v", cluster) + return "", fmt.Errorf("ARCHIVE/CLUSTERCONFIG > unkown cluster: %#v", cluster) } if c.SubClusters[0].Nodes == "" { return c.SubClusters[0].Name, nil } - return "", fmt.Errorf("no subcluster found for cluster %#v and host %#v", cluster, hostname) + return "", fmt.Errorf("ARCHIVE/CLUSTERCONFIG > no subcluster found for cluster %#v and host %#v", cluster, hostname) } diff --git a/pkg/archive/fsBackend.go b/pkg/archive/fsBackend.go index d2726a8..2bdbfce 100644 --- a/pkg/archive/fsBackend.go +++ b/pkg/archive/fsBackend.go @@ -46,7 +46,7 @@ func loadJobMeta(filename string) (*schema.JobMeta, error) { f, err := os.Open(filename) if err != nil { - log.Errorf("fsBackend loadJobMeta()- %v", err) + log.Errorf("ARCHIVE/FSBACKEND > loadJobMeta() > open file error: %v", err) return &schema.JobMeta{}, err } defer f.Close() @@ -58,19 +58,19 @@ func (fsa *FsArchive) Init(rawConfig json.RawMessage) error { var config FsArchiveConfig if err := json.Unmarshal(rawConfig, &config); err != nil { - log.Errorf("fsBackend Init()- %v", err) + log.Errorf("ARCHIVE/FSBACKEND > Init() > Unmarshal error: %v", err) return err } if config.Path == "" { - err := fmt.Errorf("fsBackend Init()- empty path") - log.Errorf("fsBackend Init()- %v", err) + err := fmt.Errorf("ARCHIVE/FSBACKEND > Init() : empty config.Path") + log.Errorf("ARCHIVE/FSBACKEND > Init() > config.Path error: %v", err) return err } fsa.path = config.Path entries, err := os.ReadDir(fsa.path) if err != nil { - log.Errorf("fsBackend Init()- %v", err) + log.Errorf("ARCHIVE/FSBACKEND > Init() > ReadDir() error: %v", err) return err } @@ -86,7 +86,7 @@ func (fsa *FsArchive) LoadJobData(job *schema.Job) (schema.JobData, error) { filename := getPath(job, fsa.path, "data.json") f, err := os.Open(filename) if err != nil { - log.Errorf("fsBackend LoadJobData()- %v", err) + log.Errorf("ARCHIVE/FSBACKEND > LoadJobData() > open file error: %v", err) return nil, err } defer f.Close() @@ -104,12 +104,12 @@ func (fsa *FsArchive) LoadClusterCfg(name string) (*schema.Cluster, error) { b, err := os.ReadFile(filepath.Join(fsa.path, name, "cluster.json")) if err != nil { - log.Errorf("fsBackend LoadClusterCfg()- %v", err) + log.Errorf("ARCHIVE/FSBACKEND > LoadClusterCfg() > open file error: %v", err) return &schema.Cluster{}, err } if config.Keys.Validate { if err := schema.Validate(schema.ClusterCfg, bytes.NewReader(b)); err != nil { - return &schema.Cluster{}, fmt.Errorf("Validate cluster config: %v\n", err) + return &schema.Cluster{}, fmt.Errorf("ARCHIVE/FSBACKEND > Validate cluster config: %v\n", err) } } return DecodeCluster(bytes.NewReader(b)) @@ -121,13 +121,13 @@ func (fsa *FsArchive) Iter() <-chan *schema.JobMeta { go func() { clustersDir, err := os.ReadDir(fsa.path) if err != nil { - log.Fatalf("Reading clusters failed: %s", err.Error()) + log.Fatalf("ARCHIVE/FSBACKEND > Reading clusters failed @ cluster dirs: %s", err.Error()) } for _, clusterDir := range clustersDir { lvl1Dirs, err := os.ReadDir(filepath.Join(fsa.path, clusterDir.Name())) if err != nil { - log.Fatalf("Reading jobs failed: %s", err.Error()) + log.Fatalf("ARCHIVE/FSBACKEND > Reading jobs failed @ lvl1 dirs: %s", err.Error()) } for _, lvl1Dir := range lvl1Dirs { @@ -138,21 +138,21 @@ func (fsa *FsArchive) Iter() <-chan *schema.JobMeta { lvl2Dirs, err := os.ReadDir(filepath.Join(fsa.path, clusterDir.Name(), lvl1Dir.Name())) if err != nil { - log.Fatalf("Reading jobs failed: %s", err.Error()) + log.Fatalf("ARCHIVE/FSBACKEND > Reading jobs failed @ lvl2 dirs: %s", err.Error()) } for _, lvl2Dir := range lvl2Dirs { dirpath := filepath.Join(fsa.path, clusterDir.Name(), lvl1Dir.Name(), lvl2Dir.Name()) startTimeDirs, err := os.ReadDir(dirpath) if err != nil { - log.Fatalf("Reading jobs failed: %s", err.Error()) + log.Fatalf("ARCHIVE/FSBACKEND > Reading jobs failed @ starttime dirs: %s", err.Error()) } for _, startTimeDir := range startTimeDirs { if startTimeDir.IsDir() { job, err := loadJobMeta(filepath.Join(dirpath, startTimeDir.Name(), "meta.json")) if err != nil { - log.Errorf("in %s: %s", filepath.Join(dirpath, startTimeDir.Name()), err.Error()) + log.Errorf("ARCHIVE/FSBACKEND > error in %s: %s", filepath.Join(dirpath, startTimeDir.Name()), err.Error()) } else { ch <- job } diff --git a/pkg/archive/nodelist.go b/pkg/archive/nodelist.go index 744ce8d..4daecff 100644 --- a/pkg/archive/nodelist.go +++ b/pkg/archive/nodelist.go @@ -64,7 +64,7 @@ type NLExprIntRange struct { func (nle NLExprIntRange) consume(input string) (next string, ok bool) { if !nle.zeroPadded || nle.digits < 1 { - log.Error("node list: only zero-padded ranges are allowed") + log.Error("ARCHIVE/NODELIST > only zero-padded ranges are allowed") return "", false } @@ -102,7 +102,7 @@ func ParseNodeList(raw string) (NodeList, error) { i++ } if i == len(raw) { - return nil, fmt.Errorf("node list: unclosed '['") + return nil, fmt.Errorf("ARCHIVE/NODELIST > unclosed '['") } } else if raw[i] == ',' { rawterms = append(rawterms, raw[prevterm:i]) @@ -135,7 +135,7 @@ func ParseNodeList(raw string) (NodeList, error) { end := strings.Index(rawterm[i:], "]") if end == -1 { - return nil, fmt.Errorf("node list: unclosed '['") + return nil, fmt.Errorf("ARCHIVE/NODELIST > unclosed '['") } parts := strings.Split(rawterm[i+1:i+end], ",") @@ -144,21 +144,21 @@ func ParseNodeList(raw string) (NodeList, error) { for _, part := range parts { minus := strings.Index(part, "-") if minus == -1 { - return nil, fmt.Errorf("node list: no '-' found inside '[...]'") + return nil, fmt.Errorf("ARCHIVE/NODELIST > no '-' found inside '[...]'") } s1, s2 := part[0:minus], part[minus+1:] if len(s1) != len(s2) || len(s1) == 0 { - return nil, fmt.Errorf("node list: %#v and %#v are not of equal length or of length zero", s1, s2) + return nil, fmt.Errorf("ARCHIVE/NODELIST > %#v and %#v are not of equal length or of length zero", s1, s2) } x1, err := strconv.ParseInt(s1, 10, 32) if err != nil { - return nil, fmt.Errorf("node list: %w", err) + return nil, fmt.Errorf("ARCHIVE/NODELIST > could not parse int: %w", err) } x2, err := strconv.ParseInt(s2, 10, 32) if err != nil { - return nil, fmt.Errorf("node list: %w", err) + return nil, fmt.Errorf("ARCHIVE/NODELIST > could not parse int: %w", err) } nles = append(nles, NLExprIntRange{ @@ -172,7 +172,7 @@ func ParseNodeList(raw string) (NodeList, error) { exprs = append(exprs, nles) i += end } else { - return nil, fmt.Errorf("node list: invalid character: %#v", rune(c)) + return nil, fmt.Errorf("ARCHIVE/NODELIST > invalid character: %#v", rune(c)) } } nl = append(nl, exprs) diff --git a/pkg/lrucache/cache.go b/pkg/lrucache/cache.go index 679bd2e..874028e 100644 --- a/pkg/lrucache/cache.go +++ b/pkg/lrucache/cache.go @@ -69,7 +69,7 @@ func (c *Cache) Get(key string, computeValue ComputeValue) interface{} { if now.After(entry.expiration) { if !c.evictEntry(entry) { if entry.expiration.IsZero() { - panic("cache entry that shoud have been waited for could not be evicted.") + panic("LRUCACHE/CACHE > cache entry that shoud have been waited for could not be evicted.") } c.mutex.Unlock() return entry.value @@ -208,7 +208,7 @@ func (c *Cache) Keys(f func(key string, val interface{})) { size := 0 for key, e := range c.entries { if key != e.key { - panic("key mismatch") + panic("LRUCACHE/CACHE > key mismatch") } if now.After(e.expiration) { @@ -219,13 +219,13 @@ func (c *Cache) Keys(f func(key string, val interface{})) { if e.prev != nil { if e.prev.next != e { - panic("list corrupted") + panic("LRUCACHE/CACHE > list corrupted") } } if e.next != nil { if e.next.prev != e { - panic("list corrupted") + panic("LRUCACHE/CACHE > list corrupted") } } @@ -234,18 +234,18 @@ func (c *Cache) Keys(f func(key string, val interface{})) { } if size != c.usedmemory { - panic("size calculations failed") + panic("LRUCACHE/CACHE > size calculations failed") } if c.head != nil { if c.tail == nil || c.head.prev != nil { - panic("head/tail corrupted") + panic("LRUCACHE/CACHE > head/tail corrupted") } } if c.tail != nil { if c.head == nil || c.tail.next != nil { - panic("head/tail corrupted") + panic("LRUCACHE/CACHE > head/tail corrupted") } } } @@ -281,7 +281,7 @@ func (c *Cache) unlinkEntry(e *cacheEntry) { func (c *Cache) evictEntry(e *cacheEntry) bool { if e.waitingForComputation != 0 { - // panic("cannot evict this entry as other goroutines need the value") + // panic("LRUCACHE/CACHE > cannot evict this entry as other goroutines need the value") return false } diff --git a/pkg/schema/job.go b/pkg/schema/job.go index c87d906..9b99d99 100644 --- a/pkg/schema/job.go +++ b/pkg/schema/job.go @@ -133,12 +133,12 @@ const ( func (e *JobState) UnmarshalGQL(v interface{}) error { str, ok := v.(string) if !ok { - return fmt.Errorf("enums must be strings") + return fmt.Errorf("SCHEMA/JOB > enums must be strings") } *e = JobState(str) if !e.Valid() { - return errors.New("invalid job state") + return errors.New("SCHEMA/JOB > invalid job state") } return nil diff --git a/pkg/schema/metrics.go b/pkg/schema/metrics.go index 171901c..9f41cf4 100644 --- a/pkg/schema/metrics.go +++ b/pkg/schema/metrics.go @@ -92,12 +92,12 @@ func (e *MetricScope) Max(other MetricScope) MetricScope { func (e *MetricScope) UnmarshalGQL(v interface{}) error { str, ok := v.(string) if !ok { - return fmt.Errorf("enums must be strings") + return fmt.Errorf("SCHEMA/METRICS > enums must be strings") } *e = MetricScope(str) if !e.Valid() { - return fmt.Errorf("%s is not a valid MetricScope", str) + return fmt.Errorf("SCHEMA/METRICS > %s is not a valid MetricScope", str) } return nil } @@ -303,7 +303,7 @@ func (jm *JobMetric) AddPercentiles(ps []int) bool { for _, p := range ps { if p < 1 || p > 99 { - panic("invalid percentile") + panic("SCHEMA/METRICS > invalid percentile") } if _, ok := jm.StatisticsSeries.Percentiles[p]; ok { diff --git a/pkg/schema/validate.go b/pkg/schema/validate.go index 7201c9a..52730c0 100644 --- a/pkg/schema/validate.go +++ b/pkg/schema/validate.go @@ -45,7 +45,7 @@ func Validate(k Kind, r io.Reader) (err error) { case Config: s, err = jsonschema.Compile("embedfs://config.schema.json") default: - return fmt.Errorf("unkown schema kind ") + return fmt.Errorf("SCHEMA/VALIDATE > unkown schema kind: %v", k) } if err != nil { @@ -54,12 +54,12 @@ func Validate(k Kind, r io.Reader) (err error) { var v interface{} if err := json.NewDecoder(r).Decode(&v); err != nil { - log.Errorf("schema.Validate() - Failed to decode %v", err) + log.Errorf("SCHEMA/VALIDATE > Failed to decode %v", err) return err } if err = s.Validate(v); err != nil { - return fmt.Errorf("%#v", err) + return fmt.Errorf("SCHEMA/VALIDATE > %#v", err) } return nil diff --git a/pkg/units/units.go b/pkg/units/units.go index 9cca4df..e7560a1 100644 --- a/pkg/units/units.go +++ b/pkg/units/units.go @@ -192,7 +192,7 @@ func GetUnitUnitFactor(in Unit, out Unit) (func(value interface{}) interface{}, } else if in.getMeasure() == TemperatureF && out.getMeasure() == TemperatureC { return convertTempF2TempC, nil } else if in.getMeasure() != out.getMeasure() || in.getUnitDenominator() != out.getUnitDenominator() { - return func(value interface{}) interface{} { return 1.0 }, fmt.Errorf("invalid measures in in and out Unit") + return func(value interface{}) interface{} { return 1.0 }, fmt.Errorf("UNITS/UNITS > invalid measures in in and out Unit") } return GetPrefixPrefixFactor(in.getPrefix(), out.getPrefix()), nil } diff --git a/web/web.go b/web/web.go index 61e0128..fc807c1 100644 --- a/web/web.go +++ b/web/web.go @@ -24,6 +24,7 @@ var frontendFiles embed.FS func ServeFiles() http.Handler { publicFiles, err := fs.Sub(frontendFiles, "frontend/public") if err != nil { + log.Fatalf("WEB/WEB > cannot find frontend public files") panic(err) } return http.FileServer(http.FS(publicFiles)) @@ -47,6 +48,7 @@ func init() { templates[strings.TrimPrefix(path, "templates/")] = template.Must(template.Must(base.Clone()).ParseFS(templateFiles, path)) return nil }); err != nil { + log.Fatalf("WEB/WEB > cannot find frontend template files") panic(err) } @@ -80,6 +82,7 @@ type Page struct { func RenderTemplate(rw http.ResponseWriter, r *http.Request, file string, page *Page) { t, ok := templates[file] if !ok { + log.Fatalf("WEB/WEB > template '%s' not found", file) panic("template not found") } @@ -91,6 +94,6 @@ func RenderTemplate(rw http.ResponseWriter, r *http.Request, file string, page * log.Infof("%v\n", page.Config) if err := t.Execute(rw, page); err != nil { - log.Errorf("template error: %s", err.Error()) + log.Errorf("WEB/WEB > template error: %s", err.Error()) } } From 1ec543daf6f9cf1ec7b120d2d71a7ec0de64c7e3 Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Fri, 20 Jan 2023 09:49:17 +0100 Subject: [PATCH 02/12] Add log location string to main.go - Adresses issue #26 --- cmd/cc-backend/main.go | 56 +++++++++++++++++++++--------------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/cmd/cc-backend/main.go b/cmd/cc-backend/main.go index 1113159..9d69748 100644 --- a/cmd/cc-backend/main.go +++ b/cmd/cc-backend/main.go @@ -88,12 +88,12 @@ func main() { // See https://github.com/google/gops (Runtime overhead is almost zero) if flagGops { if err := agent.Listen(agent.Options{}); err != nil { - log.Fatalf("gops/agent.Listen failed: %s", err.Error()) + log.Fatalf("MAIN > gops/agent.Listen failed: %s", err.Error()) } } if err := runtimeEnv.LoadEnv("./.env"); err != nil && !os.IsNotExist(err) { - log.Fatalf("parsing './.env' file failed: %s", err.Error()) + log.Fatalf("MAIN > parsing './.env' file failed: %s", err.Error()) } // Initialize sub-modules and handle command line flags. @@ -118,7 +118,7 @@ func main() { "ldap": config.Keys.LdapConfig, "jwt": config.Keys.JwtConfig, }); err != nil { - log.Fatalf("auth initialization failed: %v", err) + log.Fatalf("MAIN > auth initialization failed: %v", err) } if d, err := time.ParseDuration(config.Keys.SessionMaxAge); err != nil { @@ -128,70 +128,70 @@ func main() { if flagNewUser != "" { parts := strings.SplitN(flagNewUser, ":", 3) if len(parts) != 3 || len(parts[0]) == 0 { - log.Fatal("invalid argument format for user creation") + log.Fatal("MAIN > invalid argument format for user creation") } if err := authentication.AddUser(&auth.User{ Username: parts[0], Password: parts[2], Roles: strings.Split(parts[1], ","), }); err != nil { - log.Fatalf("adding '%s' user authentication failed: %v", parts[0], err) + log.Fatalf("MAIN > adding '%s' user authentication failed: %v", parts[0], err) } } if flagDelUser != "" { if err := authentication.DelUser(flagDelUser); err != nil { - log.Fatalf("deleting user failed: %v", err) + log.Fatalf("MAIN > deleting user failed: %v", err) } } if flagSyncLDAP { if authentication.LdapAuth == nil { - log.Fatal("cannot sync: LDAP authentication is not configured") + log.Fatal("MAIN > cannot sync: LDAP authentication is not configured") } if err := authentication.LdapAuth.Sync(); err != nil { - log.Fatalf("LDAP sync failed: %v", err) + log.Fatalf("MAIN > LDAP sync failed: %v", err) } - log.Info("LDAP sync successfull") + log.Info("MAIN > LDAP sync successfull") } if flagGenJWT != "" { user, err := authentication.GetUser(flagGenJWT) if err != nil { - log.Fatalf("could not get user from JWT: %v", err) + log.Fatalf("MAIN > could not get user from JWT: %v", err) } if !user.HasRole(auth.RoleApi) { - log.Warnf("user '%s' does not have the API role", user.Username) + log.Warnf("MAIN > user '%s' does not have the API role", user.Username) } jwt, err := authentication.JwtAuth.ProvideJWT(user) if err != nil { - log.Fatalf("failed to provide JWT to user '%s': %v", user.Username, err) + log.Fatalf("MAIN > failed to provide JWT to user '%s': %v", user.Username, err) } - fmt.Printf("JWT for '%s': %s\n", user.Username, jwt) + fmt.Printf("MAIN > JWT for '%s': %s\n", user.Username, jwt) } } else if flagNewUser != "" || flagDelUser != "" { - log.Fatal("arguments --add-user and --del-user can only be used if authentication is enabled") + log.Fatal("MAIN > arguments --add-user and --del-user can only be used if authentication is enabled") } if err := archive.Init(config.Keys.Archive, config.Keys.DisableArchive); err != nil { - log.Fatalf("failed to initialize archive: %s", err.Error()) + log.Fatalf("MAIN > failed to initialize archive: %s", err.Error()) } if err := metricdata.Init(config.Keys.DisableArchive); err != nil { - log.Fatalf("failed to initialize metricdata repository: %s", err.Error()) + log.Fatalf("MAIN > failed to initialize metricdata repository: %s", err.Error()) } if flagReinitDB { if err := repository.InitDB(); err != nil { - log.Fatal("failed to re-initialize repository DB: %s", err.Error()) + log.Fatal("MAIN > failed to re-initialize repository DB: %s", err.Error()) } } if flagImportJob != "" { if err := repository.HandleImportFlag(flagImportJob); err != nil { - log.Fatalf("import failed: %s", err.Error()) + log.Fatalf("MAIN > job import failed: %s", err.Error()) } } @@ -209,12 +209,12 @@ func main() { graphQLEndpoint.SetRecoverFunc(func(ctx context.Context, err interface{}) error { switch e := err.(type) { case string: - return fmt.Errorf("panic: %s", e) + return fmt.Errorf("MAIN > Panic: %s", e) case error: - return fmt.Errorf("panic caused by: %w", e) + return fmt.Errorf("MAIN > Panic caused by: %w", e) } - return errors.New("internal server error (panic)") + return errors.New("MAIN > Internal server error (panic)") }) } @@ -361,7 +361,7 @@ func main() { // Start http or https server listener, err := net.Listen("tcp", config.Keys.Addr) if err != nil { - log.Fatalf("starting http listener failed: %v", err) + log.Fatalf("MAIN > starting http listener failed: %v", err) } if !strings.HasSuffix(config.Keys.Addr, ":80") && config.Keys.RedirectHttpTo != "" { @@ -373,7 +373,7 @@ func main() { if config.Keys.HttpsCertFile != "" && config.Keys.HttpsKeyFile != "" { cert, err := tls.LoadX509KeyPair(config.Keys.HttpsCertFile, config.Keys.HttpsKeyFile) if err != nil { - log.Fatalf("loading X509 keypair failed: %v", err) + log.Fatalf("MAIN > loading X509 keypair failed: %v", err) } listener = tls.NewListener(listener, &tls.Config{ Certificates: []tls.Certificate{cert}, @@ -391,16 +391,16 @@ func main() { // Because this program will want to bind to a privileged port (like 80), the listener must // be established first, then the user can be changed, and after that, - // the actuall http server can be started. + // the actual http server can be started. if err := runtimeEnv.DropPrivileges(config.Keys.Group, config.Keys.User); err != nil { - log.Fatalf("error while changing user: %s", err.Error()) + log.Fatalf("MAIN > error while preparing server start: %s", err.Error()) } wg.Add(1) go func() { defer wg.Done() if err := server.Serve(listener); err != nil && err != http.ErrServerClosed { - log.Fatalf("starting server failed: %v", err) + log.Fatalf("MAIN > starting server failed: %v", err) } }() @@ -410,7 +410,7 @@ func main() { go func() { defer wg.Done() <-sigs - runtimeEnv.SystemdNotifiy(false, "shutting down") + runtimeEnv.SystemdNotifiy(false, "Shutting down ...") // First shut down the server gracefully (waiting for all ongoing requests) server.Shutdown(context.Background()) @@ -424,7 +424,7 @@ func main() { for range time.Tick(30 * time.Minute) { err := jobRepo.StopJobsExceedingWalltimeBy(config.Keys.StopJobsExceedingWalltime) if err != nil { - log.Errorf("error while looking for jobs exceeding their walltime: %s", err.Error()) + log.Errorf("MAIN > error while looking for jobs exceeding their walltime: %s", err.Error()) } runtime.GC() } From 25286ff068ff3804342b1210784d04d1a6089068 Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Fri, 20 Jan 2023 09:58:16 +0100 Subject: [PATCH 03/12] Small addition to page-config infolog - Adresses issue #26 --- web/web.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/web.go b/web/web.go index fc807c1..85021c0 100644 --- a/web/web.go +++ b/web/web.go @@ -92,7 +92,7 @@ func RenderTemplate(rw http.ResponseWriter, r *http.Request, file string, page * } } - log.Infof("%v\n", page.Config) + log.Infof("WEB/WEB > page config : %v\n", page.Config) if err := t.Execute(rw, page); err != nil { log.Errorf("WEB/WEB > template error: %s", err.Error()) } From 79a949b55e3c18b837375738a72600166b4676b6 Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Mon, 23 Jan 2023 18:48:06 +0100 Subject: [PATCH 04/12] Rework pkg/log, add 'loglevel' and 'logdate' flags, streamline - removes some previously added manual location strings: now handled by pkg/log depending on loglevel - kept manual string locations on fmt print functions - add 'notice' and 'critical' loglevels - add 'Panic' and 'Panicf' functions to log panics - adresses issue #26 --- cmd/cc-backend/main.go | 52 ++++--- internal/api/rest.go | 120 +++++++-------- internal/auth/auth.go | 18 +-- internal/auth/jwt.go | 18 +-- internal/auth/ldap.go | 12 +- internal/auth/users.go | 2 +- internal/config/config.go | 8 +- internal/graph/schema.resolvers.go | 5 +- internal/metricdata/influxdb-v2.go | 10 +- internal/metricdata/metricdata.go | 4 +- internal/metricdata/prometheus.go | 22 +-- internal/repository/dbConnection.go | 6 +- internal/repository/init.go | 14 +- internal/repository/job.go | 18 +-- internal/repository/query.go | 4 +- internal/repository/user.go | 4 +- internal/routerConfig/routes.go | 8 +- internal/runtimeEnv/setup.go | 2 +- pkg/archive/fsBackend.go | 22 +-- pkg/archive/nodelist.go | 2 +- pkg/log/log.go | 224 +++++++++++++++++++++++----- pkg/schema/validate.go | 2 +- 22 files changed, 368 insertions(+), 209 deletions(-) diff --git a/cmd/cc-backend/main.go b/cmd/cc-backend/main.go index 9d69748..a314990 100644 --- a/cmd/cc-backend/main.go +++ b/cmd/cc-backend/main.go @@ -62,19 +62,21 @@ var ( ) func main() { - var flagReinitDB, flagServer, flagSyncLDAP, flagGops, flagDev, flagVersion bool - var flagNewUser, flagDelUser, flagGenJWT, flagConfigFile, flagImportJob string + var flagReinitDB, flagServer, flagSyncLDAP, flagGops, flagDev, flagVersion, flagLogDateTime bool + var flagNewUser, flagDelUser, flagGenJWT, flagConfigFile, flagImportJob, flagLogLevel string flag.BoolVar(&flagReinitDB, "init-db", false, "Go through job-archive and re-initialize the 'job', 'tag', and 'jobtag' tables (all running jobs will be lost!)") flag.BoolVar(&flagSyncLDAP, "sync-ldap", false, "Sync the 'user' table with ldap") flag.BoolVar(&flagServer, "server", false, "Start a server, continues listening on port after initialization and argument handling") flag.BoolVar(&flagGops, "gops", false, "Listen via github.com/google/gops/agent (for debugging)") flag.BoolVar(&flagDev, "dev", false, "Enable development components: GraphQL Playground and Swagger UI") flag.BoolVar(&flagVersion, "version", false, "Show version information and exit") + flag.BoolVar(&flagLogDateTime, "logdate", false, "Set true to add date and time to log messages (Default: false)") flag.StringVar(&flagConfigFile, "config", "./config.json", "Specify alternative path to `config.json`") flag.StringVar(&flagNewUser, "add-user", "", "Add a new user. Argument format: `:[admin,support,api,user]:`") flag.StringVar(&flagDelUser, "del-user", "", "Remove user by `username`") flag.StringVar(&flagGenJWT, "jwt", "", "Generate and print a JWT for the user specified by its `username`") flag.StringVar(&flagImportJob, "import-job", "", "Import a job. Argument format: `:,...`") + flag.StringVar(&flagLogLevel, "loglevel", "debug", "Sets the logging level: `[debug (default),info,notice,warn,err,fatal,crit]`") flag.Parse() if flagVersion { @@ -85,15 +87,19 @@ func main() { os.Exit(0) } + // Apply config flags for pkg/log + log.SetLogLevel(flagLogLevel) + log.SetLogDateTime(flagLogDateTime) + // See https://github.com/google/gops (Runtime overhead is almost zero) if flagGops { if err := agent.Listen(agent.Options{}); err != nil { - log.Fatalf("MAIN > gops/agent.Listen failed: %s", err.Error()) + log.Fatalf("gops/agent.Listen failed: %s", err.Error()) } } if err := runtimeEnv.LoadEnv("./.env"); err != nil && !os.IsNotExist(err) { - log.Fatalf("MAIN > parsing './.env' file failed: %s", err.Error()) + log.Fatalf("parsing './.env' file failed: %s", err.Error()) } // Initialize sub-modules and handle command line flags. @@ -118,7 +124,7 @@ func main() { "ldap": config.Keys.LdapConfig, "jwt": config.Keys.JwtConfig, }); err != nil { - log.Fatalf("MAIN > auth initialization failed: %v", err) + log.Fatalf("auth initialization failed: %v", err) } if d, err := time.ParseDuration(config.Keys.SessionMaxAge); err != nil { @@ -128,70 +134,70 @@ func main() { if flagNewUser != "" { parts := strings.SplitN(flagNewUser, ":", 3) if len(parts) != 3 || len(parts[0]) == 0 { - log.Fatal("MAIN > invalid argument format for user creation") + log.Fatal("invalid argument format for user creation") } if err := authentication.AddUser(&auth.User{ Username: parts[0], Password: parts[2], Roles: strings.Split(parts[1], ","), }); err != nil { - log.Fatalf("MAIN > adding '%s' user authentication failed: %v", parts[0], err) + log.Fatalf("adding '%s' user authentication failed: %v", parts[0], err) } } if flagDelUser != "" { if err := authentication.DelUser(flagDelUser); err != nil { - log.Fatalf("MAIN > deleting user failed: %v", err) + log.Fatalf("deleting user failed: %v", err) } } if flagSyncLDAP { if authentication.LdapAuth == nil { - log.Fatal("MAIN > cannot sync: LDAP authentication is not configured") + log.Fatal("cannot sync: LDAP authentication is not configured") } if err := authentication.LdapAuth.Sync(); err != nil { - log.Fatalf("MAIN > LDAP sync failed: %v", err) + log.Fatalf("LDAP sync failed: %v", err) } - log.Info("MAIN > LDAP sync successfull") + log.Info("LDAP sync successfull") } if flagGenJWT != "" { user, err := authentication.GetUser(flagGenJWT) if err != nil { - log.Fatalf("MAIN > could not get user from JWT: %v", err) + log.Fatalf("could not get user from JWT: %v", err) } if !user.HasRole(auth.RoleApi) { - log.Warnf("MAIN > user '%s' does not have the API role", user.Username) + log.Warnf("user '%s' does not have the API role", user.Username) } jwt, err := authentication.JwtAuth.ProvideJWT(user) if err != nil { - log.Fatalf("MAIN > failed to provide JWT to user '%s': %v", user.Username, err) + log.Fatalf("failed to provide JWT to user '%s': %v", user.Username, err) } fmt.Printf("MAIN > JWT for '%s': %s\n", user.Username, jwt) } } else if flagNewUser != "" || flagDelUser != "" { - log.Fatal("MAIN > arguments --add-user and --del-user can only be used if authentication is enabled") + log.Fatal("arguments --add-user and --del-user can only be used if authentication is enabled") } if err := archive.Init(config.Keys.Archive, config.Keys.DisableArchive); err != nil { - log.Fatalf("MAIN > failed to initialize archive: %s", err.Error()) + log.Fatalf("failed to initialize archive: %s", err.Error()) } if err := metricdata.Init(config.Keys.DisableArchive); err != nil { - log.Fatalf("MAIN > failed to initialize metricdata repository: %s", err.Error()) + log.Fatalf("failed to initialize metricdata repository: %s", err.Error()) } if flagReinitDB { if err := repository.InitDB(); err != nil { - log.Fatal("MAIN > failed to re-initialize repository DB: %s", err.Error()) + log.Fatalf("failed to re-initialize repository DB: %s", err.Error()) } } if flagImportJob != "" { if err := repository.HandleImportFlag(flagImportJob); err != nil { - log.Fatalf("MAIN > job import failed: %s", err.Error()) + log.Fatalf("job import failed: %s", err.Error()) } } @@ -361,7 +367,7 @@ func main() { // Start http or https server listener, err := net.Listen("tcp", config.Keys.Addr) if err != nil { - log.Fatalf("MAIN > starting http listener failed: %v", err) + log.Fatalf("starting http listener failed: %v", err) } if !strings.HasSuffix(config.Keys.Addr, ":80") && config.Keys.RedirectHttpTo != "" { @@ -373,7 +379,7 @@ func main() { if config.Keys.HttpsCertFile != "" && config.Keys.HttpsKeyFile != "" { cert, err := tls.LoadX509KeyPair(config.Keys.HttpsCertFile, config.Keys.HttpsKeyFile) if err != nil { - log.Fatalf("MAIN > loading X509 keypair failed: %v", err) + log.Fatalf("loading X509 keypair failed: %v", err) } listener = tls.NewListener(listener, &tls.Config{ Certificates: []tls.Certificate{cert}, @@ -393,14 +399,14 @@ func main() { // be established first, then the user can be changed, and after that, // the actual http server can be started. if err := runtimeEnv.DropPrivileges(config.Keys.Group, config.Keys.User); err != nil { - log.Fatalf("MAIN > error while preparing server start: %s", err.Error()) + log.Fatalf("error while preparing server start: %s", err.Error()) } wg.Add(1) go func() { defer wg.Done() if err := server.Serve(listener); err != nil && err != http.ErrServerClosed { - log.Fatalf("MAIN > starting server failed: %v", err) + log.Fatalf("starting server failed: %v", err) } }() diff --git a/internal/api/rest.go b/internal/api/rest.go index 004d183..75be29f 100644 --- a/internal/api/rest.go +++ b/internal/api/rest.go @@ -134,7 +134,7 @@ type ApiTag struct { type TagJobApiRequest []*ApiTag func handleError(err error, statusCode int, rw http.ResponseWriter) { - log.Warnf("API/REST > ERROR : %s", err.Error()) + log.Warnf("REST ERROR : %s", err.Error()) rw.Header().Add("Content-Type", "application/json") rw.WriteHeader(statusCode) json.NewEncoder(rw).Encode(ErrorResponse{ @@ -169,7 +169,7 @@ func decode(r io.Reader, val interface{}) error { // @router /jobs/ [get] func (api *RestApi) getJobs(rw http.ResponseWriter, r *http.Request) { if user := auth.GetUser(r.Context()); user != nil && !user.HasRole(auth.RoleApi) { - handleError(fmt.Errorf("API/REST > missing role: %#v", auth.RoleApi), http.StatusForbidden, rw) + handleError(fmt.Errorf("missing role: %#v", auth.RoleApi), http.StatusForbidden, rw) return } @@ -184,7 +184,7 @@ func (api *RestApi) getJobs(rw http.ResponseWriter, r *http.Request) { for _, s := range vals { state := schema.JobState(s) if !state.Valid() { - http.Error(rw, "invalid query parameter value: state", http.StatusBadRequest) + http.Error(rw, "REST > invalid query parameter value: state", http.StatusBadRequest) return } filter.State = append(filter.State, state) @@ -194,7 +194,7 @@ func (api *RestApi) getJobs(rw http.ResponseWriter, r *http.Request) { case "start-time": st := strings.Split(vals[0], "-") if len(st) != 2 { - http.Error(rw, "invalid query parameter value: startTime", http.StatusBadRequest) + http.Error(rw, "REST > invalid query parameter value: startTime", http.StatusBadRequest) return } from, err := strconv.ParseInt(st[0], 10, 64) @@ -226,7 +226,7 @@ func (api *RestApi) getJobs(rw http.ResponseWriter, r *http.Request) { case "with-metadata": withMetadata = true default: - http.Error(rw, "invalid query parameter: "+key, http.StatusBadRequest) + http.Error(rw, "REST > invalid query parameter: "+key, http.StatusBadRequest) return } } @@ -271,7 +271,7 @@ func (api *RestApi) getJobs(rw http.ResponseWriter, r *http.Request) { results = append(results, res) } - log.Debugf("API/REST > /api/jobs: %d jobs returned", len(results)) + log.Debugf("/api/jobs: %d jobs returned", len(results)) bw := bufio.NewWriter(rw) defer bw.Flush() if err := json.NewEncoder(bw).Encode(map[string]interface{}{ @@ -300,7 +300,7 @@ func (api *RestApi) getJobs(rw http.ResponseWriter, r *http.Request) { // @router /jobs/tag_job/{id} [post] func (api *RestApi) tagJob(rw http.ResponseWriter, r *http.Request) { if user := auth.GetUser(r.Context()); user != nil && !user.HasRole(auth.RoleApi) { - handleError(fmt.Errorf("API/REST > missing role: %#v", auth.RoleApi), http.StatusForbidden, rw) + handleError(fmt.Errorf("missing role: %#v", auth.RoleApi), http.StatusForbidden, rw) return } @@ -365,13 +365,13 @@ func (api *RestApi) tagJob(rw http.ResponseWriter, r *http.Request) { // @router /jobs/start_job/ [post] func (api *RestApi) startJob(rw http.ResponseWriter, r *http.Request) { if user := auth.GetUser(r.Context()); user != nil && !user.HasRole(auth.RoleApi) { - handleError(fmt.Errorf("API/REST > missing role: %#v", auth.RoleApi), http.StatusForbidden, rw) + handleError(fmt.Errorf("missing role: %#v", auth.RoleApi), http.StatusForbidden, rw) return } req := schema.JobMeta{BaseJob: schema.JobDefaults} if err := decode(r.Body, &req); err != nil { - handleError(fmt.Errorf("API/REST > parsing request body failed: %w", err), http.StatusBadRequest, rw) + handleError(fmt.Errorf("parsing request body failed: %w", err), http.StatusBadRequest, rw) return } @@ -391,12 +391,12 @@ func (api *RestApi) startJob(rw http.ResponseWriter, r *http.Request) { // Check if combination of (job_id, cluster_id, start_time) already exists: jobs, err := api.JobRepository.FindAll(&req.JobID, &req.Cluster, nil) if err != nil && err != sql.ErrNoRows { - handleError(fmt.Errorf("API/REST > checking for duplicate failed: %w", err), http.StatusInternalServerError, rw) + handleError(fmt.Errorf("checking for duplicate failed: %w", err), http.StatusInternalServerError, rw) return } else if err == nil { for _, job := range jobs { if (req.StartTime - job.StartTimeUnix) < 86400 { - handleError(fmt.Errorf("API/REST > a job with that jobId, cluster and startTime already exists: dbid: %d", job.ID), http.StatusUnprocessableEntity, rw) + handleError(fmt.Errorf("a job with that jobId, cluster and startTime already exists: dbid: %d", job.ID), http.StatusUnprocessableEntity, rw) return } } @@ -404,7 +404,7 @@ func (api *RestApi) startJob(rw http.ResponseWriter, r *http.Request) { id, err := api.JobRepository.Start(&req) if err != nil { - handleError(fmt.Errorf("API/REST > insert into database failed: %w", err), http.StatusInternalServerError, rw) + handleError(fmt.Errorf("insert into database failed: %w", err), http.StatusInternalServerError, rw) return } // unlock here, adding Tags can be async @@ -413,12 +413,12 @@ func (api *RestApi) startJob(rw http.ResponseWriter, r *http.Request) { for _, tag := range req.Tags { if _, err := api.JobRepository.AddTagOrCreate(id, tag.Type, tag.Name); err != nil { http.Error(rw, err.Error(), http.StatusInternalServerError) - handleError(fmt.Errorf("API/REST > adding tag to new job %d failed: %w", id, err), http.StatusInternalServerError, rw) + handleError(fmt.Errorf("adding tag to new job %d failed: %w", id, err), http.StatusInternalServerError, rw) return } } - log.Printf("API/REST > new job (id: %d): cluster=%s, jobId=%d, user=%s, startTime=%d", id, req.Cluster, req.JobID, req.User, req.StartTime) + log.Printf("new job (id: %d): cluster=%s, jobId=%d, user=%s, startTime=%d", id, req.Cluster, req.JobID, req.User, req.StartTime) rw.Header().Add("Content-Type", "application/json") rw.WriteHeader(http.StatusCreated) json.NewEncoder(rw).Encode(StartJobApiResponse{ @@ -446,14 +446,14 @@ func (api *RestApi) startJob(rw http.ResponseWriter, r *http.Request) { // @router /jobs/stop_job/{id} [post] func (api *RestApi) stopJobById(rw http.ResponseWriter, r *http.Request) { if user := auth.GetUser(r.Context()); user != nil && !user.HasRole(auth.RoleApi) { - handleError(fmt.Errorf("API/REST > missing role: %#v", auth.RoleApi), http.StatusForbidden, rw) + handleError(fmt.Errorf("missing role: %#v", auth.RoleApi), http.StatusForbidden, rw) return } // Parse request body: Only StopTime and State req := StopJobApiRequest{} if err := decode(r.Body, &req); err != nil { - handleError(fmt.Errorf("API/REST > parsing request body failed: %w", err), http.StatusBadRequest, rw) + handleError(fmt.Errorf("parsing request body failed: %w", err), http.StatusBadRequest, rw) return } @@ -464,17 +464,17 @@ func (api *RestApi) stopJobById(rw http.ResponseWriter, r *http.Request) { if ok { id, e := strconv.ParseInt(id, 10, 64) if e != nil { - handleError(fmt.Errorf("API/REST > integer expected in path for id: %w", e), http.StatusBadRequest, rw) + handleError(fmt.Errorf("integer expected in path for id: %w", e), http.StatusBadRequest, rw) return } job, err = api.JobRepository.FindById(id) } else { - handleError(errors.New("API/REST > the parameter 'id' is required"), http.StatusBadRequest, rw) + handleError(errors.New("the parameter 'id' is required"), http.StatusBadRequest, rw) return } if err != nil { - handleError(fmt.Errorf("API/REST > finding job failed: %w", err), http.StatusUnprocessableEntity, rw) + handleError(fmt.Errorf("finding job failed: %w", err), http.StatusUnprocessableEntity, rw) return } @@ -499,14 +499,14 @@ func (api *RestApi) stopJobById(rw http.ResponseWriter, r *http.Request) { // @router /jobs/stop_job/ [post] func (api *RestApi) stopJobByRequest(rw http.ResponseWriter, r *http.Request) { if user := auth.GetUser(r.Context()); user != nil && !user.HasRole(auth.RoleApi) { - handleError(fmt.Errorf("API/REST > missing role: %#v", auth.RoleApi), http.StatusForbidden, rw) + handleError(fmt.Errorf("missing role: %#v", auth.RoleApi), http.StatusForbidden, rw) return } // Parse request body req := StopJobApiRequest{} if err := decode(r.Body, &req); err != nil { - handleError(fmt.Errorf("API/REST > parsing request body failed: %w", err), http.StatusBadRequest, rw) + handleError(fmt.Errorf("parsing request body failed: %w", err), http.StatusBadRequest, rw) return } @@ -514,14 +514,14 @@ func (api *RestApi) stopJobByRequest(rw http.ResponseWriter, r *http.Request) { var job *schema.Job var err error if req.JobId == nil { - handleError(errors.New("API/REST > the field 'jobId' is required"), http.StatusBadRequest, rw) + handleError(errors.New("the field 'jobId' is required"), http.StatusBadRequest, rw) return } job, err = api.JobRepository.Find(req.JobId, req.Cluster, req.StartTime) if err != nil { - handleError(fmt.Errorf("API/REST > finding job failed: %w", err), http.StatusUnprocessableEntity, rw) + handleError(fmt.Errorf("finding job failed: %w", err), http.StatusUnprocessableEntity, rw) return } @@ -545,7 +545,7 @@ func (api *RestApi) stopJobByRequest(rw http.ResponseWriter, r *http.Request) { // @router /jobs/delete_job/{id} [delete] func (api *RestApi) deleteJobById(rw http.ResponseWriter, r *http.Request) { if user := auth.GetUser(r.Context()); user != nil && !user.HasRole(auth.RoleApi) { - handleError(fmt.Errorf("API/REST > missing role: %#v", auth.RoleApi), http.StatusForbidden, rw) + handleError(fmt.Errorf("missing role: %#v", auth.RoleApi), http.StatusForbidden, rw) return } @@ -555,23 +555,23 @@ func (api *RestApi) deleteJobById(rw http.ResponseWriter, r *http.Request) { if ok { id, e := strconv.ParseInt(id, 10, 64) if e != nil { - handleError(fmt.Errorf("API/REST > integer expected in path for id: %w", e), http.StatusBadRequest, rw) + handleError(fmt.Errorf("integer expected in path for id: %w", e), http.StatusBadRequest, rw) return } err = api.JobRepository.DeleteJobById(id) } else { - handleError(errors.New("API/REST > the parameter 'id' is required"), http.StatusBadRequest, rw) + handleError(errors.New("the parameter 'id' is required"), http.StatusBadRequest, rw) return } if err != nil { - handleError(fmt.Errorf("API/REST > deleting job failed: %w", err), http.StatusUnprocessableEntity, rw) + handleError(fmt.Errorf("deleting job failed: %w", err), http.StatusUnprocessableEntity, rw) return } rw.Header().Add("Content-Type", "application/json") rw.WriteHeader(http.StatusOK) json.NewEncoder(rw).Encode(DeleteJobApiResponse{ - Message: fmt.Sprintf("API/REST > Successfully deleted job %s", id), + Message: fmt.Sprintf("Successfully deleted job %s", id), }) } @@ -593,14 +593,14 @@ func (api *RestApi) deleteJobById(rw http.ResponseWriter, r *http.Request) { // @router /jobs/delete_job/ [delete] func (api *RestApi) deleteJobByRequest(rw http.ResponseWriter, r *http.Request) { if user := auth.GetUser(r.Context()); user != nil && !user.HasRole(auth.RoleApi) { - handleError(fmt.Errorf("API/REST > missing role: %#v", auth.RoleApi), http.StatusForbidden, rw) + handleError(fmt.Errorf("missing role: %#v", auth.RoleApi), http.StatusForbidden, rw) return } // Parse request body req := DeleteJobApiRequest{} if err := decode(r.Body, &req); err != nil { - handleError(fmt.Errorf("API/REST > parsing request body failed: %w", err), http.StatusBadRequest, rw) + handleError(fmt.Errorf("parsing request body failed: %w", err), http.StatusBadRequest, rw) return } @@ -608,27 +608,27 @@ func (api *RestApi) deleteJobByRequest(rw http.ResponseWriter, r *http.Request) var job *schema.Job var err error if req.JobId == nil { - handleError(errors.New("API/REST > the field 'jobId' is required"), http.StatusBadRequest, rw) + handleError(errors.New("the field 'jobId' is required"), http.StatusBadRequest, rw) return } job, err = api.JobRepository.Find(req.JobId, req.Cluster, req.StartTime) if err != nil { - handleError(fmt.Errorf("API/REST > finding job failed: %w", err), http.StatusUnprocessableEntity, rw) + handleError(fmt.Errorf("finding job failed: %w", err), http.StatusUnprocessableEntity, rw) return } err = api.JobRepository.DeleteJobById(job.ID) if err != nil { - handleError(fmt.Errorf("API/REST > deleting job failed: %w", err), http.StatusUnprocessableEntity, rw) + handleError(fmt.Errorf("deleting job failed: %w", err), http.StatusUnprocessableEntity, rw) return } rw.Header().Add("Content-Type", "application/json") rw.WriteHeader(http.StatusOK) json.NewEncoder(rw).Encode(DeleteJobApiResponse{ - Message: fmt.Sprintf("API/REST > Successfully deleted job %d", job.ID), + Message: fmt.Sprintf("Successfully deleted job %d", job.ID), }) } @@ -649,7 +649,7 @@ func (api *RestApi) deleteJobByRequest(rw http.ResponseWriter, r *http.Request) // @router /jobs/delete_job_before/{ts} [delete] func (api *RestApi) deleteJobBefore(rw http.ResponseWriter, r *http.Request) { if user := auth.GetUser(r.Context()); user != nil && !user.HasRole(auth.RoleApi) { - handleError(fmt.Errorf("API/REST > missing role: %#v", auth.RoleApi), http.StatusForbidden, rw) + handleError(fmt.Errorf("missing role: %#v", auth.RoleApi), http.StatusForbidden, rw) return } @@ -660,24 +660,24 @@ func (api *RestApi) deleteJobBefore(rw http.ResponseWriter, r *http.Request) { if ok { ts, e := strconv.ParseInt(id, 10, 64) if e != nil { - handleError(fmt.Errorf("API/REST > integer expected in path for ts: %w", e), http.StatusBadRequest, rw) + handleError(fmt.Errorf("integer expected in path for ts: %w", e), http.StatusBadRequest, rw) return } cnt, err = api.JobRepository.DeleteJobsBefore(ts) } else { - handleError(errors.New("API/REST > the parameter 'ts' is required"), http.StatusBadRequest, rw) + handleError(errors.New("the parameter 'ts' is required"), http.StatusBadRequest, rw) return } if err != nil { - handleError(fmt.Errorf("API/REST > deleting jobs failed: %w", err), http.StatusUnprocessableEntity, rw) + handleError(fmt.Errorf("deleting jobs failed: %w", err), http.StatusUnprocessableEntity, rw) return } rw.Header().Add("Content-Type", "application/json") rw.WriteHeader(http.StatusOK) json.NewEncoder(rw).Encode(DeleteJobApiResponse{ - Message: fmt.Sprintf("API/REST > Successfully deleted %d jobs", cnt), + Message: fmt.Sprintf("Successfully deleted %d jobs", cnt), }) } @@ -685,12 +685,12 @@ func (api *RestApi) checkAndHandleStopJob(rw http.ResponseWriter, job *schema.Jo // Sanity checks if job == nil || job.StartTime.Unix() >= req.StopTime || job.State != schema.JobStateRunning { - handleError(errors.New("API/REST > stopTime must be larger than startTime and only running jobs can be stopped"), http.StatusBadRequest, rw) + handleError(errors.New("stopTime must be larger than startTime and only running jobs can be stopped"), http.StatusBadRequest, rw) return } if req.State != "" && !req.State.Valid() { - handleError(fmt.Errorf("API/REST > invalid job state: %#v", req.State), http.StatusBadRequest, rw) + handleError(fmt.Errorf("invalid job state: %#v", req.State), http.StatusBadRequest, rw) return } else if req.State == "" { req.State = schema.JobStateCompleted @@ -700,11 +700,11 @@ func (api *RestApi) checkAndHandleStopJob(rw http.ResponseWriter, job *schema.Jo job.Duration = int32(req.StopTime - job.StartTime.Unix()) job.State = req.State if err := api.JobRepository.Stop(job.ID, job.Duration, job.State, job.MonitoringStatus); err != nil { - handleError(fmt.Errorf("API/REST > marking job as stopped failed: %w", err), http.StatusInternalServerError, rw) + handleError(fmt.Errorf("marking job as stopped failed: %w", err), http.StatusInternalServerError, rw) return } - log.Printf("API/REST > archiving job... (dbid: %d): cluster=%s, jobId=%d, user=%s, startTime=%s", job.ID, job.Cluster, job.JobID, job.User, job.StartTime) + log.Printf("archiving job... (dbid: %d): cluster=%s, jobId=%d, user=%s, startTime=%s", job.ID, job.Cluster, job.JobID, job.User, job.StartTime) // Send a response (with status OK). This means that erros that happen from here on forward // can *NOT* be communicated to the client. If reading from a MetricDataRepository or @@ -724,7 +724,7 @@ func (api *RestApi) checkAndHandleStopJob(rw http.ResponseWriter, job *schema.Jo // func (api *RestApi) importJob(rw http.ResponseWriter, r *http.Request) { // if user := auth.GetUser(r.Context()); user != nil && !user.HasRole(auth.RoleApi) { -// handleError(fmt.Errorf("API/REST > missing role: %#v", auth.RoleApi), http.StatusForbidden, rw) +// handleError(fmt.Errorf("missing role: %#v", auth.RoleApi), http.StatusForbidden, rw) // return // } @@ -733,12 +733,12 @@ func (api *RestApi) checkAndHandleStopJob(rw http.ResponseWriter, job *schema.Jo // Data *schema.JobData `json:"data"` // } // if err := decode(r.Body, &body); err != nil { -// handleError(fmt.Errorf("API/REST > import failed: %s", err.Error()), http.StatusBadRequest, rw) +// handleError(fmt.Errorf("import failed: %s", err.Error()), http.StatusBadRequest, rw) // return // } // if err := api.JobRepository.ImportJob(body.Meta, body.Data); err != nil { -// handleError(fmt.Errorf("API/REST > import failed: %s", err.Error()), http.StatusUnprocessableEntity, rw) +// handleError(fmt.Errorf("import failed: %s", err.Error()), http.StatusUnprocessableEntity, rw) // return // } @@ -793,7 +793,7 @@ func (api *RestApi) getJWT(rw http.ResponseWriter, r *http.Request) { me := auth.GetUser(r.Context()) if !me.HasRole(auth.RoleAdmin) { if username != me.Username { - http.Error(rw, "API/REST > only admins are allowed to sign JWTs not for themselves", http.StatusForbidden) + http.Error(rw, "REST > only admins are allowed to sign JWTs not for themselves", http.StatusForbidden) return } } @@ -818,13 +818,13 @@ func (api *RestApi) createUser(rw http.ResponseWriter, r *http.Request) { rw.Header().Set("Content-Type", "text/plain") me := auth.GetUser(r.Context()) if !me.HasRole(auth.RoleAdmin) { - http.Error(rw, "API/REST > only admins are allowed to create new users", http.StatusForbidden) + http.Error(rw, "REST > only admins are allowed to create new users", http.StatusForbidden) return } username, password, role, name, email := r.FormValue("username"), r.FormValue("password"), r.FormValue("role"), r.FormValue("name"), r.FormValue("email") if len(password) == 0 && role != auth.RoleApi { - http.Error(rw, "API/REST > only API users are allowed to have a blank password (login will be impossible)", http.StatusBadRequest) + http.Error(rw, "REST > only API users are allowed to have a blank password (login will be impossible)", http.StatusBadRequest) return } @@ -838,12 +838,12 @@ func (api *RestApi) createUser(rw http.ResponseWriter, r *http.Request) { return } - rw.Write([]byte(fmt.Sprintf("API/REST > User %#v successfully created!\n", username))) + rw.Write([]byte(fmt.Sprintf("User %#v successfully created!\n", username))) } func (api *RestApi) deleteUser(rw http.ResponseWriter, r *http.Request) { if user := auth.GetUser(r.Context()); !user.HasRole(auth.RoleAdmin) { - http.Error(rw, "API/REST > only admins are allowed to delete a user", http.StatusForbidden) + http.Error(rw, "REST > only admins are allowed to delete a user", http.StatusForbidden) return } @@ -858,7 +858,7 @@ func (api *RestApi) deleteUser(rw http.ResponseWriter, r *http.Request) { func (api *RestApi) getUsers(rw http.ResponseWriter, r *http.Request) { if user := auth.GetUser(r.Context()); !user.HasRole(auth.RoleAdmin) { - http.Error(rw, "API/REST > only admins are allowed to fetch a list of users", http.StatusForbidden) + http.Error(rw, "REST > only admins are allowed to fetch a list of users", http.StatusForbidden) return } @@ -873,7 +873,7 @@ func (api *RestApi) getUsers(rw http.ResponseWriter, r *http.Request) { func (api *RestApi) updateUser(rw http.ResponseWriter, r *http.Request) { if user := auth.GetUser(r.Context()); !user.HasRole(auth.RoleAdmin) { - http.Error(rw, "API/REST > only admins are allowed to update a user", http.StatusForbidden) + http.Error(rw, "REST > only admins are allowed to update a user", http.StatusForbidden) return } @@ -893,9 +893,9 @@ func (api *RestApi) updateUser(rw http.ResponseWriter, r *http.Request) { http.Error(rw, err.Error(), http.StatusUnprocessableEntity) return } - rw.Write([]byte("Remove Role Success")) + rw.Write([]byte("REST > Remove Role Success")) } else { - http.Error(rw, "Not Add or Del?", http.StatusInternalServerError) + http.Error(rw, "REST > Not Add or Del?", http.StatusInternalServerError) } } @@ -903,7 +903,7 @@ func (api *RestApi) updateConfiguration(rw http.ResponseWriter, r *http.Request) rw.Header().Set("Content-Type", "text/plain") key, value := r.FormValue("key"), r.FormValue("value") - fmt.Printf("API/REST > KEY: %#v\nVALUE: %#v\n", key, value) + fmt.Printf("REST > KEY: %#v\nVALUE: %#v\n", key, value) if err := repository.GetUserCfgRepo().UpdateConfig(key, value, auth.GetUser(r.Context())); err != nil { http.Error(rw, err.Error(), http.StatusUnprocessableEntity) @@ -915,7 +915,7 @@ func (api *RestApi) updateConfiguration(rw http.ResponseWriter, r *http.Request) func (api *RestApi) putMachineState(rw http.ResponseWriter, r *http.Request) { if api.MachineStateDir == "" { - http.Error(rw, "API/REST > machine state not enabled", http.StatusNotFound) + http.Error(rw, "REST > machine state not enabled", http.StatusNotFound) return } @@ -928,7 +928,7 @@ func (api *RestApi) putMachineState(rw http.ResponseWriter, r *http.Request) { return } - filename := filepath.Join(dir, fmt.Sprintf("API/REST > %s.json", host)) + filename := filepath.Join(dir, fmt.Sprintf("%s.json", host)) f, err := os.Create(filename) if err != nil { http.Error(rw, err.Error(), http.StatusInternalServerError) @@ -946,12 +946,12 @@ func (api *RestApi) putMachineState(rw http.ResponseWriter, r *http.Request) { func (api *RestApi) getMachineState(rw http.ResponseWriter, r *http.Request) { if api.MachineStateDir == "" { - http.Error(rw, "API/REST > machine state not enabled", http.StatusNotFound) + http.Error(rw, "REST > machine state not enabled", http.StatusNotFound) return } vars := mux.Vars(r) - filename := filepath.Join(api.MachineStateDir, vars["cluster"], fmt.Sprintf("API/REST > %s.json", vars["host"])) + filename := filepath.Join(api.MachineStateDir, vars["cluster"], fmt.Sprintf("%s.json", vars["host"])) // Sets the content-type and 'Last-Modified' Header and so on automatically http.ServeFile(rw, r, filename) diff --git a/internal/auth/auth.go b/internal/auth/auth.go index c119417..4d5fe5e 100644 --- a/internal/auth/auth.go +++ b/internal/auth/auth.go @@ -99,7 +99,7 @@ func Init(db *sqlx.DB, sessKey := os.Getenv("SESSION_KEY") if sessKey == "" { - log.Warn("AUTH/AUTH > environment variable 'SESSION_KEY' not set (will use non-persistent random key)") + log.Warn("environment variable 'SESSION_KEY' not set (will use non-persistent random key)") bytes := make([]byte, 32) if _, err := rand.Read(bytes); err != nil { return nil, err @@ -169,7 +169,7 @@ func (auth *Authentication) Login( user := (*User)(nil) if username != "" { if user, _ = auth.GetUser(username); err != nil { - // log.Warnf("AUTH/AUTH > login of unkown user %#v", username) + // log.Warnf("login of unkown user %#v", username) _ = err } } @@ -181,14 +181,14 @@ func (auth *Authentication) Login( user, err = authenticator.Login(user, rw, r) if err != nil { - log.Warnf("AUTH/AUTH > user '%s' login failed: %s", user.Username, err.Error()) + log.Warnf("user '%s' login failed: %s", user.Username, err.Error()) onfailure(rw, r, err) return } session, err := auth.sessionStore.New(r, "session") if err != nil { - log.Errorf("AUTH/AUTH > session creation failed: %s", err.Error()) + log.Errorf("session creation failed: %s", err.Error()) http.Error(rw, err.Error(), http.StatusInternalServerError) return } @@ -199,18 +199,18 @@ func (auth *Authentication) Login( session.Values["username"] = user.Username session.Values["roles"] = user.Roles if err := auth.sessionStore.Save(r, rw, session); err != nil { - log.Errorf("AUTH/AUTH > session save failed: %s", err.Error()) + log.Errorf("session save failed: %s", err.Error()) http.Error(rw, err.Error(), http.StatusInternalServerError) return } - log.Infof("AUTH/AUTH > login successfull: user: %#v (roles: %v)", user.Username, user.Roles) + log.Infof("login successfull: user: %#v (roles: %v)", user.Username, user.Roles) ctx := context.WithValue(r.Context(), ContextUserKey, user) onsuccess.ServeHTTP(rw, r.WithContext(ctx)) return } - log.Warn("AUTH/AUTH > login failed: no authenticator applied") + log.Warn("login failed: no authenticator applied") onfailure(rw, r, err) }) } @@ -226,7 +226,7 @@ func (auth *Authentication) Auth( for _, authenticator := range auth.authenticators { user, err := authenticator.Auth(rw, r) if err != nil { - log.Warnf("AUTH/AUTH > authentication failed: %s", err.Error()) + log.Warnf("authentication failed: %s", err.Error()) http.Error(rw, err.Error(), http.StatusUnauthorized) return } @@ -239,7 +239,7 @@ func (auth *Authentication) Auth( return } - log.Warnf("AUTH/AUTH > authentication failed: %s", "no authenticator applied") + log.Warnf("authentication failed: %s", "no authenticator applied") // http.Error(rw, http.StatusText(http.StatusUnauthorized), http.StatusUnauthorized) onfailure(rw, r, errors.New("unauthorized (login first or use a token)")) }) diff --git a/internal/auth/jwt.go b/internal/auth/jwt.go index ed8e413..798023e 100644 --- a/internal/auth/jwt.go +++ b/internal/auth/jwt.go @@ -41,7 +41,7 @@ func (ja *JWTAuthenticator) Init(auth *Authentication, conf interface{}) error { pubKey, privKey := os.Getenv("JWT_PUBLIC_KEY"), os.Getenv("JWT_PRIVATE_KEY") if pubKey == "" || privKey == "" { - log.Warn("AUTH/JWT > environment variables 'JWT_PUBLIC_KEY' or 'JWT_PRIVATE_KEY' not set (token based authentication will not work)") + log.Warn("environment variables 'JWT_PUBLIC_KEY' or 'JWT_PRIVATE_KEY' not set (token based authentication will not work)") } else { bytes, err := base64.StdEncoding.DecodeString(pubKey) if err != nil { @@ -75,20 +75,20 @@ func (ja *JWTAuthenticator) Init(auth *Authentication, conf interface{}) error { // Warn if other necessary settings are not configured if ja.config != nil { if ja.config.CookieName == "" { - log.Warn("AUTH/JWT > cookieName for JWTs not configured (cross login via JWT cookie will fail)") + log.Warn("cookieName for JWTs not configured (cross login via JWT cookie will fail)") } if !ja.config.ForceJWTValidationViaDatabase { - log.Warn("AUTH/JWT > forceJWTValidationViaDatabase not set to true: CC will accept users and roles defined in JWTs regardless of its own database!") + log.Warn("forceJWTValidationViaDatabase not set to true: CC will accept users and roles defined in JWTs regardless of its own database!") } if ja.config.TrustedExternalIssuer == "" { - log.Warn("AUTH/JWT > trustedExternalIssuer for JWTs not configured (cross login via JWT cookie will fail)") + log.Warn("trustedExternalIssuer for JWTs not configured (cross login via JWT cookie will fail)") } } else { - log.Warn("AUTH/JWT > cookieName and trustedExternalIssuer for JWTs not configured (cross login via JWT cookie will fail)") + log.Warn("cookieName and trustedExternalIssuer for JWTs not configured (cross login via JWT cookie will fail)") } } else { ja.publicKeyCrossLogin = nil - log.Warn("AUTH/JWT > environment variable 'CROSS_LOGIN_JWT_PUBLIC_KEY' not set (cross login token based authentication will not work)") + log.Warn("environment variable 'CROSS_LOGIN_JWT_PUBLIC_KEY' not set (cross login token based authentication will not work)") } return nil @@ -243,7 +243,7 @@ func (ja *JWTAuthenticator) Auth( // Deny any logins for unknown usernames if err != nil { - log.Warn("AUTH/JWT > Could not find user from JWT in internal database.") + log.Warn("Could not find user from JWT in internal database.") return nil, errors.New("unknown user") } @@ -264,7 +264,7 @@ func (ja *JWTAuthenticator) Auth( // Create a session so that we no longer need the JTW Cookie session, err := ja.auth.sessionStore.New(r, "session") if err != nil { - log.Errorf("AUTH/JWT > session creation failed: %s", err.Error()) + log.Errorf("session creation failed: %s", err.Error()) http.Error(rw, err.Error(), http.StatusInternalServerError) return nil, err } @@ -276,7 +276,7 @@ func (ja *JWTAuthenticator) Auth( session.Values["roles"] = roles if err := ja.auth.sessionStore.Save(r, rw, session); err != nil { - log.Errorf("AUTH/JWT > session save failed: %s", err.Error()) + log.Errorf("session save failed: %s", err.Error()) http.Error(rw, err.Error(), http.StatusInternalServerError) return nil, err } diff --git a/internal/auth/ldap.go b/internal/auth/ldap.go index 6f974b5..1e0235c 100644 --- a/internal/auth/ldap.go +++ b/internal/auth/ldap.go @@ -33,7 +33,7 @@ func (la *LdapAuthenticator) Init( la.syncPassword = os.Getenv("LDAP_ADMIN_PASSWORD") if la.syncPassword == "" { - log.Warn("AUTH/LDAP > environment variable 'LDAP_ADMIN_PASSWORD' not set (ldap sync will not work)") + log.Warn("environment variable 'LDAP_ADMIN_PASSWORD' not set (ldap sync will not work)") } if la.config != nil && la.config.SyncInterval != "" { @@ -49,11 +49,11 @@ func (la *LdapAuthenticator) Init( go func() { ticker := time.NewTicker(interval) for t := range ticker.C { - log.Printf("AUTH/LDAP > sync started at %s", t.Format(time.RFC3339)) + log.Printf("sync started at %s", t.Format(time.RFC3339)) if err := la.Sync(); err != nil { - log.Errorf("AUTH/LDAP > sync failed: %s", err.Error()) + log.Errorf("sync failed: %s", err.Error()) } - log.Print("AUTH/LDAP > sync done") + log.Print("sync done") } }() } @@ -147,13 +147,13 @@ func (la *LdapAuthenticator) Sync() error { for username, where := range users { if where == IN_DB && la.config.SyncDelOldUsers { - log.Debugf("AUTH/LDAP > sync: remove %#v (does not show up in LDAP anymore)", username) + log.Debugf("sync: remove %#v (does not show up in LDAP anymore)", username) if _, err := la.auth.db.Exec(`DELETE FROM user WHERE user.username = ?`, username); err != nil { return err } } else if where == IN_LDAP { name := newnames[username] - log.Debugf("AUTH/LDAP > sync: add %#v (name: %#v, roles: [user], ldap: true)", username, name) + log.Debugf("sync: add %#v (name: %#v, roles: [user], ldap: true)", username, name) if _, err := la.auth.db.Exec(`INSERT INTO user (username, ldap, name, roles) VALUES (?, ?, ?, ?)`, username, 1, name, "[\""+RoleUser+"\"]"); err != nil { return err diff --git a/internal/auth/users.go b/internal/auth/users.go index 009fe35..f890c1b 100644 --- a/internal/auth/users.go +++ b/internal/auth/users.go @@ -67,7 +67,7 @@ func (auth *Authentication) AddUser(user *User) error { return err } - log.Infof("AUTH/USERS > new user %#v created (roles: %s, auth-source: %d)", user.Username, rolesJson, user.AuthSource) + log.Infof("new user %#v created (roles: %s, auth-source: %d)", user.Username, rolesJson, user.AuthSource) return nil } diff --git a/internal/config/config.go b/internal/config/config.go index 700413a..b8afc9a 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -49,20 +49,20 @@ func Init(flagConfigFile string) { raw, err := os.ReadFile(flagConfigFile) if err != nil { if !os.IsNotExist(err) { - log.Fatalf("CONFIG/CONFIG > ERROR: %v", err) + log.Fatalf("CONFIG ERROR: %v", err) } } else { if err := schema.Validate(schema.Config, bytes.NewReader(raw)); err != nil { - log.Fatalf("CONFIG/CONFIG > Validate config: %v\n", err) + log.Fatalf("Validate config: %v\n", err) } dec := json.NewDecoder(bytes.NewReader(raw)) dec.DisallowUnknownFields() if err := dec.Decode(&Keys); err != nil { - log.Fatalf("CONFIG/CONFIG > could not decode: %v", err) + log.Fatalf("could not decode: %v", err) } if Keys.Clusters == nil || len(Keys.Clusters) < 1 { - log.Fatal("CONFIG/CONFIG > At least one cluster required in config!") + log.Fatal("At least one cluster required in config!") } } } diff --git a/internal/graph/schema.resolvers.go b/internal/graph/schema.resolvers.go index 12d316c..5c927f1 100644 --- a/internal/graph/schema.resolvers.go +++ b/internal/graph/schema.resolvers.go @@ -17,6 +17,7 @@ import ( "github.com/ClusterCockpit/cc-backend/internal/repository" "github.com/ClusterCockpit/cc-backend/pkg/archive" "github.com/ClusterCockpit/cc-backend/pkg/schema" + "github.com/ClusterCockpit/cc-backend/pkg/log" ) // Partitions is the resolver for the partitions field. @@ -51,7 +52,7 @@ func (r *mutationResolver) CreateTag(ctx context.Context, typeArg string, name s // DeleteTag is the resolver for the deleteTag field. func (r *mutationResolver) DeleteTag(ctx context.Context, id string) (string, error) { - panic(fmt.Errorf("GRAPH/RESOLVERS > not implemented: DeleteTag - deleteTag")) + log.Panic(fmt.Errorf("not implemented: DeleteTag - deleteTag")) } // AddTagsToJob is the resolver for the addTagsToJob field. @@ -175,7 +176,7 @@ func (r *queryResolver) JobMetrics(ctx context.Context, id string, metrics []str for name, md := range data { for scope, metric := range md { if metric.Scope != schema.MetricScope(scope) { - panic("GRAPH/RESOLVERS > metric.Scope != schema.MetricScope(scope) : Should not happen!") + log.Panic("metric.Scope != schema.MetricScope(scope) : Should not happen!") } res = append(res, &model.JobMetricWithName{ diff --git a/internal/metricdata/influxdb-v2.go b/internal/metricdata/influxdb-v2.go index 51a7dbb..8c1e428 100644 --- a/internal/metricdata/influxdb-v2.go +++ b/internal/metricdata/influxdb-v2.go @@ -84,7 +84,7 @@ func (idb *InfluxDBv2DataRepository) LoadData( switch scope { case "node": // Get Finest Granularity, Groupy By Measurement and Hostname (== Metric / Node), Calculate Mean for 60s windows - // log.Info("METRICDATA/INFLUXV2 > Scope 'node' requested. ") + // log.Info("Scope 'node' requested. ") query = fmt.Sprintf(` from(bucket: "%s") |> range(start: %s, stop: %s) @@ -97,10 +97,10 @@ func (idb *InfluxDBv2DataRepository) LoadData( idb.formatTime(job.StartTime), idb.formatTime(idb.epochToTime(job.StartTimeUnix+int64(job.Duration)+int64(1))), measurementsCond, hostsCond) case "socket": - log.Info("METRICDATA/INFLUXV2 > Scope 'socket' requested, but not yet supported: Will return 'node' scope only. ") + log.Info("Scope 'socket' requested, but not yet supported: Will return 'node' scope only. ") continue case "core": - log.Info("METRICDATA/INFLUXV2 > Scope 'core' requested, but not yet supported: Will return 'node' scope only. ") + log.Info(" Scope 'core' requested, but not yet supported: Will return 'node' scope only. ") continue // Get Finest Granularity only, Set NULL to 0.0 // query = fmt.Sprintf(` @@ -114,7 +114,7 @@ func (idb *InfluxDBv2DataRepository) LoadData( // idb.formatTime(job.StartTime), idb.formatTime(idb.epochToTime(job.StartTimeUnix + int64(job.Duration) + int64(1) )), // measurementsCond, hostsCond) default: - log.Info("METRICDATA/INFLUXV2 > Unknown Scope requested: Will return 'node' scope. ") + log.Info("Unknown Scope requested: Will return 'node' scope. ") continue // return nil, errors.New("METRICDATA/INFLUXV2 > the InfluxDB metric data repository does not yet support other scopes than 'node'") } @@ -319,7 +319,7 @@ func (idb *InfluxDBv2DataRepository) LoadNodeData( ctx context.Context) (map[string]map[string][]*schema.JobMetric, error) { // TODO : Implement to be used in Analysis- und System/Node-View - log.Infof("METRICDATA/INFLUXV2 > LoadNodeData unimplemented for InfluxDBv2DataRepository, Args: cluster %s, metrics %v, nodes %v, scopes %v", cluster, metrics, nodes, scopes) + log.Infof("LoadNodeData unimplemented for InfluxDBv2DataRepository, Args: cluster %s, metrics %v, nodes %v, scopes %v", cluster, metrics, nodes, scopes) return nil, errors.New("METRICDATA/INFLUXV2 > unimplemented for InfluxDBv2DataRepository") } diff --git a/internal/metricdata/metricdata.go b/internal/metricdata/metricdata.go index 2b68f55..56bcdc6 100644 --- a/internal/metricdata/metricdata.go +++ b/internal/metricdata/metricdata.go @@ -107,7 +107,7 @@ func LoadData(job *schema.Job, jd, err = repo.LoadData(job, metrics, scopes, ctx) if err != nil { if len(jd) != 0 { - log.Errorf("METRICDATA/METRICDATA > partial error: %s", err.Error()) + log.Errorf("partial error: %s", err.Error()) } else { return err, 0, 0 } @@ -229,7 +229,7 @@ func LoadNodeData( data, err := repo.LoadNodeData(cluster, metrics, nodes, scopes, from, to, ctx) if err != nil { if len(data) != 0 { - log.Errorf("METRICDATA/METRICDATA > partial error: %s", err.Error()) + log.Errorf("partial error: %s", err.Error()) } else { return nil, err } diff --git a/internal/metricdata/prometheus.go b/internal/metricdata/prometheus.go index db2bc6f..7d51218 100644 --- a/internal/metricdata/prometheus.go +++ b/internal/metricdata/prometheus.go @@ -184,9 +184,9 @@ func (pdb *PrometheusDataRepository) Init(rawConfig json.RawMessage) error { for metric, templ := range config.Templates { pdb.templates[metric], err = template.New(metric).Parse(templ) if err == nil { - log.Debugf("METRICDATA/PROMETHEUS > Added PromQL template for %s: %s", metric, templ) + log.Debugf("Added PromQL template for %s: %s", metric, templ) } else { - log.Errorf("METRICDATA/PROMETHEUS > Failed to parse PromQL template %s for metric %s", templ, metric) + log.Errorf("Failed to parse PromQL template %s for metric %s", templ, metric) } } return nil @@ -216,7 +216,7 @@ func (pdb *PrometheusDataRepository) FormatQuery( return "", errors.New(fmt.Sprintf("METRICDATA/PROMETHEUS > Error compiling template %v", templ)) } else { query := buf.String() - log.Debugf("METRICDATA/PROMETHEUS > PromQL: %s", query) + log.Debugf("PromQL: %s", query) return query, nil } } else { @@ -283,14 +283,14 @@ func (pdb *PrometheusDataRepository) LoadData( for _, scope := range scopes { if scope != schema.MetricScopeNode { - logOnce.Do(func(){log.Infof("METRICDATA/PROMETHEUS > Scope '%s' requested, but not yet supported: Will return 'node' scope only.", scope)}) + logOnce.Do(func(){log.Infof("Scope '%s' requested, but not yet supported: Will return 'node' scope only.", scope)}) continue } for _, metric := range metrics { metricConfig := archive.GetMetricConfig(job.Cluster, metric) if metricConfig == nil { - log.Errorf("METRICDATA/PROMETHEUS > Error in LoadData: Metric %s for cluster %s not configured", metric, job.Cluster) + log.Errorf("Error in LoadData: Metric %s for cluster %s not configured", metric, job.Cluster) return nil, errors.New("METRICDATA/PROMETHEUS > Prometheus query error") } query, err := pdb.FormatQuery(metric, scope, nodes, job.Cluster) @@ -307,7 +307,7 @@ func (pdb *PrometheusDataRepository) LoadData( result, warnings, err := pdb.queryClient.QueryRange(ctx, query, r) if err != nil { - log.Errorf("METRICDATA/PROMETHEUS > Prometheus query error in LoadData: %v\nQuery: %s", err, query) + log.Errorf("Prometheus query error in LoadData: %v\nQuery: %s", err, query) return nil, errors.New("METRICDATA/PROMETHEUS > Prometheus query error") } if len(warnings) > 0 { @@ -389,13 +389,13 @@ func (pdb *PrometheusDataRepository) LoadNodeData( } for _, scope := range scopes { if scope != schema.MetricScopeNode { - logOnce.Do(func(){log.Infof("METRICDATA/PROMETHEUS > Note: Scope '%s' requested, but not yet supported: Will return 'node' scope only.", scope)}) + logOnce.Do(func(){log.Infof("Note: Scope '%s' requested, but not yet supported: Will return 'node' scope only.", scope)}) continue } for _, metric := range metrics { metricConfig := archive.GetMetricConfig(cluster, metric) if metricConfig == nil { - log.Errorf("METRICDATA/PROMETHEUS > Error in LoadNodeData: Metric %s for cluster %s not configured", metric, cluster) + log.Errorf("Error in LoadNodeData: Metric %s for cluster %s not configured", metric, cluster) return nil, errors.New("METRICDATA/PROMETHEUS > Prometheus querry error") } query, err := pdb.FormatQuery(metric, scope, nodes, cluster) @@ -412,11 +412,11 @@ func (pdb *PrometheusDataRepository) LoadNodeData( result, warnings, err := pdb.queryClient.QueryRange(ctx, query, r) if err != nil { - log.Errorf("METRICDATA/PROMETHEUS > Prometheus query error in LoadNodeData: %v\n", err) + log.Errorf("Prometheus query error in LoadNodeData: %v\n", err) return nil, errors.New("METRICDATA/PROMETHEUS > Prometheus querry error") } if len(warnings) > 0 { - log.Warnf("METRICDATA/PROMETHEUS > Warnings: %v\n", warnings) + log.Warnf("Warnings: %v\n", warnings) } step := int64(metricConfig.Timestep) @@ -442,6 +442,6 @@ func (pdb *PrometheusDataRepository) LoadNodeData( } } t1 := time.Since(t0) - log.Debugf("METRICDATA/PROMETHEUS > LoadNodeData of %v nodes took %s", len(data), t1) + log.Debugf("LoadNodeData of %v nodes took %s", len(data), t1) return data, nil } diff --git a/internal/repository/dbConnection.go b/internal/repository/dbConnection.go index 987bf20..86f0762 100644 --- a/internal/repository/dbConnection.go +++ b/internal/repository/dbConnection.go @@ -39,14 +39,14 @@ func Connect(driver string, db string) { } else if driver == "mysql" { dbHandle, err = sqlx.Open("mysql", fmt.Sprintf("%s?multiStatements=true", db)) if err != nil { - log.Fatalf("REPOSITORY/DBCONNECTION > sqlx.Open() error: %v", err) + log.Fatalf("sqlx.Open() error: %v", err) } dbHandle.SetConnMaxLifetime(time.Minute * 3) dbHandle.SetMaxOpenConns(10) dbHandle.SetMaxIdleConns(10) } else { - log.Fatalf("REPOSITORY/DBCONNECTION > unsupported database driver: %s", driver) + log.Fatalf("unsupported database driver: %s", driver) } dbConnInstance = &DBConnection{DB: dbHandle} @@ -55,7 +55,7 @@ func Connect(driver string, db string) { func GetConnection() *DBConnection { if dbConnInstance == nil { - log.Fatalf("REPOSITORY/DBCONNECTION > Database connection not initialized!") + log.Fatalf("Database connection not initialized!") } return dbConnInstance diff --git a/internal/repository/init.go b/internal/repository/init.go index 1358100..08b5565 100644 --- a/internal/repository/init.go +++ b/internal/repository/init.go @@ -186,7 +186,7 @@ func HandleImportFlag(flag string) error { } } - log.Infof("REPOSITORY/INIT > successfully imported a new job (jobId: %d, cluster: %s, dbid: %d)", job.JobID, job.Cluster, id) + log.Infof("successfully imported a new job (jobId: %d, cluster: %s, dbid: %d)", job.JobID, job.Cluster, id) } return nil } @@ -260,34 +260,34 @@ func InitDB() error { job.RawResources, err = json.Marshal(job.Resources) if err != nil { - log.Errorf("REPOSITORY/INIT > repository initDB(): %v", err) + log.Errorf("repository initDB(): %v", err) errorOccured++ continue } job.RawMetaData, err = json.Marshal(job.MetaData) if err != nil { - log.Errorf("REPOSITORY/INIT > repository initDB(): %v", err) + log.Errorf("repository initDB(): %v", err) errorOccured++ continue } if err := SanityChecks(&job.BaseJob); err != nil { - log.Errorf("REPOSITORY/INIT > repository initDB(): %v", err) + log.Errorf("repository initDB(): %v", err) errorOccured++ continue } res, err := stmt.Exec(job) if err != nil { - log.Errorf("REPOSITORY/INIT > repository initDB(): %v", err) + log.Errorf("repository initDB(): %v", err) errorOccured++ continue } id, err := res.LastInsertId() if err != nil { - log.Errorf("REPOSITORY/INIT > repository initDB(): %v", err) + log.Errorf("repository initDB(): %v", err) errorOccured++ continue } @@ -318,7 +318,7 @@ func InitDB() error { } if errorOccured > 0 { - log.Errorf("REPOSITORY/INIT > Error in import of %d jobs!", errorOccured) + log.Errorf("Error in import of %d jobs!", errorOccured) } if err := tx.Commit(); err != nil { diff --git a/internal/repository/job.go b/internal/repository/job.go index ca7c639..76db377 100644 --- a/internal/repository/job.go +++ b/internal/repository/job.go @@ -259,9 +259,9 @@ func (r *JobRepository) DeleteJobsBefore(startTime int64) (int, error) { err := r.DB.Get(&cnt, qs) //ignore error as it will also occur in delete statement _, err = r.DB.Exec(`DELETE FROM job WHERE job.start_time < ?`, startTime) if err != nil { - log.Warnf("REPOSITORY/JOB > DeleteJobsBefore(%d): error %v", startTime, err) + log.Warnf(" DeleteJobsBefore(%d): error %v", startTime, err) } else { - log.Infof("REPOSITORY/JOB > DeleteJobsBefore(%d): Deleted %d jobs", startTime, cnt) + log.Infof("DeleteJobsBefore(%d): Deleted %d jobs", startTime, cnt) } return cnt, err } @@ -269,9 +269,9 @@ func (r *JobRepository) DeleteJobsBefore(startTime int64) (int, error) { func (r *JobRepository) DeleteJobById(id int64) error { _, err := r.DB.Exec(`DELETE FROM job WHERE job.id = ?`, id) if err != nil { - log.Warnf("REPOSITORY/JOB > DeleteJobById(%d): error %v", id, err) + log.Warnf("DeleteJobById(%d): error %v", id, err) } else { - log.Infof("REPOSITORY/JOB > DeleteJobById(%d): Success", id) + log.Infof("DeleteJobById(%d): Success", id) } return err } @@ -376,7 +376,7 @@ func (r *JobRepository) archivingWorker(){ // not using meta data, called to load JobMeta into Cache? // will fail if job meta not in repository if _, err := r.FetchMetadata(job); err != nil { - log.Errorf("REPOSITORY/JOB > archiving job (dbid: %d) failed: %s", job.ID, err.Error()) + log.Errorf("archiving job (dbid: %d) failed: %s", job.ID, err.Error()) r.UpdateMonitoringStatus(job.ID, schema.MonitoringStatusArchivingFailed) continue } @@ -385,18 +385,18 @@ func (r *JobRepository) archivingWorker(){ // TODO: Maybe use context with cancel/timeout here jobMeta, err := metricdata.ArchiveJob(job, context.Background()) if err != nil { - log.Errorf("REPOSITORY/JOB > archiving job (dbid: %d) failed: %s", job.ID, err.Error()) + log.Errorf("archiving job (dbid: %d) failed: %s", job.ID, err.Error()) r.UpdateMonitoringStatus(job.ID, schema.MonitoringStatusArchivingFailed) continue } // Update the jobs database entry one last time: if err := r.MarkArchived(job.ID, schema.MonitoringStatusArchivingSuccessful, jobMeta.Statistics); err != nil { - log.Errorf("REPOSITORY/JOB > archiving job (dbid: %d) failed: %s", job.ID, err.Error()) + log.Errorf("archiving job (dbid: %d) failed: %s", job.ID, err.Error()) continue } - log.Printf("REPOSITORY/JOB > archiving job (dbid: %d) successful", job.ID) + log.Printf("archiving job (dbid: %d) successful", job.ID) r.archivePending.Done() } } @@ -523,7 +523,7 @@ func (r *JobRepository) StopJobsExceedingWalltimeBy(seconds int) error { } if rowsAffected > 0 { - log.Warnf("REPOSITORY/JOB > %d jobs have been marked as failed due to running too long", rowsAffected) + log.Warnf("%d jobs have been marked as failed due to running too long", rowsAffected) } return nil } diff --git a/internal/repository/query.go b/internal/repository/query.go index bc2d108..9b86c59 100644 --- a/internal/repository/query.go +++ b/internal/repository/query.go @@ -54,7 +54,7 @@ func (r *JobRepository) QueryJobs( return nil, err } - log.Debugf("REPOSITORY/QUERY > SQL query: `%s`, args: %#v", sql, args) + log.Debugf("SQL query: `%s`, args: %#v", sql, args) rows, err := query.RunWith(r.stmtCache).Query() if err != nil { return nil, err @@ -209,7 +209,7 @@ var matchAllCap = regexp.MustCompile("([a-z0-9])([A-Z])") func toSnakeCase(str string) string { for _, c := range str { if c == '\'' || c == '\\' { - panic("REPOSITORY/QUERY > toSnakeCase() attack vector!") + log.Panic("toSnakeCase() attack vector!") } } diff --git a/internal/repository/user.go b/internal/repository/user.go index 80adfc8..b96641a 100644 --- a/internal/repository/user.go +++ b/internal/repository/user.go @@ -42,12 +42,12 @@ func GetUserCfgRepo() *UserCfgRepo { FOREIGN KEY (username) REFERENCES user (username) ON DELETE CASCADE ON UPDATE NO ACTION);`) if err != nil { - log.Fatalf("REPOSITORY/USER > db.DB.exec() error: %v", err) + log.Fatalf("db.DB.exec() error: %v", err) } lookupConfigStmt, err := db.DB.Preparex(`SELECT confkey, value FROM configuration WHERE configuration.username = ?`) if err != nil { - log.Fatalf("REPOSITORY/USER > db.DB.Preparex() error: %v", err) + log.Fatalf("db.DB.Preparex() error: %v", err) } userCfgRepoInstance = &UserCfgRepo{ diff --git a/internal/routerConfig/routes.go b/internal/routerConfig/routes.go index fa03b26..9424df7 100644 --- a/internal/routerConfig/routes.go +++ b/internal/routerConfig/routes.go @@ -61,12 +61,12 @@ func setupHomeRoute(i InfoType, r *http.Request) InfoType { State: []schema.JobState{schema.JobStateRunning}, }}, nil, nil) if err != nil { - log.Errorf("ROUTERCONFIG/ROUTES > failed to count jobs: %s", err.Error()) + log.Errorf("failed to count jobs: %s", err.Error()) runningJobs = map[string]int{} } totalJobs, err := jobRepo.CountGroupedJobs(r.Context(), model.AggregateCluster, nil, nil, nil) if err != nil { - log.Errorf("ROUTERCONFIG/ROUTES > failed to count jobs: %s", err.Error()) + log.Errorf("failed to count jobs: %s", err.Error()) totalJobs = map[string]int{} } from := time.Now().Add(-24 * time.Hour) @@ -75,7 +75,7 @@ func setupHomeRoute(i InfoType, r *http.Request) InfoType { Duration: &schema.IntRange{From: 0, To: graph.ShortJobDuration}, }}, nil, nil) if err != nil { - log.Errorf("ROUTERCONFIG/ROUTES > failed to count jobs: %s", err.Error()) + log.Errorf("failed to count jobs: %s", err.Error()) recentShortJobs = map[string]int{} } @@ -150,7 +150,7 @@ func setupTaglistRoute(i InfoType, r *http.Request) InfoType { tags, counts, err := jobRepo.CountTags(username) tagMap := make(map[string][]map[string]interface{}) if err != nil { - log.Errorf("ROUTERCONFIG/ROUTES > GetTags failed: %s", err.Error()) + log.Errorf("GetTags failed: %s", err.Error()) i["tagmap"] = tagMap return i } diff --git a/internal/runtimeEnv/setup.go b/internal/runtimeEnv/setup.go index 7ac7138..a4d37be 100644 --- a/internal/runtimeEnv/setup.go +++ b/internal/runtimeEnv/setup.go @@ -65,7 +65,7 @@ func LoadEnv(file string) error { case '"': sb.WriteRune('"') default: - return fmt.Errorf("RUNTIME/SETUP > unsupprorted escape sequence in quoted string: backslash %#v", runes[i]) + return fmt.Errorf("RUNTIME/SETUP > unsupported escape sequence in quoted string: backslash %#v", runes[i]) } continue } diff --git a/pkg/archive/fsBackend.go b/pkg/archive/fsBackend.go index 2bdbfce..608573d 100644 --- a/pkg/archive/fsBackend.go +++ b/pkg/archive/fsBackend.go @@ -46,7 +46,7 @@ func loadJobMeta(filename string) (*schema.JobMeta, error) { f, err := os.Open(filename) if err != nil { - log.Errorf("ARCHIVE/FSBACKEND > loadJobMeta() > open file error: %v", err) + log.Errorf("loadJobMeta() > open file error: %v", err) return &schema.JobMeta{}, err } defer f.Close() @@ -58,19 +58,19 @@ func (fsa *FsArchive) Init(rawConfig json.RawMessage) error { var config FsArchiveConfig if err := json.Unmarshal(rawConfig, &config); err != nil { - log.Errorf("ARCHIVE/FSBACKEND > Init() > Unmarshal error: %v", err) + log.Errorf("Init() > Unmarshal error: %v", err) return err } if config.Path == "" { err := fmt.Errorf("ARCHIVE/FSBACKEND > Init() : empty config.Path") - log.Errorf("ARCHIVE/FSBACKEND > Init() > config.Path error: %v", err) + log.Errorf("Init() > config.Path error: %v", err) return err } fsa.path = config.Path entries, err := os.ReadDir(fsa.path) if err != nil { - log.Errorf("ARCHIVE/FSBACKEND > Init() > ReadDir() error: %v", err) + log.Errorf("Init() > ReadDir() error: %v", err) return err } @@ -86,7 +86,7 @@ func (fsa *FsArchive) LoadJobData(job *schema.Job) (schema.JobData, error) { filename := getPath(job, fsa.path, "data.json") f, err := os.Open(filename) if err != nil { - log.Errorf("ARCHIVE/FSBACKEND > LoadJobData() > open file error: %v", err) + log.Errorf("LoadJobData() > open file error: %v", err) return nil, err } defer f.Close() @@ -104,7 +104,7 @@ func (fsa *FsArchive) LoadClusterCfg(name string) (*schema.Cluster, error) { b, err := os.ReadFile(filepath.Join(fsa.path, name, "cluster.json")) if err != nil { - log.Errorf("ARCHIVE/FSBACKEND > LoadClusterCfg() > open file error: %v", err) + log.Errorf("LoadClusterCfg() > open file error: %v", err) return &schema.Cluster{}, err } if config.Keys.Validate { @@ -121,13 +121,13 @@ func (fsa *FsArchive) Iter() <-chan *schema.JobMeta { go func() { clustersDir, err := os.ReadDir(fsa.path) if err != nil { - log.Fatalf("ARCHIVE/FSBACKEND > Reading clusters failed @ cluster dirs: %s", err.Error()) + log.Fatalf("Reading clusters failed @ cluster dirs: %s", err.Error()) } for _, clusterDir := range clustersDir { lvl1Dirs, err := os.ReadDir(filepath.Join(fsa.path, clusterDir.Name())) if err != nil { - log.Fatalf("ARCHIVE/FSBACKEND > Reading jobs failed @ lvl1 dirs: %s", err.Error()) + log.Fatalf("Reading jobs failed @ lvl1 dirs: %s", err.Error()) } for _, lvl1Dir := range lvl1Dirs { @@ -138,21 +138,21 @@ func (fsa *FsArchive) Iter() <-chan *schema.JobMeta { lvl2Dirs, err := os.ReadDir(filepath.Join(fsa.path, clusterDir.Name(), lvl1Dir.Name())) if err != nil { - log.Fatalf("ARCHIVE/FSBACKEND > Reading jobs failed @ lvl2 dirs: %s", err.Error()) + log.Fatalf("Reading jobs failed @ lvl2 dirs: %s", err.Error()) } for _, lvl2Dir := range lvl2Dirs { dirpath := filepath.Join(fsa.path, clusterDir.Name(), lvl1Dir.Name(), lvl2Dir.Name()) startTimeDirs, err := os.ReadDir(dirpath) if err != nil { - log.Fatalf("ARCHIVE/FSBACKEND > Reading jobs failed @ starttime dirs: %s", err.Error()) + log.Fatalf("Reading jobs failed @ starttime dirs: %s", err.Error()) } for _, startTimeDir := range startTimeDirs { if startTimeDir.IsDir() { job, err := loadJobMeta(filepath.Join(dirpath, startTimeDir.Name(), "meta.json")) if err != nil { - log.Errorf("ARCHIVE/FSBACKEND > error in %s: %s", filepath.Join(dirpath, startTimeDir.Name()), err.Error()) + log.Errorf("error in %s: %s", filepath.Join(dirpath, startTimeDir.Name()), err.Error()) } else { ch <- job } diff --git a/pkg/archive/nodelist.go b/pkg/archive/nodelist.go index 4daecff..c22d99d 100644 --- a/pkg/archive/nodelist.go +++ b/pkg/archive/nodelist.go @@ -64,7 +64,7 @@ type NLExprIntRange struct { func (nle NLExprIntRange) consume(input string) (next string, ok bool) { if !nle.zeroPadded || nle.digits < 1 { - log.Error("ARCHIVE/NODELIST > only zero-padded ranges are allowed") + log.Error("only zero-padded ranges are allowed") return "", false } diff --git a/pkg/log/log.go b/pkg/log/log.go index 0fb8bf7..e7cabb0 100644 --- a/pkg/log/log.go +++ b/pkg/log/log.go @@ -9,122 +9,274 @@ import ( "io" "log" "os" + "time" ) // Provides a simple way of logging with different levels. -// Time/Data are not logged on purpose because systemd adds -// them for us. +// Time/Date are not logged because systemd adds +// them for us (Default, can be changed by flag '--logdate true'). // // Uses these prefixes: https://www.freedesktop.org/software/systemd/man/sd-daemon.html +var logDateTime bool +var logLevel string + var ( DebugWriter io.Writer = os.Stderr + NoteWriter io.Writer = os.Stderr InfoWriter io.Writer = os.Stderr WarnWriter io.Writer = os.Stderr ErrWriter io.Writer = os.Stderr + CritWriter io.Writer = os.Stderr ) var ( - DebugPrefix string = "<7>[DEBUG] " - InfoPrefix string = "<6>[INFO] " - WarnPrefix string = "<4>[WARNING] " - ErrPrefix string = "<3>[ERROR] " + DebugPrefix string = "<7>[DEBUG] " + InfoPrefix string = "<6>[INFO] " + NotePrefix string = "<5>[NOTICE] " + WarnPrefix string = "<4>[WARNING] " + ErrPrefix string = "<3>[ERROR] " + CritPrefix string = "<2>[CRITICAL] " ) var ( + // No Time/Date DebugLog *log.Logger = log.New(DebugWriter, DebugPrefix, 0) - InfoLog *log.Logger = log.New(InfoWriter, InfoPrefix, 0) - WarnLog *log.Logger = log.New(WarnWriter, WarnPrefix, 0) - ErrLog *log.Logger = log.New(ErrWriter, ErrPrefix, 0) + InfoLog *log.Logger = log.New(InfoWriter, InfoPrefix, 0) + NoteLog *log.Logger = log.New(NoteWriter, NotePrefix, log.Lshortfile) + WarnLog *log.Logger = log.New(WarnWriter, WarnPrefix, log.Lshortfile) + ErrLog *log.Logger = log.New(ErrWriter, ErrPrefix, log.Llongfile) + CritLog *log.Logger = log.New(CritWriter, CritPrefix, log.Llongfile) + // Log Time/Date + DebugTimeLog *log.Logger = log.New(DebugWriter, DebugPrefix, log.LstdFlags) + InfoTimeLog *log.Logger = log.New(InfoWriter, InfoPrefix, log.LstdFlags) + NoteTimeLog *log.Logger = log.New(NoteWriter, NotePrefix, log.LstdFlags|log.Lshortfile) + WarnTimeLog *log.Logger = log.New(WarnWriter, WarnPrefix, log.LstdFlags|log.Lshortfile) + ErrTimeLog *log.Logger = log.New(ErrWriter, ErrPrefix, log.LstdFlags|log.Llongfile) + CritTimeLog *log.Logger = log.New(CritWriter, CritPrefix, log.LstdFlags|log.Llongfile) ) -func init() { - if lvl, ok := os.LookupEnv("LOGLEVEL"); ok { - switch lvl { +/* CONFIG */ + +func SetLogLevel(lvl string) { + // fmt.Printf("pkg/log: Set LOGLEVEL -> %s\n", lvl) + switch lvl { + case "crit": + ErrWriter = io.Discard + fallthrough case "err", "fatal": WarnWriter = io.Discard fallthrough case "warn": InfoWriter = io.Discard fallthrough + case "notice": + NoteWriter = io.Discard + fallthrough case "info": DebugWriter = io.Discard + break case "debug": // Nothing to do... + break default: - Warnf("environment variable LOGLEVEL has invalid value %#v", lvl) - } + fmt.Printf("pkg/log: Flag 'loglevel' has invalid value %#v\npkg/log: Will use default loglevel 'debug'\n", lvl) + SetLogLevel("debug") } } -func Debug(v ...interface{}) { - if DebugWriter != io.Discard { - DebugLog.Print(v...) - } +func SetLogDateTime(logdate bool) { + //fmt.Printf("pkg/log: Set DATEBOOL -> %v\n", logdate) + logDateTime = logdate } -func Info(v ...interface{}) { - if InfoWriter != io.Discard { - InfoLog.Print(v...) - } +/* PRINT */ + +// Private helper +func printStr(v ...interface{}) string { + return fmt.Sprint(v...) } func Print(v ...interface{}) { Info(v...) } +func Debug(v ...interface{}) { + if DebugWriter != io.Discard { + out := printStr(v...) + if logDateTime { + DebugTimeLog.Output(2, out) + } else { + DebugLog.Output(2, out) + } + } +} + +func Info(v ...interface{}) { + if InfoWriter != io.Discard { + out := printStr(v...) + if logDateTime { + InfoTimeLog.Output(2, out) + } else { + InfoLog.Output(2, out) + } + } +} + +func Note(v ...interface{}) { + if NoteWriter != io.Discard { + out := printStr(v...) + if logDateTime { + NoteTimeLog.Output(2, out) + } else { + NoteLog.Output(2, out) + } + } +} + func Warn(v ...interface{}) { if WarnWriter != io.Discard { - WarnLog.Print(v...) + out := printStr(v...) + if logDateTime { + WarnTimeLog.Output(2, out) + } else { + WarnLog.Output(2, out) + } } } func Error(v ...interface{}) { if ErrWriter != io.Discard { - ErrLog.Print(v...) + out := printStr(v...) + if logDateTime { + ErrTimeLog.Output(2, out) + } else { + ErrLog.Output(2, out) + } } } +// Writes panic stacktrace, keeps application alive +func Panic(v ...interface{}) { + Error(v...) + panic("Panic triggered ...") +} + +// Writes error log, stops application func Fatal(v ...interface{}) { Error(v...) os.Exit(1) } -func Debugf(format string, v ...interface{}) { - if DebugWriter != io.Discard { - DebugLog.Printf(format, v...) +func Crit(v ...interface{}) { + if CritWriter != io.Discard { + out := printStr(v...) + if logDateTime { + CritTimeLog.Output(2, out) + } else { + CritLog.Output(2, out) + } } } -func Infof(format string, v ...interface{}) { - if InfoWriter != io.Discard { - InfoLog.Printf(format, v...) - } +/* PRINT FORMAT*/ + +// Private helper +func printfStr(format string, v ...interface{}) string { + return fmt.Sprintf(format, v...) } func Printf(format string, v ...interface{}) { Infof(format, v...) } -func Finfof(w io.Writer, format string, v ...interface{}) { - if w != io.Discard { - fmt.Fprintf(InfoWriter, InfoPrefix+format+"\n", v...) +func Debugf(format string, v ...interface{}) { + if DebugWriter != io.Discard { + out := printfStr(format, v...) + if logDateTime { + DebugTimeLog.Output(2, out) + } else { + DebugLog.Output(2, out) + } + } +} + +func Infof(format string, v ...interface{}) { + if InfoWriter != io.Discard { + out := printfStr(format, v...) + if logDateTime { + InfoTimeLog.Output(2, out) + } else { + InfoLog.Output(2, out) + } + } +} + +func Notef(format string, v ...interface{}) { + if NoteWriter != io.Discard { + out := printfStr(format, v...) + if logDateTime { + NoteTimeLog.Output(2, out) + } else { + NoteLog.Output(2, out) + } } } func Warnf(format string, v ...interface{}) { if WarnWriter != io.Discard { - WarnLog.Printf(format, v...) + out := printfStr(format, v...) + if logDateTime { + WarnTimeLog.Output(2, out) + } else { + WarnLog.Output(2, out) + } } } func Errorf(format string, v ...interface{}) { if ErrWriter != io.Discard { - ErrLog.Printf(format, v...) + out := printfStr(format, v...) + if logDateTime { + ErrTimeLog.Output(2, out) + } else { + ErrLog.Output(2, out) + } } } +// Writes panic stacktrace, keeps application alive +func Panicf(format string, v ...interface{}) { + Errorf(format, v...) + panic("Panic triggered ...") +} + +// Writes error log, stops application func Fatalf(format string, v ...interface{}) { Errorf(format, v...) os.Exit(1) } + +func Critf(format string, v ...interface{}) { + if CritWriter != io.Discard { + out := printfStr(format, v...) + if logDateTime { + CritTimeLog.Output(2, out) + } else { + CritLog.Output(2, out) + } + } +} + +/* SPECIAL */ + +func Finfof(w io.Writer, format string, v ...interface{}) { + if w != io.Discard { + if logDateTime { + currentTime := time.Now() + fmt.Fprintf(InfoWriter, currentTime.String()+InfoPrefix+format+"\n", v...) + } else { + fmt.Fprintf(InfoWriter, InfoPrefix+format+"\n", v...) + } + } +} diff --git a/pkg/schema/validate.go b/pkg/schema/validate.go index 52730c0..b3eee6e 100644 --- a/pkg/schema/validate.go +++ b/pkg/schema/validate.go @@ -54,7 +54,7 @@ func Validate(k Kind, r io.Reader) (err error) { var v interface{} if err := json.NewDecoder(r).Decode(&v); err != nil { - log.Errorf("SCHEMA/VALIDATE > Failed to decode %v", err) + log.Errorf("Failed to decode %v", err) return err } From bd770d0e32600a832c0c562a68790b35c30498a6 Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Tue, 24 Jan 2023 12:02:29 +0100 Subject: [PATCH 05/12] Fix return error, fix wrong log path, add notice log where applicable - Adresses issue #26 --- internal/graph/schema.resolvers.go | 2 +- internal/metricdata/cc-metric-store.go | 2 +- internal/metricdata/influxdb-v2.go | 9 ++-- internal/metricdata/prometheus.go | 4 +- internal/repository/job.go | 4 ++ pkg/log/log.go | 66 ++++++++++++++++++++------ 6 files changed, 65 insertions(+), 22 deletions(-) diff --git a/internal/graph/schema.resolvers.go b/internal/graph/schema.resolvers.go index 5c927f1..139da17 100644 --- a/internal/graph/schema.resolvers.go +++ b/internal/graph/schema.resolvers.go @@ -52,7 +52,7 @@ func (r *mutationResolver) CreateTag(ctx context.Context, typeArg string, name s // DeleteTag is the resolver for the deleteTag field. func (r *mutationResolver) DeleteTag(ctx context.Context, id string) (string, error) { - log.Panic(fmt.Errorf("not implemented: DeleteTag - deleteTag")) + panic(fmt.Errorf("not implemented: DeleteTag - deleteTag")) } // AddTagsToJob is the resolver for the addTagsToJob field. diff --git a/internal/metricdata/cc-metric-store.go b/internal/metricdata/cc-metric-store.go index 027c0e0..65d0742 100644 --- a/internal/metricdata/cc-metric-store.go +++ b/internal/metricdata/cc-metric-store.go @@ -273,7 +273,7 @@ func (ccms *CCMetricStore) buildQueries( mc := archive.GetMetricConfig(job.Cluster, metric) if mc == nil { // return nil, fmt.Errorf("METRICDATA/CCMS > metric '%s' is not specified for cluster '%s'", metric, job.Cluster) - // log.Printf("METRICDATA/CCMS > metric '%s' is not specified for cluster '%s'", metric, job.Cluster) + log.Notef("metric '%s' is not specified for cluster '%s'", metric, job.Cluster) continue } diff --git a/internal/metricdata/influxdb-v2.go b/internal/metricdata/influxdb-v2.go index 8c1e428..349f8a8 100644 --- a/internal/metricdata/influxdb-v2.go +++ b/internal/metricdata/influxdb-v2.go @@ -97,10 +97,10 @@ func (idb *InfluxDBv2DataRepository) LoadData( idb.formatTime(job.StartTime), idb.formatTime(idb.epochToTime(job.StartTimeUnix+int64(job.Duration)+int64(1))), measurementsCond, hostsCond) case "socket": - log.Info("Scope 'socket' requested, but not yet supported: Will return 'node' scope only. ") + log.Note("Scope 'socket' requested, but not yet supported: Will return 'node' scope only. ") continue case "core": - log.Info(" Scope 'core' requested, but not yet supported: Will return 'node' scope only. ") + log.Note(" Scope 'core' requested, but not yet supported: Will return 'node' scope only. ") continue // Get Finest Granularity only, Set NULL to 0.0 // query = fmt.Sprintf(` @@ -114,7 +114,7 @@ func (idb *InfluxDBv2DataRepository) LoadData( // idb.formatTime(job.StartTime), idb.formatTime(idb.epochToTime(job.StartTimeUnix + int64(job.Duration) + int64(1) )), // measurementsCond, hostsCond) default: - log.Info("Unknown Scope requested: Will return 'node' scope. ") + log.Notef("Unknown scope '%s' requested: Will return 'node' scope.", scope) continue // return nil, errors.New("METRICDATA/INFLUXV2 > the InfluxDB metric data repository does not yet support other scopes than 'node'") } @@ -192,6 +192,7 @@ func (idb *InfluxDBv2DataRepository) LoadData( // hostSeries.Data = append(hostSeries.Data, schema.Float(val)) // } default: + log.Notef("Unknown scope '%s' requested: Will return 'node' scope.", scope) continue // return nil, errors.New("the InfluxDB metric data repository does not yet support other scopes than 'node, core'") } @@ -319,7 +320,7 @@ func (idb *InfluxDBv2DataRepository) LoadNodeData( ctx context.Context) (map[string]map[string][]*schema.JobMetric, error) { // TODO : Implement to be used in Analysis- und System/Node-View - log.Infof("LoadNodeData unimplemented for InfluxDBv2DataRepository, Args: cluster %s, metrics %v, nodes %v, scopes %v", cluster, metrics, nodes, scopes) + log.Notef("LoadNodeData unimplemented for InfluxDBv2DataRepository, Args: cluster %s, metrics %v, nodes %v, scopes %v", cluster, metrics, nodes, scopes) return nil, errors.New("METRICDATA/INFLUXV2 > unimplemented for InfluxDBv2DataRepository") } diff --git a/internal/metricdata/prometheus.go b/internal/metricdata/prometheus.go index 7d51218..3c79866 100644 --- a/internal/metricdata/prometheus.go +++ b/internal/metricdata/prometheus.go @@ -283,7 +283,7 @@ func (pdb *PrometheusDataRepository) LoadData( for _, scope := range scopes { if scope != schema.MetricScopeNode { - logOnce.Do(func(){log.Infof("Scope '%s' requested, but not yet supported: Will return 'node' scope only.", scope)}) + logOnce.Do(func(){log.Notef("Scope '%s' requested, but not yet supported: Will return 'node' scope only.", scope)}) continue } @@ -389,7 +389,7 @@ func (pdb *PrometheusDataRepository) LoadNodeData( } for _, scope := range scopes { if scope != schema.MetricScopeNode { - logOnce.Do(func(){log.Infof("Note: Scope '%s' requested, but not yet supported: Will return 'node' scope only.", scope)}) + logOnce.Do(func(){log.Notef("Note: Scope '%s' requested, but not yet supported: Will return 'node' scope only.", scope)}) continue } for _, metric := range metrics { diff --git a/internal/repository/job.go b/internal/repository/job.go index 76db377..0ade90b 100644 --- a/internal/repository/job.go +++ b/internal/repository/job.go @@ -292,6 +292,8 @@ func (r *JobRepository) CountGroupedJobs(ctx context.Context, aggreg model.Aggre now := time.Now().Unix() count = fmt.Sprintf(`sum(job.num_nodes * (CASE WHEN job.job_state = "running" THEN %d - job.start_time ELSE job.duration END)) as count`, now) runner = r.DB + default: + log.Notef("CountGroupedJobs() Weight %v unknown.", *weight) } } @@ -356,6 +358,8 @@ func (r *JobRepository) MarkArchived( stmt = stmt.Set("net_bw_avg", stats.Avg) case "file_bw": stmt = stmt.Set("file_bw_avg", stats.Avg) + default: + log.Notef("MarkArchived() Metric %s unknown.", metric) } } diff --git a/pkg/log/log.go b/pkg/log/log.go index e7cabb0..c223566 100644 --- a/pkg/log/log.go +++ b/pkg/log/log.go @@ -97,6 +97,8 @@ func printStr(v ...interface{}) string { return fmt.Sprint(v...) } +// Uses Info() -> If errorpath required at some point: +// Will need own writer with 'Output(2, out)' to correctly render path func Print(v ...interface{}) { Info(v...) } @@ -156,17 +158,20 @@ func Error(v ...interface{}) { } } -// Writes panic stacktrace, keeps application alive +// Writes panic stacktrace, but keeps application alive func Panic(v ...interface{}) { - Error(v...) + if ErrWriter != io.Discard { + out := printStr(v...) + if logDateTime { + ErrTimeLog.Output(2, out) + } else { + ErrLog.Output(2, out) + } + } + panic("Panic triggered ...") } -// Writes error log, stops application -func Fatal(v ...interface{}) { - Error(v...) - os.Exit(1) -} func Crit(v ...interface{}) { if CritWriter != io.Discard { @@ -179,6 +184,20 @@ func Crit(v ...interface{}) { } } +// Writes critical log, stops application +func Fatal(v ...interface{}) { + if CritWriter != io.Discard { + out := printStr(v...) + if logDateTime { + CritTimeLog.Output(2, out) + } else { + CritLog.Output(2, out) + } + } + + os.Exit(1) +} + /* PRINT FORMAT*/ // Private helper @@ -186,6 +205,8 @@ func printfStr(format string, v ...interface{}) string { return fmt.Sprintf(format, v...) } +// Uses Infof() -> If errorpath required at some point: +// Will need own writer with 'Output(2, out)' to correctly render path func Printf(format string, v ...interface{}) { Infof(format, v...) } @@ -245,17 +266,20 @@ func Errorf(format string, v ...interface{}) { } } -// Writes panic stacktrace, keeps application alive +// Writes panic stacktrace, but keeps application alive func Panicf(format string, v ...interface{}) { - Errorf(format, v...) + if ErrWriter != io.Discard { + out := printfStr(format, v...) + if logDateTime { + ErrTimeLog.Output(2, out) + } else { + ErrLog.Output(2, out) + } + } + panic("Panic triggered ...") } -// Writes error log, stops application -func Fatalf(format string, v ...interface{}) { - Errorf(format, v...) - os.Exit(1) -} func Critf(format string, v ...interface{}) { if CritWriter != io.Discard { @@ -268,6 +292,20 @@ func Critf(format string, v ...interface{}) { } } +// Writes crit log, stops application +func Fatalf(format string, v ...interface{}) { + if CritWriter != io.Discard { + out := printfStr(format, v...) + if logDateTime { + CritTimeLog.Output(2, out) + } else { + CritLog.Output(2, out) + } + } + + os.Exit(1) +} + /* SPECIAL */ func Finfof(w io.Writer, format string, v ...interface{}) { From 25eb3bb481d72106912870e2fb0d05cc1bbec7d4 Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Tue, 24 Jan 2023 12:03:36 +0100 Subject: [PATCH 06/12] Add missing dependency --- internal/metricdata/cc-metric-store.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/metricdata/cc-metric-store.go b/internal/metricdata/cc-metric-store.go index 65d0742..9875390 100644 --- a/internal/metricdata/cc-metric-store.go +++ b/internal/metricdata/cc-metric-store.go @@ -17,6 +17,7 @@ import ( "github.com/ClusterCockpit/cc-backend/pkg/archive" "github.com/ClusterCockpit/cc-backend/pkg/schema" + "github.com/ClusterCockpit/cc-backend/pkg/log" ) type CCMetricStoreConfig struct { From b77bd078e59ccb27112fba123b90d3f39510e1a8 Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Tue, 31 Jan 2023 18:28:44 +0100 Subject: [PATCH 07/12] Add log messages to error events w/o log message, primaryly error level - "log spam" to be controlled via loglevel flag on startup --- internal/auth/auth.go | 7 +++++++ internal/auth/jwt.go | 10 ++++++++++ internal/auth/ldap.go | 12 ++++++++++++ internal/auth/users.go | 14 +++++++++++++ internal/graph/schema.resolvers.go | 17 ++++++++++++++++ internal/graph/stats.go | 14 +++++++++++++ internal/metricdata/cc-metric-store.go | 10 ++++++++++ internal/metricdata/influxdb-v2.go | 4 ++++ internal/metricdata/metricdata.go | 8 ++++++++ internal/metricdata/prometheus.go | 5 +++++ internal/repository/init.go | 23 ++++++++++++++++++++++ internal/repository/job.go | 27 +++++++++++++++++++++----- internal/repository/query.go | 3 +++ internal/repository/tags.go | 9 +++++++++ internal/repository/user.go | 8 +++++++- internal/runtimeEnv/setup.go | 7 +++++++ pkg/archive/archive.go | 6 ++++++ pkg/archive/clusterConfig.go | 2 ++ pkg/archive/fsBackend.go | 27 ++++++++++++++++++++------ pkg/archive/json.go | 7 +++++++ pkg/schema/float.go | 3 +++ pkg/schema/validate.go | 5 +++-- 22 files changed, 214 insertions(+), 14 deletions(-) diff --git a/internal/auth/auth.go b/internal/auth/auth.go index 4d5fe5e..1b3ac38 100644 --- a/internal/auth/auth.go +++ b/internal/auth/auth.go @@ -94,6 +94,7 @@ func Init(db *sqlx.DB, roles varchar(255) NOT NULL DEFAULT "[]", email varchar(255) DEFAULT NULL);`) if err != nil { + log.Error("Error while initializing authentication -> create user table failed") return nil, err } @@ -102,12 +103,14 @@ func Init(db *sqlx.DB, log.Warn("environment variable 'SESSION_KEY' not set (will use non-persistent random key)") bytes := make([]byte, 32) if _, err := rand.Read(bytes); err != nil { + log.Error("Error while initializing authentication -> failed to generate random bytes for session key") return nil, err } auth.sessionStore = sessions.NewCookieStore(bytes) } else { bytes, err := base64.StdEncoding.DecodeString(sessKey) if err != nil { + log.Error("Error while initializing authentication -> decoding session key failed") return nil, err } auth.sessionStore = sessions.NewCookieStore(bytes) @@ -115,12 +118,14 @@ func Init(db *sqlx.DB, auth.LocalAuth = &LocalAuthenticator{} if err := auth.LocalAuth.Init(auth, nil); err != nil { + log.Error("Error while initializing authentication -> localAuth init failed") return nil, err } auth.authenticators = append(auth.authenticators, auth.LocalAuth) auth.JwtAuth = &JWTAuthenticator{} if err := auth.JwtAuth.Init(auth, configs["jwt"]); err != nil { + log.Error("Error while initializing authentication -> jwtAuth init failed") return nil, err } auth.authenticators = append(auth.authenticators, auth.JwtAuth) @@ -128,6 +133,7 @@ func Init(db *sqlx.DB, if config, ok := configs["ldap"]; ok { auth.LdapAuth = &LdapAuthenticator{} if err := auth.LdapAuth.Init(auth, config); err != nil { + log.Error("Error while initializing authentication -> ldapAuth init failed") return nil, err } auth.authenticators = append(auth.authenticators, auth.LdapAuth) @@ -142,6 +148,7 @@ func (auth *Authentication) AuthViaSession( session, err := auth.sessionStore.Get(r, "session") if err != nil { + log.Error("Error while getting session store") return nil, err } diff --git a/internal/auth/jwt.go b/internal/auth/jwt.go index 798023e..45b1473 100644 --- a/internal/auth/jwt.go +++ b/internal/auth/jwt.go @@ -45,11 +45,13 @@ func (ja *JWTAuthenticator) Init(auth *Authentication, conf interface{}) error { } else { bytes, err := base64.StdEncoding.DecodeString(pubKey) if err != nil { + log.Error("Could not decode JWT public key") return err } ja.publicKey = ed25519.PublicKey(bytes) bytes, err = base64.StdEncoding.DecodeString(privKey) if err != nil { + log.Error("Could not decode JWT private key") return err } ja.privateKey = ed25519.PrivateKey(bytes) @@ -58,6 +60,7 @@ func (ja *JWTAuthenticator) Init(auth *Authentication, conf interface{}) error { if pubKey = os.Getenv("CROSS_LOGIN_JWT_HS512_KEY"); pubKey != "" { bytes, err := base64.StdEncoding.DecodeString(pubKey) if err != nil { + log.Error("Could not decode cross login JWT HS512 key") return err } ja.loginTokenKey = bytes @@ -68,6 +71,7 @@ func (ja *JWTAuthenticator) Init(auth *Authentication, conf interface{}) error { if keyFound && pubKeyCrossLogin != "" { bytes, err := base64.StdEncoding.DecodeString(pubKeyCrossLogin) if err != nil { + log.Error("Could not decode cross login JWT public key") return err } ja.publicKeyCrossLogin = ed25519.PublicKey(bytes) @@ -126,10 +130,12 @@ func (ja *JWTAuthenticator) Login( return nil, fmt.Errorf("AUTH/JWT > unkown signing method for login token: %s (known: HS256, HS512, EdDSA)", t.Method.Alg()) }) if err != nil { + log.Error("Error while parsing jwt token") return nil, err } if err := token.Claims.Valid(); err != nil { + log.Warn("jwt token claims are not valid") return nil, err } @@ -151,6 +157,7 @@ func (ja *JWTAuthenticator) Login( if user == nil { user, err = ja.auth.GetUser(sub) if err != nil && err != sql.ErrNoRows { + log.Errorf("Error while loading user '%#v'", sub) return nil, err } else if user == nil { user = &User{ @@ -159,6 +166,7 @@ func (ja *JWTAuthenticator) Login( AuthSource: AuthViaToken, } if err := ja.auth.AddUser(user); err != nil { + log.Errorf("Error while adding user '%#v' to auth from token", user.Username) return nil, err } } @@ -223,11 +231,13 @@ func (ja *JWTAuthenticator) Auth( return ja.publicKey, nil }) if err != nil { + log.Error("Error while parsing token") return nil, err } // Check token validity if err := token.Claims.Valid(); err != nil { + log.Warn("jwt token claims are not valid") return nil, err } diff --git a/internal/auth/ldap.go b/internal/auth/ldap.go index 1e0235c..f1b8fe8 100644 --- a/internal/auth/ldap.go +++ b/internal/auth/ldap.go @@ -39,10 +39,12 @@ func (la *LdapAuthenticator) Init( if la.config != nil && la.config.SyncInterval != "" { interval, err := time.ParseDuration(la.config.SyncInterval) if err != nil { + log.Errorf("Could not parse duration for sync interval: %#v", la.config.SyncInterval) return err } if interval == 0 { + log.Note("Sync interval is zero") return nil } @@ -76,12 +78,14 @@ func (la *LdapAuthenticator) Login( l, err := la.getLdapConnection(false) if err != nil { + log.Error("Error while getting ldap connection") return nil, err } defer l.Close() userDn := strings.Replace(la.config.UserBind, "{username}", user.Username, -1) if err := l.Bind(userDn, r.FormValue("password")); err != nil { + log.Error("Error while binding to ldap connection") return nil, err } @@ -104,12 +108,14 @@ func (la *LdapAuthenticator) Sync() error { users := map[string]int{} rows, err := la.auth.db.Query(`SELECT username FROM user WHERE user.ldap = 1`) if err != nil { + log.Error("Error while querying LDAP users") return err } for rows.Next() { var username string if err := rows.Scan(&username); err != nil { + log.Errorf("Error while scanning for user '%s'", username) return err } @@ -118,6 +124,7 @@ func (la *LdapAuthenticator) Sync() error { l, err := la.getLdapConnection(true) if err != nil { + log.Error("LDAP connection error") return err } defer l.Close() @@ -126,6 +133,7 @@ func (la *LdapAuthenticator) Sync() error { la.config.UserBase, ldap.ScopeWholeSubtree, ldap.NeverDerefAliases, 0, 0, false, la.config.UserFilter, []string{"dn", "uid", "gecos"}, nil)) if err != nil { + log.Error("LDAP search error") return err } @@ -149,6 +157,7 @@ func (la *LdapAuthenticator) Sync() error { if where == IN_DB && la.config.SyncDelOldUsers { log.Debugf("sync: remove %#v (does not show up in LDAP anymore)", username) if _, err := la.auth.db.Exec(`DELETE FROM user WHERE user.username = ?`, username); err != nil { + log.Errorf("User '%s' not in LDAP anymore: Delete from DB failed", username) return err } } else if where == IN_LDAP { @@ -156,6 +165,7 @@ func (la *LdapAuthenticator) Sync() error { log.Debugf("sync: add %#v (name: %#v, roles: [user], ldap: true)", username, name) if _, err := la.auth.db.Exec(`INSERT INTO user (username, ldap, name, roles) VALUES (?, ?, ?, ?)`, username, 1, name, "[\""+RoleUser+"\"]"); err != nil { + log.Errorf("User '%s' new in LDAP: Insert into DB failed", username) return err } } @@ -170,12 +180,14 @@ func (la *LdapAuthenticator) getLdapConnection(admin bool) (*ldap.Conn, error) { conn, err := ldap.DialURL(la.config.Url) if err != nil { + log.Error("LDAP URL dial failed") return nil, err } if admin { if err := conn.Bind(la.config.SearchDN, la.syncPassword); err != nil { conn.Close() + log.Error("LDAP connection bind failed") return nil, err } } diff --git a/internal/auth/users.go b/internal/auth/users.go index f890c1b..093fb14 100644 --- a/internal/auth/users.go +++ b/internal/auth/users.go @@ -25,6 +25,7 @@ func (auth *Authentication) GetUser(username string) (*User, error) { if err := sq.Select("password", "ldap", "name", "roles", "email").From("user"). Where("user.username = ?", username).RunWith(auth.db). QueryRow().Scan(&hashedPassword, &user.AuthSource, &name, &rawRoles, &email); err != nil { + log.Errorf("Error while querying user '%#v' from database", username) return nil, err } @@ -33,6 +34,7 @@ func (auth *Authentication) GetUser(username string) (*User, error) { user.Email = email.String if rawRoles.Valid { if err := json.Unmarshal([]byte(rawRoles.String), &user.Roles); err != nil { + log.Error("Error while unmarshaling raw roles from DB") return nil, err } } @@ -57,6 +59,7 @@ func (auth *Authentication) AddUser(user *User) error { if user.Password != "" { password, err := bcrypt.GenerateFromPassword([]byte(user.Password), bcrypt.DefaultCost) if err != nil { + log.Error("Error while encrypting new user password") return err } cols = append(cols, "password") @@ -64,6 +67,7 @@ func (auth *Authentication) AddUser(user *User) error { } if _, err := sq.Insert("user").Columns(cols...).Values(vals...).RunWith(auth.db).Exec(); err != nil { + log.Errorf("Error while inserting new user '%#v' into DB", user.Username) return err } @@ -74,6 +78,7 @@ func (auth *Authentication) AddUser(user *User) error { func (auth *Authentication) DelUser(username string) error { _, err := auth.db.Exec(`DELETE FROM user WHERE user.username = ?`, username) + log.Errorf("Error while deleting user '%s' from DB", username) return err } @@ -86,6 +91,7 @@ func (auth *Authentication) ListUsers(specialsOnly bool) ([]*User, error) { rows, err := q.RunWith(auth.db).Query() if err != nil { + log.Error("Error while querying user list") return nil, err } @@ -96,10 +102,12 @@ func (auth *Authentication) ListUsers(specialsOnly bool) ([]*User, error) { user := &User{} var name, email sql.NullString if err := rows.Scan(&user.Username, &name, &email, &rawroles); err != nil { + log.Error("Error while scanning user list") return nil, err } if err := json.Unmarshal([]byte(rawroles), &user.Roles); err != nil { + log.Error("Error while unmarshaling raw role list") return nil, err } @@ -117,6 +125,7 @@ func (auth *Authentication) AddRole( user, err := auth.GetUser(username) if err != nil { + log.Errorf("Could not load user '%s'", username) return err } @@ -132,6 +141,7 @@ func (auth *Authentication) AddRole( roles, _ := json.Marshal(append(user.Roles, role)) if _, err := sq.Update("user").Set("roles", roles).Where("user.username = ?", username).RunWith(auth.db).Exec(); err != nil { + log.Errorf("Error while adding new role for user '%s'", user.Username) return err } return nil @@ -140,6 +150,7 @@ func (auth *Authentication) AddRole( func (auth *Authentication) RemoveRole(ctx context.Context, username string, role string) error { user, err := auth.GetUser(username) if err != nil { + log.Errorf("Could not load user '%s'", username) return err } @@ -160,6 +171,7 @@ func (auth *Authentication) RemoveRole(ctx context.Context, username string, rol if (exists == true) { var mroles, _ = json.Marshal(newroles) if _, err := sq.Update("user").Set("roles", mroles).Where("user.username = ?", username).RunWith(auth.db).Exec(); err != nil { + log.Errorf("Error while removing role for user '%s'", user.Username) return err } return nil @@ -179,9 +191,11 @@ func FetchUser(ctx context.Context, db *sqlx.DB, username string) (*model.User, if err := sq.Select("name", "email").From("user").Where("user.username = ?", username). RunWith(db).QueryRow().Scan(&name, &email); err != nil { if err == sql.ErrNoRows { + log.Errorf("User '%s' Not found in DB", username) return nil, nil } + log.Errorf("Error while fetching user '%s'", username) return nil, err } diff --git a/internal/graph/schema.resolvers.go b/internal/graph/schema.resolvers.go index 139da17..f2653a6 100644 --- a/internal/graph/schema.resolvers.go +++ b/internal/graph/schema.resolvers.go @@ -44,6 +44,7 @@ func (r *jobResolver) UserData(ctx context.Context, obj *schema.Job) (*model.Use func (r *mutationResolver) CreateTag(ctx context.Context, typeArg string, name string) (*schema.Tag, error) { id, err := r.Repo.CreateTag(typeArg, name) if err != nil { + log.Warn("Error while creating tag") return nil, err } @@ -59,6 +60,7 @@ func (r *mutationResolver) DeleteTag(ctx context.Context, id string) (string, er func (r *mutationResolver) AddTagsToJob(ctx context.Context, job string, tagIds []string) ([]*schema.Tag, error) { jid, err := strconv.ParseInt(job, 10, 64) if err != nil { + log.Warn("Error while adding tag to job") return nil, err } @@ -66,10 +68,12 @@ func (r *mutationResolver) AddTagsToJob(ctx context.Context, job string, tagIds for _, tagId := range tagIds { tid, err := strconv.ParseInt(tagId, 10, 64) if err != nil { + log.Warn("Error while parsing tag id") return nil, err } if tags, err = r.Repo.AddTag(jid, tid); err != nil { + log.Warn("Error while adding tag") return nil, err } } @@ -81,6 +85,7 @@ func (r *mutationResolver) AddTagsToJob(ctx context.Context, job string, tagIds func (r *mutationResolver) RemoveTagsFromJob(ctx context.Context, job string, tagIds []string) ([]*schema.Tag, error) { jid, err := strconv.ParseInt(job, 10, 64) if err != nil { + log.Warn("Error while parsing job id") return nil, err } @@ -88,10 +93,12 @@ func (r *mutationResolver) RemoveTagsFromJob(ctx context.Context, job string, ta for _, tagId := range tagIds { tid, err := strconv.ParseInt(tagId, 10, 64) if err != nil { + log.Warn("Error while parsing tag id") return nil, err } if tags, err = r.Repo.RemoveTag(jid, tid); err != nil { + log.Warn("Error while removing tag") return nil, err } } @@ -102,6 +109,7 @@ func (r *mutationResolver) RemoveTagsFromJob(ctx context.Context, job string, ta // UpdateConfiguration is the resolver for the updateConfiguration field. func (r *mutationResolver) UpdateConfiguration(ctx context.Context, name string, value string) (*string, error) { if err := repository.GetUserCfgRepo().UpdateConfig(name, value, auth.GetUser(ctx)); err != nil { + log.Warn("Error while updating user config") return nil, err } @@ -127,6 +135,7 @@ func (r *queryResolver) User(ctx context.Context, username string) (*model.User, func (r *queryResolver) AllocatedNodes(ctx context.Context, cluster string) ([]*model.Count, error) { data, err := r.Repo.AllocatedNodes(cluster) if err != nil { + log.Warn("Error while fetching allocated nodes") return nil, err } @@ -145,11 +154,13 @@ func (r *queryResolver) AllocatedNodes(ctx context.Context, cluster string) ([]* func (r *queryResolver) Job(ctx context.Context, id string) (*schema.Job, error) { numericId, err := strconv.ParseInt(id, 10, 64) if err != nil { + log.Warn("Error while parsing job id") return nil, err } job, err := r.Repo.FindById(numericId) if err != nil { + log.Warn("Error while finding job by id") return nil, err } @@ -164,11 +175,13 @@ func (r *queryResolver) Job(ctx context.Context, id string) (*schema.Job, error) func (r *queryResolver) JobMetrics(ctx context.Context, id string, metrics []string, scopes []schema.MetricScope) ([]*model.JobMetricWithName, error) { job, err := r.Query().Job(ctx, id) if err != nil { + log.Warn("Error while querying job for metrics") return nil, err } data, err := metricdata.LoadData(job, metrics, scopes, ctx) if err != nil { + log.Warn("Error while loading job data") return nil, err } @@ -205,11 +218,13 @@ func (r *queryResolver) Jobs(ctx context.Context, filter []*model.JobFilter, pag jobs, err := r.Repo.QueryJobs(ctx, filter, page, order) if err != nil { + log.Warn("Error while querying jobs") return nil, err } count, err := r.Repo.CountJobs(ctx, filter) if err != nil { + log.Warn("Error while counting jobs") return nil, err } @@ -225,6 +240,7 @@ func (r *queryResolver) JobsStatistics(ctx context.Context, filter []*model.JobF func (r *queryResolver) JobsCount(ctx context.Context, filter []*model.JobFilter, groupBy model.Aggregate, weight *model.Weights, limit *int) ([]*model.Count, error) { counts, err := r.Repo.CountGroupedJobs(ctx, groupBy, filter, weight, limit) if err != nil { + log.Warn("Error while counting grouped jobs") return nil, err } @@ -258,6 +274,7 @@ func (r *queryResolver) NodeMetrics(ctx context.Context, cluster string, nodes [ data, err := metricdata.LoadNodeData(cluster, metrics, nodes, scopes, from, to, ctx) if err != nil { + log.Warn("Error while loading node data") return nil, err } diff --git a/internal/graph/stats.go b/internal/graph/stats.go index 5d8c4a3..2bdb59e 100644 --- a/internal/graph/stats.go +++ b/internal/graph/stats.go @@ -18,6 +18,7 @@ import ( "github.com/ClusterCockpit/cc-backend/internal/repository" "github.com/ClusterCockpit/cc-backend/pkg/archive" "github.com/ClusterCockpit/cc-backend/pkg/schema" + "github.com/ClusterCockpit/cc-backend/pkg/log" sq "github.com/Masterminds/squirrel" ) @@ -68,6 +69,7 @@ func (r *queryResolver) jobsStatistics(ctx context.Context, filter []*model.JobF rows, err := query.RunWith(r.DB).Query() if err != nil { + log.Error("Error while querying DB for job statistics") return nil, err } @@ -75,6 +77,7 @@ func (r *queryResolver) jobsStatistics(ctx context.Context, filter []*model.JobF var id sql.NullString var jobs, walltime, corehours sql.NullInt64 if err := rows.Scan(&id, &jobs, &walltime, &corehours); err != nil { + log.Error("Error while scanning rows") return nil, err } @@ -103,6 +106,7 @@ func (r *queryResolver) jobsStatistics(ctx context.Context, filter []*model.JobF query = repository.BuildWhereClause(f, query) } if err := query.RunWith(r.DB).QueryRow().Scan(&(stats[""].ShortJobs)); err != nil { + log.Error("Error while scanning rows for short job stats") return nil, err } } else { @@ -114,6 +118,7 @@ func (r *queryResolver) jobsStatistics(ctx context.Context, filter []*model.JobF } rows, err := query.RunWith(r.DB).Query() if err != nil { + log.Error("Error while querying jobs for short jobs") return nil, err } @@ -121,6 +126,7 @@ func (r *queryResolver) jobsStatistics(ctx context.Context, filter []*model.JobF var id sql.NullString var shortJobs sql.NullInt64 if err := rows.Scan(&id, &shortJobs); err != nil { + log.Error("Error while scanning rows for short jobs") return nil, err } @@ -154,11 +160,13 @@ func (r *queryResolver) jobsStatistics(ctx context.Context, filter []*model.JobF value := fmt.Sprintf(`CAST(ROUND((CASE WHEN job.job_state = "running" THEN %d - job.start_time ELSE job.duration END) / 3600) as int) as value`, time.Now().Unix()) stat.HistDuration, err = r.jobsStatisticsHistogram(ctx, value, filter, id, col) if err != nil { + log.Error("Error while loading job statistics histogram: running jobs") return nil, err } stat.HistNumNodes, err = r.jobsStatisticsHistogram(ctx, "job.num_nodes as value", filter, id, col) if err != nil { + log.Error("Error while loading job statistics histogram: num nodes") return nil, err } } @@ -182,6 +190,7 @@ func (r *queryResolver) jobsStatisticsHistogram(ctx context.Context, value strin rows, err := query.GroupBy("value").RunWith(r.DB).Query() if err != nil { + log.Error("Error while running query") return nil, err } @@ -189,6 +198,7 @@ func (r *queryResolver) jobsStatisticsHistogram(ctx context.Context, value strin for rows.Next() { point := model.HistoPoint{} if err := rows.Scan(&point.Value, &point.Count); err != nil { + log.Error("Error while scanning rows") return nil, err } @@ -208,6 +218,7 @@ func (r *queryResolver) rooflineHeatmap( jobs, err := r.Repo.QueryJobs(ctx, filter, &model.PageRequest{Page: 1, ItemsPerPage: MAX_JOBS_FOR_ANALYSIS + 1}, nil) if err != nil { + log.Error("Error while querying jobs for roofline") return nil, err } if len(jobs) > MAX_JOBS_FOR_ANALYSIS { @@ -228,6 +239,7 @@ func (r *queryResolver) rooflineHeatmap( jobdata, err := metricdata.LoadData(job, []string{"flops_any", "mem_bw"}, []schema.MetricScope{schema.MetricScopeNode}, ctx) if err != nil { + log.Error("Error while loading metrics for roofline") return nil, err } @@ -272,6 +284,7 @@ func (r *queryResolver) rooflineHeatmap( func (r *queryResolver) jobsFootprints(ctx context.Context, filter []*model.JobFilter, metrics []string) (*model.Footprints, error) { jobs, err := r.Repo.QueryJobs(ctx, filter, &model.PageRequest{Page: 1, ItemsPerPage: MAX_JOBS_FOR_ANALYSIS + 1}, nil) if err != nil { + log.Error("Error while querying jobs for footprint") return nil, err } if len(jobs) > MAX_JOBS_FOR_ANALYSIS { @@ -290,6 +303,7 @@ func (r *queryResolver) jobsFootprints(ctx context.Context, filter []*model.JobF } if err := metricdata.LoadAverages(job, metrics, avgs, ctx); err != nil { + log.Error("Error while loading averages for footprint") return nil, err } diff --git a/internal/metricdata/cc-metric-store.go b/internal/metricdata/cc-metric-store.go index 9875390..31e675a 100644 --- a/internal/metricdata/cc-metric-store.go +++ b/internal/metricdata/cc-metric-store.go @@ -79,6 +79,7 @@ func (ccms *CCMetricStore) Init(rawConfig json.RawMessage) error { var config CCMetricStoreConfig if err := json.Unmarshal(rawConfig, &config); err != nil { + log.Error("Error while unmarshaling raw json config") return err } @@ -125,11 +126,13 @@ func (ccms *CCMetricStore) doRequest( buf := &bytes.Buffer{} if err := json.NewEncoder(buf).Encode(body); err != nil { + log.Error("Error while encoding request body") return nil, err } req, err := http.NewRequestWithContext(ctx, http.MethodPost, ccms.queryEndpoint, buf) if err != nil { + log.Error("Error while building request body") return nil, err } if ccms.jwt != "" { @@ -138,6 +141,7 @@ func (ccms *CCMetricStore) doRequest( res, err := ccms.client.Do(req) if err != nil { + log.Error("Error while performing request") return nil, err } @@ -147,6 +151,7 @@ func (ccms *CCMetricStore) doRequest( var resBody ApiQueryResponse if err := json.NewDecoder(bufio.NewReader(res.Body)).Decode(&resBody); err != nil { + log.Error("Error while decoding result body") return nil, err } @@ -162,6 +167,7 @@ func (ccms *CCMetricStore) LoadData( topology := archive.GetSubCluster(job.Cluster, job.SubCluster).Topology queries, assignedScope, err := ccms.buildQueries(job, metrics, scopes) if err != nil { + log.Error("Error while building queries") return nil, err } @@ -176,6 +182,7 @@ func (ccms *CCMetricStore) LoadData( resBody, err := ccms.doRequest(ctx, &req) if err != nil { + log.Error("Error while performing request") return nil, err } @@ -499,6 +506,7 @@ func (ccms *CCMetricStore) LoadStats( queries, _, err := ccms.buildQueries(job, metrics, []schema.MetricScope{schema.MetricScopeNode}) if err != nil { + log.Error("Error while building query") return nil, err } @@ -513,6 +521,7 @@ func (ccms *CCMetricStore) LoadStats( resBody, err := ccms.doRequest(ctx, &req) if err != nil { + log.Error("Error while performing request") return nil, err } @@ -578,6 +587,7 @@ func (ccms *CCMetricStore) LoadNodeData( resBody, err := ccms.doRequest(ctx, &req) if err != nil { + log.Error("Error while performing request") return nil, err } diff --git a/internal/metricdata/influxdb-v2.go b/internal/metricdata/influxdb-v2.go index 349f8a8..62da698 100644 --- a/internal/metricdata/influxdb-v2.go +++ b/internal/metricdata/influxdb-v2.go @@ -37,6 +37,7 @@ type InfluxDBv2DataRepository struct { func (idb *InfluxDBv2DataRepository) Init(rawConfig json.RawMessage) error { var config InfluxDBv2DataRepositoryConfig if err := json.Unmarshal(rawConfig, &config); err != nil { + log.Error("Error while unmarshaling raw json config") return err } @@ -121,6 +122,7 @@ func (idb *InfluxDBv2DataRepository) LoadData( rows, err := idb.queryClient.Query(ctx, query) if err != nil { + log.Error("Error while performing query") return nil, err } @@ -203,6 +205,7 @@ func (idb *InfluxDBv2DataRepository) LoadData( // Get Stats stats, err := idb.LoadStats(job, metrics, ctx) if err != nil { + log.Error("Error while loading statistics") return nil, err } @@ -276,6 +279,7 @@ func (idb *InfluxDBv2DataRepository) LoadStats( rows, err := idb.queryClient.Query(ctx, query) if err != nil { + log.Error("Error while performing query") return nil, err } diff --git a/internal/metricdata/metricdata.go b/internal/metricdata/metricdata.go index 56bcdc6..d022f5e 100644 --- a/internal/metricdata/metricdata.go +++ b/internal/metricdata/metricdata.go @@ -46,6 +46,7 @@ func Init(disableArchive bool) error { Kind string `json:"kind"` } if err := json.Unmarshal(cluster.MetricDataRepository, &kind); err != nil { + log.Error("Error while unmarshaling raw json MetricDataRepository") return err } @@ -64,6 +65,7 @@ func Init(disableArchive bool) error { } if err := mdr.Init(cluster.MetricDataRepository); err != nil { + log.Error("Error initializing the MetricDataRepository") return err } metricDataRepos[cluster.Name] = mdr @@ -109,6 +111,7 @@ func LoadData(job *schema.Job, if len(jd) != 0 { log.Errorf("partial error: %s", err.Error()) } else { + log.Error("Error while loading job data from metric repository") return err, 0, 0 } } @@ -116,6 +119,7 @@ func LoadData(job *schema.Job, } else { jd, err = archive.GetHandle().LoadJobData(job) if err != nil { + log.Error("Error while loading job data from archive") return err, 0, 0 } @@ -163,6 +167,7 @@ func LoadData(job *schema.Job, }) if err, ok := data.(error); ok { + log.Error("Error in returned dataset") return nil, err } @@ -187,6 +192,7 @@ func LoadAverages( stats, err := repo.LoadStats(job, metrics, ctx) if err != nil { + log.Errorf("Error while loading statistics for job %#v (User %#v, Project %#v)", job.JobID, job.User, job.Project) return err } @@ -231,6 +237,7 @@ func LoadNodeData( if len(data) != 0 { log.Errorf("partial error: %s", err.Error()) } else { + log.Error("Error while loading node data from metric repository") return nil, err } } @@ -303,6 +310,7 @@ func ArchiveJob(job *schema.Job, ctx context.Context) (*schema.JobMeta, error) { jobData, err := LoadData(job, allMetrics, scopes, ctx) if err != nil { + log.Error("Error wile loading job data for archiving") return nil, err } diff --git a/internal/metricdata/prometheus.go b/internal/metricdata/prometheus.go index 3c79866..dd8ceb8 100644 --- a/internal/metricdata/prometheus.go +++ b/internal/metricdata/prometheus.go @@ -154,6 +154,7 @@ func (pdb *PrometheusDataRepository) Init(rawConfig json.RawMessage) error { var config PrometheusDataRepositoryConfig // parse config if err := json.Unmarshal(rawConfig, &config); err != nil { + log.Error("Error while unmarshaling raw json config") return err } // support basic authentication @@ -172,6 +173,7 @@ func (pdb *PrometheusDataRepository) Init(rawConfig json.RawMessage) error { RoundTripper: rt, }) if err != nil { + log.Error("Error while initializing new prometheus client") return err } // init query client @@ -295,6 +297,7 @@ func (pdb *PrometheusDataRepository) LoadData( } query, err := pdb.FormatQuery(metric, scope, nodes, job.Cluster) if err != nil { + log.Error("Error while formatting prometheus query") return nil, err } @@ -358,6 +361,7 @@ func (pdb *PrometheusDataRepository) LoadStats( data, err := pdb.LoadData(job, metrics, []schema.MetricScope{schema.MetricScopeNode}, ctx) if err != nil { + log.Error("Error while loading job for stats") return nil, err } for metric, metricData := range data { @@ -400,6 +404,7 @@ func (pdb *PrometheusDataRepository) LoadNodeData( } query, err := pdb.FormatQuery(metric, scope, nodes, cluster) if err != nil { + log.Error("Error while formatting prometheus query") return nil, err } diff --git a/internal/repository/init.go b/internal/repository/init.go index 08b5565..1d3f403 100644 --- a/internal/repository/init.go +++ b/internal/repository/init.go @@ -100,6 +100,7 @@ func HandleImportFlag(flag string) error { raw, err := os.ReadFile(files[0]) if err != nil { + log.Error("Error while reading metadata file for import") return err } @@ -112,11 +113,13 @@ func HandleImportFlag(flag string) error { dec.DisallowUnknownFields() jobMeta := schema.JobMeta{BaseJob: schema.JobDefaults} if err := dec.Decode(&jobMeta); err != nil { + log.Error("Error while decoding raw json metadata for import") return err } raw, err = os.ReadFile(files[1]) if err != nil { + log.Error("Error while reading jobdata file for import") return err } @@ -129,6 +132,7 @@ func HandleImportFlag(flag string) error { dec.DisallowUnknownFields() jobData := schema.JobData{} if err := dec.Decode(&jobData); err != nil { + log.Error("Error while decoding raw json jobdata for import") return err } @@ -136,6 +140,7 @@ func HandleImportFlag(flag string) error { jobMeta.MonitoringStatus = schema.MonitoringStatusArchivingSuccessful if job, err := GetJobRepository().Find(&jobMeta.JobID, &jobMeta.Cluster, &jobMeta.StartTime); err != sql.ErrNoRows { if err != nil { + log.Error("Error while finding job in jobRepository") return err } @@ -155,33 +160,40 @@ func HandleImportFlag(flag string) error { job.FileBwAvg = loadJobStat(&jobMeta, "file_bw") job.RawResources, err = json.Marshal(job.Resources) if err != nil { + log.Error("Error while marshaling job resources") return err } job.RawMetaData, err = json.Marshal(job.MetaData) if err != nil { + log.Error("Error while marshaling job metadata") return err } if err := SanityChecks(&job.BaseJob); err != nil { + log.Error("BaseJob SanityChecks failed") return err } if err := archive.GetHandle().ImportJob(&jobMeta, &jobData); err != nil { + log.Error("Error while importing job") return err } res, err := GetConnection().DB.NamedExec(NamedJobInsert, job) if err != nil { + log.Error("Error while NamedJobInsert") return err } id, err := res.LastInsertId() if err != nil { + log.Error("Error while getting last insert ID") return err } for _, tag := range job.Tags { if _, err := GetJobRepository().AddTagOrCreate(id, tag.Type, tag.Name); err != nil { + log.Error("Error while adding or creating tag") return err } } @@ -201,6 +213,7 @@ func InitDB() error { // Basic database structure: _, err := db.DB.Exec(JobsDBSchema) if err != nil { + log.Error("Error while initializing basic DB structure") return err } @@ -208,11 +221,13 @@ func InitDB() error { // that speeds up inserts A LOT. tx, err := db.DB.Beginx() if err != nil { + log.Error("Error while bundling transactions") return err } stmt, err := tx.PrepareNamed(NamedJobInsert) if err != nil { + log.Error("Error while preparing namedJobInsert") return err } tags := make(map[string]int64) @@ -232,12 +247,14 @@ func InitDB() error { if i%10 == 0 { if tx != nil { if err := tx.Commit(); err != nil { + log.Error("Error while committing transactions for jobMeta") return err } } tx, err = db.DB.Beginx() if err != nil { + log.Error("Error while bundling transactions for jobMeta") return err } @@ -298,16 +315,19 @@ func InitDB() error { if !ok { res, err := tx.Exec(`INSERT INTO tag (tag_name, tag_type) VALUES (?, ?)`, tag.Name, tag.Type) if err != nil { + log.Errorf("Error while inserting tag into tag table: %#v %#v", tag.Name, tag.Type) return err } tagId, err = res.LastInsertId() if err != nil { + log.Error("Error while getting last insert ID") return err } tags[tagstr] = tagId } if _, err := tx.Exec(`INSERT INTO jobtag (job_id, tag_id) VALUES (?, ?)`, id, tagId); err != nil { + log.Errorf("Error while inserting jobtag into jobtag table: %#v %#v", id, tagId) return err } } @@ -322,12 +342,14 @@ func InitDB() error { } if err := tx.Commit(); err != nil { + log.Error("Error while committing SQL transactions") return err } // Create indexes after inserts so that they do not // need to be continually updated. if _, err := db.DB.Exec(JobsDbIndexes); err != nil { + log.Error("Error while creating indices after inserts") return err } @@ -341,6 +363,7 @@ func SanityChecks(job *schema.BaseJob) error { return fmt.Errorf("no such cluster: %#v", job.Cluster) } if err := archive.AssignSubCluster(job); err != nil { + log.Error("Error while assigning subcluster to job") return err } if !job.State.Valid() { diff --git a/internal/repository/job.go b/internal/repository/job.go index 0ade90b..202b678 100644 --- a/internal/repository/job.go +++ b/internal/repository/job.go @@ -68,10 +68,12 @@ func scanJob(row interface{ Scan(...interface{}) error }) (*schema.Job, error) { &job.ID, &job.JobID, &job.User, &job.Project, &job.Cluster, &job.SubCluster, &job.StartTimeUnix, &job.Partition, &job.ArrayJobId, &job.NumNodes, &job.NumHWThreads, &job.NumAcc, &job.Exclusive, &job.MonitoringStatus, &job.SMT, &job.State, &job.Duration, &job.Walltime, &job.RawResources /*&job.MetaData*/); err != nil { + log.Error("Error while scanning rows") return nil, err } if err := json.Unmarshal(job.RawResources, &job.Resources); err != nil { + log.Error("Error while unmarhsaling raw resources json") return nil, err } @@ -93,6 +95,7 @@ func (r *JobRepository) FetchMetadata(job *schema.Job) (map[string]string, error if err := sq.Select("job.meta_data").From("job").Where("job.id = ?", job.ID). RunWith(r.stmtCache).QueryRow().Scan(&job.RawMetaData); err != nil { + log.Error("Error while scanning for job metadata") return nil, err } @@ -101,6 +104,7 @@ func (r *JobRepository) FetchMetadata(job *schema.Job) (map[string]string, error } if err := json.Unmarshal(job.RawMetaData, &job.MetaData); err != nil { + log.Error("Error while unmarshaling raw metadata json") return nil, err } @@ -113,6 +117,7 @@ func (r *JobRepository) UpdateMetadata(job *schema.Job, key, val string) (err er r.cache.Del(cachekey) if job.MetaData == nil { if _, err = r.FetchMetadata(job); err != nil { + log.Errorf("Error while fetching metadata for job, DB ID '%#v'", job.ID) return err } } @@ -129,10 +134,12 @@ func (r *JobRepository) UpdateMetadata(job *schema.Job, key, val string) (err er } if job.RawMetaData, err = json.Marshal(job.MetaData); err != nil { + log.Errorf("Error while marshaling metadata for job, DB ID '%#v'", job.ID) return err } if _, err = sq.Update("job").Set("meta_data", job.RawMetaData).Where("job.id = ?", job.ID).RunWith(r.stmtCache).Exec(); err != nil { + log.Errorf("Error while updating metadata for job, DB ID '%#v'", job.ID) return err } @@ -185,6 +192,7 @@ func (r *JobRepository) FindAll( rows, err := q.RunWith(r.stmtCache).Query() if err != nil { + log.Error("Error while running query") return nil, err } @@ -192,6 +200,7 @@ func (r *JobRepository) FindAll( for rows.Next() { job, err := scanJob(rows) if err != nil { + log.Error("Error while scanning rows") return nil, err } jobs = append(jobs, job) @@ -259,7 +268,7 @@ func (r *JobRepository) DeleteJobsBefore(startTime int64) (int, error) { err := r.DB.Get(&cnt, qs) //ignore error as it will also occur in delete statement _, err = r.DB.Exec(`DELETE FROM job WHERE job.start_time < ?`, startTime) if err != nil { - log.Warnf(" DeleteJobsBefore(%d): error %v", startTime, err) + log.Errorf(" DeleteJobsBefore(%d): error %#v", startTime, err) } else { log.Infof("DeleteJobsBefore(%d): Deleted %d jobs", startTime, cnt) } @@ -269,7 +278,7 @@ func (r *JobRepository) DeleteJobsBefore(startTime int64) (int, error) { func (r *JobRepository) DeleteJobById(id int64) error { _, err := r.DB.Exec(`DELETE FROM job WHERE job.id = ?`, id) if err != nil { - log.Warnf("DeleteJobById(%d): error %v", id, err) + log.Errorf("DeleteJobById(%d): error %#v", id, err) } else { log.Infof("DeleteJobById(%d): Success", id) } @@ -293,7 +302,7 @@ func (r *JobRepository) CountGroupedJobs(ctx context.Context, aggreg model.Aggre count = fmt.Sprintf(`sum(job.num_nodes * (CASE WHEN job.job_state = "running" THEN %d - job.start_time ELSE job.duration END)) as count`, now) runner = r.DB default: - log.Notef("CountGroupedJobs() Weight %v unknown.", *weight) + log.Notef("CountGroupedJobs() Weight %#v unknown.", *weight) } } @@ -309,6 +318,7 @@ func (r *JobRepository) CountGroupedJobs(ctx context.Context, aggreg model.Aggre counts := map[string]int{} rows, err := q.RunWith(runner).Query() if err != nil { + log.Error("Error while running query") return nil, err } @@ -316,6 +326,7 @@ func (r *JobRepository) CountGroupedJobs(ctx context.Context, aggreg model.Aggre var group string var count int if err := rows.Scan(&group, &count); err != nil { + log.Error("Error while scanning rows") return nil, err } @@ -359,11 +370,12 @@ func (r *JobRepository) MarkArchived( case "file_bw": stmt = stmt.Set("file_bw_avg", stats.Avg) default: - log.Notef("MarkArchived() Metric %s unknown.", metric) + log.Notef("MarkArchived() Metric '%#v' unknown", metric) } } if _, err := stmt.RunWith(r.stmtCache).Exec(); err != nil { + log.Error("Error while marking job as archived") return err } return nil @@ -478,6 +490,7 @@ func (r *JobRepository) AllocatedNodes(cluster string) (map[string]map[string]in Where("job.cluster = ?", cluster). RunWith(r.stmtCache).Query() if err != nil { + log.Error("Error while running query") return nil, err } @@ -488,9 +501,11 @@ func (r *JobRepository) AllocatedNodes(cluster string) (map[string]map[string]in var resources []*schema.Resource var subcluster string if err := rows.Scan(&raw, &subcluster); err != nil { + log.Error("Error while scanning rows") return nil, err } if err := json.Unmarshal(raw, &resources); err != nil { + log.Error("Error while unmarshaling raw resources json") return nil, err } @@ -518,16 +533,18 @@ func (r *JobRepository) StopJobsExceedingWalltimeBy(seconds int) error { Where(fmt.Sprintf("(%d - job.start_time) > (job.walltime + %d)", time.Now().Unix(), seconds)). RunWith(r.DB).Exec() if err != nil { + log.Error("Error while stopping jobs exceeding walltime") return err } rowsAffected, err := res.RowsAffected() if err != nil { + log.Error("Error while fetching affected rows after stopping due to exceeded walltime") return err } if rowsAffected > 0 { - log.Warnf("%d jobs have been marked as failed due to running too long", rowsAffected) + log.Notef("%d jobs have been marked as failed due to running too long", rowsAffected) } return nil } diff --git a/internal/repository/query.go b/internal/repository/query.go index 9b86c59..f98b702 100644 --- a/internal/repository/query.go +++ b/internal/repository/query.go @@ -51,12 +51,14 @@ func (r *JobRepository) QueryJobs( sql, args, err := query.ToSql() if err != nil { + log.Error("Error while converting query to sql") return nil, err } log.Debugf("SQL query: `%s`, args: %#v", sql, args) rows, err := query.RunWith(r.stmtCache).Query() if err != nil { + log.Error("Error while running query") return nil, err } @@ -65,6 +67,7 @@ func (r *JobRepository) QueryJobs( job, err := scanJob(rows) if err != nil { rows.Close() + log.Error("Error while scanning rows") return nil, err } jobs = append(jobs, job) diff --git a/internal/repository/tags.go b/internal/repository/tags.go index 07c9910..6ab92e5 100644 --- a/internal/repository/tags.go +++ b/internal/repository/tags.go @@ -7,22 +7,26 @@ package repository import ( "github.com/ClusterCockpit/cc-backend/pkg/archive" "github.com/ClusterCockpit/cc-backend/pkg/schema" + "github.com/ClusterCockpit/cc-backend/pkg/log" sq "github.com/Masterminds/squirrel" ) // Add the tag with id `tagId` to the job with the database id `jobId`. func (r *JobRepository) AddTag(job int64, tag int64) ([]*schema.Tag, error) { if _, err := r.stmtCache.Exec(`INSERT INTO jobtag (job_id, tag_id) VALUES ($1, $2)`, job, tag); err != nil { + log.Error("Error while running query") return nil, err } j, err := r.FindById(job) if err != nil { + log.Error("Error while finding job by id") return nil, err } tags, err := r.GetTags(&job) if err != nil { + log.Error("Error while getting tags for job") return nil, err } @@ -32,16 +36,19 @@ func (r *JobRepository) AddTag(job int64, tag int64) ([]*schema.Tag, error) { // Removes a tag from a job func (r *JobRepository) RemoveTag(job, tag int64) ([]*schema.Tag, error) { if _, err := r.stmtCache.Exec("DELETE FROM jobtag WHERE jobtag.job_id = $1 AND jobtag.tag_id = $2", job, tag); err != nil { + log.Error("Error while running query") return nil, err } j, err := r.FindById(job) if err != nil { + log.Error("Error while finding job by id") return nil, err } tags, err := r.GetTags(&job) if err != nil { + log.Error("Error while getting tags for job") return nil, err } @@ -138,6 +145,7 @@ func (r *JobRepository) GetTags(job *int64) ([]*schema.Tag, error) { rows, err := q.RunWith(r.stmtCache).Query() if err != nil { + log.Error("Error while running query") return nil, err } @@ -145,6 +153,7 @@ func (r *JobRepository) GetTags(job *int64) ([]*schema.Tag, error) { for rows.Next() { tag := &schema.Tag{} if err := rows.Scan(&tag.ID, &tag.Type, &tag.Name); err != nil { + log.Error("Error while scanning rows") return nil, err } tags = append(tags, tag) diff --git a/internal/repository/user.go b/internal/repository/user.go index b96641a..01ecb32 100644 --- a/internal/repository/user.go +++ b/internal/repository/user.go @@ -6,13 +6,13 @@ package repository import ( "encoding/json" - "log" "sync" "time" "github.com/ClusterCockpit/cc-backend/internal/auth" "github.com/ClusterCockpit/cc-backend/internal/config" "github.com/ClusterCockpit/cc-backend/pkg/lrucache" + "github.com/ClusterCockpit/cc-backend/pkg/log" "github.com/jmoiron/sqlx" ) @@ -82,6 +82,7 @@ func (uCfg *UserCfgRepo) GetUIConfig(user *auth.User) (map[string]interface{}, e rows, err := uCfg.Lookup.Query(user.Username) if err != nil { + log.Errorf("Error while looking up user config for user '%#v'", user.Username) return err, 0, 0 } @@ -90,11 +91,13 @@ func (uCfg *UserCfgRepo) GetUIConfig(user *auth.User) (map[string]interface{}, e for rows.Next() { var key, rawval string if err := rows.Scan(&key, &rawval); err != nil { + log.Error("Error while scanning user config values") return err, 0, 0 } var val interface{} if err := json.Unmarshal([]byte(rawval), &val); err != nil { + log.Error("Error while unmarshaling raw user config json") return err, 0, 0 } @@ -106,6 +109,7 @@ func (uCfg *UserCfgRepo) GetUIConfig(user *auth.User) (map[string]interface{}, e return config, 24 * time.Hour, size }) if err, ok := data.(error); ok { + log.Error("Error in data set") return nil, err } @@ -122,6 +126,7 @@ func (uCfg *UserCfgRepo) UpdateConfig( if user == nil { var val interface{} if err := json.Unmarshal([]byte(value), &val); err != nil { + log.Error("Error while unmarshaling raw user config json") return err } @@ -133,6 +138,7 @@ func (uCfg *UserCfgRepo) UpdateConfig( if _, err := uCfg.DB.Exec(`REPLACE INTO configuration (username, confkey, value) VALUES (?, ?, ?)`, user, key, value); err != nil { + log.Errorf("Error while replacing user config in DB for user '$#v'", user) return err } diff --git a/internal/runtimeEnv/setup.go b/internal/runtimeEnv/setup.go index a4d37be..2d0fc61 100644 --- a/internal/runtimeEnv/setup.go +++ b/internal/runtimeEnv/setup.go @@ -14,6 +14,8 @@ import ( "strconv" "strings" "syscall" + + "github.com/ClusterCockpit/cc-backend/pkg/log" ) // Very simple and limited .env file reader. @@ -22,6 +24,7 @@ import ( func LoadEnv(file string) error { f, err := os.Open(file) if err != nil { + log.Error("Error while opening file") return err } @@ -89,11 +92,13 @@ func DropPrivileges(username string, group string) error { if group != "" { g, err := user.LookupGroup(group) if err != nil { + log.Error("Error while looking up group") return err } gid, _ := strconv.Atoi(g.Gid) if err := syscall.Setgid(gid); err != nil { + log.Error("Error while setting gid") return err } } @@ -101,11 +106,13 @@ func DropPrivileges(username string, group string) error { if username != "" { u, err := user.Lookup(username) if err != nil { + log.Error("Error while looking up user") return err } uid, _ := strconv.Atoi(u.Uid) if err := syscall.Setuid(uid); err != nil { + log.Error("Error while setting uid") return err } } diff --git a/pkg/archive/archive.go b/pkg/archive/archive.go index f0a7027..051cc8b 100644 --- a/pkg/archive/archive.go +++ b/pkg/archive/archive.go @@ -10,6 +10,7 @@ import ( "github.com/ClusterCockpit/cc-backend/pkg/lrucache" "github.com/ClusterCockpit/cc-backend/pkg/schema" + "github.com/ClusterCockpit/cc-backend/pkg/log" ) type ArchiveBackend interface { @@ -40,6 +41,7 @@ func Init(rawConfig json.RawMessage, disableArchive bool) error { Kind string `json:"kind"` } if err := json.Unmarshal(rawConfig, &kind); err != nil { + log.Error("Error while unmarshaling raw config json") return err } @@ -53,6 +55,7 @@ func Init(rawConfig json.RawMessage, disableArchive bool) error { } if err := ar.Init(rawConfig); err != nil { + log.Error("Error while initializing archiveBackend") return err } return initClusterConfig() @@ -70,6 +73,7 @@ func LoadAveragesFromArchive( metaFile, err := ar.LoadJobMeta(job) if err != nil { + log.Error("Error while loading job metadata from archiveBackend") return err } @@ -88,6 +92,7 @@ func GetStatistics(job *schema.Job) (map[string]schema.JobStatistics, error) { metaFile, err := ar.LoadJobMeta(job) if err != nil { + log.Error("Error while loading job metadata from archiveBackend") return nil, err } @@ -104,6 +109,7 @@ func UpdateTags(job *schema.Job, tags []*schema.Tag) error { jobMeta, err := ar.LoadJobMeta(job) if err != nil { + log.Error("Error while loading job metadata from archiveBackend") return err } diff --git a/pkg/archive/clusterConfig.go b/pkg/archive/clusterConfig.go index 317dfb7..86635cc 100644 --- a/pkg/archive/clusterConfig.go +++ b/pkg/archive/clusterConfig.go @@ -9,6 +9,7 @@ import ( "fmt" "github.com/ClusterCockpit/cc-backend/pkg/schema" + "github.com/ClusterCockpit/cc-backend/pkg/log" ) var Clusters []*schema.Cluster @@ -23,6 +24,7 @@ func initClusterConfig() error { cluster, err := ar.LoadClusterCfg(c) if err != nil { + log.Errorf("Error while loading cluster config for cluster '%#v'", c) return err } diff --git a/pkg/archive/fsBackend.go b/pkg/archive/fsBackend.go index 608573d..1e363bb 100644 --- a/pkg/archive/fsBackend.go +++ b/pkg/archive/fsBackend.go @@ -46,7 +46,7 @@ func loadJobMeta(filename string) (*schema.JobMeta, error) { f, err := os.Open(filename) if err != nil { - log.Errorf("loadJobMeta() > open file error: %v", err) + log.Errorf("loadJobMeta() > open file error: %#v", err) return &schema.JobMeta{}, err } defer f.Close() @@ -58,19 +58,19 @@ func (fsa *FsArchive) Init(rawConfig json.RawMessage) error { var config FsArchiveConfig if err := json.Unmarshal(rawConfig, &config); err != nil { - log.Errorf("Init() > Unmarshal error: %v", err) + log.Errorf("Init() > Unmarshal error: %#v", err) return err } if config.Path == "" { err := fmt.Errorf("ARCHIVE/FSBACKEND > Init() : empty config.Path") - log.Errorf("Init() > config.Path error: %v", err) + log.Errorf("Init() > config.Path error: %#v", err) return err } fsa.path = config.Path entries, err := os.ReadDir(fsa.path) if err != nil { - log.Errorf("Init() > ReadDir() error: %v", err) + log.Errorf("Init() > ReadDir() error: %#v", err) return err } @@ -86,7 +86,7 @@ func (fsa *FsArchive) LoadJobData(job *schema.Job) (schema.JobData, error) { filename := getPath(job, fsa.path, "data.json") f, err := os.Open(filename) if err != nil { - log.Errorf("LoadJobData() > open file error: %v", err) + log.Errorf("LoadJobData() > open file error: %#v", err) return nil, err } defer f.Close() @@ -175,12 +175,15 @@ func (fsa *FsArchive) StoreJobMeta(jobMeta *schema.JobMeta) error { } f, err := os.Create(getPath(&job, fsa.path, "meta.json")) if err != nil { + log.Error("Error while creating filepath for meta.json") return err } if err := EncodeJobMeta(f, jobMeta); err != nil { + log.Error("Error while encoding job metadata to meta.json file") return err } if err := f.Close(); err != nil { + log.Error("Error while closing meta.json file") return err } @@ -203,26 +206,38 @@ func (fsa *FsArchive) ImportJob( } dir := getPath(&job, fsa.path, "") if err := os.MkdirAll(dir, 0777); err != nil { + log.Error("Error while creating job archive path") return err } f, err := os.Create(path.Join(dir, "meta.json")) if err != nil { + log.Error("Error while creating filepath for meta.json") return err } if err := EncodeJobMeta(f, jobMeta); err != nil { + log.Error("Error while encoding job metadata to meta.json file") return err } if err := f.Close(); err != nil { + log.Error("Error while closing meta.json file") return err } f, err = os.Create(path.Join(dir, "data.json")) if err != nil { + log.Error("Error while creating filepath for data.json") return err } if err := EncodeJobData(f, jobData); err != nil { + log.Error("Error while encoding job metricdata to data.json file") return err } - return f.Close() + if err := f.Close(); err != nil { + log.Error("Error while closing data.json file") + return err + } + + // no error: final return is nil + return nil } diff --git a/pkg/archive/json.go b/pkg/archive/json.go index 69db584..0fd3bce 100644 --- a/pkg/archive/json.go +++ b/pkg/archive/json.go @@ -10,12 +10,14 @@ import ( "time" "github.com/ClusterCockpit/cc-backend/pkg/schema" + "github.com/ClusterCockpit/cc-backend/pkg/log" ) func DecodeJobData(r io.Reader, k string) (schema.JobData, error) { data := cache.Get(k, func() (value interface{}, ttl time.Duration, size int) { var d schema.JobData if err := json.NewDecoder(r).Decode(&d); err != nil { + log.Error("Error while decoding raw job data json") return err, 0, 1000 } @@ -23,6 +25,7 @@ func DecodeJobData(r io.Reader, k string) (schema.JobData, error) { }) if err, ok := data.(error); ok { + log.Error("Error in decoded job data set") return nil, err } @@ -32,6 +35,7 @@ func DecodeJobData(r io.Reader, k string) (schema.JobData, error) { func DecodeJobMeta(r io.Reader) (*schema.JobMeta, error) { var d schema.JobMeta if err := json.NewDecoder(r).Decode(&d); err != nil { + log.Error("Error while decoding raw job meta json") return &d, err } @@ -43,6 +47,7 @@ func DecodeJobMeta(r io.Reader) (*schema.JobMeta, error) { func DecodeCluster(r io.Reader) (*schema.Cluster, error) { var c schema.Cluster if err := json.NewDecoder(r).Decode(&c); err != nil { + log.Error("Error while decoding raw cluster json") return &c, err } @@ -54,6 +59,7 @@ func DecodeCluster(r io.Reader) (*schema.Cluster, error) { func EncodeJobData(w io.Writer, d *schema.JobData) error { // Sanitize parameters if err := json.NewEncoder(w).Encode(d); err != nil { + log.Error("Error while encoding new job data json") return err } @@ -63,6 +69,7 @@ func EncodeJobData(w io.Writer, d *schema.JobData) error { func EncodeJobMeta(w io.Writer, d *schema.JobMeta) error { // Sanitize parameters if err := json.NewEncoder(w).Encode(d); err != nil { + log.Error("Error while encoding new job meta json") return err } diff --git a/pkg/schema/float.go b/pkg/schema/float.go index df084fa..52197f2 100644 --- a/pkg/schema/float.go +++ b/pkg/schema/float.go @@ -9,6 +9,8 @@ import ( "io" "math" "strconv" + + "github.com/ClusterCockpit/cc-backend/pkg/log" ) // A custom float type is used so that (Un)MarshalJSON and @@ -43,6 +45,7 @@ func (f *Float) UnmarshalJSON(input []byte) error { val, err := strconv.ParseFloat(s, 64) if err != nil { + log.Error("Error while parsing custom float") return err } *f = Float(val) diff --git a/pkg/schema/validate.go b/pkg/schema/validate.go index b3eee6e..dafe892 100644 --- a/pkg/schema/validate.go +++ b/pkg/schema/validate.go @@ -45,16 +45,17 @@ func Validate(k Kind, r io.Reader) (err error) { case Config: s, err = jsonschema.Compile("embedfs://config.schema.json") default: - return fmt.Errorf("SCHEMA/VALIDATE > unkown schema kind: %v", k) + return fmt.Errorf("SCHEMA/VALIDATE > unkown schema kind: %#v", k) } if err != nil { + log.Errorf("Error while compiling json schema for kind '%#v'", k) return err } var v interface{} if err := json.NewDecoder(r).Decode(&v); err != nil { - log.Errorf("Failed to decode %v", err) + log.Errorf("Error while decoding raw json schema: %#v", err) return err } From a885e69125b261b44efe1b6794f4cbc3e63c3b2f Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Wed, 1 Feb 2023 11:58:27 +0100 Subject: [PATCH 08/12] Adapt loglevel for logs, shorten strings, fix formats, streamline - Switched to Warn for most errors, reduces bloat, improves log control --- cmd/cc-backend/main.go | 2 +- internal/api/rest.go | 42 +++++++++++------------ internal/auth/auth.go | 6 ++-- internal/auth/jwt.go | 18 +++++----- internal/auth/ldap.go | 18 +++++----- internal/auth/users.go | 32 +++++++++--------- internal/graph/stats.go | 16 ++++----- internal/metricdata/cc-metric-store.go | 22 +++++++----- internal/metricdata/influxdb-v2.go | 4 +-- internal/metricdata/metricdata.go | 12 +++---- internal/metricdata/prometheus.go | 22 ++++++------ internal/repository/init.go | 46 +++++++++++++------------- internal/repository/job.go | 32 +++++++++--------- internal/repository/query.go | 4 +-- internal/repository/tags.go | 10 +++--- internal/repository/user.go | 12 +++---- internal/routerConfig/routes.go | 8 ++--- internal/runtimeEnv/setup.go | 8 ++--- pkg/archive/archive.go | 8 ++--- pkg/archive/clusterConfig.go | 12 +++---- pkg/archive/fsBackend.go | 23 +++++++------ pkg/archive/json.go | 12 +++---- pkg/archive/nodelist.go | 2 +- pkg/schema/float.go | 2 +- pkg/schema/validate.go | 2 +- web/web.go | 4 +-- 26 files changed, 193 insertions(+), 186 deletions(-) diff --git a/cmd/cc-backend/main.go b/cmd/cc-backend/main.go index a314990..979a99d 100644 --- a/cmd/cc-backend/main.go +++ b/cmd/cc-backend/main.go @@ -430,7 +430,7 @@ func main() { for range time.Tick(30 * time.Minute) { err := jobRepo.StopJobsExceedingWalltimeBy(config.Keys.StopJobsExceedingWalltime) if err != nil { - log.Errorf("MAIN > error while looking for jobs exceeding their walltime: %s", err.Error()) + log.Warnf("Error while looking for jobs exceeding their walltime: %s", err.Error()) } runtime.GC() } diff --git a/internal/api/rest.go b/internal/api/rest.go index 75be29f..0937dce 100644 --- a/internal/api/rest.go +++ b/internal/api/rest.go @@ -169,7 +169,7 @@ func decode(r io.Reader, val interface{}) error { // @router /jobs/ [get] func (api *RestApi) getJobs(rw http.ResponseWriter, r *http.Request) { if user := auth.GetUser(r.Context()); user != nil && !user.HasRole(auth.RoleApi) { - handleError(fmt.Errorf("missing role: %#v", auth.RoleApi), http.StatusForbidden, rw) + handleError(fmt.Errorf("missing role: %v", auth.RoleApi), http.StatusForbidden, rw) return } @@ -184,7 +184,7 @@ func (api *RestApi) getJobs(rw http.ResponseWriter, r *http.Request) { for _, s := range vals { state := schema.JobState(s) if !state.Valid() { - http.Error(rw, "REST > invalid query parameter value: state", http.StatusBadRequest) + http.Error(rw, "invalid query parameter value: state", http.StatusBadRequest) return } filter.State = append(filter.State, state) @@ -194,7 +194,7 @@ func (api *RestApi) getJobs(rw http.ResponseWriter, r *http.Request) { case "start-time": st := strings.Split(vals[0], "-") if len(st) != 2 { - http.Error(rw, "REST > invalid query parameter value: startTime", http.StatusBadRequest) + http.Error(rw, "invalid query parameter value: startTime", http.StatusBadRequest) return } from, err := strconv.ParseInt(st[0], 10, 64) @@ -226,7 +226,7 @@ func (api *RestApi) getJobs(rw http.ResponseWriter, r *http.Request) { case "with-metadata": withMetadata = true default: - http.Error(rw, "REST > invalid query parameter: "+key, http.StatusBadRequest) + http.Error(rw, "invalid query parameter: "+key, http.StatusBadRequest) return } } @@ -300,7 +300,7 @@ func (api *RestApi) getJobs(rw http.ResponseWriter, r *http.Request) { // @router /jobs/tag_job/{id} [post] func (api *RestApi) tagJob(rw http.ResponseWriter, r *http.Request) { if user := auth.GetUser(r.Context()); user != nil && !user.HasRole(auth.RoleApi) { - handleError(fmt.Errorf("missing role: %#v", auth.RoleApi), http.StatusForbidden, rw) + handleError(fmt.Errorf("missing role: %v", auth.RoleApi), http.StatusForbidden, rw) return } @@ -365,7 +365,7 @@ func (api *RestApi) tagJob(rw http.ResponseWriter, r *http.Request) { // @router /jobs/start_job/ [post] func (api *RestApi) startJob(rw http.ResponseWriter, r *http.Request) { if user := auth.GetUser(r.Context()); user != nil && !user.HasRole(auth.RoleApi) { - handleError(fmt.Errorf("missing role: %#v", auth.RoleApi), http.StatusForbidden, rw) + handleError(fmt.Errorf("missing role: %v", auth.RoleApi), http.StatusForbidden, rw) return } @@ -446,7 +446,7 @@ func (api *RestApi) startJob(rw http.ResponseWriter, r *http.Request) { // @router /jobs/stop_job/{id} [post] func (api *RestApi) stopJobById(rw http.ResponseWriter, r *http.Request) { if user := auth.GetUser(r.Context()); user != nil && !user.HasRole(auth.RoleApi) { - handleError(fmt.Errorf("missing role: %#v", auth.RoleApi), http.StatusForbidden, rw) + handleError(fmt.Errorf("missing role: %v", auth.RoleApi), http.StatusForbidden, rw) return } @@ -499,7 +499,7 @@ func (api *RestApi) stopJobById(rw http.ResponseWriter, r *http.Request) { // @router /jobs/stop_job/ [post] func (api *RestApi) stopJobByRequest(rw http.ResponseWriter, r *http.Request) { if user := auth.GetUser(r.Context()); user != nil && !user.HasRole(auth.RoleApi) { - handleError(fmt.Errorf("missing role: %#v", auth.RoleApi), http.StatusForbidden, rw) + handleError(fmt.Errorf("missing role: %v", auth.RoleApi), http.StatusForbidden, rw) return } @@ -545,7 +545,7 @@ func (api *RestApi) stopJobByRequest(rw http.ResponseWriter, r *http.Request) { // @router /jobs/delete_job/{id} [delete] func (api *RestApi) deleteJobById(rw http.ResponseWriter, r *http.Request) { if user := auth.GetUser(r.Context()); user != nil && !user.HasRole(auth.RoleApi) { - handleError(fmt.Errorf("missing role: %#v", auth.RoleApi), http.StatusForbidden, rw) + handleError(fmt.Errorf("missing role: %v", auth.RoleApi), http.StatusForbidden, rw) return } @@ -593,7 +593,7 @@ func (api *RestApi) deleteJobById(rw http.ResponseWriter, r *http.Request) { // @router /jobs/delete_job/ [delete] func (api *RestApi) deleteJobByRequest(rw http.ResponseWriter, r *http.Request) { if user := auth.GetUser(r.Context()); user != nil && !user.HasRole(auth.RoleApi) { - handleError(fmt.Errorf("missing role: %#v", auth.RoleApi), http.StatusForbidden, rw) + handleError(fmt.Errorf("missing role: %v", auth.RoleApi), http.StatusForbidden, rw) return } @@ -649,7 +649,7 @@ func (api *RestApi) deleteJobByRequest(rw http.ResponseWriter, r *http.Request) // @router /jobs/delete_job_before/{ts} [delete] func (api *RestApi) deleteJobBefore(rw http.ResponseWriter, r *http.Request) { if user := auth.GetUser(r.Context()); user != nil && !user.HasRole(auth.RoleApi) { - handleError(fmt.Errorf("missing role: %#v", auth.RoleApi), http.StatusForbidden, rw) + handleError(fmt.Errorf("missing role: %v", auth.RoleApi), http.StatusForbidden, rw) return } @@ -724,7 +724,7 @@ func (api *RestApi) checkAndHandleStopJob(rw http.ResponseWriter, job *schema.Jo // func (api *RestApi) importJob(rw http.ResponseWriter, r *http.Request) { // if user := auth.GetUser(r.Context()); user != nil && !user.HasRole(auth.RoleApi) { -// handleError(fmt.Errorf("missing role: %#v", auth.RoleApi), http.StatusForbidden, rw) +// handleError(fmt.Errorf("missing role: %v", auth.RoleApi), http.StatusForbidden, rw) // return // } @@ -793,7 +793,7 @@ func (api *RestApi) getJWT(rw http.ResponseWriter, r *http.Request) { me := auth.GetUser(r.Context()) if !me.HasRole(auth.RoleAdmin) { if username != me.Username { - http.Error(rw, "REST > only admins are allowed to sign JWTs not for themselves", http.StatusForbidden) + http.Error(rw, "Only admins are allowed to sign JWTs not for themselves", http.StatusForbidden) return } } @@ -818,13 +818,13 @@ func (api *RestApi) createUser(rw http.ResponseWriter, r *http.Request) { rw.Header().Set("Content-Type", "text/plain") me := auth.GetUser(r.Context()) if !me.HasRole(auth.RoleAdmin) { - http.Error(rw, "REST > only admins are allowed to create new users", http.StatusForbidden) + http.Error(rw, "Only admins are allowed to create new users", http.StatusForbidden) return } username, password, role, name, email := r.FormValue("username"), r.FormValue("password"), r.FormValue("role"), r.FormValue("name"), r.FormValue("email") if len(password) == 0 && role != auth.RoleApi { - http.Error(rw, "REST > only API users are allowed to have a blank password (login will be impossible)", http.StatusBadRequest) + http.Error(rw, "Only API users are allowed to have a blank password (login will be impossible)", http.StatusBadRequest) return } @@ -838,12 +838,12 @@ func (api *RestApi) createUser(rw http.ResponseWriter, r *http.Request) { return } - rw.Write([]byte(fmt.Sprintf("User %#v successfully created!\n", username))) + rw.Write([]byte(fmt.Sprintf("User %v successfully created!\n", username))) } func (api *RestApi) deleteUser(rw http.ResponseWriter, r *http.Request) { if user := auth.GetUser(r.Context()); !user.HasRole(auth.RoleAdmin) { - http.Error(rw, "REST > only admins are allowed to delete a user", http.StatusForbidden) + http.Error(rw, "Only admins are allowed to delete a user", http.StatusForbidden) return } @@ -858,7 +858,7 @@ func (api *RestApi) deleteUser(rw http.ResponseWriter, r *http.Request) { func (api *RestApi) getUsers(rw http.ResponseWriter, r *http.Request) { if user := auth.GetUser(r.Context()); !user.HasRole(auth.RoleAdmin) { - http.Error(rw, "REST > only admins are allowed to fetch a list of users", http.StatusForbidden) + http.Error(rw, "Only admins are allowed to fetch a list of users", http.StatusForbidden) return } @@ -873,7 +873,7 @@ func (api *RestApi) getUsers(rw http.ResponseWriter, r *http.Request) { func (api *RestApi) updateUser(rw http.ResponseWriter, r *http.Request) { if user := auth.GetUser(r.Context()); !user.HasRole(auth.RoleAdmin) { - http.Error(rw, "REST > only admins are allowed to update a user", http.StatusForbidden) + http.Error(rw, "Only admins are allowed to update a user", http.StatusForbidden) return } @@ -893,9 +893,9 @@ func (api *RestApi) updateUser(rw http.ResponseWriter, r *http.Request) { http.Error(rw, err.Error(), http.StatusUnprocessableEntity) return } - rw.Write([]byte("REST > Remove Role Success")) + rw.Write([]byte("Remove Role Success")) } else { - http.Error(rw, "REST > Not Add or Del?", http.StatusInternalServerError) + http.Error(rw, "Not Add or Del?", http.StatusInternalServerError) } } diff --git a/internal/auth/auth.go b/internal/auth/auth.go index 1b3ac38..c50c618 100644 --- a/internal/auth/auth.go +++ b/internal/auth/auth.go @@ -176,7 +176,7 @@ func (auth *Authentication) Login( user := (*User)(nil) if username != "" { if user, _ = auth.GetUser(username); err != nil { - // log.Warnf("login of unkown user %#v", username) + // log.Warnf("login of unkown user %v", username) _ = err } } @@ -206,12 +206,12 @@ func (auth *Authentication) Login( session.Values["username"] = user.Username session.Values["roles"] = user.Roles if err := auth.sessionStore.Save(r, rw, session); err != nil { - log.Errorf("session save failed: %s", err.Error()) + log.Warnf("session save failed: %s", err.Error()) http.Error(rw, err.Error(), http.StatusInternalServerError) return } - log.Infof("login successfull: user: %#v (roles: %v)", user.Username, user.Roles) + log.Infof("login successfull: user: %v (roles: %v)", user.Username, user.Roles) ctx := context.WithValue(r.Context(), ContextUserKey, user) onsuccess.ServeHTTP(rw, r.WithContext(ctx)) return diff --git a/internal/auth/jwt.go b/internal/auth/jwt.go index 45b1473..64fe671 100644 --- a/internal/auth/jwt.go +++ b/internal/auth/jwt.go @@ -45,13 +45,13 @@ func (ja *JWTAuthenticator) Init(auth *Authentication, conf interface{}) error { } else { bytes, err := base64.StdEncoding.DecodeString(pubKey) if err != nil { - log.Error("Could not decode JWT public key") + log.Warn("Could not decode JWT public key") return err } ja.publicKey = ed25519.PublicKey(bytes) bytes, err = base64.StdEncoding.DecodeString(privKey) if err != nil { - log.Error("Could not decode JWT private key") + log.Warn("Could not decode JWT private key") return err } ja.privateKey = ed25519.PrivateKey(bytes) @@ -60,7 +60,7 @@ func (ja *JWTAuthenticator) Init(auth *Authentication, conf interface{}) error { if pubKey = os.Getenv("CROSS_LOGIN_JWT_HS512_KEY"); pubKey != "" { bytes, err := base64.StdEncoding.DecodeString(pubKey) if err != nil { - log.Error("Could not decode cross login JWT HS512 key") + log.Warn("Could not decode cross login JWT HS512 key") return err } ja.loginTokenKey = bytes @@ -71,7 +71,7 @@ func (ja *JWTAuthenticator) Init(auth *Authentication, conf interface{}) error { if keyFound && pubKeyCrossLogin != "" { bytes, err := base64.StdEncoding.DecodeString(pubKeyCrossLogin) if err != nil { - log.Error("Could not decode cross login JWT public key") + log.Warn("Could not decode cross login JWT public key") return err } ja.publicKeyCrossLogin = ed25519.PublicKey(bytes) @@ -130,7 +130,7 @@ func (ja *JWTAuthenticator) Login( return nil, fmt.Errorf("AUTH/JWT > unkown signing method for login token: %s (known: HS256, HS512, EdDSA)", t.Method.Alg()) }) if err != nil { - log.Error("Error while parsing jwt token") + log.Warn("Error while parsing jwt token") return nil, err } @@ -157,7 +157,7 @@ func (ja *JWTAuthenticator) Login( if user == nil { user, err = ja.auth.GetUser(sub) if err != nil && err != sql.ErrNoRows { - log.Errorf("Error while loading user '%#v'", sub) + log.Errorf("Error while loading user '%v'", sub) return nil, err } else if user == nil { user = &User{ @@ -166,7 +166,7 @@ func (ja *JWTAuthenticator) Login( AuthSource: AuthViaToken, } if err := ja.auth.AddUser(user); err != nil { - log.Errorf("Error while adding user '%#v' to auth from token", user.Username) + log.Errorf("Error while adding user '%v' to auth from token", user.Username) return nil, err } } @@ -231,7 +231,7 @@ func (ja *JWTAuthenticator) Auth( return ja.publicKey, nil }) if err != nil { - log.Error("Error while parsing token") + log.Warn("Error while parsing token") return nil, err } @@ -286,7 +286,7 @@ func (ja *JWTAuthenticator) Auth( session.Values["roles"] = roles if err := ja.auth.sessionStore.Save(r, rw, session); err != nil { - log.Errorf("session save failed: %s", err.Error()) + log.Warnf("session save failed: %s", err.Error()) http.Error(rw, err.Error(), http.StatusInternalServerError) return nil, err } diff --git a/internal/auth/ldap.go b/internal/auth/ldap.go index f1b8fe8..ae9c5f0 100644 --- a/internal/auth/ldap.go +++ b/internal/auth/ldap.go @@ -39,7 +39,7 @@ func (la *LdapAuthenticator) Init( if la.config != nil && la.config.SyncInterval != "" { interval, err := time.ParseDuration(la.config.SyncInterval) if err != nil { - log.Errorf("Could not parse duration for sync interval: %#v", la.config.SyncInterval) + log.Warnf("Could not parse duration for sync interval: %v", la.config.SyncInterval) return err } @@ -78,7 +78,7 @@ func (la *LdapAuthenticator) Login( l, err := la.getLdapConnection(false) if err != nil { - log.Error("Error while getting ldap connection") + log.Warn("Error while getting ldap connection") return nil, err } defer l.Close() @@ -108,14 +108,14 @@ func (la *LdapAuthenticator) Sync() error { users := map[string]int{} rows, err := la.auth.db.Query(`SELECT username FROM user WHERE user.ldap = 1`) if err != nil { - log.Error("Error while querying LDAP users") + log.Warn("Error while querying LDAP users") return err } for rows.Next() { var username string if err := rows.Scan(&username); err != nil { - log.Errorf("Error while scanning for user '%s'", username) + log.Warnf("Error while scanning for user '%s'", username) return err } @@ -133,7 +133,7 @@ func (la *LdapAuthenticator) Sync() error { la.config.UserBase, ldap.ScopeWholeSubtree, ldap.NeverDerefAliases, 0, 0, false, la.config.UserFilter, []string{"dn", "uid", "gecos"}, nil)) if err != nil { - log.Error("LDAP search error") + log.Warn("LDAP search error") return err } @@ -155,14 +155,14 @@ func (la *LdapAuthenticator) Sync() error { for username, where := range users { if where == IN_DB && la.config.SyncDelOldUsers { - log.Debugf("sync: remove %#v (does not show up in LDAP anymore)", username) + log.Debugf("sync: remove %v (does not show up in LDAP anymore)", username) if _, err := la.auth.db.Exec(`DELETE FROM user WHERE user.username = ?`, username); err != nil { log.Errorf("User '%s' not in LDAP anymore: Delete from DB failed", username) return err } } else if where == IN_LDAP { name := newnames[username] - log.Debugf("sync: add %#v (name: %#v, roles: [user], ldap: true)", username, name) + log.Debugf("sync: add %v (name: %v, roles: [user], ldap: true)", username, name) if _, err := la.auth.db.Exec(`INSERT INTO user (username, ldap, name, roles) VALUES (?, ?, ?, ?)`, username, 1, name, "[\""+RoleUser+"\"]"); err != nil { log.Errorf("User '%s' new in LDAP: Insert into DB failed", username) @@ -180,14 +180,14 @@ func (la *LdapAuthenticator) getLdapConnection(admin bool) (*ldap.Conn, error) { conn, err := ldap.DialURL(la.config.Url) if err != nil { - log.Error("LDAP URL dial failed") + log.Warn("LDAP URL dial failed") return nil, err } if admin { if err := conn.Bind(la.config.SearchDN, la.syncPassword); err != nil { conn.Close() - log.Error("LDAP connection bind failed") + log.Warn("LDAP connection bind failed") return nil, err } } diff --git a/internal/auth/users.go b/internal/auth/users.go index 093fb14..0b358d0 100644 --- a/internal/auth/users.go +++ b/internal/auth/users.go @@ -25,7 +25,7 @@ func (auth *Authentication) GetUser(username string) (*User, error) { if err := sq.Select("password", "ldap", "name", "roles", "email").From("user"). Where("user.username = ?", username).RunWith(auth.db). QueryRow().Scan(&hashedPassword, &user.AuthSource, &name, &rawRoles, &email); err != nil { - log.Errorf("Error while querying user '%#v' from database", username) + log.Warnf("Error while querying user '%v' from database", username) return nil, err } @@ -34,7 +34,7 @@ func (auth *Authentication) GetUser(username string) (*User, error) { user.Email = email.String if rawRoles.Valid { if err := json.Unmarshal([]byte(rawRoles.String), &user.Roles); err != nil { - log.Error("Error while unmarshaling raw roles from DB") + log.Warn("Error while unmarshaling raw roles from DB") return nil, err } } @@ -67,11 +67,11 @@ func (auth *Authentication) AddUser(user *User) error { } if _, err := sq.Insert("user").Columns(cols...).Values(vals...).RunWith(auth.db).Exec(); err != nil { - log.Errorf("Error while inserting new user '%#v' into DB", user.Username) + log.Errorf("Error while inserting new user '%v' into DB", user.Username) return err } - log.Infof("new user %#v created (roles: %s, auth-source: %d)", user.Username, rolesJson, user.AuthSource) + log.Infof("new user %v created (roles: %s, auth-source: %d)", user.Username, rolesJson, user.AuthSource) return nil } @@ -91,7 +91,7 @@ func (auth *Authentication) ListUsers(specialsOnly bool) ([]*User, error) { rows, err := q.RunWith(auth.db).Query() if err != nil { - log.Error("Error while querying user list") + log.Warn("Error while querying user list") return nil, err } @@ -102,12 +102,12 @@ func (auth *Authentication) ListUsers(specialsOnly bool) ([]*User, error) { user := &User{} var name, email sql.NullString if err := rows.Scan(&user.Username, &name, &email, &rawroles); err != nil { - log.Error("Error while scanning user list") + log.Warn("Error while scanning user list") return nil, err } if err := json.Unmarshal([]byte(rawroles), &user.Roles); err != nil { - log.Error("Error while unmarshaling raw role list") + log.Warn("Error while unmarshaling raw role list") return nil, err } @@ -125,17 +125,17 @@ func (auth *Authentication) AddRole( user, err := auth.GetUser(username) if err != nil { - log.Errorf("Could not load user '%s'", username) + log.Warnf("Could not load user '%s'", username) return err } if role != RoleAdmin && role != RoleApi && role != RoleUser && role != RoleSupport { - return fmt.Errorf("AUTH/USERS > invalid user role: %#v", role) + return fmt.Errorf("Invalid user role: %v", role) } for _, r := range user.Roles { if r == role { - return fmt.Errorf("AUTH/USERS > user %#v already has role %#v", username, role) + return fmt.Errorf("User %v already has role %v", username, role) } } @@ -150,12 +150,12 @@ func (auth *Authentication) AddRole( func (auth *Authentication) RemoveRole(ctx context.Context, username string, role string) error { user, err := auth.GetUser(username) if err != nil { - log.Errorf("Could not load user '%s'", username) + log.Warnf("Could not load user '%s'", username) return err } if role != RoleAdmin && role != RoleApi && role != RoleUser && role != RoleSupport { - return fmt.Errorf("AUTH/USERS > invalid user role: %#v", role) + return fmt.Errorf("Invalid user role: %v", role) } var exists bool @@ -176,7 +176,7 @@ func (auth *Authentication) RemoveRole(ctx context.Context, username string, rol } return nil } else { - return fmt.Errorf("AUTH/USERS > user %#v already does not have role %#v", username, role) + return fmt.Errorf("User '%v' already does not have role: %v", username, role) } } @@ -191,11 +191,13 @@ func FetchUser(ctx context.Context, db *sqlx.DB, username string) (*model.User, if err := sq.Select("name", "email").From("user").Where("user.username = ?", username). RunWith(db).QueryRow().Scan(&name, &email); err != nil { if err == sql.ErrNoRows { - log.Errorf("User '%s' Not found in DB", username) + /* This warning will be logged *often* for non-local users, i.e. users mentioned only in job-table or archive, */ + /* since FetchUser will be called to retrieve full name and mail for every job in query/list */ + // log.Warnf("User '%s' Not found in DB", username) return nil, nil } - log.Errorf("Error while fetching user '%s'", username) + log.Warnf("Error while fetching user '%s'", username) return nil, err } diff --git a/internal/graph/stats.go b/internal/graph/stats.go index 2bdb59e..3678ea4 100644 --- a/internal/graph/stats.go +++ b/internal/graph/stats.go @@ -69,7 +69,7 @@ func (r *queryResolver) jobsStatistics(ctx context.Context, filter []*model.JobF rows, err := query.RunWith(r.DB).Query() if err != nil { - log.Error("Error while querying DB for job statistics") + log.Warn("Error while querying DB for job statistics") return nil, err } @@ -77,7 +77,7 @@ func (r *queryResolver) jobsStatistics(ctx context.Context, filter []*model.JobF var id sql.NullString var jobs, walltime, corehours sql.NullInt64 if err := rows.Scan(&id, &jobs, &walltime, &corehours); err != nil { - log.Error("Error while scanning rows") + log.Warn("Error while scanning rows") return nil, err } @@ -106,7 +106,7 @@ func (r *queryResolver) jobsStatistics(ctx context.Context, filter []*model.JobF query = repository.BuildWhereClause(f, query) } if err := query.RunWith(r.DB).QueryRow().Scan(&(stats[""].ShortJobs)); err != nil { - log.Error("Error while scanning rows for short job stats") + log.Warn("Error while scanning rows for short job stats") return nil, err } } else { @@ -118,7 +118,7 @@ func (r *queryResolver) jobsStatistics(ctx context.Context, filter []*model.JobF } rows, err := query.RunWith(r.DB).Query() if err != nil { - log.Error("Error while querying jobs for short jobs") + log.Warn("Error while querying jobs for short jobs") return nil, err } @@ -126,7 +126,7 @@ func (r *queryResolver) jobsStatistics(ctx context.Context, filter []*model.JobF var id sql.NullString var shortJobs sql.NullInt64 if err := rows.Scan(&id, &shortJobs); err != nil { - log.Error("Error while scanning rows for short jobs") + log.Warn("Error while scanning rows for short jobs") return nil, err } @@ -160,13 +160,13 @@ func (r *queryResolver) jobsStatistics(ctx context.Context, filter []*model.JobF value := fmt.Sprintf(`CAST(ROUND((CASE WHEN job.job_state = "running" THEN %d - job.start_time ELSE job.duration END) / 3600) as int) as value`, time.Now().Unix()) stat.HistDuration, err = r.jobsStatisticsHistogram(ctx, value, filter, id, col) if err != nil { - log.Error("Error while loading job statistics histogram: running jobs") + log.Warn("Error while loading job statistics histogram: running jobs") return nil, err } stat.HistNumNodes, err = r.jobsStatisticsHistogram(ctx, "job.num_nodes as value", filter, id, col) if err != nil { - log.Error("Error while loading job statistics histogram: num nodes") + log.Warn("Error while loading job statistics histogram: num nodes") return nil, err } } @@ -198,7 +198,7 @@ func (r *queryResolver) jobsStatisticsHistogram(ctx context.Context, value strin for rows.Next() { point := model.HistoPoint{} if err := rows.Scan(&point.Value, &point.Count); err != nil { - log.Error("Error while scanning rows") + log.Warn("Error while scanning rows") return nil, err } diff --git a/internal/metricdata/cc-metric-store.go b/internal/metricdata/cc-metric-store.go index 31e675a..f1b5ffe 100644 --- a/internal/metricdata/cc-metric-store.go +++ b/internal/metricdata/cc-metric-store.go @@ -79,7 +79,7 @@ func (ccms *CCMetricStore) Init(rawConfig json.RawMessage) error { var config CCMetricStoreConfig if err := json.Unmarshal(rawConfig, &config); err != nil { - log.Error("Error while unmarshaling raw json config") + log.Warn("Error while unmarshaling raw json config") return err } @@ -126,13 +126,13 @@ func (ccms *CCMetricStore) doRequest( buf := &bytes.Buffer{} if err := json.NewEncoder(buf).Encode(body); err != nil { - log.Error("Error while encoding request body") + log.Warn("Error while encoding request body") return nil, err } req, err := http.NewRequestWithContext(ctx, http.MethodPost, ccms.queryEndpoint, buf) if err != nil { - log.Error("Error while building request body") + log.Warn("Error while building request body") return nil, err } if ccms.jwt != "" { @@ -151,7 +151,7 @@ func (ccms *CCMetricStore) doRequest( var resBody ApiQueryResponse if err := json.NewDecoder(bufio.NewReader(res.Body)).Decode(&resBody); err != nil { - log.Error("Error while decoding result body") + log.Warn("Error while decoding result body") return nil, err } @@ -167,7 +167,7 @@ func (ccms *CCMetricStore) LoadData( topology := archive.GetSubCluster(job.Cluster, job.SubCluster).Topology queries, assignedScope, err := ccms.buildQueries(job, metrics, scopes) if err != nil { - log.Error("Error while building queries") + log.Warn("Error while building queries") return nil, err } @@ -210,7 +210,8 @@ func (ccms *CCMetricStore) LoadData( for _, res := range row { if res.Error != nil { - errors = append(errors, fmt.Sprintf("METRICDATA/CCMS > failed to fetch '%s' from host '%s': %s", query.Metric, query.Hostname, *res.Error)) + /* Build list for "partial errors", if any */ + errors = append(errors, fmt.Sprintf("failed to fetch '%s' from host '%s': %s", query.Metric, query.Hostname, *res.Error)) continue } @@ -253,6 +254,7 @@ func (ccms *CCMetricStore) LoadData( } if len(errors) != 0 { + /* Returns list for "partial errors" */ return jobData, fmt.Errorf("METRICDATA/CCMS > Errors: %s", strings.Join(errors, ", ")) } @@ -506,7 +508,7 @@ func (ccms *CCMetricStore) LoadStats( queries, _, err := ccms.buildQueries(job, metrics, []schema.MetricScope{schema.MetricScopeNode}) if err != nil { - log.Error("Error while building query") + log.Warn("Error while building query") return nil, err } @@ -604,7 +606,8 @@ func (ccms *CCMetricStore) LoadNodeData( metric := ccms.toLocalName(query.Metric) qdata := res[0] if qdata.Error != nil { - errors = append(errors, fmt.Sprintf("METRICDATA/CCMS > fetching %s for node %s failed: %s", metric, query.Hostname, *qdata.Error)) + /* Build list for "partial errors", if any */ + errors = append(errors, fmt.Sprintf("fetching %s for node %s failed: %s", metric, query.Hostname, *qdata.Error)) } if qdata.Avg.IsNaN() || qdata.Min.IsNaN() || qdata.Max.IsNaN() { @@ -638,7 +641,8 @@ func (ccms *CCMetricStore) LoadNodeData( } if len(errors) != 0 { - return data, fmt.Errorf("METRICDATA/CCMS > errors: %s", strings.Join(errors, ", ")) + /* Returns list of "partial errors" */ + return data, fmt.Errorf("METRICDATA/CCMS > Errors: %s", strings.Join(errors, ", ")) } return data, nil diff --git a/internal/metricdata/influxdb-v2.go b/internal/metricdata/influxdb-v2.go index 62da698..17a2165 100644 --- a/internal/metricdata/influxdb-v2.go +++ b/internal/metricdata/influxdb-v2.go @@ -37,7 +37,7 @@ type InfluxDBv2DataRepository struct { func (idb *InfluxDBv2DataRepository) Init(rawConfig json.RawMessage) error { var config InfluxDBv2DataRepositoryConfig if err := json.Unmarshal(rawConfig, &config); err != nil { - log.Error("Error while unmarshaling raw json config") + log.Warn("Error while unmarshaling raw json config") return err } @@ -205,7 +205,7 @@ func (idb *InfluxDBv2DataRepository) LoadData( // Get Stats stats, err := idb.LoadStats(job, metrics, ctx) if err != nil { - log.Error("Error while loading statistics") + log.Warn("Error while loading statistics") return nil, err } diff --git a/internal/metricdata/metricdata.go b/internal/metricdata/metricdata.go index d022f5e..0c0c5d6 100644 --- a/internal/metricdata/metricdata.go +++ b/internal/metricdata/metricdata.go @@ -46,7 +46,7 @@ func Init(disableArchive bool) error { Kind string `json:"kind"` } if err := json.Unmarshal(cluster.MetricDataRepository, &kind); err != nil { - log.Error("Error while unmarshaling raw json MetricDataRepository") + log.Warn("Error while unmarshaling raw json MetricDataRepository") return err } @@ -61,11 +61,11 @@ func Init(disableArchive bool) error { case "test": mdr = &TestMetricDataRepository{} default: - return fmt.Errorf("METRICDATA/METRICDATA > unkown metric data repository '%s' for cluster '%s'", kind.Kind, cluster.Name) + return fmt.Errorf("METRICDATA/METRICDATA > Unknown MetricDataRepository %v for cluster %v", kind.Kind, cluster.Name) } if err := mdr.Init(cluster.MetricDataRepository); err != nil { - log.Error("Error initializing the MetricDataRepository") + log.Errorf("Error initializing MetricDataRepository %v for cluster %v", kind.Kind, cluster.Name) return err } metricDataRepos[cluster.Name] = mdr @@ -109,7 +109,7 @@ func LoadData(job *schema.Job, jd, err = repo.LoadData(job, metrics, scopes, ctx) if err != nil { if len(jd) != 0 { - log.Errorf("partial error: %s", err.Error()) + log.Warnf("partial error: %s", err.Error()) } else { log.Error("Error while loading job data from metric repository") return err, 0, 0 @@ -192,7 +192,7 @@ func LoadAverages( stats, err := repo.LoadStats(job, metrics, ctx) if err != nil { - log.Errorf("Error while loading statistics for job %#v (User %#v, Project %#v)", job.JobID, job.User, job.Project) + log.Errorf("Error while loading statistics for job %v (User %v, Project %v)", job.JobID, job.User, job.Project) return err } @@ -235,7 +235,7 @@ func LoadNodeData( data, err := repo.LoadNodeData(cluster, metrics, nodes, scopes, from, to, ctx) if err != nil { if len(data) != 0 { - log.Errorf("partial error: %s", err.Error()) + log.Warnf("partial error: %s", err.Error()) } else { log.Error("Error while loading node data from metric repository") return nil, err diff --git a/internal/metricdata/prometheus.go b/internal/metricdata/prometheus.go index dd8ceb8..dd23024 100644 --- a/internal/metricdata/prometheus.go +++ b/internal/metricdata/prometheus.go @@ -154,7 +154,7 @@ func (pdb *PrometheusDataRepository) Init(rawConfig json.RawMessage) error { var config PrometheusDataRepositoryConfig // parse config if err := json.Unmarshal(rawConfig, &config); err != nil { - log.Error("Error while unmarshaling raw json config") + log.Warn("Error while unmarshaling raw json config") return err } // support basic authentication @@ -188,7 +188,7 @@ func (pdb *PrometheusDataRepository) Init(rawConfig json.RawMessage) error { if err == nil { log.Debugf("Added PromQL template for %s: %s", metric, templ) } else { - log.Errorf("Failed to parse PromQL template %s for metric %s", templ, metric) + log.Warnf("Failed to parse PromQL template %s for metric %s", templ, metric) } } return nil @@ -292,12 +292,12 @@ func (pdb *PrometheusDataRepository) LoadData( for _, metric := range metrics { metricConfig := archive.GetMetricConfig(job.Cluster, metric) if metricConfig == nil { - log.Errorf("Error in LoadData: Metric %s for cluster %s not configured", metric, job.Cluster) - return nil, errors.New("METRICDATA/PROMETHEUS > Prometheus query error") + log.Warnf("Error in LoadData: Metric %s for cluster %s not configured", metric, job.Cluster) + return nil, errors.New("Prometheus config error") } query, err := pdb.FormatQuery(metric, scope, nodes, job.Cluster) if err != nil { - log.Error("Error while formatting prometheus query") + log.Warn("Error while formatting prometheus query") return nil, err } @@ -311,7 +311,7 @@ func (pdb *PrometheusDataRepository) LoadData( if err != nil { log.Errorf("Prometheus query error in LoadData: %v\nQuery: %s", err, query) - return nil, errors.New("METRICDATA/PROMETHEUS > Prometheus query error") + return nil, errors.New("Prometheus query error") } if len(warnings) > 0 { log.Warnf("Warnings: %v\n", warnings) @@ -361,7 +361,7 @@ func (pdb *PrometheusDataRepository) LoadStats( data, err := pdb.LoadData(job, metrics, []schema.MetricScope{schema.MetricScopeNode}, ctx) if err != nil { - log.Error("Error while loading job for stats") + log.Warn("Error while loading job for stats") return nil, err } for metric, metricData := range data { @@ -399,12 +399,12 @@ func (pdb *PrometheusDataRepository) LoadNodeData( for _, metric := range metrics { metricConfig := archive.GetMetricConfig(cluster, metric) if metricConfig == nil { - log.Errorf("Error in LoadNodeData: Metric %s for cluster %s not configured", metric, cluster) - return nil, errors.New("METRICDATA/PROMETHEUS > Prometheus querry error") + log.Warnf("Error in LoadNodeData: Metric %s for cluster %s not configured", metric, cluster) + return nil, errors.New("Prometheus config error") } query, err := pdb.FormatQuery(metric, scope, nodes, cluster) if err != nil { - log.Error("Error while formatting prometheus query") + log.Warn("Error while formatting prometheus query") return nil, err } @@ -418,7 +418,7 @@ func (pdb *PrometheusDataRepository) LoadNodeData( if err != nil { log.Errorf("Prometheus query error in LoadNodeData: %v\n", err) - return nil, errors.New("METRICDATA/PROMETHEUS > Prometheus querry error") + return nil, errors.New("Prometheus query error") } if len(warnings) > 0 { log.Warnf("Warnings: %v\n", warnings) diff --git a/internal/repository/init.go b/internal/repository/init.go index 1d3f403..1281c43 100644 --- a/internal/repository/init.go +++ b/internal/repository/init.go @@ -100,7 +100,7 @@ func HandleImportFlag(flag string) error { raw, err := os.ReadFile(files[0]) if err != nil { - log.Error("Error while reading metadata file for import") + log.Warn("Error while reading metadata file for import") return err } @@ -113,13 +113,13 @@ func HandleImportFlag(flag string) error { dec.DisallowUnknownFields() jobMeta := schema.JobMeta{BaseJob: schema.JobDefaults} if err := dec.Decode(&jobMeta); err != nil { - log.Error("Error while decoding raw json metadata for import") + log.Warn("Error while decoding raw json metadata for import") return err } raw, err = os.ReadFile(files[1]) if err != nil { - log.Error("Error while reading jobdata file for import") + log.Warn("Error while reading jobdata file for import") return err } @@ -132,7 +132,7 @@ func HandleImportFlag(flag string) error { dec.DisallowUnknownFields() jobData := schema.JobData{} if err := dec.Decode(&jobData); err != nil { - log.Error("Error while decoding raw json jobdata for import") + log.Warn("Error while decoding raw json jobdata for import") return err } @@ -140,7 +140,7 @@ func HandleImportFlag(flag string) error { jobMeta.MonitoringStatus = schema.MonitoringStatusArchivingSuccessful if job, err := GetJobRepository().Find(&jobMeta.JobID, &jobMeta.Cluster, &jobMeta.StartTime); err != sql.ErrNoRows { if err != nil { - log.Error("Error while finding job in jobRepository") + log.Warn("Error while finding job in jobRepository") return err } @@ -160,17 +160,17 @@ func HandleImportFlag(flag string) error { job.FileBwAvg = loadJobStat(&jobMeta, "file_bw") job.RawResources, err = json.Marshal(job.Resources) if err != nil { - log.Error("Error while marshaling job resources") + log.Warn("Error while marshaling job resources") return err } job.RawMetaData, err = json.Marshal(job.MetaData) if err != nil { - log.Error("Error while marshaling job metadata") + log.Warn("Error while marshaling job metadata") return err } if err := SanityChecks(&job.BaseJob); err != nil { - log.Error("BaseJob SanityChecks failed") + log.Warn("BaseJob SanityChecks failed") return err } @@ -181,13 +181,13 @@ func HandleImportFlag(flag string) error { res, err := GetConnection().DB.NamedExec(NamedJobInsert, job) if err != nil { - log.Error("Error while NamedJobInsert") + log.Warn("Error while NamedJobInsert") return err } id, err := res.LastInsertId() if err != nil { - log.Error("Error while getting last insert ID") + log.Warn("Error while getting last insert ID") return err } @@ -221,13 +221,13 @@ func InitDB() error { // that speeds up inserts A LOT. tx, err := db.DB.Beginx() if err != nil { - log.Error("Error while bundling transactions") + log.Warn("Error while bundling transactions") return err } stmt, err := tx.PrepareNamed(NamedJobInsert) if err != nil { - log.Error("Error while preparing namedJobInsert") + log.Warn("Error while preparing namedJobInsert") return err } tags := make(map[string]int64) @@ -247,14 +247,14 @@ func InitDB() error { if i%10 == 0 { if tx != nil { if err := tx.Commit(); err != nil { - log.Error("Error while committing transactions for jobMeta") + log.Warn("Error while committing transactions for jobMeta") return err } } tx, err = db.DB.Beginx() if err != nil { - log.Error("Error while bundling transactions for jobMeta") + log.Warn("Error while bundling transactions for jobMeta") return err } @@ -315,19 +315,19 @@ func InitDB() error { if !ok { res, err := tx.Exec(`INSERT INTO tag (tag_name, tag_type) VALUES (?, ?)`, tag.Name, tag.Type) if err != nil { - log.Errorf("Error while inserting tag into tag table: %#v %#v", tag.Name, tag.Type) + log.Errorf("Error while inserting tag into tag table: %v (Type %v)", tag.Name, tag.Type) return err } tagId, err = res.LastInsertId() if err != nil { - log.Error("Error while getting last insert ID") + log.Warn("Error while getting last insert ID") return err } tags[tagstr] = tagId } if _, err := tx.Exec(`INSERT INTO jobtag (job_id, tag_id) VALUES (?, ?)`, id, tagId); err != nil { - log.Errorf("Error while inserting jobtag into jobtag table: %#v %#v", id, tagId) + log.Errorf("Error while inserting jobtag into jobtag table: %v (TagID %v)", id, tagId) return err } } @@ -338,18 +338,18 @@ func InitDB() error { } if errorOccured > 0 { - log.Errorf("Error in import of %d jobs!", errorOccured) + log.Warnf("Error in import of %d jobs!", errorOccured) } if err := tx.Commit(); err != nil { - log.Error("Error while committing SQL transactions") + log.Warn("Error while committing SQL transactions") return err } // Create indexes after inserts so that they do not // need to be continually updated. if _, err := db.DB.Exec(JobsDbIndexes); err != nil { - log.Error("Error while creating indices after inserts") + log.Warn("Error while creating indices after inserts") return err } @@ -360,14 +360,14 @@ func InitDB() error { // This function also sets the subcluster if necessary! func SanityChecks(job *schema.BaseJob) error { if c := archive.GetCluster(job.Cluster); c == nil { - return fmt.Errorf("no such cluster: %#v", job.Cluster) + return fmt.Errorf("no such cluster: %v", job.Cluster) } if err := archive.AssignSubCluster(job); err != nil { - log.Error("Error while assigning subcluster to job") + log.Warn("Error while assigning subcluster to job") return err } if !job.State.Valid() { - return fmt.Errorf("not a valid job state: %#v", job.State) + return fmt.Errorf("not a valid job state: %v", job.State) } if len(job.Resources) == 0 || len(job.User) == 0 { return fmt.Errorf("'resources' and 'user' should not be empty") diff --git a/internal/repository/job.go b/internal/repository/job.go index 202b678..53316ad 100644 --- a/internal/repository/job.go +++ b/internal/repository/job.go @@ -68,12 +68,12 @@ func scanJob(row interface{ Scan(...interface{}) error }) (*schema.Job, error) { &job.ID, &job.JobID, &job.User, &job.Project, &job.Cluster, &job.SubCluster, &job.StartTimeUnix, &job.Partition, &job.ArrayJobId, &job.NumNodes, &job.NumHWThreads, &job.NumAcc, &job.Exclusive, &job.MonitoringStatus, &job.SMT, &job.State, &job.Duration, &job.Walltime, &job.RawResources /*&job.MetaData*/); err != nil { - log.Error("Error while scanning rows") + log.Warn("Error while scanning rows") return nil, err } if err := json.Unmarshal(job.RawResources, &job.Resources); err != nil { - log.Error("Error while unmarhsaling raw resources json") + log.Warn("Error while unmarhsaling raw resources json") return nil, err } @@ -95,7 +95,7 @@ func (r *JobRepository) FetchMetadata(job *schema.Job) (map[string]string, error if err := sq.Select("job.meta_data").From("job").Where("job.id = ?", job.ID). RunWith(r.stmtCache).QueryRow().Scan(&job.RawMetaData); err != nil { - log.Error("Error while scanning for job metadata") + log.Warn("Error while scanning for job metadata") return nil, err } @@ -104,7 +104,7 @@ func (r *JobRepository) FetchMetadata(job *schema.Job) (map[string]string, error } if err := json.Unmarshal(job.RawMetaData, &job.MetaData); err != nil { - log.Error("Error while unmarshaling raw metadata json") + log.Warn("Error while unmarshaling raw metadata json") return nil, err } @@ -117,7 +117,7 @@ func (r *JobRepository) UpdateMetadata(job *schema.Job, key, val string) (err er r.cache.Del(cachekey) if job.MetaData == nil { if _, err = r.FetchMetadata(job); err != nil { - log.Errorf("Error while fetching metadata for job, DB ID '%#v'", job.ID) + log.Warnf("Error while fetching metadata for job, DB ID '%v'", job.ID) return err } } @@ -134,12 +134,12 @@ func (r *JobRepository) UpdateMetadata(job *schema.Job, key, val string) (err er } if job.RawMetaData, err = json.Marshal(job.MetaData); err != nil { - log.Errorf("Error while marshaling metadata for job, DB ID '%#v'", job.ID) + log.Warnf("Error while marshaling metadata for job, DB ID '%v'", job.ID) return err } if _, err = sq.Update("job").Set("meta_data", job.RawMetaData).Where("job.id = ?", job.ID).RunWith(r.stmtCache).Exec(); err != nil { - log.Errorf("Error while updating metadata for job, DB ID '%#v'", job.ID) + log.Warnf("Error while updating metadata for job, DB ID '%v'", job.ID) return err } @@ -200,7 +200,7 @@ func (r *JobRepository) FindAll( for rows.Next() { job, err := scanJob(rows) if err != nil { - log.Error("Error while scanning rows") + log.Warn("Error while scanning rows") return nil, err } jobs = append(jobs, job) @@ -302,7 +302,7 @@ func (r *JobRepository) CountGroupedJobs(ctx context.Context, aggreg model.Aggre count = fmt.Sprintf(`sum(job.num_nodes * (CASE WHEN job.job_state = "running" THEN %d - job.start_time ELSE job.duration END)) as count`, now) runner = r.DB default: - log.Notef("CountGroupedJobs() Weight %#v unknown.", *weight) + log.Notef("CountGroupedJobs() Weight %v unknown.", *weight) } } @@ -326,7 +326,7 @@ func (r *JobRepository) CountGroupedJobs(ctx context.Context, aggreg model.Aggre var group string var count int if err := rows.Scan(&group, &count); err != nil { - log.Error("Error while scanning rows") + log.Warn("Error while scanning rows") return nil, err } @@ -370,12 +370,12 @@ func (r *JobRepository) MarkArchived( case "file_bw": stmt = stmt.Set("file_bw_avg", stats.Avg) default: - log.Notef("MarkArchived() Metric '%#v' unknown", metric) + log.Notef("MarkArchived() Metric '%v' unknown", metric) } } if _, err := stmt.RunWith(r.stmtCache).Exec(); err != nil { - log.Error("Error while marking job as archived") + log.Warn("Error while marking job as archived") return err } return nil @@ -501,11 +501,11 @@ func (r *JobRepository) AllocatedNodes(cluster string) (map[string]map[string]in var resources []*schema.Resource var subcluster string if err := rows.Scan(&raw, &subcluster); err != nil { - log.Error("Error while scanning rows") + log.Warn("Error while scanning rows") return nil, err } if err := json.Unmarshal(raw, &resources); err != nil { - log.Error("Error while unmarshaling raw resources json") + log.Warn("Error while unmarshaling raw resources json") return nil, err } @@ -533,13 +533,13 @@ func (r *JobRepository) StopJobsExceedingWalltimeBy(seconds int) error { Where(fmt.Sprintf("(%d - job.start_time) > (job.walltime + %d)", time.Now().Unix(), seconds)). RunWith(r.DB).Exec() if err != nil { - log.Error("Error while stopping jobs exceeding walltime") + log.Warn("Error while stopping jobs exceeding walltime") return err } rowsAffected, err := res.RowsAffected() if err != nil { - log.Error("Error while fetching affected rows after stopping due to exceeded walltime") + log.Warn("Error while fetching affected rows after stopping due to exceeded walltime") return err } diff --git a/internal/repository/query.go b/internal/repository/query.go index f98b702..3a5ae42 100644 --- a/internal/repository/query.go +++ b/internal/repository/query.go @@ -51,7 +51,7 @@ func (r *JobRepository) QueryJobs( sql, args, err := query.ToSql() if err != nil { - log.Error("Error while converting query to sql") + log.Warn("Error while converting query to sql") return nil, err } @@ -67,7 +67,7 @@ func (r *JobRepository) QueryJobs( job, err := scanJob(rows) if err != nil { rows.Close() - log.Error("Error while scanning rows") + log.Warn("Error while scanning rows") return nil, err } jobs = append(jobs, job) diff --git a/internal/repository/tags.go b/internal/repository/tags.go index 6ab92e5..c8795e1 100644 --- a/internal/repository/tags.go +++ b/internal/repository/tags.go @@ -20,13 +20,13 @@ func (r *JobRepository) AddTag(job int64, tag int64) ([]*schema.Tag, error) { j, err := r.FindById(job) if err != nil { - log.Error("Error while finding job by id") + log.Warn("Error while finding job by id") return nil, err } tags, err := r.GetTags(&job) if err != nil { - log.Error("Error while getting tags for job") + log.Warn("Error while getting tags for job") return nil, err } @@ -42,13 +42,13 @@ func (r *JobRepository) RemoveTag(job, tag int64) ([]*schema.Tag, error) { j, err := r.FindById(job) if err != nil { - log.Error("Error while finding job by id") + log.Warn("Error while finding job by id") return nil, err } tags, err := r.GetTags(&job) if err != nil { - log.Error("Error while getting tags for job") + log.Warn("Error while getting tags for job") return nil, err } @@ -153,7 +153,7 @@ func (r *JobRepository) GetTags(job *int64) ([]*schema.Tag, error) { for rows.Next() { tag := &schema.Tag{} if err := rows.Scan(&tag.ID, &tag.Type, &tag.Name); err != nil { - log.Error("Error while scanning rows") + log.Warn("Error while scanning rows") return nil, err } tags = append(tags, tag) diff --git a/internal/repository/user.go b/internal/repository/user.go index 01ecb32..759ed5b 100644 --- a/internal/repository/user.go +++ b/internal/repository/user.go @@ -82,7 +82,7 @@ func (uCfg *UserCfgRepo) GetUIConfig(user *auth.User) (map[string]interface{}, e rows, err := uCfg.Lookup.Query(user.Username) if err != nil { - log.Errorf("Error while looking up user config for user '%#v'", user.Username) + log.Warnf("Error while looking up user config for user '%v'", user.Username) return err, 0, 0 } @@ -91,13 +91,13 @@ func (uCfg *UserCfgRepo) GetUIConfig(user *auth.User) (map[string]interface{}, e for rows.Next() { var key, rawval string if err := rows.Scan(&key, &rawval); err != nil { - log.Error("Error while scanning user config values") + log.Warn("Error while scanning user config values") return err, 0, 0 } var val interface{} if err := json.Unmarshal([]byte(rawval), &val); err != nil { - log.Error("Error while unmarshaling raw user config json") + log.Warn("Error while unmarshaling raw user config json") return err, 0, 0 } @@ -109,7 +109,7 @@ func (uCfg *UserCfgRepo) GetUIConfig(user *auth.User) (map[string]interface{}, e return config, 24 * time.Hour, size }) if err, ok := data.(error); ok { - log.Error("Error in data set") + log.Error("Error in returned dataset") return nil, err } @@ -126,7 +126,7 @@ func (uCfg *UserCfgRepo) UpdateConfig( if user == nil { var val interface{} if err := json.Unmarshal([]byte(value), &val); err != nil { - log.Error("Error while unmarshaling raw user config json") + log.Warn("Error while unmarshaling raw user config json") return err } @@ -138,7 +138,7 @@ func (uCfg *UserCfgRepo) UpdateConfig( if _, err := uCfg.DB.Exec(`REPLACE INTO configuration (username, confkey, value) VALUES (?, ?, ?)`, user, key, value); err != nil { - log.Errorf("Error while replacing user config in DB for user '$#v'", user) + log.Warnf("Error while replacing user config in DB for user '$#v'", user) return err } diff --git a/internal/routerConfig/routes.go b/internal/routerConfig/routes.go index 9424df7..c872a69 100644 --- a/internal/routerConfig/routes.go +++ b/internal/routerConfig/routes.go @@ -61,12 +61,12 @@ func setupHomeRoute(i InfoType, r *http.Request) InfoType { State: []schema.JobState{schema.JobStateRunning}, }}, nil, nil) if err != nil { - log.Errorf("failed to count jobs: %s", err.Error()) + log.Warnf("failed to count jobs: %s", err.Error()) runningJobs = map[string]int{} } totalJobs, err := jobRepo.CountGroupedJobs(r.Context(), model.AggregateCluster, nil, nil, nil) if err != nil { - log.Errorf("failed to count jobs: %s", err.Error()) + log.Warnf("failed to count jobs: %s", err.Error()) totalJobs = map[string]int{} } from := time.Now().Add(-24 * time.Hour) @@ -75,7 +75,7 @@ func setupHomeRoute(i InfoType, r *http.Request) InfoType { Duration: &schema.IntRange{From: 0, To: graph.ShortJobDuration}, }}, nil, nil) if err != nil { - log.Errorf("failed to count jobs: %s", err.Error()) + log.Warnf("failed to count jobs: %s", err.Error()) recentShortJobs = map[string]int{} } @@ -150,7 +150,7 @@ func setupTaglistRoute(i InfoType, r *http.Request) InfoType { tags, counts, err := jobRepo.CountTags(username) tagMap := make(map[string][]map[string]interface{}) if err != nil { - log.Errorf("GetTags failed: %s", err.Error()) + log.Warnf("GetTags failed: %s", err.Error()) i["tagmap"] = tagMap return i } diff --git a/internal/runtimeEnv/setup.go b/internal/runtimeEnv/setup.go index 2d0fc61..a98bf39 100644 --- a/internal/runtimeEnv/setup.go +++ b/internal/runtimeEnv/setup.go @@ -92,13 +92,13 @@ func DropPrivileges(username string, group string) error { if group != "" { g, err := user.LookupGroup(group) if err != nil { - log.Error("Error while looking up group") + log.Warn("Error while looking up group") return err } gid, _ := strconv.Atoi(g.Gid) if err := syscall.Setgid(gid); err != nil { - log.Error("Error while setting gid") + log.Warn("Error while setting gid") return err } } @@ -106,13 +106,13 @@ func DropPrivileges(username string, group string) error { if username != "" { u, err := user.Lookup(username) if err != nil { - log.Error("Error while looking up user") + log.Warn("Error while looking up user") return err } uid, _ := strconv.Atoi(u.Uid) if err := syscall.Setuid(uid); err != nil { - log.Error("Error while setting uid") + log.Warn("Error while setting uid") return err } } diff --git a/pkg/archive/archive.go b/pkg/archive/archive.go index 051cc8b..94fa1e6 100644 --- a/pkg/archive/archive.go +++ b/pkg/archive/archive.go @@ -41,7 +41,7 @@ func Init(rawConfig json.RawMessage, disableArchive bool) error { Kind string `json:"kind"` } if err := json.Unmarshal(rawConfig, &kind); err != nil { - log.Error("Error while unmarshaling raw config json") + log.Warn("Error while unmarshaling raw config json") return err } @@ -73,7 +73,7 @@ func LoadAveragesFromArchive( metaFile, err := ar.LoadJobMeta(job) if err != nil { - log.Error("Error while loading job metadata from archiveBackend") + log.Warn("Error while loading job metadata from archiveBackend") return err } @@ -92,7 +92,7 @@ func GetStatistics(job *schema.Job) (map[string]schema.JobStatistics, error) { metaFile, err := ar.LoadJobMeta(job) if err != nil { - log.Error("Error while loading job metadata from archiveBackend") + log.Warn("Error while loading job metadata from archiveBackend") return nil, err } @@ -109,7 +109,7 @@ func UpdateTags(job *schema.Job, tags []*schema.Tag) error { jobMeta, err := ar.LoadJobMeta(job) if err != nil { - log.Error("Error while loading job metadata from archiveBackend") + log.Warn("Error while loading job metadata from archiveBackend") return err } diff --git a/pkg/archive/clusterConfig.go b/pkg/archive/clusterConfig.go index 86635cc..d77e9c3 100644 --- a/pkg/archive/clusterConfig.go +++ b/pkg/archive/clusterConfig.go @@ -24,7 +24,7 @@ func initClusterConfig() error { cluster, err := ar.LoadClusterCfg(c) if err != nil { - log.Errorf("Error while loading cluster config for cluster '%#v'", c) + log.Warnf("Error while loading cluster config for cluster '%v'", c) return err } @@ -114,7 +114,7 @@ func AssignSubCluster(job *schema.BaseJob) error { cluster := GetCluster(job.Cluster) if cluster == nil { - return fmt.Errorf("ARCHIVE/CLUSTERCONFIG > unkown cluster: %#v", job.Cluster) + return fmt.Errorf("ARCHIVE/CLUSTERCONFIG > unkown cluster: %v", job.Cluster) } if job.SubCluster != "" { @@ -123,7 +123,7 @@ func AssignSubCluster(job *schema.BaseJob) error { return nil } } - return fmt.Errorf("ARCHIVE/CLUSTERCONFIG > already assigned subcluster %#v unkown (cluster: %#v)", job.SubCluster, job.Cluster) + return fmt.Errorf("ARCHIVE/CLUSTERCONFIG > already assigned subcluster %v unkown (cluster: %v)", job.SubCluster, job.Cluster) } if len(job.Resources) == 0 { @@ -143,7 +143,7 @@ func AssignSubCluster(job *schema.BaseJob) error { return nil } - return fmt.Errorf("ARCHIVE/CLUSTERCONFIG > no subcluster found for cluster %#v and host %#v", job.Cluster, host0) + return fmt.Errorf("ARCHIVE/CLUSTERCONFIG > no subcluster found for cluster %v and host %v", job.Cluster, host0) } func GetSubClusterByNode(cluster, hostname string) (string, error) { @@ -156,12 +156,12 @@ func GetSubClusterByNode(cluster, hostname string) (string, error) { c := GetCluster(cluster) if c == nil { - return "", fmt.Errorf("ARCHIVE/CLUSTERCONFIG > unkown cluster: %#v", cluster) + return "", fmt.Errorf("ARCHIVE/CLUSTERCONFIG > unkown cluster: %v", cluster) } if c.SubClusters[0].Nodes == "" { return c.SubClusters[0].Name, nil } - return "", fmt.Errorf("ARCHIVE/CLUSTERCONFIG > no subcluster found for cluster %#v and host %#v", cluster, hostname) + return "", fmt.Errorf("ARCHIVE/CLUSTERCONFIG > no subcluster found for cluster %v and host %v", cluster, hostname) } diff --git a/pkg/archive/fsBackend.go b/pkg/archive/fsBackend.go index 1e363bb..40e9502 100644 --- a/pkg/archive/fsBackend.go +++ b/pkg/archive/fsBackend.go @@ -46,7 +46,7 @@ func loadJobMeta(filename string) (*schema.JobMeta, error) { f, err := os.Open(filename) if err != nil { - log.Errorf("loadJobMeta() > open file error: %#v", err) + log.Errorf("loadJobMeta() > open file error: %v", err) return &schema.JobMeta{}, err } defer f.Close() @@ -58,19 +58,19 @@ func (fsa *FsArchive) Init(rawConfig json.RawMessage) error { var config FsArchiveConfig if err := json.Unmarshal(rawConfig, &config); err != nil { - log.Errorf("Init() > Unmarshal error: %#v", err) + log.Warnf("Init() > Unmarshal error: %#v", err) return err } if config.Path == "" { - err := fmt.Errorf("ARCHIVE/FSBACKEND > Init() : empty config.Path") - log.Errorf("Init() > config.Path error: %#v", err) + err := fmt.Errorf("Init() : empty config.Path") + log.Errorf("Init() > config.Path error: %v", err) return err } fsa.path = config.Path entries, err := os.ReadDir(fsa.path) if err != nil { - log.Errorf("Init() > ReadDir() error: %#v", err) + log.Errorf("Init() > ReadDir() error: %v", err) return err } @@ -86,7 +86,7 @@ func (fsa *FsArchive) LoadJobData(job *schema.Job) (schema.JobData, error) { filename := getPath(job, fsa.path, "data.json") f, err := os.Open(filename) if err != nil { - log.Errorf("LoadJobData() > open file error: %#v", err) + log.Errorf("LoadJobData() > open file error: %v", err) return nil, err } defer f.Close() @@ -109,7 +109,8 @@ func (fsa *FsArchive) LoadClusterCfg(name string) (*schema.Cluster, error) { } if config.Keys.Validate { if err := schema.Validate(schema.ClusterCfg, bytes.NewReader(b)); err != nil { - return &schema.Cluster{}, fmt.Errorf("ARCHIVE/FSBACKEND > Validate cluster config: %v\n", err) + log.Warnf("Validate cluster config: %v\n", err) + return &schema.Cluster{}, fmt.Errorf("Validate cluster config: %v\n", err) } } return DecodeCluster(bytes.NewReader(b)) @@ -183,7 +184,7 @@ func (fsa *FsArchive) StoreJobMeta(jobMeta *schema.JobMeta) error { return err } if err := f.Close(); err != nil { - log.Error("Error while closing meta.json file") + log.Warn("Error while closing meta.json file") return err } @@ -220,7 +221,7 @@ func (fsa *FsArchive) ImportJob( return err } if err := f.Close(); err != nil { - log.Error("Error while closing meta.json file") + log.Warn("Error while closing meta.json file") return err } @@ -234,10 +235,10 @@ func (fsa *FsArchive) ImportJob( return err } if err := f.Close(); err != nil { - log.Error("Error while closing data.json file") + log.Warn("Error while closing data.json file") return err } - + // no error: final return is nil return nil } diff --git a/pkg/archive/json.go b/pkg/archive/json.go index 0fd3bce..d4409a6 100644 --- a/pkg/archive/json.go +++ b/pkg/archive/json.go @@ -17,7 +17,7 @@ func DecodeJobData(r io.Reader, k string) (schema.JobData, error) { data := cache.Get(k, func() (value interface{}, ttl time.Duration, size int) { var d schema.JobData if err := json.NewDecoder(r).Decode(&d); err != nil { - log.Error("Error while decoding raw job data json") + log.Warn("Error while decoding raw job data json") return err, 0, 1000 } @@ -25,7 +25,7 @@ func DecodeJobData(r io.Reader, k string) (schema.JobData, error) { }) if err, ok := data.(error); ok { - log.Error("Error in decoded job data set") + log.Warn("Error in decoded job data set") return nil, err } @@ -35,7 +35,7 @@ func DecodeJobData(r io.Reader, k string) (schema.JobData, error) { func DecodeJobMeta(r io.Reader) (*schema.JobMeta, error) { var d schema.JobMeta if err := json.NewDecoder(r).Decode(&d); err != nil { - log.Error("Error while decoding raw job meta json") + log.Warn("Error while decoding raw job meta json") return &d, err } @@ -47,7 +47,7 @@ func DecodeJobMeta(r io.Reader) (*schema.JobMeta, error) { func DecodeCluster(r io.Reader) (*schema.Cluster, error) { var c schema.Cluster if err := json.NewDecoder(r).Decode(&c); err != nil { - log.Error("Error while decoding raw cluster json") + log.Warn("Error while decoding raw cluster json") return &c, err } @@ -59,7 +59,7 @@ func DecodeCluster(r io.Reader) (*schema.Cluster, error) { func EncodeJobData(w io.Writer, d *schema.JobData) error { // Sanitize parameters if err := json.NewEncoder(w).Encode(d); err != nil { - log.Error("Error while encoding new job data json") + log.Warn("Error while encoding new job data json") return err } @@ -69,7 +69,7 @@ func EncodeJobData(w io.Writer, d *schema.JobData) error { func EncodeJobMeta(w io.Writer, d *schema.JobMeta) error { // Sanitize parameters if err := json.NewEncoder(w).Encode(d); err != nil { - log.Error("Error while encoding new job meta json") + log.Warn("Error while encoding new job meta json") return err } diff --git a/pkg/archive/nodelist.go b/pkg/archive/nodelist.go index c22d99d..d31a65e 100644 --- a/pkg/archive/nodelist.go +++ b/pkg/archive/nodelist.go @@ -149,7 +149,7 @@ func ParseNodeList(raw string) (NodeList, error) { s1, s2 := part[0:minus], part[minus+1:] if len(s1) != len(s2) || len(s1) == 0 { - return nil, fmt.Errorf("ARCHIVE/NODELIST > %#v and %#v are not of equal length or of length zero", s1, s2) + return nil, fmt.Errorf("ARCHIVE/NODELIST > %v and %v are not of equal length or of length zero", s1, s2) } x1, err := strconv.ParseInt(s1, 10, 32) diff --git a/pkg/schema/float.go b/pkg/schema/float.go index 52197f2..45d0750 100644 --- a/pkg/schema/float.go +++ b/pkg/schema/float.go @@ -45,7 +45,7 @@ func (f *Float) UnmarshalJSON(input []byte) error { val, err := strconv.ParseFloat(s, 64) if err != nil { - log.Error("Error while parsing custom float") + log.Warn("Error while parsing custom float") return err } *f = Float(val) diff --git a/pkg/schema/validate.go b/pkg/schema/validate.go index dafe892..aec234c 100644 --- a/pkg/schema/validate.go +++ b/pkg/schema/validate.go @@ -55,7 +55,7 @@ func Validate(k Kind, r io.Reader) (err error) { var v interface{} if err := json.NewDecoder(r).Decode(&v); err != nil { - log.Errorf("Error while decoding raw json schema: %#v", err) + log.Warnf("Error while decoding raw json schema: %#v", err) return err } diff --git a/web/web.go b/web/web.go index 85021c0..3210e0d 100644 --- a/web/web.go +++ b/web/web.go @@ -92,8 +92,8 @@ func RenderTemplate(rw http.ResponseWriter, r *http.Request, file string, page * } } - log.Infof("WEB/WEB > page config : %v\n", page.Config) + log.Infof("Page config : %v\n", page.Config) if err := t.Execute(rw, page); err != nil { - log.Errorf("WEB/WEB > template error: %s", err.Error()) + log.Errorf("Template error: %s", err.Error()) } } From ef1b2d7a81718b96b288b996c177df8010094cc9 Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Wed, 1 Feb 2023 12:29:58 +0100 Subject: [PATCH 09/12] Fix typo --- internal/repository/user.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/internal/repository/user.go b/internal/repository/user.go index 759ed5b..d8f86a1 100644 --- a/internal/repository/user.go +++ b/internal/repository/user.go @@ -136,9 +136,8 @@ func (uCfg *UserCfgRepo) UpdateConfig( return nil } - if _, err := uCfg.DB.Exec(`REPLACE INTO configuration (username, confkey, value) VALUES (?, ?, ?)`, - user, key, value); err != nil { - log.Warnf("Error while replacing user config in DB for user '$#v'", user) + if _, err := uCfg.DB.Exec(`REPLACE INTO configuration (username, confkey, value) VALUES (?, ?, ?)`, user, key, value); err != nil { + log.Warnf("Error while replacing user config in DB for user '%v'", user) return err } From c123a87ece1594a9de2e82ccf1ff15e73cc9c3d6 Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Thu, 9 Feb 2023 14:14:58 +0100 Subject: [PATCH 10/12] Simplify log wrapper implementation --- cmd/cc-backend/main.go | 5 +- pkg/log/log.go | 255 +++++++++++------------------------------ 2 files changed, 70 insertions(+), 190 deletions(-) diff --git a/cmd/cc-backend/main.go b/cmd/cc-backend/main.go index 979a99d..ad0f432 100644 --- a/cmd/cc-backend/main.go +++ b/cmd/cc-backend/main.go @@ -88,8 +88,7 @@ func main() { } // Apply config flags for pkg/log - log.SetLogLevel(flagLogLevel) - log.SetLogDateTime(flagLogDateTime) + log.Init(flagLogLevel, flagLogDateTime) // See https://github.com/google/gops (Runtime overhead is almost zero) if flagGops { @@ -297,7 +296,7 @@ func main() { if flagDev { r.Handle("/playground", playground.Handler("GraphQL playground", "/query")) r.PathPrefix("/swagger/").Handler(httpSwagger.Handler( - httpSwagger.URL("http://" + config.Keys.Addr + "/swagger/doc.json"))).Methods(http.MethodGet) + httpSwagger.URL("http://" + config.Keys.Addr + "/swagger/doc.json"))).Methods(http.MethodGet) } secured.Handle("/query", graphQLEndpoint) diff --git a/pkg/log/log.go b/pkg/log/log.go index c223566..aa8ad56 100644 --- a/pkg/log/log.go +++ b/pkg/log/log.go @@ -9,7 +9,6 @@ import ( "io" "log" "os" - "time" ) // Provides a simple way of logging with different levels. @@ -18,9 +17,6 @@ import ( // // Uses these prefixes: https://www.freedesktop.org/software/systemd/man/sd-daemon.html -var logDateTime bool -var logLevel string - var ( DebugWriter io.Writer = os.Stderr NoteWriter io.Writer = os.Stderr @@ -40,54 +36,57 @@ var ( ) var ( - // No Time/Date DebugLog *log.Logger = log.New(DebugWriter, DebugPrefix, 0) - InfoLog *log.Logger = log.New(InfoWriter, InfoPrefix, 0) - NoteLog *log.Logger = log.New(NoteWriter, NotePrefix, log.Lshortfile) - WarnLog *log.Logger = log.New(WarnWriter, WarnPrefix, log.Lshortfile) - ErrLog *log.Logger = log.New(ErrWriter, ErrPrefix, log.Llongfile) - CritLog *log.Logger = log.New(CritWriter, CritPrefix, log.Llongfile) - // Log Time/Date - DebugTimeLog *log.Logger = log.New(DebugWriter, DebugPrefix, log.LstdFlags) - InfoTimeLog *log.Logger = log.New(InfoWriter, InfoPrefix, log.LstdFlags) - NoteTimeLog *log.Logger = log.New(NoteWriter, NotePrefix, log.LstdFlags|log.Lshortfile) - WarnTimeLog *log.Logger = log.New(WarnWriter, WarnPrefix, log.LstdFlags|log.Lshortfile) - ErrTimeLog *log.Logger = log.New(ErrWriter, ErrPrefix, log.LstdFlags|log.Llongfile) - CritTimeLog *log.Logger = log.New(CritWriter, CritPrefix, log.LstdFlags|log.Llongfile) + InfoLog *log.Logger = log.New(InfoWriter, InfoPrefix, 0) + NoteLog *log.Logger = log.New(NoteWriter, NotePrefix, log.Lshortfile) + WarnLog *log.Logger = log.New(WarnWriter, WarnPrefix, log.Lshortfile) + ErrLog *log.Logger = log.New(ErrWriter, ErrPrefix, log.Llongfile) + CritLog *log.Logger = log.New(CritWriter, CritPrefix, log.Llongfile) ) /* CONFIG */ -func SetLogLevel(lvl string) { +func Init(lvl string, logdate bool) { // fmt.Printf("pkg/log: Set LOGLEVEL -> %s\n", lvl) switch lvl { - case "crit": - ErrWriter = io.Discard - fallthrough - case "err", "fatal": - WarnWriter = io.Discard - fallthrough - case "warn": - InfoWriter = io.Discard - fallthrough - case "notice": - NoteWriter = io.Discard - fallthrough - case "info": - DebugWriter = io.Discard - break - case "debug": - // Nothing to do... - break - default: - fmt.Printf("pkg/log: Flag 'loglevel' has invalid value %#v\npkg/log: Will use default loglevel 'debug'\n", lvl) - SetLogLevel("debug") + case "crit": + ErrWriter = io.Discard + fallthrough + case "err", "fatal": + WarnWriter = io.Discard + fallthrough + case "warn": + InfoWriter = io.Discard + fallthrough + case "notice": + NoteWriter = io.Discard + fallthrough + case "info": + DebugWriter = io.Discard + break + case "debug": + // Nothing to do... + break + default: + fmt.Printf("pkg/log: Flag 'loglevel' has invalid value %#v\npkg/log: Will use default loglevel 'debug'\n", lvl) + //SetLogLevel("debug") } -} -func SetLogDateTime(logdate bool) { - //fmt.Printf("pkg/log: Set DATEBOOL -> %v\n", logdate) - logDateTime = logdate + if logdate == false { + DebugLog = log.New(DebugWriter, DebugPrefix, 0) + InfoLog = log.New(InfoWriter, InfoPrefix, 0) + NoteLog = log.New(NoteWriter, NotePrefix, log.Lshortfile) + WarnLog = log.New(WarnWriter, WarnPrefix, log.Lshortfile) + ErrLog = log.New(ErrWriter, ErrPrefix, log.Llongfile) + CritLog = log.New(CritWriter, CritPrefix, log.Llongfile) + } else { + DebugLog = log.New(DebugWriter, DebugPrefix, log.LstdFlags) + InfoLog = log.New(InfoWriter, InfoPrefix, log.LstdFlags) + NoteLog = log.New(NoteWriter, NotePrefix, log.LstdFlags|log.Lshortfile) + WarnLog = log.New(WarnWriter, WarnPrefix, log.LstdFlags|log.Lshortfile) + ErrLog = log.New(ErrWriter, ErrPrefix, log.LstdFlags|log.Llongfile) + CritLog = log.New(CritWriter, CritPrefix, log.LstdFlags|log.Llongfile) + } } /* PRINT */ @@ -104,97 +103,38 @@ func Print(v ...interface{}) { } func Debug(v ...interface{}) { - if DebugWriter != io.Discard { - out := printStr(v...) - if logDateTime { - DebugTimeLog.Output(2, out) - } else { - DebugLog.Output(2, out) - } - } + DebugLog.Output(2, printStr(v...)) } func Info(v ...interface{}) { - if InfoWriter != io.Discard { - out := printStr(v...) - if logDateTime { - InfoTimeLog.Output(2, out) - } else { - InfoLog.Output(2, out) - } - } + InfoLog.Output(2, printStr(v...)) } func Note(v ...interface{}) { - if NoteWriter != io.Discard { - out := printStr(v...) - if logDateTime { - NoteTimeLog.Output(2, out) - } else { - NoteLog.Output(2, out) - } - } + NoteLog.Output(2, printStr(v...)) } func Warn(v ...interface{}) { - if WarnWriter != io.Discard { - out := printStr(v...) - if logDateTime { - WarnTimeLog.Output(2, out) - } else { - WarnLog.Output(2, out) - } - } + WarnLog.Output(2, printStr(v...)) } func Error(v ...interface{}) { - if ErrWriter != io.Discard { - out := printStr(v...) - if logDateTime { - ErrTimeLog.Output(2, out) - } else { - ErrLog.Output(2, out) - } - } + ErrLog.Output(2, printStr(v...)) } // Writes panic stacktrace, but keeps application alive func Panic(v ...interface{}) { - if ErrWriter != io.Discard { - out := printStr(v...) - if logDateTime { - ErrTimeLog.Output(2, out) - } else { - ErrLog.Output(2, out) - } - } - + ErrLog.Output(2, printStr(v...)) panic("Panic triggered ...") } - func Crit(v ...interface{}) { - if CritWriter != io.Discard { - out := printStr(v...) - if logDateTime { - CritTimeLog.Output(2, out) - } else { - CritLog.Output(2, out) - } - } + CritLog.Output(2, printStr(v...)) } // Writes critical log, stops application func Fatal(v ...interface{}) { - if CritWriter != io.Discard { - out := printStr(v...) - if logDateTime { - CritTimeLog.Output(2, out) - } else { - CritLog.Output(2, out) - } - } - + CritLog.Output(2, printStr(v...)) os.Exit(1) } @@ -212,109 +152,50 @@ func Printf(format string, v ...interface{}) { } func Debugf(format string, v ...interface{}) { - if DebugWriter != io.Discard { - out := printfStr(format, v...) - if logDateTime { - DebugTimeLog.Output(2, out) - } else { - DebugLog.Output(2, out) - } - } + DebugLog.Output(2, printStr(v...)) } func Infof(format string, v ...interface{}) { - if InfoWriter != io.Discard { - out := printfStr(format, v...) - if logDateTime { - InfoTimeLog.Output(2, out) - } else { - InfoLog.Output(2, out) - } - } + InfoLog.Output(2, printStr(v...)) } func Notef(format string, v ...interface{}) { - if NoteWriter != io.Discard { - out := printfStr(format, v...) - if logDateTime { - NoteTimeLog.Output(2, out) - } else { - NoteLog.Output(2, out) - } - } + NoteLog.Output(2, printStr(v...)) } func Warnf(format string, v ...interface{}) { - if WarnWriter != io.Discard { - out := printfStr(format, v...) - if logDateTime { - WarnTimeLog.Output(2, out) - } else { - WarnLog.Output(2, out) - } - } + WarnLog.Output(2, printStr(v...)) } func Errorf(format string, v ...interface{}) { - if ErrWriter != io.Discard { - out := printfStr(format, v...) - if logDateTime { - ErrTimeLog.Output(2, out) - } else { - ErrLog.Output(2, out) - } - } + ErrLog.Output(2, printStr(v...)) } // Writes panic stacktrace, but keeps application alive func Panicf(format string, v ...interface{}) { - if ErrWriter != io.Discard { - out := printfStr(format, v...) - if logDateTime { - ErrTimeLog.Output(2, out) - } else { - ErrLog.Output(2, out) - } - } - + ErrLog.Output(2, printStr(v...)) panic("Panic triggered ...") } - func Critf(format string, v ...interface{}) { - if CritWriter != io.Discard { - out := printfStr(format, v...) - if logDateTime { - CritTimeLog.Output(2, out) - } else { - CritLog.Output(2, out) - } - } + CritLog.Output(2, printStr(v...)) } // Writes crit log, stops application func Fatalf(format string, v ...interface{}) { - if CritWriter != io.Discard { - out := printfStr(format, v...) - if logDateTime { - CritTimeLog.Output(2, out) - } else { - CritLog.Output(2, out) - } - } - + CritLog.Output(2, printStr(v...)) os.Exit(1) } /* SPECIAL */ -func Finfof(w io.Writer, format string, v ...interface{}) { - if w != io.Discard { - if logDateTime { - currentTime := time.Now() - fmt.Fprintf(InfoWriter, currentTime.String()+InfoPrefix+format+"\n", v...) - } else { - fmt.Fprintf(InfoWriter, InfoPrefix+format+"\n", v...) - } - } -} +// func Finfof(w io.Writer, format string, v ...interface{}) { +// if w != io.Discard { +// if logDateTime { +// currentTime := time.Now() +// fmt.Fprintf(InfoWriter, currentTime.String()+InfoPrefix+format+"\n", v...) +// } else { +// fmt.Fprintf(InfoWriter, InfoPrefix+format+"\n", v...) +// } +// } +// } From 96ad865b53adff897692638fdd5b89ab090b560b Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Thu, 9 Feb 2023 15:43:38 +0100 Subject: [PATCH 11/12] Cleanup and linting --- pkg/log/log.go | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/pkg/log/log.go b/pkg/log/log.go index aa8ad56..91c7259 100644 --- a/pkg/log/log.go +++ b/pkg/log/log.go @@ -47,7 +47,6 @@ var ( /* CONFIG */ func Init(lvl string, logdate bool) { - // fmt.Printf("pkg/log: Set LOGLEVEL -> %s\n", lvl) switch lvl { case "crit": ErrWriter = io.Discard @@ -63,7 +62,6 @@ func Init(lvl string, logdate bool) { fallthrough case "info": DebugWriter = io.Discard - break case "debug": // Nothing to do... break @@ -72,7 +70,7 @@ func Init(lvl string, logdate bool) { //SetLogLevel("debug") } - if logdate == false { + if !logdate { DebugLog = log.New(DebugWriter, DebugPrefix, 0) InfoLog = log.New(InfoWriter, InfoPrefix, 0) NoteLog = log.New(NoteWriter, NotePrefix, log.Lshortfile) @@ -141,9 +139,9 @@ func Fatal(v ...interface{}) { /* PRINT FORMAT*/ // Private helper -func printfStr(format string, v ...interface{}) string { - return fmt.Sprintf(format, v...) -} +// func printfStr(format string, v ...interface{}) string { +// return fmt.Sprintf(format, v...) +// } // Uses Infof() -> If errorpath required at some point: // Will need own writer with 'Output(2, out)' to correctly render path From b8528b42d2c415d461e942bdfc44a200b8d13fb3 Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Thu, 9 Feb 2023 15:46:07 +0100 Subject: [PATCH 12/12] Fix flaginfo for --logdate --- cmd/cc-backend/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/cc-backend/main.go b/cmd/cc-backend/main.go index d8315f6..62912b5 100644 --- a/cmd/cc-backend/main.go +++ b/cmd/cc-backend/main.go @@ -69,7 +69,7 @@ func main() { flag.BoolVar(&flagGops, "gops", false, "Listen via github.com/google/gops/agent (for debugging)") flag.BoolVar(&flagDev, "dev", false, "Enable development components: GraphQL Playground and Swagger UI") flag.BoolVar(&flagVersion, "version", false, "Show version information and exit") - flag.BoolVar(&flagLogDateTime, "logdate", false, "Set true to add date and time to log messages (Default: false)") + flag.BoolVar(&flagLogDateTime, "logdate", false, "Set this flag to add date and time to log messages") flag.StringVar(&flagConfigFile, "config", "./config.json", "Specify alternative path to `config.json`") flag.StringVar(&flagNewUser, "add-user", "", "Add a new user. Argument format: `:[admin,support,api,user]:`") flag.StringVar(&flagDelUser, "del-user", "", "Remove user by `username`")