diff --git a/internal/auth/auth.go b/internal/auth/auth.go index b6e4cbe..270989f 100644 --- a/internal/auth/auth.go +++ b/internal/auth/auth.go @@ -143,19 +143,36 @@ func GetAuthInstance() *Authentication { return authInstance } -func persistUser(user *schema.User) { +func handleTokenUser(tokenUser *schema.User) { r := repository.GetUserRepository() - dbUser, err := r.GetUser(user.Username) + dbUser, err := r.GetUser(tokenUser.Username) if err != nil && err != sql.ErrNoRows { - log.Errorf("Error while loading user '%s': %v", user.Username, err) - } else if err == sql.ErrNoRows { // Adds New User - if err := r.AddUser(user); err != nil { - log.Errorf("Error while adding user '%s' to DB: %v", user.Username, err) + log.Errorf("Error while loading user '%s': %v", tokenUser.Username, err) + } else if err == sql.ErrNoRows && config.Keys.JwtConfig.SyncUserOnLogin { // Adds New User + if err := r.AddUser(tokenUser); err != nil { + log.Errorf("Error while adding user '%s' to DB: %v", tokenUser.Username, err) } - } else { // Update Existing - if err := r.UpdateUser(dbUser, user); err != nil { - log.Errorf("Error while updating user '%s' to DB: %v", user.Username, err) + } else if err == nil && config.Keys.JwtConfig.UpdateUserOnLogin { // Update Existing User + if err := r.UpdateUser(dbUser, tokenUser); err != nil { + log.Errorf("Error while updating user '%s' to DB: %v", dbUser.Username, err) + } + } +} + +func handleOIDCUser(OIDCUser *schema.User) { + r := repository.GetUserRepository() + dbUser, err := r.GetUser(OIDCUser.Username) + + if err != nil && err != sql.ErrNoRows { + log.Errorf("Error while loading user '%s': %v", OIDCUser.Username, err) + } else if err == sql.ErrNoRows && config.Keys.OpenIDConfig.SyncUserOnLogin { // Adds New User + if err := r.AddUser(OIDCUser); err != nil { + log.Errorf("Error while adding user '%s' to DB: %v", OIDCUser.Username, err) + } + } else if err == nil && config.Keys.OpenIDConfig.UpdateUserOnLogin { // Update Existing User + if err := r.UpdateUser(dbUser, OIDCUser); err != nil { + log.Errorf("Error while updating user '%s' to DB: %v", dbUser.Username, err) } } } diff --git a/internal/auth/jwtCookieSession.go b/internal/auth/jwtCookieSession.go index 926f7ba..7e0e045 100644 --- a/internal/auth/jwtCookieSession.go +++ b/internal/auth/jwtCookieSession.go @@ -198,8 +198,8 @@ func (ja *JWTCookieSessionAuthenticator) Login( AuthSource: schema.AuthViaToken, } - if jc.SyncUserOnLogin { - persistUser(user) + if jc.SyncUserOnLogin || jc.UpdateUserOnLogin { + handleTokenUser(user) } } diff --git a/internal/auth/jwtSession.go b/internal/auth/jwtSession.go index 765a9fd..67457ee 100644 --- a/internal/auth/jwtSession.go +++ b/internal/auth/jwtSession.go @@ -138,8 +138,8 @@ func (ja *JWTSessionAuthenticator) Login( AuthSource: schema.AuthViaToken, } - if config.Keys.JwtConfig.SyncUserOnLogin { - persistUser(user) + if config.Keys.JwtConfig.SyncUserOnLogin || config.Keys.JwtConfig.UpdateUserOnLogin { + handleTokenUser(user) } } diff --git a/internal/auth/oidc.go b/internal/auth/oidc.go index 5cfb563..ba1c9da 100644 --- a/internal/auth/oidc.go +++ b/internal/auth/oidc.go @@ -168,8 +168,8 @@ func (oa *OIDC) OAuth2Callback(rw http.ResponseWriter, r *http.Request) { AuthSource: schema.AuthViaOIDC, } - if config.Keys.OpenIDConfig.SyncUserOnLogin { - persistUser(user) + if config.Keys.OpenIDConfig.SyncUserOnLogin || config.Keys.OpenIDConfig.UpdateUserOnLogin { + handleOIDCUser(user) } oa.authentication.SaveSession(rw, r, user) diff --git a/internal/repository/job.go b/internal/repository/job.go index d408341..d1c54a1 100644 --- a/internal/repository/job.go +++ b/internal/repository/job.go @@ -621,13 +621,12 @@ func (r *JobRepository) UpdateEnergy( } var rawFootprint []byte - if rawFootprint, err = json.Marshal(energyFootprint); err != nil { - log.Warnf("Error while marshaling energy footprint for job, DB ID '%v'", jobMeta.ID) + log.Warnf("Error while marshaling energy footprint for job INTO BYTES, DB ID '%v'", jobMeta.ID) return stmt, err } - return stmt.Set("energy_footprint", rawFootprint).Set("energy", (math.Round(totalEnergy*100) / 100)), nil + return stmt.Set("energy_footprint", string(rawFootprint)).Set("energy", (math.Round(totalEnergy*100) / 100)), nil } func (r *JobRepository) UpdateFootprint( @@ -643,7 +642,17 @@ func (r *JobRepository) UpdateFootprint( footprint := make(map[string]float64) for _, fp := range sc.Footprint { - statType := "avg" + var statType string + for _, gm := range archive.GlobalMetricList { + if gm.Name == fp { + statType = gm.Footprint + } + } + + if statType != "avg" && statType != "min" && statType != "max" { + log.Warnf("unknown statType for footprint update: %s", statType) + return stmt, fmt.Errorf("unknown statType for footprint update: %s", statType) + } if i, err := archive.MetricIndex(sc.MetricConfig, fp); err != nil { statType = sc.MetricConfig[i].Footprint @@ -654,11 +663,10 @@ func (r *JobRepository) UpdateFootprint( } var rawFootprint []byte - if rawFootprint, err = json.Marshal(footprint); err != nil { - log.Warnf("Error while marshaling footprint for job, DB ID '%v'", jobMeta.ID) + log.Warnf("Error while marshaling footprint for job INTO BYTES, DB ID '%v'", jobMeta.ID) return stmt, err } - return stmt.Set("footprint", rawFootprint), nil + return stmt.Set("footprint", string(rawFootprint)), nil } diff --git a/internal/repository/transaction.go b/internal/repository/transaction.go index 8c5d357..603d505 100644 --- a/internal/repository/transaction.go +++ b/internal/repository/transaction.go @@ -49,7 +49,6 @@ func (r *JobRepository) TransactionEnd(t *Transaction) error { log.Warn("Error while committing SQL transactions") return err } - return nil } @@ -74,11 +73,16 @@ func (r *JobRepository) TransactionAddNamed( } func (r *JobRepository) TransactionAdd(t *Transaction, query string, args ...interface{}) (int64, error) { - res := t.tx.MustExec(query, args) + + res, err := t.tx.Exec(query, args...) + if err != nil { + log.Errorf("TransactionAdd(), Exec() Error: %v", err) + return 0, err + } id, err := res.LastInsertId() if err != nil { - log.Errorf("repository initDB(): %v", err) + log.Errorf("TransactionAdd(), LastInsertId() Error: %v", err) return 0, err } diff --git a/internal/taskManager/updateDurationService.go b/internal/taskManager/updateDurationService.go index 6023547..fc7f446 100644 --- a/internal/taskManager/updateDurationService.go +++ b/internal/taskManager/updateDurationService.go @@ -7,14 +7,21 @@ package taskManager import ( "time" + "github.com/ClusterCockpit/cc-backend/internal/config" "github.com/ClusterCockpit/cc-backend/pkg/log" "github.com/go-co-op/gocron/v2" ) func RegisterUpdateDurationWorker() { - log.Info("Register duration update service") + var frequency string + if config.Keys.CronFrequency.DurationWorker != "" { + frequency = config.Keys.CronFrequency.DurationWorker + } else { + frequency = "5m" + } + d, _ := time.ParseDuration(frequency) + log.Infof("Register Duration Update service with %s interval", frequency) - d, _ := time.ParseDuration("5m") s.NewJob(gocron.DurationJob(d), gocron.NewTask( func() { diff --git a/internal/taskManager/updateFootprintService.go b/internal/taskManager/updateFootprintService.go index b21502a..a0eccc8 100644 --- a/internal/taskManager/updateFootprintService.go +++ b/internal/taskManager/updateFootprintService.go @@ -9,6 +9,7 @@ import ( "math" "time" + "github.com/ClusterCockpit/cc-backend/internal/config" "github.com/ClusterCockpit/cc-backend/internal/metricDataDispatcher" "github.com/ClusterCockpit/cc-backend/pkg/archive" "github.com/ClusterCockpit/cc-backend/pkg/log" @@ -18,18 +19,28 @@ import ( ) func RegisterFootprintWorker() { - log.Info("Register Footprint Update service") - d, _ := time.ParseDuration("10m") + var frequency string + if config.Keys.CronFrequency.FootprintWorker != "" { + frequency = config.Keys.CronFrequency.FootprintWorker + } else { + frequency = "10m" + } + d, _ := time.ParseDuration(frequency) + log.Infof("Register Footprint Update service with %s interval", frequency) + s.NewJob(gocron.DurationJob(d), gocron.NewTask( func() { s := time.Now() - log.Printf("Update Footprints started at %s using direct query execution", s.Format(time.RFC3339)) + c := 0 + ce := 0 + cl := 0 + log.Printf("Update Footprints started at %s", s.Format(time.RFC3339)) - // t, err := jobRepo.TransactionInit() - // if err != nil { - // log.Errorf("Failed TransactionInit %v", err) - // } + t, err := jobRepo.TransactionInit() + if err != nil { + log.Errorf("Failed TransactionInit %v", err) + } for _, cluster := range archive.Clusters { jobs, err := jobRepo.FindRunningJobs(cluster.Name) @@ -47,10 +58,12 @@ func RegisterFootprintWorker() { scopes = append(scopes, schema.MetricScopeAccelerator) for _, job := range jobs { - // log.Debugf("Try job %d", job.JobID) + log.Debugf("Try job %d", job.JobID) + cl++ jobData, err := metricDataDispatcher.LoadData(job, allMetrics, scopes, context.Background(), 0) // 0 Resolution-Value retrieves highest res if err != nil { log.Errorf("Error wile loading job data for footprint update: %v", err) + ce++ continue } @@ -65,6 +78,7 @@ func RegisterFootprintWorker() { nodeData, ok := data["node"] if !ok { // This should never happen ? + ce++ continue } @@ -92,33 +106,38 @@ func RegisterFootprintWorker() { stmt, err = jobRepo.UpdateFootprint(stmt, jobMeta) if err != nil { log.Errorf("Update job (dbid: %d) failed at update Footprint step: %s", job.ID, err.Error()) + ce++ continue } stmt, err = jobRepo.UpdateEnergy(stmt, jobMeta) if err != nil { log.Errorf("Update job (dbid: %d) failed at update Energy step: %s", job.ID, err.Error()) + ce++ continue } // Add WHERE Filter stmt = stmt.Where("job.id = ?", job.ID) - // query, args, err := stmt.ToSql() - // if err != nil { - // log.Errorf("Failed in ToSQL conversion: %v", err) - // continue - // } - - // jobRepo.TransactionAdd(t, query, args) - if err := jobRepo.Execute(stmt); err != nil { - log.Errorf("Update job footprint (dbid: %d) failed at db execute: %s", job.ID, err.Error()) + query, args, err := stmt.ToSql() + if err != nil { + log.Errorf("Failed in ToSQL conversion: %v", err) + ce++ continue } + + // Args: JSON, JSON, ENERGY, JOBID + jobRepo.TransactionAdd(t, query, args...) + // if err := jobRepo.Execute(stmt); err != nil { + // log.Errorf("Update job footprint (dbid: %d) failed at db execute: %s", job.ID, err.Error()) + // continue + // } + c++ log.Debugf("Finish Job %d", job.JobID) } + jobRepo.TransactionCommit(t) log.Debugf("Finish Cluster %s", cluster.Name) - // jobRepo.TransactionCommit(t) } - // jobRepo.TransactionEnd(t) - log.Printf("Update Footprints is done and took %s", time.Since(s)) + jobRepo.TransactionEnd(t) + log.Printf("Updating %d (of %d; Skipped %d) Footprints is done and took %s", c, cl, ce, time.Since(s)) })) } diff --git a/pkg/schema/config.go b/pkg/schema/config.go index ccd848b..b87841c 100644 --- a/pkg/schema/config.go +++ b/pkg/schema/config.go @@ -24,8 +24,9 @@ type LdapConfig struct { } type OpenIDConfig struct { - Provider string `json:"provider"` - SyncUserOnLogin bool `json:"syncUserOnLogin"` + Provider string `json:"provider"` + SyncUserOnLogin bool `json:"syncUserOnLogin"` + UpdateUserOnLogin bool `json:"updateUserOnLogin"` } type JWTAuthConfig struct { @@ -45,6 +46,9 @@ type JWTAuthConfig struct { // Should an non-existent user be added to the DB based on the information in the token SyncUserOnLogin bool `json:"syncUserOnLogin"` + + // Should an existent user be updated in the DB based on the information in the token + UpdateUserOnLogin bool `json:"updateUserOnLogin"` } type IntRange struct { @@ -84,6 +88,13 @@ type ResampleConfig struct { Resolutions []int `json:"resolutions"` } +type CronFrequency struct { + // Duration Update Worker [Defaults to '5m'] + DurationWorker string `json:"duration-worker"` + // Metric- and Energy Footprint Update Worker [Defaults to '10m'] + FootprintWorker string `json:"footprint-worker"` +} + // Format of the configuration (file). See below for the defaults. type ProgramConfig struct { // Address where the http (or https) server will listen on (for example: 'localhost:80'). @@ -159,4 +170,7 @@ type ProgramConfig struct { // Energy Mix CO2 Emission Constant [g/kWh] // If entered, displays estimated CO2 emission for job based on jobs totalEnergy EmissionConstant int `json:"emission-constant"` + + // Frequency of cron job workers + CronFrequency *CronFrequency `json:"cron-frequency"` } diff --git a/web/frontend/package-lock.json b/web/frontend/package-lock.json index 2e7e8aa..e4c1d1e 100644 --- a/web/frontend/package-lock.json +++ b/web/frontend/package-lock.json @@ -1,34 +1,34 @@ { "name": "cc-frontend", - "version": "1.0.0", + "version": "1.0.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "cc-frontend", - "version": "1.0.0", + "version": "1.0.1", "license": "MIT", "dependencies": { - "@rollup/plugin-replace": "^5.0.5", - "@sveltestrap/sveltestrap": "^6.2.6", - "@urql/svelte": "^4.1.0", - "chart.js": "^4.4.2", + "@rollup/plugin-replace": "^5.0.7", + "@sveltestrap/sveltestrap": "^6.2.7", + "@urql/svelte": "^4.2.1", + "chart.js": "^4.4.5", "date-fns": "^2.30.0", - "graphql": "^16.8.1", - "mathjs": "^12.4.0", + "graphql": "^16.9.0", + "mathjs": "^12.4.3", "svelte-chartjs": "^3.1.5", - "uplot": "^1.6.30", + "uplot": "^1.6.31", "wonka": "^6.3.4" }, "devDependencies": { - "@rollup/plugin-commonjs": "^25.0.7", - "@rollup/plugin-node-resolve": "^15.2.3", + "@rollup/plugin-commonjs": "^25.0.8", + "@rollup/plugin-node-resolve": "^15.3.0", "@rollup/plugin-terser": "^0.4.4", "@timohausmann/quadtree-js": "^1.2.6", - "rollup": "^4.12.1", + "rollup": "^4.24.0", "rollup-plugin-css-only": "^4.5.2", - "rollup-plugin-svelte": "^7.1.6", - "svelte": "^4.2.12" + "rollup-plugin-svelte": "^7.2.2", + "svelte": "^4.2.19" } }, "node_modules/@0no-co/graphql.web": { @@ -543,9 +543,9 @@ } }, "node_modules/acorn": { - "version": "8.12.1", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.12.1.tgz", - "integrity": "sha512-tcpGyI9zbizT9JbV6oYE477V6mTlXvvi0T0G3SNIYE2apm/G5huBa1+K89VGeovbg+jycCrfhl3ADxErOuO6Jg==", + "version": "8.13.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.13.0.tgz", + "integrity": "sha512-8zSiw54Oxrdym50NlZ9sUusyO1Z1ZchgRLWRaK6c86XJFClyCgFKetdowBg5bKxyp/u+CDBJG4Mpp0m3HLZl9w==", "license": "MIT", "bin": { "acorn": "bin/acorn" @@ -597,9 +597,9 @@ "license": "MIT" }, "node_modules/chart.js": { - "version": "4.4.4", - "resolved": "https://registry.npmjs.org/chart.js/-/chart.js-4.4.4.tgz", - "integrity": "sha512-emICKGBABnxhMjUjlYRR12PmOXhJ2eJjEHL2/dZlWjxRAZT1D8xplLFq5M0tMQK8ja+wBS/tuVEJB5C6r7VxJA==", + "version": "4.4.5", + "resolved": "https://registry.npmjs.org/chart.js/-/chart.js-4.4.5.tgz", + "integrity": "sha512-CVVjg1RYTJV9OCC8WeJPMx8gsV8K6WIyIEQUE3ui4AR9Hfgls9URri6Ja3hyMVBbTF8Q2KFa19PE815gWcWhng==", "license": "MIT", "dependencies": { "@kurkle/color": "^0.3.0" @@ -645,9 +645,9 @@ "license": "MIT" }, "node_modules/complex.js": { - "version": "2.2.3", - "resolved": "https://registry.npmjs.org/complex.js/-/complex.js-2.2.3.tgz", - "integrity": "sha512-XnOksGXxhQTvL3LjUgwiOPqL7vF7uikCQE/jpuylNpXmG2LZ+l0z1t6qIlJ2TJVDteXPHhlYd3+mhHOGeTFfsg==", + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/complex.js/-/complex.js-2.3.0.tgz", + "integrity": "sha512-wWHzifVdUPbPBhh+ObvpVGIzrAQjTvmnnEJKBfLW5YbyAB6OXQ0r+Q92fByMIrSSlxUuCujqxriJSR6R/kVxPA==", "license": "MIT", "engines": { "node": "*" @@ -867,9 +867,9 @@ "license": "MIT" }, "node_modules/magic-string": { - "version": "0.30.11", - "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.11.tgz", - "integrity": "sha512-+Wri9p0QHMy+545hKww7YAu5NyzF8iomPL/RQazugQ9+Ez4Ic3mERMd8ZTX5rfK944j+560ZJi8iAwgak1Ac7A==", + "version": "0.30.12", + "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.12.tgz", + "integrity": "sha512-Ea8I3sQMVXr8JhN4z+H/d8zwo+tYDgHE9+5G4Wnrwhs0gaK9fXTKx0Tw5Xwsd/bCPTTZNRAdpyzvoeORe9LYpw==", "license": "MIT", "dependencies": { "@jridgewell/sourcemap-codec": "^1.5.0" @@ -1244,9 +1244,9 @@ } }, "node_modules/terser": { - "version": "5.34.1", - "resolved": "https://registry.npmjs.org/terser/-/terser-5.34.1.tgz", - "integrity": "sha512-FsJZ7iZLd/BXkz+4xrRTGJ26o/6VTjQytUk8b8OxkwcD2I+79VPJlz7qss1+zE7h8GNIScFqXcDyJ/KqBYZFVA==", + "version": "5.36.0", + "resolved": "https://registry.npmjs.org/terser/-/terser-5.36.0.tgz", + "integrity": "sha512-IYV9eNMuFAV4THUspIRXkLakHnV6XO7FEdtKjf/mDyrnqUg9LnlOn6/RwRvM9SZjR4GUq8Nk8zj67FzVARr74w==", "dev": true, "license": "BSD-2-Clause", "dependencies": { diff --git a/web/frontend/package.json b/web/frontend/package.json index c70e57a..1079440 100644 --- a/web/frontend/package.json +++ b/web/frontend/package.json @@ -7,25 +7,25 @@ "dev": "rollup -c -w" }, "devDependencies": { - "@rollup/plugin-commonjs": "^25.0.7", - "@rollup/plugin-node-resolve": "^15.2.3", + "@rollup/plugin-commonjs": "^25.0.8", + "@rollup/plugin-node-resolve": "^15.3.0", "@rollup/plugin-terser": "^0.4.4", "@timohausmann/quadtree-js": "^1.2.6", - "rollup": "^4.12.1", + "rollup": "^4.24.0", "rollup-plugin-css-only": "^4.5.2", - "rollup-plugin-svelte": "^7.1.6", - "svelte": "^4.2.12" + "rollup-plugin-svelte": "^7.2.2", + "svelte": "^4.2.19" }, "dependencies": { - "@rollup/plugin-replace": "^5.0.5", - "@sveltestrap/sveltestrap": "^6.2.6", - "@urql/svelte": "^4.1.0", - "chart.js": "^4.4.2", + "@rollup/plugin-replace": "^5.0.7", + "@sveltestrap/sveltestrap": "^6.2.7", + "@urql/svelte": "^4.2.1", + "chart.js": "^4.4.5", "date-fns": "^2.30.0", - "graphql": "^16.8.1", - "mathjs": "^12.4.0", + "graphql": "^16.9.0", + "mathjs": "^12.4.3", "svelte-chartjs": "^3.1.5", - "uplot": "^1.6.30", + "uplot": "^1.6.31", "wonka": "^6.3.4" } } diff --git a/web/frontend/src/generic/joblist/JobInfo.svelte b/web/frontend/src/generic/joblist/JobInfo.svelte index 93b43c3..adacd4f 100644 --- a/web/frontend/src/generic/joblist/JobInfo.svelte +++ b/web/frontend/src/generic/joblist/JobInfo.svelte @@ -7,7 +7,7 @@ -->

- {job.jobId} - ({job.cluster}) + + + {job.jobId} + ({job.cluster}) + + + {#if job.metaData?.jobName} -
{#if job.metaData?.jobName.length <= 25}

{job.metaData.jobName}
{:else}
{job.metaData.jobName}