ClusterCockpit/cc-backend
1
0
Fork 0
mirror of https://github.com/ClusterCockpit/cc-backend synced 2025-01-24 02:19:05 +01:00
Code Issues Packages Releases Activity

Merge branch 'dev' into add_detailed_nodelist

Browse Source
This commit is contained in:
Christoph Kluge 2024-10-23 17:09:33 +02:00
commit 447b8d3372
12 changed files with 185 additions and 97 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
internal/auth/auth.go
View File

@ -143,19 +143,36 @@ func GetAuthInstance() *Authentication {
return authInstance
}
func persistUser(user *schema.User) {
func handleTokenUser(tokenUser *schema.User) {
r := repository.GetUserRepository()
dbUser, err := r.GetUser(user.Username)
dbUser, err := r.GetUser(tokenUser.Username)
if err != nil && err != sql.ErrNoRows {
log.Errorf("Error while loading user '%s': %v", user.Username, err)
} else if err == sql.ErrNoRows { // Adds New User
if err := r.AddUser(user); err != nil {
log.Errorf("Error while adding user '%s' to DB: %v", user.Username, err)
log.Errorf("Error while loading user '%s': %v", tokenUser.Username, err)
} else if err == sql.ErrNoRows && config.Keys.JwtConfig.SyncUserOnLogin { // Adds New User
if err := r.AddUser(tokenUser); err != nil {
log.Errorf("Error while adding user '%s' to DB: %v", tokenUser.Username, err)
}
} else { // Update Existing
if err := r.UpdateUser(dbUser, user); err != nil {
log.Errorf("Error while updating user '%s' to DB: %v", user.Username, err)
} else if err == nil && config.Keys.JwtConfig.UpdateUserOnLogin { // Update Existing User
if err := r.UpdateUser(dbUser, tokenUser); err != nil {
log.Errorf("Error while updating user '%s' to DB: %v", dbUser.Username, err)
}
}
}
func handleOIDCUser(OIDCUser *schema.User) {
r := repository.GetUserRepository()
dbUser, err := r.GetUser(OIDCUser.Username)
if err != nil && err != sql.ErrNoRows {
log.Errorf("Error while loading user '%s': %v", OIDCUser.Username, err)
} else if err == sql.ErrNoRows && config.Keys.OpenIDConfig.SyncUserOnLogin { // Adds New User
if err := r.AddUser(OIDCUser); err != nil {
log.Errorf("Error while adding user '%s' to DB: %v", OIDCUser.Username, err)
}
} else if err == nil && config.Keys.OpenIDConfig.UpdateUserOnLogin { // Update Existing User
if err := r.UpdateUser(dbUser, OIDCUser); err != nil {
log.Errorf("Error while updating user '%s' to DB: %v", dbUser.Username, err)
}
}
}

4
internal/auth/jwtCookieSession.go
View File

@ -198,8 +198,8 @@ func (ja *JWTCookieSessionAuthenticator) Login(
AuthSource: schema.AuthViaToken,
}
if jc.SyncUserOnLogin {
persistUser(user)
if jc.SyncUserOnLogin || jc.UpdateUserOnLogin {
handleTokenUser(user)
}
}

4
internal/auth/jwtSession.go
View File

@ -138,8 +138,8 @@ func (ja *JWTSessionAuthenticator) Login(
AuthSource: schema.AuthViaToken,
}
if config.Keys.JwtConfig.SyncUserOnLogin {
persistUser(user)
if config.Keys.JwtConfig.SyncUserOnLogin || config.Keys.JwtConfig.UpdateUserOnLogin {
handleTokenUser(user)
}
}

4
internal/auth/oidc.go
View File

@ -168,8 +168,8 @@ func (oa *OIDC) OAuth2Callback(rw http.ResponseWriter, r *http.Request) {
AuthSource: schema.AuthViaOIDC,
}
if config.Keys.OpenIDConfig.SyncUserOnLogin {
persistUser(user)
if config.Keys.OpenIDConfig.SyncUserOnLogin || config.Keys.OpenIDConfig.UpdateUserOnLogin {
handleOIDCUser(user)
}
oa.authentication.SaveSession(rw, r, user)

22
internal/repository/job.go
View File

@ -621,13 +621,12 @@ func (r *JobRepository) UpdateEnergy(
}
var rawFootprint []byte
if rawFootprint, err = json.Marshal(energyFootprint); err != nil {
log.Warnf("Error while marshaling energy footprint for job, DB ID '%v'", jobMeta.ID)
log.Warnf("Error while marshaling energy footprint for job INTO BYTES, DB ID '%v'", jobMeta.ID)
return stmt, err
}
return stmt.Set("energy_footprint", rawFootprint).Set("energy", (math.Round(totalEnergy*100) / 100)), nil
return stmt.Set("energy_footprint", string(rawFootprint)).Set("energy", (math.Round(totalEnergy*100) / 100)), nil
}
func (r *JobRepository) UpdateFootprint(
@ -643,7 +642,17 @@ func (r *JobRepository) UpdateFootprint(
footprint := make(map[string]float64)
for _, fp := range sc.Footprint {
statType := "avg"
var statType string
for _, gm := range archive.GlobalMetricList {
if gm.Name == fp {
statType = gm.Footprint
}
}
if statType != "avg" && statType != "min" && statType != "max" {
log.Warnf("unknown statType for footprint update: %s", statType)
return stmt, fmt.Errorf("unknown statType for footprint update: %s", statType)
}
if i, err := archive.MetricIndex(sc.MetricConfig, fp); err != nil {
statType = sc.MetricConfig[i].Footprint
@ -654,11 +663,10 @@ func (r *JobRepository) UpdateFootprint(
}
var rawFootprint []byte
if rawFootprint, err = json.Marshal(footprint); err != nil {
log.Warnf("Error while marshaling footprint for job, DB ID '%v'", jobMeta.ID)
log.Warnf("Error while marshaling footprint for job INTO BYTES, DB ID '%v'", jobMeta.ID)
return stmt, err
}
return stmt.Set("footprint", rawFootprint), nil
return stmt.Set("footprint", string(rawFootprint)), nil
}

10
internal/repository/transaction.go
View File

@ -49,7 +49,6 @@ func (r *JobRepository) TransactionEnd(t *Transaction) error {
log.Warn("Error while committing SQL transactions")
return err
}
return nil
}
@ -74,11 +73,16 @@ func (r *JobRepository) TransactionAddNamed(
}
func (r *JobRepository) TransactionAdd(t *Transaction, query string, args ...interface{}) (int64, error) {
res := t.tx.MustExec(query, args)
res, err := t.tx.Exec(query, args...)
if err != nil {
log.Errorf("TransactionAdd(), Exec() Error: %v", err)
return 0, err
}
id, err := res.LastInsertId()
if err != nil {
log.Errorf("repository initDB(): %v", err)
log.Errorf("TransactionAdd(), LastInsertId() Error: %v", err)
return 0, err
}

11
internal/taskManager/updateDurationService.go
View File

@ -7,14 +7,21 @@ package taskManager
import (
"time"
"github.com/ClusterCockpit/cc-backend/internal/config"
"github.com/ClusterCockpit/cc-backend/pkg/log"
"github.com/go-co-op/gocron/v2"
)
func RegisterUpdateDurationWorker() {
log.Info("Register duration update service")
var frequency string
if config.Keys.CronFrequency.DurationWorker != "" {
frequency = config.Keys.CronFrequency.DurationWorker
} else {
frequency = "5m"
}
d, _ := time.ParseDuration(frequency)
log.Infof("Register Duration Update service with %s interval", frequency)
d, _ := time.ParseDuration("5m")
s.NewJob(gocron.DurationJob(d),
gocron.NewTask(
func() {

59
internal/taskManager/updateFootprintService.go
View File

@ -9,6 +9,7 @@ import (
"math"
"time"
"github.com/ClusterCockpit/cc-backend/internal/config"
"github.com/ClusterCockpit/cc-backend/internal/metricDataDispatcher"
"github.com/ClusterCockpit/cc-backend/pkg/archive"
"github.com/ClusterCockpit/cc-backend/pkg/log"
@ -18,18 +19,28 @@ import (
)
func RegisterFootprintWorker() {
log.Info("Register Footprint Update service")
d, _ := time.ParseDuration("10m")
var frequency string
if config.Keys.CronFrequency.FootprintWorker != "" {
frequency = config.Keys.CronFrequency.FootprintWorker
} else {
frequency = "10m"
}
d, _ := time.ParseDuration(frequency)
log.Infof("Register Footprint Update service with %s interval", frequency)
s.NewJob(gocron.DurationJob(d),
gocron.NewTask(
func() {
s := time.Now()
log.Printf("Update Footprints started at %s using direct query execution", s.Format(time.RFC3339))
c := 0
ce := 0
cl := 0
log.Printf("Update Footprints started at %s", s.Format(time.RFC3339))
// t, err := jobRepo.TransactionInit()
// if err != nil {
// log.Errorf("Failed TransactionInit %v", err)
// }
t, err := jobRepo.TransactionInit()
if err != nil {
log.Errorf("Failed TransactionInit %v", err)
}
for _, cluster := range archive.Clusters {
jobs, err := jobRepo.FindRunningJobs(cluster.Name)
@ -47,10 +58,12 @@ func RegisterFootprintWorker() {
scopes = append(scopes, schema.MetricScopeAccelerator)
for _, job := range jobs {
// log.Debugf("Try job %d", job.JobID)
log.Debugf("Try job %d", job.JobID)
cl++
jobData, err := metricDataDispatcher.LoadData(job, allMetrics, scopes, context.Background(), 0) // 0 Resolution-Value retrieves highest res
if err != nil {
log.Errorf("Error wile loading job data for footprint update: %v", err)
ce++
continue
}
@ -65,6 +78,7 @@ func RegisterFootprintWorker() {
nodeData, ok := data["node"]
if !ok {
// This should never happen ?
ce++
continue
}
@ -92,33 +106,38 @@ func RegisterFootprintWorker() {
stmt, err = jobRepo.UpdateFootprint(stmt, jobMeta)
if err != nil {
log.Errorf("Update job (dbid: %d) failed at update Footprint step: %s", job.ID, err.Error())
ce++
continue
}
stmt, err = jobRepo.UpdateEnergy(stmt, jobMeta)
if err != nil {
log.Errorf("Update job (dbid: %d) failed at update Energy step: %s", job.ID, err.Error())
ce++
continue
}
// Add WHERE Filter
stmt = stmt.Where("job.id = ?", job.ID)
// query, args, err := stmt.ToSql()
// if err != nil {
// log.Errorf("Failed in ToSQL conversion: %v", err)
// continue
// }
// jobRepo.TransactionAdd(t, query, args)
if err := jobRepo.Execute(stmt); err != nil {
log.Errorf("Update job footprint (dbid: %d) failed at db execute: %s", job.ID, err.Error())
query, args, err := stmt.ToSql()
if err != nil {
log.Errorf("Failed in ToSQL conversion: %v", err)
ce++
continue
}
// Args: JSON, JSON, ENERGY, JOBID
jobRepo.TransactionAdd(t, query, args...)
// if err := jobRepo.Execute(stmt); err != nil {
// log.Errorf("Update job footprint (dbid: %d) failed at db execute: %s", job.ID, err.Error())
// continue
// }
c++
log.Debugf("Finish Job %d", job.JobID)
}
jobRepo.TransactionCommit(t)
log.Debugf("Finish Cluster %s", cluster.Name)
// jobRepo.TransactionCommit(t)
}
// jobRepo.TransactionEnd(t)
log.Printf("Update Footprints is done and took %s", time.Since(s))
jobRepo.TransactionEnd(t)
log.Printf("Updating %d (of %d; Skipped %d) Footprints is done and took %s", c, cl, ce, time.Since(s))
}))
}

18
pkg/schema/config.go
View File

@ -24,8 +24,9 @@ type LdapConfig struct {
}
type OpenIDConfig struct {
Provider string `json:"provider"`
SyncUserOnLogin bool `json:"syncUserOnLogin"`
Provider string `json:"provider"`
SyncUserOnLogin bool `json:"syncUserOnLogin"`
UpdateUserOnLogin bool `json:"updateUserOnLogin"`
}
type JWTAuthConfig struct {
@ -45,6 +46,9 @@ type JWTAuthConfig struct {
// Should an non-existent user be added to the DB based on the information in the token
SyncUserOnLogin bool `json:"syncUserOnLogin"`
// Should an existent user be updated in the DB based on the information in the token
UpdateUserOnLogin bool `json:"updateUserOnLogin"`
}
type IntRange struct {
@ -84,6 +88,13 @@ type ResampleConfig struct {
Resolutions []int `json:"resolutions"`
}
type CronFrequency struct {
// Duration Update Worker [Defaults to '5m']
DurationWorker string `json:"duration-worker"`
// Metric- and Energy Footprint Update Worker [Defaults to '10m']
FootprintWorker string `json:"footprint-worker"`
}
// Format of the configuration (file). See below for the defaults.
type ProgramConfig struct {
// Address where the http (or https) server will listen on (for example: 'localhost:80').
@ -159,4 +170,7 @@ type ProgramConfig struct {
// Energy Mix CO2 Emission Constant [g/kWh]
// If entered, displays estimated CO2 emission for job based on jobs totalEnergy
EmissionConstant int `json:"emission-constant"`
// Frequency of cron job workers
CronFrequency *CronFrequency `json:"cron-frequency"`
}

58
web/frontend/package-lock.json generated
View File

@ -1,34 +1,34 @@
{
"name": "cc-frontend",
"version": "1.0.0",
"version": "1.0.1",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "cc-frontend",
"version": "1.0.0",
"version": "1.0.1",
"license": "MIT",
"dependencies": {
"@rollup/plugin-replace": "^5.0.5",
"@sveltestrap/sveltestrap": "^6.2.6",
"@urql/svelte": "^4.1.0",
"chart.js": "^4.4.2",
"@rollup/plugin-replace": "^5.0.7",
"@sveltestrap/sveltestrap": "^6.2.7",
"@urql/svelte": "^4.2.1",
"chart.js": "^4.4.5",
"date-fns": "^2.30.0",
"graphql": "^16.8.1",
"mathjs": "^12.4.0",
"graphql": "^16.9.0",
"mathjs": "^12.4.3",
"svelte-chartjs": "^3.1.5",
"uplot": "^1.6.30",
"uplot": "^1.6.31",
"wonka": "^6.3.4"
},
"devDependencies": {
"@rollup/plugin-commonjs": "^25.0.7",
"@rollup/plugin-node-resolve": "^15.2.3",
"@rollup/plugin-commonjs": "^25.0.8",
"@rollup/plugin-node-resolve": "^15.3.0",
"@rollup/plugin-terser": "^0.4.4",
"@timohausmann/quadtree-js": "^1.2.6",
"rollup": "^4.12.1",
"rollup": "^4.24.0",
"rollup-plugin-css-only": "^4.5.2",
"rollup-plugin-svelte": "^7.1.6",
"svelte": "^4.2.12"
"rollup-plugin-svelte": "^7.2.2",
"svelte": "^4.2.19"
}
},
"node_modules/@0no-co/graphql.web": {
@ -543,9 +543,9 @@
}
},
"node_modules/acorn": {
"version": "8.12.1",
"resolved": "https://registry.npmjs.org/acorn/-/acorn-8.12.1.tgz",
"integrity": "sha512-tcpGyI9zbizT9JbV6oYE477V6mTlXvvi0T0G3SNIYE2apm/G5huBa1+K89VGeovbg+jycCrfhl3ADxErOuO6Jg==",
"version": "8.13.0",
"resolved": "https://registry.npmjs.org/acorn/-/acorn-8.13.0.tgz",
"integrity": "sha512-8zSiw54Oxrdym50NlZ9sUusyO1Z1ZchgRLWRaK6c86XJFClyCgFKetdowBg5bKxyp/u+CDBJG4Mpp0m3HLZl9w==",
"license": "MIT",
"bin": {
"acorn": "bin/acorn"
@ -597,9 +597,9 @@
"license": "MIT"
},
"node_modules/chart.js": {
"version": "4.4.4",
"resolved": "https://registry.npmjs.org/chart.js/-/chart.js-4.4.4.tgz",
"integrity": "sha512-emICKGBABnxhMjUjlYRR12PmOXhJ2eJjEHL2/dZlWjxRAZT1D8xplLFq5M0tMQK8ja+wBS/tuVEJB5C6r7VxJA==",
"version": "4.4.5",
"resolved": "https://registry.npmjs.org/chart.js/-/chart.js-4.4.5.tgz",
"integrity": "sha512-CVVjg1RYTJV9OCC8WeJPMx8gsV8K6WIyIEQUE3ui4AR9Hfgls9URri6Ja3hyMVBbTF8Q2KFa19PE815gWcWhng==",
"license": "MIT",
"dependencies": {
"@kurkle/color": "^0.3.0"
@ -645,9 +645,9 @@
"license": "MIT"
},
"node_modules/complex.js": {
"version": "2.2.3",
"resolved": "https://registry.npmjs.org/complex.js/-/complex.js-2.2.3.tgz",
"integrity": "sha512-XnOksGXxhQTvL3LjUgwiOPqL7vF7uikCQE/jpuylNpXmG2LZ+l0z1t6qIlJ2TJVDteXPHhlYd3+mhHOGeTFfsg==",
"version": "2.3.0",
"resolved": "https://registry.npmjs.org/complex.js/-/complex.js-2.3.0.tgz",
"integrity": "sha512-wWHzifVdUPbPBhh+ObvpVGIzrAQjTvmnnEJKBfLW5YbyAB6OXQ0r+Q92fByMIrSSlxUuCujqxriJSR6R/kVxPA==",
"license": "MIT",
"engines": {
"node": "*"
@ -867,9 +867,9 @@
"license": "MIT"
},
"node_modules/magic-string": {
"version": "0.30.11",
"resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.11.tgz",
"integrity": "sha512-+Wri9p0QHMy+545hKww7YAu5NyzF8iomPL/RQazugQ9+Ez4Ic3mERMd8ZTX5rfK944j+560ZJi8iAwgak1Ac7A==",
"version": "0.30.12",
"resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.12.tgz",
"integrity": "sha512-Ea8I3sQMVXr8JhN4z+H/d8zwo+tYDgHE9+5G4Wnrwhs0gaK9fXTKx0Tw5Xwsd/bCPTTZNRAdpyzvoeORe9LYpw==",
"license": "MIT",
"dependencies": {
"@jridgewell/sourcemap-codec": "^1.5.0"
@ -1244,9 +1244,9 @@
}
},
"node_modules/terser": {
"version": "5.34.1",
"resolved": "https://registry.npmjs.org/terser/-/terser-5.34.1.tgz",
"integrity": "sha512-FsJZ7iZLd/BXkz+4xrRTGJ26o/6VTjQytUk8b8OxkwcD2I+79VPJlz7qss1+zE7h8GNIScFqXcDyJ/KqBYZFVA==",
"version": "5.36.0",
"resolved": "https://registry.npmjs.org/terser/-/terser-5.36.0.tgz",
"integrity": "sha512-IYV9eNMuFAV4THUspIRXkLakHnV6XO7FEdtKjf/mDyrnqUg9LnlOn6/RwRvM9SZjR4GUq8Nk8zj67FzVARr74w==",
"dev": true,
"license": "BSD-2-Clause",
"dependencies": {

24
web/frontend/package.json
View File

@ -7,25 +7,25 @@
"dev": "rollup -c -w"
},
"devDependencies": {
"@rollup/plugin-commonjs": "^25.0.7",
"@rollup/plugin-node-resolve": "^15.2.3",
"@rollup/plugin-commonjs": "^25.0.8",
"@rollup/plugin-node-resolve": "^15.3.0",
"@rollup/plugin-terser": "^0.4.4",
"@timohausmann/quadtree-js": "^1.2.6",
"rollup": "^4.12.1",
"rollup": "^4.24.0",
"rollup-plugin-css-only": "^4.5.2",
"rollup-plugin-svelte": "^7.1.6",
"svelte": "^4.2.12"
"rollup-plugin-svelte": "^7.2.2",
"svelte": "^4.2.19"
},
"dependencies": {
"@rollup/plugin-replace": "^5.0.5",
"@sveltestrap/sveltestrap": "^6.2.6",
"@urql/svelte": "^4.1.0",
"chart.js": "^4.4.2",
"@rollup/plugin-replace": "^5.0.7",
"@sveltestrap/sveltestrap": "^6.2.7",
"@urql/svelte": "^4.2.1",
"chart.js": "^4.4.5",
"date-fns": "^2.30.0",
"graphql": "^16.8.1",
"mathjs": "^12.4.0",
"graphql": "^16.9.0",
"mathjs": "^12.4.3",
"svelte-chartjs": "^3.1.5",
"uplot": "^1.6.30",
"uplot": "^1.6.31",
"wonka": "^6.3.4"
}
}

33
web/frontend/src/generic/joblist/JobInfo.svelte
View File

@ -7,7 +7,7 @@
-->
<script>
import { Badge, Icon } from "@sveltestrap/sveltestrap";
import { Badge, Button, Icon } from "@sveltestrap/sveltestrap";
import { scrambleNames, scramble } from "../utils.js";
import Tag from "../helper/Tag.svelte";
import TagManagement from "../helper/TagManagement.svelte";
@ -38,22 +38,41 @@
return "danger";
}
}
let displayCheck = false;
function clipJobId(jid) {
displayCheck = true;
navigator.clipboard
.writeText(jid)
.catch((reason) => console.error(reason));
setTimeout(function () {
displayCheck = false;
}, 1500);
}
</script>
<div>
<p class="mb-2">
<span class="fw-bold"
><a href="/monitoring/job/{job.id}" target="_blank">{job.jobId}</a>
({job.cluster})</span
>
<span class="d-flex justify-content-between">
<span class="align-self-center fw-bold mr-2">
<a href="/monitoring/job/{job.id}" target="_blank">{job.jobId}</a>
({job.cluster})
</span>
<Button outline color="secondary" size="sm" title="Copy JobID to Clipboard" on:click={clipJobId(job.jobId)} >
{#if displayCheck}
<Icon name="clipboard2-check-fill"/> Copied
{:else}
<Icon name="clipboard2"/> Job ID
{/if}
</Button>
</span>
{#if job.metaData?.jobName}
<br />
{#if job.metaData?.jobName.length <= 25}
<div>{job.metaData.jobName}</div>
{:else}
<div
class="truncate"
style="cursor:help; width:230px;"
style="cursor:help;"
title={job.metaData.jobName}
>
{job.metaData.jobName}