From f158eaa29cc8ec9c355f92d36c7815177ec53845 Mon Sep 17 00:00:00 2001 From: exterr2f Date: Mon, 10 Feb 2025 09:08:09 +0100 Subject: [PATCH 01/15] Add default_metrics.json which sets the defaults for job_view_selectedMetrics:cluster for new users --- configs/default_metrics.json | 12 ++++++++ internal/config/default_metrics.go | 44 ++++++++++++++++++++++++++++++ internal/repository/user.go | 25 +++++++++++++++++ 3 files changed, 81 insertions(+) create mode 100644 configs/default_metrics.json create mode 100644 internal/config/default_metrics.go diff --git a/configs/default_metrics.json b/configs/default_metrics.json new file mode 100644 index 0000000..7c392cc --- /dev/null +++ b/configs/default_metrics.json @@ -0,0 +1,12 @@ +{ + "clusters": [ + { + "name": "fritz", + "default_metrics": "cpu_load, flops_any, core_power, lustre_open, mem_used, mem_bw, net_bytes_in" + }, + { + "name": "alex", + "default_metrics": "flops_any, mem_bw, mem_used, vectorization_ratio" + } + ] +} diff --git a/internal/config/default_metrics.go b/internal/config/default_metrics.go new file mode 100644 index 0000000..83015d4 --- /dev/null +++ b/internal/config/default_metrics.go @@ -0,0 +1,44 @@ +package config + +import ( + "encoding/json" + "os" + "strings" +) + +type DefaultMetricsCluster struct { + Name string `json:"name"` + DefaultMetrics string `json:"default_metrics"` +} + +type DefaultMetricsConfig struct { + Clusters []DefaultMetricsCluster `json:"clusters"` +} + +func LoadDefaultMetricsConfig() (*DefaultMetricsConfig, error) { + filePath := "configs/default_metrics.json" + if _, err := os.Stat(filePath); os.IsNotExist(err) { + return nil, nil + } + data, err := os.ReadFile(filePath) + if err != nil { + return nil, err + } + var cfg DefaultMetricsConfig + if err := json.Unmarshal(data, &cfg); err != nil { + return nil, err + } + return &cfg, nil +} + +func ParseMetricsString(s string) []string { + parts := strings.Split(s, ",") + var metrics []string + for _, p := range parts { + trimmed := strings.TrimSpace(p) + if trimmed != "" { + metrics = append(metrics, trimmed) + } + } + return metrics +} diff --git a/internal/repository/user.go b/internal/repository/user.go index 9beca26..c411c38 100644 --- a/internal/repository/user.go +++ b/internal/repository/user.go @@ -19,6 +19,7 @@ import ( sq "github.com/Masterminds/squirrel" "github.com/jmoiron/sqlx" "golang.org/x/crypto/bcrypt" + "github.com/ClusterCockpit/cc-backend/internal/config" ) var ( @@ -127,6 +128,30 @@ func (r *UserRepository) AddUser(user *schema.User) error { } log.Infof("new user %#v created (roles: %s, auth-source: %d, projects: %s)", user.Username, rolesJson, user.AuthSource, projectsJson) + + defaultMetricsCfg, err := config.LoadDefaultMetricsConfig() + if err != nil { + log.Errorf("Error loading default metrics config: %v", err) + } else if defaultMetricsCfg != nil { + for _, cluster := range defaultMetricsCfg.Clusters { + metricsArray := config.ParseMetricsString(cluster.DefaultMetrics) + metricsJSON, err := json.Marshal(metricsArray) + if err != nil { + log.Errorf("Error marshaling default metrics for cluster %s: %v", cluster.Name, err) + continue + } + confKey := "job_view_selectedMetrics:" + cluster.Name + if _, err := sq.Insert("configuration"). + Columns("username", "confkey", "value"). + Values(user.Username, confKey, string(metricsJSON)). + RunWith(r.DB).Exec(); err != nil { + log.Errorf("Error inserting default job view metrics for user %s and cluster %s: %v", user.Username, cluster.Name, err) + } else { + log.Infof("Default job view metrics for user %s and cluster %s set to %s", user.Username, cluster.Name, string(metricsJSON)) + } + } + } + return nil } From 43cb1f1bffc0e128f250bbc134924e80c191ba73 Mon Sep 17 00:00:00 2001 From: exterr2f Date: Fri, 14 Feb 2025 11:41:34 +0100 Subject: [PATCH 02/15] Fix SessionMaxAge condition to correctly apply valid values --- internal/auth/auth.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/auth/auth.go b/internal/auth/auth.go index 15b6532..290463f 100644 --- a/internal/auth/auth.go +++ b/internal/auth/auth.go @@ -88,7 +88,7 @@ func Init() { authInstance.sessionStore = sessions.NewCookieStore(bytes) } - if d, err := time.ParseDuration(config.Keys.SessionMaxAge); err != nil { + if d, err := time.ParseDuration(config.Keys.SessionMaxAge); err == nil { authInstance.SessionMaxAge = d } From b6b37ee68bda922bb7bd8a2831debe3b44cd0494 Mon Sep 17 00:00:00 2001 From: exterr2f Date: Fri, 14 Feb 2025 12:41:28 +0100 Subject: [PATCH 03/15] Add Rate Limiting based on IP and username --- go.mod | 1 + go.sum | 2 ++ internal/auth/auth.go | 50 +++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 51 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 84cdf7d..67bf615 100644 --- a/go.mod +++ b/go.mod @@ -29,6 +29,7 @@ require ( golang.org/x/crypto v0.32.0 golang.org/x/exp v0.0.0-20240707233637-46b078467d37 golang.org/x/oauth2 v0.21.0 + golang.org/x/time v0.10.0 ) require ( diff --git a/go.sum b/go.sum index 07aaafd..c45aca5 100644 --- a/go.sum +++ b/go.sum @@ -289,6 +289,8 @@ golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= +golang.org/x/time v0.10.0 h1:3usCWA8tQn0L8+hFJQNgzpWbd89begxN66o1Ojdn5L4= +golang.org/x/time v0.10.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= diff --git a/internal/auth/auth.go b/internal/auth/auth.go index 290463f..1892b0e 100644 --- a/internal/auth/auth.go +++ b/internal/auth/auth.go @@ -10,11 +10,14 @@ import ( "database/sql" "encoding/base64" "errors" + "net" "net/http" "os" "sync" "time" + "golang.org/x/time/rate" + "github.com/ClusterCockpit/cc-backend/internal/config" "github.com/ClusterCockpit/cc-backend/internal/repository" "github.com/ClusterCockpit/cc-backend/pkg/log" @@ -32,6 +35,29 @@ var ( authInstance *Authentication ) +var ( + ipLimiters sync.Map + usernameLimiters sync.Map +) + +func getIPLimiter(ip string) *rate.Limiter { + limiter, ok := ipLimiters.Load(ip) + if !ok { + limiter = rate.NewLimiter(rate.Every(time.Minute/5), 5) + ipLimiters.Store(ip, limiter) + } + return limiter.(*rate.Limiter) +} + +func getUserLimiter(username string) *rate.Limiter { + limiter, ok := usernameLimiters.Load(username) + if !ok { + limiter = rate.NewLimiter(rate.Every(time.Hour/10), 10) + usernameLimiters.Store(username, limiter) + } + return limiter.(*rate.Limiter) +} + type Authentication struct { sessionStore *sessions.CookieStore LdapAuth *LdapAuthenticator @@ -208,9 +234,29 @@ func (auth *Authentication) Login( onfailure func(rw http.ResponseWriter, r *http.Request, loginErr error), ) http.Handler { return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) { - username := r.FormValue("username") - var dbUser *schema.User + ip, _, err := net.SplitHostPort(r.RemoteAddr) + if err != nil { + ip = r.RemoteAddr + } + username := r.FormValue("username") + + ipLimiter := getIPLimiter(ip) + userLimiter := getUserLimiter(username) + + if !ipLimiter.Allow() { + log.Warnf("AUTH/RATE > Too many login attempts from IP %s", ip) + onfailure(rw, r, errors.New("too many login attempts, please try again later")) + return + } + + if !userLimiter.Allow() { + log.Warnf("AUTH/RATE > Too many failed login attempts for user %s", username) + onfailure(rw, r, errors.New("too many login attempts for this user, please try again later")) + return + } + + var dbUser *schema.User if username != "" { var err error dbUser, err = repository.GetUserRepository().GetUser(username) From e1b992526e4218f0795aaf886c2fa15ce192313b Mon Sep 17 00:00:00 2001 From: exterr2f Date: Fri, 14 Feb 2025 20:20:42 +0100 Subject: [PATCH 04/15] Improve rate limiting to combination of IP and username --- internal/auth/auth.go | 42 ++++++++++++------------------------------ 1 file changed, 12 insertions(+), 30 deletions(-) diff --git a/internal/auth/auth.go b/internal/auth/auth.go index 1892b0e..6241585 100644 --- a/internal/auth/auth.go +++ b/internal/auth/auth.go @@ -35,25 +35,15 @@ var ( authInstance *Authentication ) -var ( - ipLimiters sync.Map - usernameLimiters sync.Map -) +var ipUserLimiters sync.Map -func getIPLimiter(ip string) *rate.Limiter { - limiter, ok := ipLimiters.Load(ip) +func getIPUserLimiter(ip, username string) *rate.Limiter { + key := ip + ":" + username + limiter, ok := ipUserLimiters.Load(key) if !ok { - limiter = rate.NewLimiter(rate.Every(time.Minute/5), 5) - ipLimiters.Store(ip, limiter) - } - return limiter.(*rate.Limiter) -} - -func getUserLimiter(username string) *rate.Limiter { - limiter, ok := usernameLimiters.Load(username) - if !ok { - limiter = rate.NewLimiter(rate.Every(time.Hour/10), 10) - usernameLimiters.Store(username, limiter) + newLimiter := rate.NewLimiter(rate.Every(time.Hour/10), 10) + ipUserLimiters.Store(key, newLimiter) + return newLimiter } return limiter.(*rate.Limiter) } @@ -241,19 +231,11 @@ func (auth *Authentication) Login( username := r.FormValue("username") - ipLimiter := getIPLimiter(ip) - userLimiter := getUserLimiter(username) - - if !ipLimiter.Allow() { - log.Warnf("AUTH/RATE > Too many login attempts from IP %s", ip) - onfailure(rw, r, errors.New("too many login attempts, please try again later")) - return - } - - if !userLimiter.Allow() { - log.Warnf("AUTH/RATE > Too many failed login attempts for user %s", username) - onfailure(rw, r, errors.New("too many login attempts for this user, please try again later")) - return + limiter := getIPUserLimiter(ip, username) + if !limiter.Allow() { + log.Warnf("AUTH/RATE > Too many login attempts for combination IP: %s, Username: %s", ip, username) + onfailure(rw, r, errors.New("Too many login attempts, try again in 1 hour")) + return } var dbUser *schema.User From 7a61bae471b5b87264948cbdb9fafdf6165df22e Mon Sep 17 00:00:00 2001 From: exterr2f Date: Mon, 17 Feb 2025 09:17:27 +0100 Subject: [PATCH 05/15] clarify error message for blocked user --- internal/auth/auth.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/auth/auth.go b/internal/auth/auth.go index 6241585..262204c 100644 --- a/internal/auth/auth.go +++ b/internal/auth/auth.go @@ -234,7 +234,7 @@ func (auth *Authentication) Login( limiter := getIPUserLimiter(ip, username) if !limiter.Allow() { log.Warnf("AUTH/RATE > Too many login attempts for combination IP: %s, Username: %s", ip, username) - onfailure(rw, r, errors.New("Too many login attempts, try again in 1 hour")) + onfailure(rw, r, errors.New("Too many login attempts, try again in a few minutes.")) return } From 6268dffff8584691bd5d0bb193ab06bd1dd54087 Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Fri, 28 Feb 2025 09:20:05 +0100 Subject: [PATCH 06/15] Readd time pkg after fixing merge conflict --- go.mod | 1 + go.sum | 2 ++ 2 files changed, 3 insertions(+) diff --git a/go.mod b/go.mod index 8e64f78..7ca48a8 100644 --- a/go.mod +++ b/go.mod @@ -29,6 +29,7 @@ require ( golang.org/x/crypto v0.35.0 golang.org/x/exp v0.0.0-20250218142911-aa4b98e5adaa golang.org/x/oauth2 v0.27.0 + golang.org/x/time v0.5.0 ) require ( diff --git a/go.sum b/go.sum index f41733a..04d165a 100644 --- a/go.sum +++ b/go.sum @@ -325,6 +325,8 @@ golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM= golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY= +golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= +golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= From 1031b3eb79caf6c6698f66e6c2c4b6edaec86e81 Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Fri, 28 Feb 2025 13:06:40 +0100 Subject: [PATCH 07/15] fix: user and status view histogram selection - correctly loads selection for selected cluster - applies availablility for selected cluster --- web/frontend/src/Status.root.svelte | 4 ++-- web/frontend/src/User.root.svelte | 4 ++-- .../generic/select/HistogramSelection.svelte | 22 +++++++++++++------ 3 files changed, 19 insertions(+), 11 deletions(-) diff --git a/web/frontend/src/Status.root.svelte b/web/frontend/src/Status.root.svelte index a44a962..63a69f5 100644 --- a/web/frontend/src/Status.root.svelte +++ b/web/frontend/src/Status.root.svelte @@ -76,8 +76,8 @@ let isHistogramSelectionOpen = false; $: metricsInHistograms = cluster - ? ccconfig[`user_view_histogramMetrics:${cluster}`] || [] - : ccconfig.user_view_histogramMetrics || []; + ? ccconfig[`user_view_histogramMetrics:${cluster}`] || ( ccconfig['user_view_histogramMetrics'] || [] ) + : ccconfig['user_view_histogramMetrics'] || []; const client = getContextClient(); // Note: nodeMetrics are requested on configured $timestep resolution diff --git a/web/frontend/src/User.root.svelte b/web/frontend/src/User.root.svelte index fae972b..77c4e01 100644 --- a/web/frontend/src/User.root.svelte +++ b/web/frontend/src/User.root.svelte @@ -69,8 +69,8 @@ let metricBinOptions = [10, 20, 50, 100]; $: metricsInHistograms = selectedCluster - ? ccconfig[`user_view_histogramMetrics:${selectedCluster}`] || [] - : ccconfig.user_view_histogramMetrics || []; + ? ccconfig[`user_view_histogramMetrics:${selectedCluster}`] || ( ccconfig['user_view_histogramMetrics'] || [] ) + : ccconfig['user_view_histogramMetrics'] || []; const client = getContextClient(); $: stats = queryStore({ diff --git a/web/frontend/src/generic/select/HistogramSelection.svelte b/web/frontend/src/generic/select/HistogramSelection.svelte index 4e38123..48971b0 100644 --- a/web/frontend/src/generic/select/HistogramSelection.svelte +++ b/web/frontend/src/generic/select/HistogramSelection.svelte @@ -27,15 +27,23 @@ const client = getContextClient(); const initialized = getContext("initialized"); - let availableMetrics = [] - function loadHistoMetrics(isInitialized) { - if (!isInitialized) return; - const rawAvailableMetrics = getContext("globalMetrics").filter((gm) => gm?.footprint).map((fgm) => { return fgm.name }) - availableMetrics = [...rawAvailableMetrics] + function loadHistoMetrics(isInitialized, thisCluster) { + if (!isInitialized) return []; + + if (!thisCluster) { + return getContext("globalMetrics") + .filter((gm) => gm?.footprint) + .map((fgm) => { return fgm.name }) + } else { + return getContext("globalMetrics") + .filter((gm) => gm?.availability.find((av) => av.cluster == thisCluster)) + .filter((agm) => agm?.footprint) + .map((afgm) => { return afgm.name }) + } } - let pendingMetrics = [...metricsInHistograms]; // Copy + $: pendingMetrics = [...metricsInHistograms]; // Copy on change from above const updateConfigurationMutation = ({ name, value }) => { return mutationStore({ @@ -71,7 +79,7 @@ }); } - $: loadHistoMetrics($initialized); + $: availableMetrics = loadHistoMetrics($initialized, cluster); From 5ce03c2db3f07a496ec7295c10a680542c87fa6d Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Fri, 28 Feb 2025 13:08:32 +0100 Subject: [PATCH 08/15] add metric selection count info to job view --- web/frontend/src/Job.root.svelte | 6 ++++-- web/frontend/src/job/StatsTable.svelte | 23 +++++++++++++++-------- 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/web/frontend/src/Job.root.svelte b/web/frontend/src/Job.root.svelte index a384e32..43d4f10 100644 --- a/web/frontend/src/Job.root.svelte +++ b/web/frontend/src/Job.root.svelte @@ -58,7 +58,8 @@ let plots = {}, statsTable - let missingMetrics = [], + let availableMetrics = new Set(), + missingMetrics = [], missingHosts = [], somethingMissing = false; @@ -293,7 +294,7 @@ {#if $initq.data} {/if} @@ -431,6 +432,7 @@ configName="job_view_selectedMetrics" bind:metrics={selectedMetrics} bind:isOpen={isMetricsSelectionOpen} + bind:allMetrics={availableMetrics} /> {/if} diff --git a/web/frontend/src/job/StatsTable.svelte b/web/frontend/src/job/StatsTable.svelte index d68d237..21d9b3b 100644 --- a/web/frontend/src/job/StatsTable.svelte +++ b/web/frontend/src/job/StatsTable.svelte @@ -18,6 +18,8 @@ InputGroup, InputGroupText, Icon, + Row, + Col } from "@sveltestrap/sveltestrap"; import { maxScope } from "../generic/utils.js"; import StatsTableEntry from "./StatsTableEntry.svelte"; @@ -26,7 +28,7 @@ export let job; export let jobMetrics; - const allMetrics = [...new Set(jobMetrics.map((m) => m.name))].sort() + const sortedJobMetrics = [...new Set(jobMetrics.map((m) => m.name))].sort() const scopesForMetric = (metric) => jobMetrics.filter((jm) => jm.name == metric).map((jm) => jm.scope); @@ -34,11 +36,12 @@ selectedScopes = {}, sorting = {}, isMetricSelectionOpen = false, + availableMetrics = new Set(), selectedMetrics = getContext("cc-config")[`job_view_nodestats_selectedMetrics:${job.cluster}`] || getContext("cc-config")["job_view_nodestats_selectedMetrics"]; - for (let metric of allMetrics) { + for (let metric of sortedJobMetrics) { // Not Exclusive or Multi-Node: get maxScope directly (mostly: node) // -> Else: Load smallest available granularity as default as per availability const availableScopes = scopesForMetric(metric); @@ -95,15 +98,19 @@ }; + + + + + +
- + {#if groupSelection.key == "user"} {:else} diff --git a/web/frontend/src/Status.root.svelte b/web/frontend/src/Status.root.svelte index 63a69f5..1249e0c 100644 --- a/web/frontend/src/Status.root.svelte +++ b/web/frontend/src/Status.root.svelte @@ -177,6 +177,7 @@ groupBy: USER ) { id + name totalJobs totalNodes totalCores @@ -518,7 +519,7 @@ From 0fe0461340aac88efb01d5ef9689856ebcdb2ab4 Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Fri, 28 Feb 2025 14:00:27 +0100 Subject: [PATCH 12/15] remove conflicting variable layer in metric histo select --- web/frontend/src/Status.root.svelte | 12 ++++++------ web/frontend/src/User.root.svelte | 12 ++++++------ .../src/generic/select/HistogramSelection.svelte | 12 ++++-------- 3 files changed, 16 insertions(+), 20 deletions(-) diff --git a/web/frontend/src/Status.root.svelte b/web/frontend/src/Status.root.svelte index 1249e0c..a310421 100644 --- a/web/frontend/src/Status.root.svelte +++ b/web/frontend/src/Status.root.svelte @@ -75,7 +75,7 @@ ); let isHistogramSelectionOpen = false; - $: metricsInHistograms = cluster + $: selectedHistograms = cluster ? ccconfig[`user_view_histogramMetrics:${cluster}`] || ( ccconfig['user_view_histogramMetrics'] || [] ) : ccconfig['user_view_histogramMetrics'] || []; @@ -90,7 +90,7 @@ $metrics: [String!] $from: Time! $to: Time! - $metricsInHistograms: [String!] + $selectedHistograms: [String!] ) { nodeMetrics( cluster: $cluster @@ -116,7 +116,7 @@ } } - stats: jobsStatistics(filter: $filter, metrics: $metricsInHistograms) { + stats: jobsStatistics(filter: $filter, metrics: $selectedHistograms) { histDuration { count value @@ -157,7 +157,7 @@ from: from.toISOString(), to: to.toISOString(), filter: [{ state: ["running"] }, { cluster: { eq: cluster } }], - metricsInHistograms: metricsInHistograms, + selectedHistograms: selectedHistograms, }, }); @@ -653,7 +653,7 @@ - {#if metricsInHistograms} + {#if selectedHistograms} {#key $mainQuery.data.stats[0].histMetrics} diff --git a/web/frontend/src/User.root.svelte b/web/frontend/src/User.root.svelte index 77c4e01..0e6a5b8 100644 --- a/web/frontend/src/User.root.svelte +++ b/web/frontend/src/User.root.svelte @@ -68,7 +68,7 @@ let durationBinOptions = ["1m","10m","1h","6h","12h"]; let metricBinOptions = [10, 20, 50, 100]; - $: metricsInHistograms = selectedCluster + $: selectedHistograms = selectedCluster ? ccconfig[`user_view_histogramMetrics:${selectedCluster}`] || ( ccconfig['user_view_histogramMetrics'] || [] ) : ccconfig['user_view_histogramMetrics'] || []; @@ -76,8 +76,8 @@ $: stats = queryStore({ client: client, query: gql` - query ($jobFilters: [JobFilter!]!, $metricsInHistograms: [String!], $numDurationBins: String, $numMetricBins: Int) { - jobsStatistics(filter: $jobFilters, metrics: $metricsInHistograms, numDurationBins: $numDurationBins , numMetricBins: $numMetricBins ) { + query ($jobFilters: [JobFilter!]!, $selectedHistograms: [String!], $numDurationBins: String, $numMetricBins: Int) { + jobsStatistics(filter: $jobFilters, metrics: $selectedHistograms, numDurationBins: $numDurationBins , numMetricBins: $numMetricBins ) { totalJobs shortJobs totalWalltime @@ -104,7 +104,7 @@ } } `, - variables: { jobFilters, metricsInHistograms, numDurationBins, numMetricBins }, + variables: { jobFilters, selectedHistograms, numDurationBins, numMetricBins }, }); onMount(() => filterComponent.updateFilters()); @@ -290,7 +290,7 @@ -{#if metricsInHistograms?.length > 0} +{#if selectedHistograms?.length > 0} {#if $stats.error} @@ -357,6 +357,6 @@ diff --git a/web/frontend/src/generic/select/HistogramSelection.svelte b/web/frontend/src/generic/select/HistogramSelection.svelte index 48971b0..604fc95 100644 --- a/web/frontend/src/generic/select/HistogramSelection.svelte +++ b/web/frontend/src/generic/select/HistogramSelection.svelte @@ -3,7 +3,7 @@ Properties: - `cluster String`: Currently selected cluster - - `metricsInHistograms [String]`: The currently selected metrics to display as histogram + - `selectedHistograms [String]`: The currently selected metrics to display as histogram - ìsOpen Bool`: Is selection opened --> @@ -21,13 +21,12 @@ import { gql, getContextClient, mutationStore } from "@urql/svelte"; export let cluster; - export let metricsInHistograms; + export let selectedHistograms; export let isOpen; const client = getContextClient(); const initialized = getContext("initialized"); - function loadHistoMetrics(isInitialized, thisCluster) { if (!isInitialized) return []; @@ -43,8 +42,6 @@ } } - $: pendingMetrics = [...metricsInHistograms]; // Copy on change from above - const updateConfigurationMutation = ({ name, value }) => { return mutationStore({ client: client, @@ -69,13 +66,12 @@ } function closeAndApply() { - metricsInHistograms = [...pendingMetrics]; // Set for parent isOpen = !isOpen; updateConfiguration({ name: cluster ? `user_view_histogramMetrics:${cluster}` : "user_view_histogramMetrics", - value: metricsInHistograms, + value: selectedHistograms, }); } @@ -89,7 +85,7 @@ {#each availableMetrics as metric (metric)} - + {metric} {/each} From c661baf058e4de122d600b51ab7ecae7a6be13bf Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Fri, 28 Feb 2025 14:36:19 +0100 Subject: [PATCH 13/15] Load new default metrics config from working directory --- internal/config/default_metrics.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/config/default_metrics.go b/internal/config/default_metrics.go index 83015d4..b0a0cc5 100644 --- a/internal/config/default_metrics.go +++ b/internal/config/default_metrics.go @@ -16,7 +16,7 @@ type DefaultMetricsConfig struct { } func LoadDefaultMetricsConfig() (*DefaultMetricsConfig, error) { - filePath := "configs/default_metrics.json" + filePath := "default_metrics.json" if _, err := os.Stat(filePath); os.IsNotExist(err) { return nil, nil } From b31aea7bc5492da69d58c00cc66742d04be17579 Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Fri, 28 Feb 2025 14:40:27 +0100 Subject: [PATCH 14/15] revert back to using globalMetrics in jobView metric default select --- web/frontend/src/Job.root.svelte | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/web/frontend/src/Job.root.svelte b/web/frontend/src/Job.root.svelte index 43d4f10..b641a43 100644 --- a/web/frontend/src/Job.root.svelte +++ b/web/frontend/src/Job.root.svelte @@ -129,7 +129,12 @@ const pendingMetrics = [ ...(ccconfig[`job_view_selectedMetrics:${job.cluster}`] || - ccconfig[`job_view_selectedMetrics`] + $initq.data.globalMetrics.reduce((names, gm) => { + if (gm.availability.find((av) => av.cluster === job.cluster)) { + names.push(gm.name); + } + return names; + }, []) ), ...(ccconfig[`job_view_nodestats_selectedMetrics:${job.cluster}`] || ccconfig[`job_view_nodestats_selectedMetrics`] From d7aefe0cf0b206a288bff8330d0814114f55d025 Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Fri, 28 Feb 2025 14:55:32 +0100 Subject: [PATCH 15/15] move user names in top lists to tooltip --- web/frontend/src/Analysis.root.svelte | 12 ++++++++++-- web/frontend/src/Status.root.svelte | 12 ++++++++++-- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/web/frontend/src/Analysis.root.svelte b/web/frontend/src/Analysis.root.svelte index 1617ccd..861c0ec 100644 --- a/web/frontend/src/Analysis.root.svelte +++ b/web/frontend/src/Analysis.root.svelte @@ -20,6 +20,7 @@ Card, Table, Icon, + Tooltip } from "@sveltestrap/sveltestrap"; import { init, @@ -425,11 +426,18 @@ {#if groupSelection.key == "user"} - + {#if te?.name} + {te.name} + {/if} {:else} - + {#if tu?.name} + {tu.name} + {/if} {/each}
- - {#each selectedMetrics as metric} @@ -163,7 +170,7 @@ From 38569f55c740fa92019cfb772f902cea073d653f Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Fri, 28 Feb 2025 13:09:04 +0100 Subject: [PATCH 09/15] add title to roofline plot - Clarify that roofline is CPU only --- web/frontend/src/generic/plots/Roofline.svelte | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/frontend/src/generic/plots/Roofline.svelte b/web/frontend/src/generic/plots/Roofline.svelte index 558d8e8..2941ecb 100644 --- a/web/frontend/src/generic/plots/Roofline.svelte +++ b/web/frontend/src/generic/plots/Roofline.svelte @@ -179,7 +179,7 @@ function render(plotData) { if (plotData) { const opts = { - title: "", + title: "CPU Roofline Diagram", mode: 2, width: width, height: height, From 42135fd26ceb2c30b02800618723026ff2426064 Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Fri, 28 Feb 2025 13:37:28 +0100 Subject: [PATCH 10/15] if disableClusterSelection is set, display info in cluster filter - instead of undocumented unresponsive cluster name select --- .../src/generic/filters/Cluster.svelte | 37 +++++++++++-------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/web/frontend/src/generic/filters/Cluster.svelte b/web/frontend/src/generic/filters/Cluster.svelte index 8606247..f886582 100644 --- a/web/frontend/src/generic/filters/Cluster.svelte +++ b/web/frontend/src/generic/filters/Cluster.svelte @@ -43,26 +43,31 @@ {#if $initialized}

Cluster

- - ((pendingCluster = null), (pendingPartition = null))} - > - Any Cluster - - {#each clusters as cluster} + {#if disableClusterSelection} + + + {:else} + ( - (pendingCluster = cluster.name), (pendingPartition = null) - )} + active={pendingCluster == null} + on:click={() => ((pendingCluster = null), (pendingPartition = null))} > - {cluster.name} + Any Cluster - {/each} - + {#each clusters as cluster} + ( + (pendingCluster = cluster.name), (pendingPartition = null) + )} + > + {cluster.name} + + {/each} + + {/if} {/if} {#if $initialized && pendingCluster != null}
From d5394c9e92de8d04a186ac18db05838c1c732c70 Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Fri, 28 Feb 2025 13:37:59 +0100 Subject: [PATCH 11/15] fix: analysis view top links fixed, add full name to topusers --- web/frontend/src/Analysis.root.svelte | 9 ++++++--- web/frontend/src/Status.root.svelte | 3 ++- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/web/frontend/src/Analysis.root.svelte b/web/frontend/src/Analysis.root.svelte index 40757d3..1617ccd 100644 --- a/web/frontend/src/Analysis.root.svelte +++ b/web/frontend/src/Analysis.root.svelte @@ -70,6 +70,8 @@ ...new Set([...metricsInHistograms, ...metricsInScatterplots.flat()]), ]; + $: clusterName = cluster?.name ? cluster.name : cluster; + const sortOptions = [ { key: "totalWalltime", label: "Walltime" }, { key: "totalNodeHours", label: "Node Hours" }, @@ -159,6 +161,7 @@ groupBy: $groupBy ) { id + name totalWalltime totalNodeHours totalCoreHours @@ -423,14 +426,14 @@
{te.id}{te.id} {te?.name ? `(${te.name})` : ''}{te.id}{tu.id}{tu.id} {tu?.name ? `(${tu.name})` : ''} {tu[topUserSelection.key]}
{te.id} {te?.name ? `(${te.name})` : ''}{te.id} {tu.id} {tu?.name ? `(${tu.name})` : ''}{tu.id}{tu[topUserSelection.key]}