From b637ddeb2825f762f0a84342603e4ff969ac7955 Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Sun, 21 Apr 2024 15:04:00 +0200 Subject: [PATCH 01/10] Refactor and reformat userConfig --- internal/repository/userConfig.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/repository/userConfig.go b/internal/repository/userConfig.go index fb8c3f5..4e5c32d 100644 --- a/internal/repository/userConfig.go +++ b/internal/repository/userConfig.go @@ -24,9 +24,9 @@ var ( type UserCfgRepo struct { DB *sqlx.DB Lookup *sqlx.Stmt - lock sync.RWMutex uiDefaults map[string]interface{} cache *lrucache.Cache + lock sync.RWMutex } func GetUserCfgRepo() *UserCfgRepo { @@ -112,8 +112,8 @@ func (uCfg *UserCfgRepo) GetUIConfig(user *schema.User) (map[string]interface{}, // configuration. func (uCfg *UserCfgRepo) UpdateConfig( key, value string, - user *schema.User) error { - + user *schema.User, +) error { if user == nil { var val interface{} if err := json.Unmarshal([]byte(value), &val); err != nil { From cbaeffde2c4013a3a9ff031fd17df74b272c621d Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Mon, 22 Apr 2024 11:29:31 +0200 Subject: [PATCH 02/10] fix: improve speed of hasNextPage query for infinite scroll --- internal/graph/schema.resolvers.go | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/internal/graph/schema.resolvers.go b/internal/graph/schema.resolvers.go index a33e041..5f55139 100644 --- a/internal/graph/schema.resolvers.go +++ b/internal/graph/schema.resolvers.go @@ -243,14 +243,21 @@ func (r *queryResolver) Jobs(ctx context.Context, filter []*model.JobFilter, pag if !config.Keys.UiDefaults["job_list_usePaging"].(bool) { hasNextPage := false - page.Page += 1 + // page.Page += 1 : Simple, but expensive + // Example Page 4 @ 10 IpP : Does item 41 exist? + // Minimal Page 41 @ 1 IpP : If len(result) is 1, Page 5 @ 10 IpP exists. + nextPage := &model.PageRequest{ + ItemsPerPage: 1, + Page: ((page.Page * page.ItemsPerPage) + 1), + } - nextJobs, err := r.Repo.QueryJobs(ctx, filter, page, order) + nextJobs, err := r.Repo.QueryJobs(ctx, filter, nextPage, order) if err != nil { log.Warn("Error while querying next jobs") return nil, err } - if len(nextJobs) > 0 { + + if len(nextJobs) == 1 { hasNextPage = true } From a22340196f388b4403accc8639b0ffa44ac148c1 Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Mon, 22 Apr 2024 12:14:40 +0200 Subject: [PATCH 03/10] Fix: Improve jobName query by parsing DB field as JSON - No DB mirgration required - SQLite internal EXTRACT function used --- internal/repository/query.go | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/internal/repository/query.go b/internal/repository/query.go index eec51a2..94aa742 100644 --- a/internal/repository/query.go +++ b/internal/repository/query.go @@ -135,7 +135,7 @@ func BuildWhereClause(filter *model.JobFilter, query sq.SelectBuilder) sq.Select query = buildStringCondition("job.project", filter.Project, query) } if filter.JobName != nil { - query = buildStringCondition("job.meta_data", filter.JobName, query) + query = buildMetaJsonCondition("jobName", filter.JobName, query) } if filter.Cluster != nil { query = buildStringCondition("job.cluster", filter.Cluster, query) @@ -235,6 +235,25 @@ func buildStringCondition(field string, cond *model.StringInput, query sq.Select return query } +func buildMetaJsonCondition(jsonField string, cond *model.StringInput, query sq.SelectBuilder) sq.SelectBuilder { + if cond.Eq != nil { + return query.Where("JSON_EXTRACT(meta_data, \"$."+jsonField+"\") = ?", *cond.Eq) + } + if cond.Neq != nil { + return query.Where("JSON_EXTRACT(meta_data, \"$."+jsonField+"\") != ?", *cond.Neq) + } + if cond.StartsWith != nil { + return query.Where("JSON_EXTRACT(meta_data, \"$."+jsonField+"\") LIKE ?", fmt.Sprint(*cond.StartsWith, "%")) + } + if cond.EndsWith != nil { + return query.Where("JSON_EXTRACT(meta_data, \"$."+jsonField+"\") LIKE ?", fmt.Sprint("%", *cond.EndsWith)) + } + if cond.Contains != nil { + return query.Where("JSON_EXTRACT(meta_data, \"$."+jsonField+"\") LIKE ?", fmt.Sprint("%", *cond.Contains, "%")) + } + return query +} + var matchFirstCap = regexp.MustCompile("(.)([A-Z][a-z]+)") var matchAllCap = regexp.MustCompile("([a-z0-9])([A-Z])") From f80123c85dbedeccb8031f9451d323b6d80f68f4 Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Wed, 24 Apr 2024 13:47:29 +0200 Subject: [PATCH 04/10] Fix: Add missing nullsafe for admin user table --- web/frontend/src/Job.root.svelte | 2 +- web/frontend/src/config/admin/ShowUsersRow.svelte | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/web/frontend/src/Job.root.svelte b/web/frontend/src/Job.root.svelte index 758cef9..b399f4d 100644 --- a/web/frontend/src/Job.root.svelte +++ b/web/frontend/src/Job.root.svelte @@ -361,7 +361,7 @@
- Missing Metrics/Reseources + Missing Metrics/Resources {#if missingMetrics.length > 0} diff --git a/web/frontend/src/config/admin/ShowUsersRow.svelte b/web/frontend/src/config/admin/ShowUsersRow.svelte index 9845241..782ea56 100644 --- a/web/frontend/src/config/admin/ShowUsersRow.svelte +++ b/web/frontend/src/config/admin/ShowUsersRow.svelte @@ -20,7 +20,7 @@ {user.name} {user.projects} {user.email} -{user.roles.join(", ")} +{user?.roles ? user.roles.join(", ") : "No Roles"} {#if !jwt} + + From b66750339ddf5cdb6b7c1d5b9413576cd6341205 Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Thu, 25 Apr 2024 16:59:04 +0200 Subject: [PATCH 06/10] add default value, remove unused argument --- web/frontend/src/Job.root.svelte | 1 - web/frontend/src/MetricSelection.svelte | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/web/frontend/src/Job.root.svelte b/web/frontend/src/Job.root.svelte index b399f4d..8d090ac 100644 --- a/web/frontend/src/Job.root.svelte +++ b/web/frontend/src/Job.root.svelte @@ -341,7 +341,6 @@ scopes={item.data.map((x) => x.scope)} {width} isShared={$initq.data.job.exclusive != 1} - resources={$initq.data.job.resources} /> {:else} Date: Thu, 25 Apr 2024 16:59:27 +0200 Subject: [PATCH 07/10] fix: fix metricPlot y zoom reset --- web/frontend/src/plots/MetricPlot.svelte | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/frontend/src/plots/MetricPlot.svelte b/web/frontend/src/plots/MetricPlot.svelte index bd44675..ba3c294 100644 --- a/web/frontend/src/plots/MetricPlot.svelte +++ b/web/frontend/src/plots/MetricPlot.svelte @@ -461,7 +461,7 @@ }, scales: { x: { time: false }, - y: maxY ? { range: [0, maxY * 1.1] } : {}, + y: maxY ? { min: 0, max: (maxY * 1.1) } : {auto: true}, // Add some space to upper render limit }, legend: { // Display legend until max 12 Y-dataseries From ef51e69ffb53e696eada560aaa0ab0c857b2c0ff Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Fri, 26 Apr 2024 11:11:55 +0200 Subject: [PATCH 08/10] feat: Add roofline color scale for time information --- web/frontend/src/plots/Roofline.svelte | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/web/frontend/src/plots/Roofline.svelte b/web/frontend/src/plots/Roofline.svelte index 1e47f6f..11d1d25 100644 --- a/web/frontend/src/plots/Roofline.svelte +++ b/web/frontend/src/plots/Roofline.svelte @@ -298,6 +298,24 @@ // Reset grid lineWidth u.ctx.lineWidth = 0.15; } + if (renderTime) { + // The Color Scale For Time Information + const posX = u.valToPos(0.1, "x", true) + const posXLimit = u.valToPos(100, "x", true) + const posY = u.valToPos(15000.0, "y", true) + u.ctx.fillStyle = 'black' + u.ctx.fillText('Start', posX, posY) + const start = posX + 10 + for (let x = start; x < posXLimit; x += 10) { + let c = (x - start) / (posXLimit - start) + u.ctx.fillStyle = getRGB(c) + u.ctx.beginPath() + u.ctx.arc(x, posY, 3, 0, Math.PI * 2, false) + u.ctx.fill() + } + u.ctx.fillStyle = 'black' + u.ctx.fillText('End', posXLimit + 23, posY) + } }, ], }, From 72557fd0bf28e0c5b456c7030542b66e55f0da2d Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Thu, 2 May 2024 16:32:01 +0200 Subject: [PATCH 09/10] feat: add statistics series render to job view metric plots --- web/frontend/src/Job.root.svelte | 3 --- web/frontend/src/Metric.svelte | 37 ++++++++++++++++++++++++++++---- 2 files changed, 33 insertions(+), 7 deletions(-) diff --git a/web/frontend/src/Job.root.svelte b/web/frontend/src/Job.root.svelte index 8d090ac..00f64e2 100644 --- a/web/frontend/src/Job.root.svelte +++ b/web/frontend/src/Job.root.svelte @@ -306,9 +306,6 @@ {/if} - diff --git a/web/frontend/src/Metric.svelte b/web/frontend/src/Metric.svelte index 6022ffb..a3bedaa 100644 --- a/web/frontend/src/Metric.svelte +++ b/web/frontend/src/Metric.svelte @@ -33,8 +33,17 @@ error = null; let selectedScope = minScope(scopes); + let statsPattern = /(.*)-stats$/ + let statsSeries = rawData.map((data) => data?.statisticsSeries ? data.statisticsSeries : null) + let selectedScopeIndex + $: availableScopes = scopes; - $: selectedScopeIndex = scopes.findIndex((s) => s == selectedScope); + $: patternMatches = statsPattern.exec(selectedScope) + $: if (!patternMatches) { + selectedScopeIndex = scopes.findIndex((s) => s == selectedScope); + } else { + selectedScopeIndex = scopes.findIndex((s) => s == patternMatches[1]); + } $: data = rawData[selectedScopeIndex]; $: series = data?.series.filter( (series) => selectedHost == null || series.hostname == selectedHost, @@ -62,6 +71,7 @@ if (jm.scope != "node") { scopes = [...scopes, jm.scope]; rawData.push(jm.metric); + statsSeries = rawData.map((data) => data?.statisticsSeries ? data.statisticsSeries : null) selectedScope = jm.scope; selectedScopeIndex = scopes.findIndex((s) => s == jm.scope); dispatch("more-loaded", jm); @@ -79,15 +89,18 @@ : "") + (metricConfig?.unit?.base ? metricConfig.unit.base : "")}) {#if job.resources.length > 1} - {#each job.resources as { hostname }} @@ -100,7 +113,7 @@ {:else if error != null} {error.message} - {:else if series != null} + {:else if series != null && !patternMatches} + {:else if statsSeries[selectedScopeIndex] != null && patternMatches} + {/if} {/key} From 597bccc080ad17478a14cb72768655573a80fa62 Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Mon, 6 May 2024 13:15:15 +0200 Subject: [PATCH 10/10] fix: add SQL JSON validity check to meta_data query --- internal/repository/query.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/internal/repository/query.go b/internal/repository/query.go index 94aa742..5ca98fb 100644 --- a/internal/repository/query.go +++ b/internal/repository/query.go @@ -236,6 +236,9 @@ func buildStringCondition(field string, cond *model.StringInput, query sq.Select } func buildMetaJsonCondition(jsonField string, cond *model.StringInput, query sq.SelectBuilder) sq.SelectBuilder { + // Verify and Search Only in Valid Jsons + query = query.Where("JSON_VALID(meta_data)") + // add "AND" Sql query Block for field match if cond.Eq != nil { return query.Where("JSON_EXTRACT(meta_data, \"$."+jsonField+"\") = ?", *cond.Eq) }