From 8a473de793278e3133ca038fff28c967c7601577 Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Thu, 3 Aug 2023 19:09:15 +0200 Subject: [PATCH 1/4] fix: core/accelerator scope in statstable on load --- web/frontend/src/Job.root.svelte | 4 +++- web/frontend/src/StatsTable.svelte | 5 ++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/web/frontend/src/Job.root.svelte b/web/frontend/src/Job.root.svelte index 21f0c7f..0806768 100644 --- a/web/frontend/src/Job.root.svelte +++ b/web/frontend/src/Job.root.svelte @@ -33,6 +33,8 @@ export let authlevel; export let roles; + const accMetrics = ['acc_utilization', 'acc_mem_used', 'acc_power', 'nv_mem_util', 'nv_sm_clock', 'nv_temp']; + const { query: initq } = init(` job(id: "${dbid}") { id, jobId, user, project, cluster, startTime, @@ -76,7 +78,6 @@ ]); // Select default Scopes to load: Check before if accelerator metrics are not on accelerator scope by default - const accMetrics = ['acc_utilization', 'acc_mem_used', 'acc_power', 'nv_mem_util', 'nv_sm_clock', 'nv_temp'] const accNodeOnly = [...toFetch].some(function(m) { if (accMetrics.includes(m)) { const mc = metrics(job.cluster, m) @@ -396,6 +397,7 @@ bind:this={statsTable} job={$initq.data.job} jobMetrics={$jobMetrics.data.jobMetrics} + accMetrics={accMetrics} /> {/key} {/if} diff --git a/web/frontend/src/StatsTable.svelte b/web/frontend/src/StatsTable.svelte index f40c204..ae15437 100644 --- a/web/frontend/src/StatsTable.svelte +++ b/web/frontend/src/StatsTable.svelte @@ -7,6 +7,7 @@ export let job export let jobMetrics + export let accMetrics const allMetrics = [...new Set(jobMetrics.map(m => m.name))].sort(), scopesForMetric = (metric) => jobMetrics @@ -21,7 +22,9 @@ || getContext('cc-config')['job_view_nodestats_selectedMetrics'] for (let metric of allMetrics) { - selectedScopes[metric] = maxScope(scopesForMetric(metric)) + selectedScopes[metric] = (job.exclusive != 1 || job.numNodes == 1) ? + (job.numAccs != 0 && accMetrics.includes(metric)) ? 'accelerator' : 'core' + : maxScope(scopesForMetric(metric)) sorting[metric] = { min: { dir: 'up', active: false }, avg: { dir: 'up', active: false }, From 298051c33489f3224a28b3a9915608cb9a4ec154 Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Fri, 4 Aug 2023 10:54:18 +0200 Subject: [PATCH 2/4] add fallback for acc metrics only on node level --- web/frontend/src/Job.root.svelte | 4 +++- web/frontend/src/StatsTable.svelte | 17 +++++++++++++---- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/web/frontend/src/Job.root.svelte b/web/frontend/src/Job.root.svelte index 0806768..9017144 100644 --- a/web/frontend/src/Job.root.svelte +++ b/web/frontend/src/Job.root.svelte @@ -34,6 +34,7 @@ export let roles; const accMetrics = ['acc_utilization', 'acc_mem_used', 'acc_power', 'nv_mem_util', 'nv_sm_clock', 'nv_temp']; + let accNodeOnly const { query: initq } = init(` job(id: "${dbid}") { @@ -78,7 +79,7 @@ ]); // Select default Scopes to load: Check before if accelerator metrics are not on accelerator scope by default - const accNodeOnly = [...toFetch].some(function(m) { + accNodeOnly = [...toFetch].some(function(m) { if (accMetrics.includes(m)) { const mc = metrics(job.cluster, m) return mc.scope !== 'accelerator' @@ -398,6 +399,7 @@ job={$initq.data.job} jobMetrics={$jobMetrics.data.jobMetrics} accMetrics={accMetrics} + accNodeOnly={accNodeOnly} /> {/key} {/if} diff --git a/web/frontend/src/StatsTable.svelte b/web/frontend/src/StatsTable.svelte index ae15437..e85e835 100644 --- a/web/frontend/src/StatsTable.svelte +++ b/web/frontend/src/StatsTable.svelte @@ -8,6 +8,7 @@ export let job export let jobMetrics export let accMetrics + export let accNodeOnly const allMetrics = [...new Set(jobMetrics.map(m => m.name))].sort(), scopesForMetric = (metric) => jobMetrics @@ -20,11 +21,19 @@ isMetricSelectionOpen = false, selectedMetrics = getContext('cc-config')[`job_view_nodestats_selectedMetrics:${job.cluster}`] || getContext('cc-config')['job_view_nodestats_selectedMetrics'] - + for (let metric of allMetrics) { - selectedScopes[metric] = (job.exclusive != 1 || job.numNodes == 1) ? - (job.numAccs != 0 && accMetrics.includes(metric)) ? 'accelerator' : 'core' - : maxScope(scopesForMetric(metric)) + // Not Exclusive or Single Node: Get maxScope() + // No Accelerators in Job and not Acc-Metric: Use 'core' + // Accelerator Metric available on accelerator scope: Use 'accelerator' + // Accelerator Metric only on node scope: Fallback to 'node' + selectedScopes[metric] = (job.exclusive != 1 || job.numNodes == 1) ? + (job.numAccs != 0 && accMetrics.includes(metric)) ? + accNodeOnly ? + 'node' + : 'accelerator' + : 'core' + : maxScope(scopesForMetric(metric)) sorting[metric] = { min: { dir: 'up', active: false }, avg: { dir: 'up', active: false }, From 163462b29c5874f24cf915b42b7b1d7cebe70c1c Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Wed, 9 Aug 2023 16:34:00 +0200 Subject: [PATCH 3/4] pre-sort numerical ids in statstable - sorting still fixed if not nodescope --- web/frontend/src/StatsTableEntry.svelte | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/web/frontend/src/StatsTableEntry.svelte b/web/frontend/src/StatsTableEntry.svelte index 6e96c1f..8871681 100644 --- a/web/frontend/src/StatsTableEntry.svelte +++ b/web/frontend/src/StatsTableEntry.svelte @@ -4,9 +4,14 @@ export let scope export let jobMetrics + function compareIds(a, b) { + return a.id - b.id; + } + $: series = jobMetrics .find(jm => jm.name == metric && jm.scope == scope) ?.metric.series.filter(s => s.hostname == host && s.statistics != null) + ?.sort(compareIds) {#if series == null || series.length == 0} From ed056b065e49f4ecd6fa65f8b33938e94ac7886e Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Wed, 9 Aug 2023 17:24:01 +0200 Subject: [PATCH 4/4] feat: add sorting in sub-node scopes in statsTable --- web/frontend/src/StatsTableEntry.svelte | 44 +++++++++++++++++++++++-- 1 file changed, 42 insertions(+), 2 deletions(-) diff --git a/web/frontend/src/StatsTableEntry.svelte b/web/frontend/src/StatsTableEntry.svelte index 8871681..5e497d4 100644 --- a/web/frontend/src/StatsTableEntry.svelte +++ b/web/frontend/src/StatsTableEntry.svelte @@ -1,17 +1,49 @@ {#if series == null || series.length == 0} @@ -29,6 +61,14 @@ {:else} + + {#each ['id', 'min', 'avg', 'max'] as field} + + {/each} + {#each series as s, i}
sortByField(field)}> + Sort + +
{s.id ?? i}