From a59df12595d0a7ae43a5a87d2f252746311fa958 Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Mon, 26 Aug 2024 17:37:23 +0200 Subject: [PATCH 1/6] init basic proof of concept --- .../src/generic/joblist/JobListRow.svelte | 4 ++++ .../src/generic/plots/MetricPlot.svelte | 20 ++++++++++++++++++- web/frontend/src/job/Metric.svelte | 8 ++++++++ 3 files changed, 31 insertions(+), 1 deletion(-) diff --git a/web/frontend/src/generic/joblist/JobListRow.svelte b/web/frontend/src/generic/joblist/JobListRow.svelte index 274e4f0..197b8b3 100644 --- a/web/frontend/src/generic/joblist/JobListRow.svelte +++ b/web/frontend/src/generic/joblist/JobListRow.svelte @@ -159,6 +159,10 @@ {#if metric.disabled == false && metric.data} { + // filterComponent.updateFilters(detail) + console.log("Upstream New Res:", detail) + }} width={plotWidth} height={plotHeight} timestep={metric.data.metric.timestep} diff --git a/web/frontend/src/generic/plots/MetricPlot.svelte b/web/frontend/src/generic/plots/MetricPlot.svelte index d092413..4fc1b77 100644 --- a/web/frontend/src/generic/plots/MetricPlot.svelte +++ b/web/frontend/src/generic/plots/MetricPlot.svelte @@ -112,7 +112,7 @@ {#each links as item} - {#if !item.perCluster} + {#if item.listOptions} + + + + {item.title} + + + + All Clusters + + + {#each clusters as cluster} + + + {cluster.name} + + + + Running Jobs + + + + {/each} + + + {:else if !item.perCluster} {item.title} From 54f3a261c5d4af0d313c54d1cae465762b6fe82b Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Mon, 2 Sep 2024 18:20:32 +0200 Subject: [PATCH 5/6] Rewrite sqlite indices from scratch for v8 migration --- .../sqlite3/08_add-footprint.up.sql | 52 ++++++++++++++++++- 1 file changed, 50 insertions(+), 2 deletions(-) diff --git a/internal/repository/migrations/sqlite3/08_add-footprint.up.sql b/internal/repository/migrations/sqlite3/08_add-footprint.up.sql index bcd6494..c101c6e 100644 --- a/internal/repository/migrations/sqlite3/08_add-footprint.up.sql +++ b/internal/repository/migrations/sqlite3/08_add-footprint.up.sql @@ -1,5 +1,11 @@ -CREATE INDEX IF NOT EXISTS job_by_project ON job (project); -CREATE INDEX IF NOT EXISTS job_list_projects ON job (project, job_state); +DROP INDEX job_stats; +DROP INDEX job_by_user; +DROP INDEX job_by_starttime; +DROP INDEX job_by_job_id; +DROP INDEX job_list; +DROP INDEX job_list_user; +DROP INDEX job_list_users; +DROP INDEX job_list_users_start; ALTER TABLE job ADD COLUMN energy REAL NOT NULL DEFAULT 0.0; ALTER TABLE job ADD COLUMN energy_footprint TEXT DEFAULT NULL; @@ -24,3 +30,45 @@ ALTER TABLE job DROP net_bw_avg; ALTER TABLE job DROP net_data_vol_total; ALTER TABLE job DROP file_bw_avg; ALTER TABLE job DROP file_data_vol_total; + +CREATE INDEX jobs_cluster IF NOT EXISTS ON job (cluster); +CREATE INDEX jobs_cluster_starttime IF NOT EXISTS ON job (cluster, start_time); +CREATE INDEX jobs_cluster_user IF NOT EXISTS ON job (cluster, user); +CREATE INDEX jobs_cluster_project IF NOT EXISTS ON job (cluster, project); +CREATE INDEX jobs_cluster_subcluster IF NOT EXISTS ON job (cluster, subcluster); + +CREATE INDEX jobs_cluster_partition IF NOT EXISTS ON job (cluster, partition); +CREATE INDEX jobs_cluster_partition_starttime IF NOT EXISTS ON job (cluster, partition, start_time); +CREATE INDEX jobs_cluster_partition_jobstate IF NOT EXISTS ON job (cluster, partition, job_state); +CREATE INDEX jobs_cluster_partition_jobstate_user IF NOT EXISTS ON job (cluster, partition, job_state, user); +CREATE INDEX jobs_cluster_partition_jobstate_project IF NOT EXISTS ON job (cluster, partition, job_state, project); +CREATE INDEX jobs_cluster_partition_jobstate_starttime IF NOT EXISTS ON job (cluster, partition, job_state, start_time); + +CREATE INDEX jobs_cluster_jobstate IF NOT EXISTS ON job (cluster, job_state); +CREATE INDEX jobs_cluster_jobstate_starttime IF NOT EXISTS ON job (cluster, job_state, starttime); +CREATE INDEX jobs_cluster_jobstate_user IF NOT EXISTS ON job (cluster, job_state, user); +CREATE INDEX jobs_cluster_jobstate_project IF NOT EXISTS ON job (cluster, job_state, project); + +CREATE INDEX jobs_user IF NOT EXISTS ON job (user); +CREATE INDEX jobs_user_starttime IF NOT EXISTS ON job (user, start_time); + +CREATE INDEX jobs_project IF NOT EXISTS ON job (project); +CREATE INDEX jobs_project_starttime IF NOT EXISTS ON job (project, start_time); +CREATE INDEX jobs_project_user IF NOT EXISTS ON job (project, user); + +CREATE INDEX jobs_jobstate IF NOT EXISTS ON job (job_state); +CREATE INDEX jobs_jobstate_user IF NOT EXISTS ON job (job_state, user); +CREATE INDEX jobs_jobstate_project IF NOT EXISTS ON job (job_state, project); +CREATE INDEX jobs_jobstate_cluster IF NOT EXISTS ON job (job_state, cluster); +CREATE INDEX jobs_jobstate_starttime IF NOT EXISTS ON job (job_state, start_time); + +CREATE INDEX jobs_arrayjobid_starttime IF NOT EXISTS ON job (array_job_id, start_time); +CREATE INDEX jobs_cluster_arrayjobid_starttime IF NOT EXISTS ON job (cluster, array_job_id, start_time); + +CREATE INDEX jobs_starttime IF NOT EXISTS ON job (start_time); +CREATE INDEX jobs_duration IF NOT EXISTS ON job (duration); +CREATE INDEX jobs_numnodes IF NOT EXISTS ON job (num_nodes); +CREATE INDEX jobs_numhwthreads IF NOT EXISTS ON job (num_hwthreads); +CREATE INDEX jobs_numacc IF NOT EXISTS ON job (num_acc); + +PRAGMA optimize; From 7602641909c15576a4a626c3069ec020ac7badf7 Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Mon, 2 Sep 2024 18:22:34 +0200 Subject: [PATCH 6/6] feat: change to resolution increase on zoom --- .../src/generic/joblist/JobListRow.svelte | 29 ++++++++--- .../src/generic/plots/MetricPlot.svelte | 52 ++++++++++++++----- web/frontend/src/job/Metric.svelte | 50 ++++++++++-------- 3 files changed, 90 insertions(+), 41 deletions(-) diff --git a/web/frontend/src/generic/joblist/JobListRow.svelte b/web/frontend/src/generic/joblist/JobListRow.svelte index 197b8b3..5581903 100644 --- a/web/frontend/src/generic/joblist/JobListRow.svelte +++ b/web/frontend/src/generic/joblist/JobListRow.svelte @@ -32,12 +32,14 @@ ? ["core", "accelerator"] : ["core"] : ["node"]; + let selectedResolution = 600; + let zoomStates = {}; const cluster = getContext("clusters").find((c) => c.name == job.cluster); const client = getContextClient(); const query = gql` - query ($id: ID!, $metrics: [String!]!, $scopes: [MetricScope!]!) { - jobMetrics(id: $id, metrics: $metrics, scopes: $scopes) { + query ($id: ID!, $metrics: [String!]!, $scopes: [MetricScope!]!, $selectedResolution: Int) { + jobMetrics(id: $id, metrics: $metrics, scopes: $scopes, resolution: $selectedResolution) { name scope metric { @@ -66,17 +68,30 @@ } `; + function handleZoom(detail, metric) { + if ( + (zoomStates[metric]?.x?.min !== detail?.lastZoomState?.x?.min) && + (zoomStates[metric]?.y?.max !== detail?.lastZoomState?.y?.max) + ) { + zoomStates[metric] = {...detail.lastZoomState} + } + + if (detail?.newRes) { // Triggers GQL + selectedResolution = detail.newRes + } + } + $: metricsQuery = queryStore({ client: client, query: query, - variables: { id, metrics, scopes }, + variables: { id, metrics, scopes, selectedResolution }, }); function refreshMetrics() { metricsQuery = queryStore({ client: client, query: query, - variables: { id, metrics, scopes }, + variables: { id, metrics, scopes, selectedResolution }, // requestPolicy: 'network-only' // use default cache-first for refresh }); } @@ -159,10 +174,7 @@ {#if metric.disabled == false && metric.data} { - // filterComponent.updateFilters(detail) - console.log("Upstream New Res:", detail) - }} + on:zoom={({detail}) => { handleZoom(detail, metric.data.name) }} width={plotWidth} height={plotHeight} timestep={metric.data.metric.timestep} @@ -175,6 +187,7 @@ isShared={job.exclusive != 1} numhwthreads={job.numHWThreads} numaccs={job.numAcc} + zoomState={zoomStates[metric.data.name]} /> {:else if metric.disabled == true && metric.data} { + u.over.addEventListener("dblclick", (e) => { + console.log('Dispatch Reset') + dispatch('zoom', { + lastZoomState: { + x: { time: false }, + y: { auto: true } + } + }); + }); + } + ], draw: [ (u) => { // Draw plot type label: @@ -437,17 +453,26 @@ setScale: [ (u, key) => { if (key === 'x') { - // Start - console.log('setScale X', key); - - // Decide which resolution to request - - // Dispatch request - const res = 1337; - dispatch('zoom-in', { - newres: res, - }); - + const numX = (u.series[0].idxs[1] - u.series[0].idxs[0]) + if (numX <= 20 && timestep !== 60) { // Zoom IN if not at MAX + console.log('Dispatch Zoom') + if (timestep == 600) { + dispatch('zoom', { + newRes: 240, + lastZoomState: u?.scales + }); + } else if (timestep === 240) { + dispatch('zoom', { + newRes: 60, + lastZoomState: u?.scales + }); + } + } else { + console.log('Dispatch Update') + dispatch('zoom', { + lastZoomState: u?.scales + }); + } }; } ] @@ -481,6 +506,10 @@ if (!uplot) { opts.width = width; opts.height = height; + if (zoomState) { + // console.log('Use last state for uPlot init:', metric, scope, zoomState) + opts.scales = {...zoomState} + } uplot = new uPlot(opts, plotData, plotWrapper); } else { uplot.setSize({ width, height }); @@ -489,7 +518,6 @@ function onSizeChange() { if (!uplot) return; - if (timeoutId != null) clearTimeout(timeoutId); timeoutId = setTimeout(() => { diff --git a/web/frontend/src/job/Metric.svelte b/web/frontend/src/job/Metric.svelte index 41e3046..ceacca5 100644 --- a/web/frontend/src/job/Metric.svelte +++ b/web/frontend/src/job/Metric.svelte @@ -27,7 +27,9 @@ Spinner, Card, } from "@sveltestrap/sveltestrap"; - import { minScope } from "../generic/utils.js"; + import { + minScope, + } from "../generic/utils.js"; import Timeseries from "../generic/plots/MetricPlot.svelte"; export let job; @@ -39,9 +41,8 @@ export let rawData; export let isShared = false; - let selectedHost = null, - plot, - error = null; + let selectedHost = null; + let error = null; let selectedScope = minScope(scopes); let selectedResolution; let pendingResolution = 600; @@ -49,11 +50,12 @@ let patternMatches = false; let nodeOnly = false; // If, after load-all, still only node scope returned let statsSeries = rawData.map((data) => data?.statisticsSeries ? data.statisticsSeries : null); + let zoomState = null; + let pendingZoomState = null; const dispatch = createEventDispatcher(); const statsPattern = /(.*)-stat$/; const unit = (metricUnit?.prefix ? metricUnit.prefix : "") + (metricUnit?.base ? metricUnit.base : ""); - const resolutions = [600, 240, 60] // DEV: Make configable const client = getContextClient(); const subQuery = gql` query ($dbid: ID!, $selectedMetrics: [String!]!, $selectedScopes: [MetricScope!]!, $selectedResolution: Int) { @@ -86,6 +88,19 @@ } `; + function handleZoom(detail) { + if ( // States have to differ, causes deathloop if just set + (pendingZoomState?.x?.min !== detail?.lastZoomState?.x?.min) && + (pendingZoomState?.y?.max !== detail?.lastZoomState?.y?.max) + ) { + pendingZoomState = {...detail.lastZoomState} + } + + if (detail?.newRes) { // Triggers GQL + pendingResolution = detail.newRes + } + } + let metricData; let selectedScopes = [...scopes] const dbid = job.id; @@ -119,11 +134,15 @@ }); if ($metricData && !$metricData.fetching) { - rawData = $metricData.data.singleUpdate.map((x) => x.metric) scopes = $metricData.data.singleUpdate.map((x) => x.scope) statsSeries = rawData.map((data) => data?.statisticsSeries ? data.statisticsSeries : null) + // Keep Zoomlevel if ResChange By Zoom + if (pendingZoomState) { + zoomState = {...pendingZoomState} + } + // Set selected scope to min of returned scopes if (selectedScope == "load-all") { selectedScope = minScope(scopes) @@ -176,11 +195,6 @@ {/each} {/if} - {#key series} {#if $metricData?.fetching == true} @@ -189,11 +203,7 @@ {error.message} {:else if series != null && !patternMatches} { - // filterComponent.updateFilters(detail) - console.log("Upstream New Res:", detail) - }} + on:zoom={({detail}) => { handleZoom(detail) }} {width} height={300} cluster={job.cluster} @@ -203,14 +213,11 @@ metric={metricName} {series} {isShared} + {zoomState} /> {:else if statsSeries[selectedScopeIndex] != null && patternMatches} { - // filterComponent.updateFilters(detail) - console.log("Upstream New Res:", detail) - }} + on:zoom={({detail}) => { handleZoom(detail) }} {width} height={300} cluster={job.cluster} @@ -220,6 +227,7 @@ metric={metricName} {series} {isShared} + {zoomState} statisticsSeries={statsSeries[selectedScopeIndex]} useStatsSeries={!!statsSeries[selectedScopeIndex]} />