From 1b8c4e293c3d1196569fee6425a46f16f8ef674d Mon Sep 17 00:00:00 2001
From: Christoph Kluge <christoph.kluge@fau.de>
Date: Tue, 5 Sep 2023 11:46:34 +0200
Subject: [PATCH] Change to prod data, allow and handle null data

- fix errors regarding render timing
- always collect time info in transFormData function
- remove size from polar plot
---
 web/frontend/src/Job.root.svelte       |  38 ++--
 web/frontend/src/Status.root.svelte    |  18 +-
 web/frontend/src/plots/Polar.svelte    |   3 +-
 web/frontend/src/plots/Roofline.svelte | 285 +++++++++++--------------
 web/frontend/src/units.js              |   4 +-
 web/frontend/src/utils.js              |  44 ++--
 6 files changed, 180 insertions(+), 212 deletions(-)

diff --git a/web/frontend/src/Job.root.svelte b/web/frontend/src/Job.root.svelte
index 153b8f9..b0db95e 100644
--- a/web/frontend/src/Job.root.svelte
+++ b/web/frontend/src/Job.root.svelte
@@ -4,6 +4,7 @@
         groupByScope,
         fetchMetricsStore,
         checkMetricDisabled,
+        transformDataForRoofline
     } from "./utils.js";
     import {
         Row,
@@ -130,8 +131,8 @@
         lazyFetchMoreMetrics();
 
     let plots = {},
+        roofWidth,
         jobTags,
-        fullWidth,
         statsTable;
     $: document.title = $initq.fetching
         ? "Loading..."
@@ -190,7 +191,6 @@
         }));
 </script>
 
-<div class="row" bind:clientWidth={fullWidth} />
 <Row>
     <Col>
         {#if $initq.error}
@@ -245,7 +245,6 @@
         {/if}
         <Col>
             <Polar
-                size={fullWidth / 4.1}
                 metrics={ccconfig[
                     `job_view_polarPlotMetrics:${$initq.data.job.cluster}`
                 ] || ccconfig[`job_view_polarPlotMetrics`]}
@@ -254,21 +253,24 @@
             />
         </Col>
         <Col>
-            <Roofline
-                width={fullWidth / 3 - 10}
-                height={fullWidth / 5}
-                cluster={clusters
-                    .find((c) => c.name == $initq.data.job.cluster)
-                    .subClusters.find(
-                        (sc) => sc.name == $initq.data.job.subCluster
-                    )}
-                flopsAny={$jobMetrics.data.jobMetrics.find(
-                    (m) => m.name == "flops_any" && m.scope == "node"
-                )}
-                memBw={$jobMetrics.data.jobMetrics.find(
-                    (m) => m.name == "mem_bw" && m.scope == "node"
-                )}
-            />
+            <div bind:clientWidth={roofWidth}>
+                <Roofline
+                    width={roofWidth - 10}
+                    height={(roofWidth / 2) - 5}
+                    renderTime={true}
+                    cluster={clusters
+                        .find((c) => c.name == $initq.data.job.cluster)
+                        .subClusters.find(
+                            (sc) => sc.name == $initq.data.job.subCluster
+                        )}
+                    data={
+                        transformDataForRoofline (
+                            $jobMetrics.data.jobMetrics.find((m) => m.name == "flops_any" && m.scope == "node").metric,
+                            $jobMetrics.data.jobMetrics.find((m) => m.name == "mem_bw" && m.scope == "node").metric
+                        )
+                    }
+                />
+            </div>
         </Col>
     {:else}
         <Col />
diff --git a/web/frontend/src/Status.root.svelte b/web/frontend/src/Status.root.svelte
index 8e162ab..73060c6 100644
--- a/web/frontend/src/Status.root.svelte
+++ b/web/frontend/src/Status.root.svelte
@@ -31,8 +31,8 @@
     export let cluster;
 
     let plotWidths = [],
-        colWidth1 = 0,
-        colWidth2 = 0
+        colWidth1,
+        colWidth2
     let from = new Date(Date.now() - 5 * 60 * 1000),
         to = new Date(Date.now());
     const topOptions = [
@@ -429,14 +429,14 @@
                         <Roofline
                             width={plotWidths[i] - 10}
                             height={300}
-                            colorDots={true}
-                            showTime={false}
                             cluster={subCluster}
-                            data={transformPerNodeDataForRoofline(
-                                $mainQuery.data.nodeMetrics.filter(
-                                    (data) => data.subCluster == subCluster.name
+                            data={
+                                transformPerNodeDataForRoofline(
+                                    $mainQuery.data.nodeMetrics.filter(
+                                        (data) => data.subCluster == subCluster.name
+                                    )
                                 )
-                            )}
+                            }
                         />
                     {/key}
                 </div>
@@ -444,7 +444,7 @@
         </Row>
     {/each}
 
-    <hr style="margin-top: -1em;" />
+    <hr/>
 
     <!-- Usage Stats as Histograms -->
 
diff --git a/web/frontend/src/plots/Polar.svelte b/web/frontend/src/plots/Polar.svelte
index e556cc2..b5c488f 100644
--- a/web/frontend/src/plots/Polar.svelte
+++ b/web/frontend/src/plots/Polar.svelte
@@ -22,7 +22,6 @@
         LineElement
     );
 
-    export let size
     export let metrics
     export let cluster
     export let jobMetrics
@@ -95,7 +94,7 @@
 </script>
 
 <div class="chart-container">
-    <Radar {data} {options} width={size} height={size}/>
+    <Radar {data} {options}/>
 </div>
 
 <style>
diff --git a/web/frontend/src/plots/Roofline.svelte b/web/frontend/src/plots/Roofline.svelte
index cc65f41..b3abd72 100644
--- a/web/frontend/src/plots/Roofline.svelte
+++ b/web/frontend/src/plots/Roofline.svelte
@@ -4,99 +4,52 @@
     import { onMount, onDestroy } from 'svelte'
     import { Card } from 'sveltestrap'
 
-    export let flopsAny = null
-    export let memBw = null
-    export let maxY = null
+    export let data = null
+    export let renderTime = false
+    export let maxY = null // Optional
     export let cluster = null
     export let width = 500
     export let height = 300
-    export let renderTime = false
-    export let data = null
 
     let plotWrapper = null
     let uplot = null
     let timeoutId = null
 
-    // Three Render-Cases:
-    // #1 Single-Job Roofline -> Has Time-Information: Use data, allow renderTime
-    // #2 MultiNode Roofline - > Has No Time-Information: Transform from nodeData, only "IST"-state of nodes, no timeInfo
-    // #3 Multi-Job Roofline as Heatmap -> Keep Original
+    /* Data Format
+     * data       = [null, [], []] // 0: null-axis required for scatter, 1: Array of XY-Array for Scatter, 2: Optional Time Info
+     * data[1][0] = [100, 200, 500, ...] // X Axis -> Intensity (Vals up to clusters' flopRateScalar value)
+     * data[1][1] = [1000, 2000, 1500, ...] // Y Axis -> Performance (Vals up to clusters' flopRateSimd value)
+     * data[2]    = [0.1, 0.15, 0.2, ...] // Color Code -> Time Information (Floats from 0 to 1) (Optional)
+    */
 
-    // Start Demo Data
-
-    function randInt(min, max) {
-        return Math.floor(Math.random() * (max - min + 1)) + min;
-    }
-
-    function randFloat(min, max) {
-        return roundTwo(((Math.random() * (max - min + 1)) + min) / randInt(1, 500));
-    }
-
-    function roundTwo(num) {
-       return  Math.round((num + Number.EPSILON) * 100) / 100
-    }
-
-    function filledArr(len, val, time) {
-        let arr = Array(len);
-
-        if (typeof val == "function") {
-            for (let i = 0; i < len; ++i)
-                arr[i] = val(i);
-        }
-        else if (time) {
-            for (let i = 0; i < len; ++i)
-                arr[i] = i / 1000;
-        }
-        else {
-            for (let i = 0; i < len; ++i)
-                arr[i] = i;
-        }
-
-        return arr;
-    }
-
-    let points = 1000;
-
-    data       = [null, []] // Null-Axis required for scatter
-    data[1][0] = filledArr(points, i => randFloat(1,5000), false) // Intensity
-    data[1][1] = filledArr(points, i => randFloat(1,5000), false) // Performance
-    // data[1][0] = filledArr(points, 0, false) // Intensity
-    // data[1][1] = filledArr(points, 0, false) // Performance
-    data[2]    = filledArr(points, 0, true) // Time Information (Optional)
-
-    // End Demo Data
+    // Check
+    // console.assert(data , "you must provide data")
 
     // Helpers
-
     function getGradientR(x) {
         if (x < 0.5) return 0
         if (x > 0.75) return 255
         x = (x - 0.5) * 4.0
         return Math.floor(x * 255.0)
     }
-
     function getGradientG(x) {
         if (x > 0.25 && x < 0.75) return 255
         if (x < 0.25) x = x * 4.0
         else          x = 1.0 - (x - 0.75) * 4.0
         return Math.floor(x * 255.0)
     }
-
     function getGradientB(x) {
         if (x < 0.25) return 255
         if (x > 0.5) return 0
         x = 1.0 - (x - 0.25) * 4.0
         return Math.floor(x * 255.0)
     }
-
     function getRGB(c) {
         return `rgb(${getGradientR(c)}, ${getGradientG(c)}, ${getGradientB(c)})`
     }
-
     function nearestThousand (num) {
         return Math.ceil(num/1000) * 1000
     }
-
     function lineIntersect(x1, y1, x2, y2, x3, y3, x4, y4) {
         let l = (y4 - y3) * (x2 - x1) - (x4 - x3) * (y2 - y1)
         let a = ((x4 - x3) * (y1 - y3) - (y4 - y3) * (x1 - x3)) / l
@@ -105,12 +58,11 @@
             y: y1 + a * (y2 - y1)
         }
     }
-
     // End Helpers
 
+    // Dot Renderers
     const drawColorPoints = (u, seriesIdx, idx0, idx1) => {
         const size = 5 * devicePixelRatio;
-
         uPlot.orient(u, seriesIdx, (series, dataX, dataY, scaleX, scaleY, valToPosX, valToPosY, xOff, yOff, xDim, yDim, moveTo, lineTo, rect, arc) => {
             let d = u.data[seriesIdx];
             let deg360 = 2 * Math.PI;
@@ -136,7 +88,6 @@
 
     const drawPoints = (u, seriesIdx, idx0, idx1) => {
         const size = 5 * devicePixelRatio;
-
         uPlot.orient(u, seriesIdx, (series, dataX, dataY, scaleX, scaleY, valToPosX, valToPosY, xOff, yOff, xDim, yDim, moveTo, lineTo, rect, arc) => {
             let d = u.data[seriesIdx];
             u.ctx.strokeStyle = getRGB(0);
@@ -158,113 +109,121 @@
         return null;
     };
 
-    function render() {
-        const opts = {
-            title: "",
-            mode: 2,
-            width: width,
-            height: height,
-            legend: {
-                show: false
-            },
-            cursor: { drag: { x: false, y: false } },
-            axes: [
-                { label: 'Intensity [FLOPS/Byte]' },
-                { label: 'Performace [GFLOPS]' }
-            ],
-            scales: {
-                x: {
-                    time: false,
-                    range: [0.01, 1000],
-                    distr: 3, // Render as log
-					log: 10, // log exp
+    // Main Function
+    function render(plotData) {
+        if (plotData) {
+            const opts = {
+                title: "",
+                mode: 2,
+                width: width,
+                height: height,
+                legend: {
+                    show: false
                 },
-                y: {
-                    range: [1.0, nearestThousand(cluster.flopRateSimd.value || maxY)],
-                    distr: 3, // Render as log
-					log: 10, // log exp
-                },
-            },
-            series: [
-                {},
-                { paths: renderTime ? drawColorPoints : drawPoints }
-            ],
-            hooks: {
-                drawClear: [
-                    u => {
-                        u.series.forEach((s, i) => {
-                            if (i > 0)
-                                s._paths = null;
-                        });
+                cursor: { drag: { x: false, y: false } },
+                axes: [
+                    { 
+                        label: 'Intensity [FLOPS/Byte]',
+                        values: (u, vals) => vals.map(v => formatNumber(v))
                     },
-                ],
-                draw: [
-                    u => { // draw roofs when cluster set
-                        // console.log(u)
-                        if (cluster != null) {
-                            const padding = u._padding // [top, right, bottom, left]
-
-                            u.ctx.strokeStyle = 'black'
-                            u.ctx.lineWidth = 2
-                            u.ctx.beginPath()
-
-                            const ycut = 0.01 * cluster.memoryBandwidth.value
-                            const scalarKnee = (cluster.flopRateScalar.value - ycut) / cluster.memoryBandwidth.value
-                            const simdKnee = (cluster.flopRateSimd.value - ycut) / cluster.memoryBandwidth.value
-                            const scalarKneeX = u.valToPos(scalarKnee, 'x', true), // Value, axis, toCanvasPixels
-                                simdKneeX = u.valToPos(simdKnee, 'x', true),
-                                flopRateScalarY = u.valToPos(cluster.flopRateScalar.value, 'y', true),
-                                flopRateSimdY = u.valToPos(cluster.flopRateSimd.value, 'y', true)
-
-                            if (scalarKneeX < width - padding[1]) { // Top horizontal roofline
-                                u.ctx.moveTo(scalarKneeX, flopRateScalarY)
-                                u.ctx.lineTo(width - padding[1], flopRateScalarY)
-                            }
-
-                            if (simdKneeX < width - padding[1]) { // Lower horitontal roofline
-                                u.ctx.moveTo(simdKneeX, flopRateSimdY)
-                                u.ctx.lineTo(width - padding[1], flopRateSimdY)
-                            }
-
-                            let x1 = u.valToPos(0.01, 'x', true),
-                                y1 = u.valToPos(ycut, 'y', true)
-                                
-                            let x2 = u.valToPos(simdKnee, 'x', true),
-                                y2 = flopRateSimdY
-
-                            let xAxisIntersect = lineIntersect(
-                                x1, y1, x2, y2,
-                                u.valToPos(0.01, 'x', true), u.valToPos(1.0, 'y', true), // X-Axis Start Coords
-                                u.valToPos(1000, 'x', true), u.valToPos(1.0, 'y', true)  // X-Axis End Coords
-                            )
-
-                            if (xAxisIntersect.x > x1) {
-                                x1 = xAxisIntersect.x
-                                y1 = xAxisIntersect.y
-                            }
-
-                            // Diagonal
-                            u.ctx.moveTo(x1, y1)
-                            u.ctx.lineTo(x2, y2)
-
-                            u.ctx.stroke()
-                            // Reset grid lineWidth
-                            u.ctx.lineWidth = 0.15
-                        }
+                    { 
+                        label: 'Performace [GFLOPS]',
+                        values: (u, vals) => vals.map(v => formatNumber(v))
                     }
-                ]
-            },
-        };
+                ],
+                scales: {
+                    x: {
+                        time: false,
+                        range: [0.01, 1000],
+                        distr: 3, // Render as log
+                        log: 10, // log exp
+                    },
+                    y: {
+                        range: [1.0, nearestThousand(cluster.flopRateSimd.value || maxY)],
+                        distr: 3, // Render as log
+                        log: 10, // log exp
+                    },
+                },
+                series: [
+                    {},
+                    { paths: renderTime ? drawColorPoints : drawPoints }
+                ],
+                hooks: {
+                    drawClear: [
+                        u => {
+                            u.series.forEach((s, i) => {
+                                if (i > 0)
+                                    s._paths = null;
+                            });
+                        },
+                    ],
+                    draw: [
+                        u => { // draw roofs when cluster set
+                            // console.log(u)
+                            if (cluster != null) {
+                                const padding = u._padding // [top, right, bottom, left]
 
-        uplot = new uPlot(opts, data, plotWrapper);
+                                u.ctx.strokeStyle = 'black'
+                                u.ctx.lineWidth = 2
+                                u.ctx.beginPath()
+
+                                const ycut = 0.01 * cluster.memoryBandwidth.value
+                                const scalarKnee = (cluster.flopRateScalar.value - ycut) / cluster.memoryBandwidth.value
+                                const simdKnee = (cluster.flopRateSimd.value - ycut) / cluster.memoryBandwidth.value
+                                const scalarKneeX = u.valToPos(scalarKnee, 'x', true), // Value, axis, toCanvasPixels
+                                    simdKneeX = u.valToPos(simdKnee, 'x', true),
+                                    flopRateScalarY = u.valToPos(cluster.flopRateScalar.value, 'y', true),
+                                    flopRateSimdY = u.valToPos(cluster.flopRateSimd.value, 'y', true)
+
+                                if (scalarKneeX < width - padding[1]) { // Top horizontal roofline
+                                    u.ctx.moveTo(scalarKneeX, flopRateScalarY)
+                                    u.ctx.lineTo(width - padding[1], flopRateScalarY)
+                                }
+
+                                if (simdKneeX < width - padding[1]) { // Lower horitontal roofline
+                                    u.ctx.moveTo(simdKneeX, flopRateSimdY)
+                                    u.ctx.lineTo(width - padding[1], flopRateSimdY)
+                                }
+
+                                let x1 = u.valToPos(0.01, 'x', true),
+                                    y1 = u.valToPos(ycut, 'y', true)
+                                    
+                                let x2 = u.valToPos(simdKnee, 'x', true),
+                                    y2 = flopRateSimdY
+
+                                let xAxisIntersect = lineIntersect(
+                                    x1, y1, x2, y2,
+                                    u.valToPos(0.01, 'x', true), u.valToPos(1.0, 'y', true), // X-Axis Start Coords
+                                    u.valToPos(1000, 'x', true), u.valToPos(1.0, 'y', true)  // X-Axis End Coords
+                                )
+
+                                if (xAxisIntersect.x > x1) {
+                                    x1 = xAxisIntersect.x
+                                    y1 = xAxisIntersect.y
+                                }
+
+                                // Diagonal
+                                u.ctx.moveTo(x1, y1)
+                                u.ctx.lineTo(x2, y2)
+
+                                u.ctx.stroke()
+                                // Reset grid lineWidth
+                                u.ctx.lineWidth = 0.15
+                            }
+                        }
+                    ]
+                },
+            };
+            uplot = new uPlot(opts, plotData, plotWrapper);
+        } else {
+            console.log('No data for roofline!')
+        }
     }
 
-    // Copied from Histogram
-    
+    // Svelte and Sizechange
     onMount(() => {
-        render()
+        render(data)
     })
-
     onDestroy(() => {
         if (uplot)
             uplot.destroy()
@@ -272,7 +231,6 @@
         if (timeoutId != null)
             clearTimeout(timeoutId)
     })
-
     function sizeChanged() {
         if (timeoutId != null)
             clearTimeout(timeoutId)
@@ -281,13 +239,10 @@
             timeoutId = null
             if (uplot)
                 uplot.destroy()
-
-            render()
+            render(data)
         }, 200)
     }
-
     $: sizeChanged(width, height)
-
 </script>
 
 {#if data != null}
diff --git a/web/frontend/src/units.js b/web/frontend/src/units.js
index d306e73..9a4defd 100644
--- a/web/frontend/src/units.js
+++ b/web/frontend/src/units.js
@@ -6,8 +6,8 @@ const power  = [1, 1e3, 1e6, 1e9, 1e12, 1e15, 1e18, 1e21]
 const prefix = ['', 'K', 'M', 'G', 'T', 'P', 'E']
 
 export function formatNumber(x) {
-    if ( isNaN(x) ) {
-        return x // Return if String , used in Histograms
+    if ( isNaN(x) || x == null) {
+        return x // Return if String or Null
     } else {
         for (let i = 0; i < prefix.length; i++)
             if (power[i] <= x && x < power[i+1])
diff --git a/web/frontend/src/utils.js b/web/frontend/src/utils.js
index c89afda..da1878a 100644
--- a/web/frontend/src/utils.js
+++ b/web/frontend/src/utils.js
@@ -6,7 +6,7 @@ import {
 } from "@urql/svelte";
 import { setContext, getContext, hasContext, onDestroy, tick } from "svelte";
 import { readable } from "svelte/store";
-import { formatNumber } from './units.js'
+// import { formatNumber } from './units.js'
 
 /*
  * Call this function only at component initialization time!
@@ -326,8 +326,11 @@ export function convert2uplot(canvasData) {
 }
 
 export function binsFromFootprint(weights, scope, values, numBins) {
-    let min = 0, max = 0
+    let min = 0, max = 0 //, median = 0
     if (values.length != 0) {
+        // Extreme, wrong peak vlaues: Filter here or backend?
+        // median = median(values)
+
         for (let x of values) {
             min = Math.min(min, x)
             max = Math.max(max, x)
@@ -364,11 +367,12 @@ export function binsFromFootprint(weights, scope, values, numBins) {
     }
 }
 
-export function transformDataForRoofline(flopsAny, memBw, renderTime) { // Uses Metric Object
+export function transformDataForRoofline(flopsAny, memBw) { // Uses Metric Objects: {series:[{},{},...], timestep:60, name:$NAME}
     const nodes = flopsAny.series.length
     const timesteps = flopsAny.series[0].data.length
 
     /* c will contain values from 0 to 1 representing the time */
+    let data = null
     const x = [], y = [], c = []
 
     if (flopsAny && memBw) {
@@ -383,24 +387,23 @@ export function transformDataForRoofline(flopsAny, memBw, renderTime) { // Uses
 
                 x.push(intensity)
                 y.push(f)
-                c.push(renderTime ? j / timesteps : 0)
+                c.push(j / timesteps)
             }
         }
     } else {
         console.warn("transformData: metrics for 'mem_bw' and/or 'flops_any' missing!")
     }
-
-    return {
-        x, y, c,
-        xLabel: 'Intensity [FLOPS/byte]',
-        yLabel: 'Performance [GFLOPS]'
+    if (x.length > 0 && y.length > 0 && c.length > 0) {
+        data = [null, [x, y], c] // for dataformat see roofline.svelte
     }
+    return data
 }
 
 //  Return something to be plotted. The argument shall be the result of the
 // `nodeMetrics` GraphQL query.
 export function transformPerNodeDataForRoofline(nodes) {
-    const x = [], y = [], c = []
+    let data = null
+    const x = [], y = []
     for (let node of nodes) {
         let flopsAny = node.metrics.find(m => m.name == 'flops_any' && m.scope == 'node')?.metric
         let memBw    = node.metrics.find(m => m.name == 'mem_bw'    && m.scope == 'node')?.metric
@@ -417,12 +420,21 @@ export function transformPerNodeDataForRoofline(nodes) {
 
         x.push(intensity)
         y.push(f)
-        c.push(0)
     }
-
-    return {
-        x, y, c,
-        xLabel: 'Intensity [FLOPS/byte]',
-        yLabel: 'Performance [GFLOPS]'
+    if (x.length > 0 && y.length > 0) {
+        data = [null, [x, y], []] // for dataformat see roofline.svelte
     }
+    return data
 }
+
+// https://stackoverflow.com/questions/45309447/calculating-median-javascript
+// function median(numbers) {
+//     const sorted = Array.from(numbers).sort((a, b) => a - b);
+//     const middle = Math.floor(sorted.length / 2);
+  
+//     if (sorted.length % 2 === 0) {
+//         return (sorted[middle - 1] + sorted[middle]) / 2;
+//     }
+  
+//     return sorted[middle];
+// }