cc-backend/web/frontend/src/generic/utils.js

import { expiringCacheExchange } from "./cache-exchange.js";
import {
    Client,
    setContextClient,
    fetchExchange,
} from "@urql/svelte";
import { setContext, getContext, hasContext, onDestroy, tick } from "svelte";
import { readable } from "svelte/store";

/*
 * Call this function only at component initialization time!
 *
 * It does several things:
 * - Initialize the GraphQL client
 * - Creates a readable store 'initialization' which indicates when the values below can be used.
 * - Adds 'tags' to the context (list of all tags)
 * - Adds 'clusters' to the context (object with cluster names as keys)
 * - Adds 'globalMetrics' to the context (list of globally available metric infos)
 * - Adds 'getMetricConfig' to the context, a function that takes a cluster, subCluster and metric name and returns the MetricConfig (or undefined)
 * - Adds 'getHardwareTopology' to the context, a function that takes a cluster nad subCluster and returns the subCluster topology (or undefined)
 */
export function init(extraInitQuery = "") {
    const jwt = hasContext("jwt")
        ? getContext("jwt")
        : getContext("cc-config")["jwt"];

    const client = new Client({
        url: `${window.location.origin}/query`,
        fetchOptions:
            jwt != null ? { headers: { Authorization: `Bearer ${jwt}` } } : {},
        exchanges: [
            expiringCacheExchange({
                ttl: 5 * 60 * 1000,
                maxSize: 150,
            }),
            fetchExchange,
        ],
    });

    setContextClient(client);

    const query = client
        .query(
            `query {
        clusters {
            name
            partitions
            subClusters {
                name
                nodes
                numberOfNodes
                processorType
                socketsPerNode
                coresPerSocket
                threadsPerCore
                flopRateScalar { unit { base, prefix }, value }
                flopRateSimd { unit { base, prefix }, value }
                memoryBandwidth { unit { base, prefix }, value }
                topology {
                    node
                    socket
                    core
                    accelerators { id }
                }
                metricConfig {
                    name
                    unit { base, prefix }
                    scope
                    aggregation
                    timestep
                    peak
                    normal
                    caution
                    alert
                    lowerIsBetter
                }
                footprint 
            }
        }
        tags { id, name, type, scope }
        globalMetrics {
            name
            scope
            footprint
            unit { base, prefix }
            availability { cluster, subClusters }
        }
        ${extraInitQuery}
    }`
        )
        .toPromise();

    let state = { fetching: true, error: null, data: null };
    let subscribers = [];
    const subscribe = (callback) => {
        callback(state);
        subscribers.push(callback);
        return () => {
            subscribers = subscribers.filter((cb) => cb != callback);
        };
    };

    const tags = []
    const clusters = []
    const globalMetrics = []

    setContext("tags", tags);
    setContext("clusters", clusters);
    setContext("globalMetrics", globalMetrics);
    setContext("getMetricConfig", (cluster, subCluster, metric) => {
        // Load objects if input is string
        if (typeof cluster !== "object")
            cluster = clusters.find((c) => c.name == cluster);
        if (typeof subCluster !== "object")
            subCluster = cluster.subClusters.find((sc) => sc.name == subCluster);

        return subCluster.metricConfig.find((m) => m.name == metric);
    });
    setContext("getHardwareTopology", (cluster, subCluster) => {
        // Load objects if input is string
        if (typeof cluster !== "object")
            cluster = clusters.find((c) => c.name == cluster);
        if (typeof subCluster !== "object")
            subCluster = cluster.subClusters.find((sc) => sc.name == subCluster);

        return subCluster?.topology;
    });
    setContext("on-init", (callback) =>
        state.fetching ? subscribers.push(callback) : callback(state)
    );
    setContext(
        "initialized",
        readable(false, (set) => subscribers.push(() => set(true)))
    );

    query.then(({ error, data }) => {
        state.fetching = false;
        if (error != null) {
            console.error(error);
            state.error = error;
            tick().then(() => subscribers.forEach((cb) => cb(state)));
            return;
        }

        for (let tag of data.tags) tags.push(tag);
        for (let cluster of data.clusters) clusters.push(cluster);
        for (let gm of data.globalMetrics) globalMetrics.push(gm);

        // Unified Sort
        globalMetrics.sort((a, b) => a.name.localeCompare(b.name))

        state.data = data;
        tick().then(() => subscribers.forEach((cb) => cb(state)));
    });

    return {
        query: { subscribe },
        tags,
        clusters,
        globalMetrics
    };
}

// Use https://developer.mozilla.org/en-US/docs/Web/API/structuredClone instead?
export function deepCopy(x) {
    return JSON.parse(JSON.stringify(x));
}

function fuzzyMatch(term, string) {
    return string.toLowerCase().includes(term);
}

// Use in filter() function to return only unique values
export function distinct(value, index, array) {
    return array.indexOf(value) === index;
}

// Load Local Bool and Handle Scrambling of input string
export const scrambleNames = window.localStorage.getItem("cc-scramble-names");
export const scramble = function (str) {
  if (str === "-") return str;
  else
    return [...str]
      .reduce((x, c, i) => x * 7 + c.charCodeAt(0) * i * 21, 5)
      .toString(32)
      .substr(0, 6);
};

export function fuzzySearchTags(term, tags) {
    if (!tags) return [];

    let results = [];
    let termparts = term
        .split(":")
        .map((s) => s.trim())
        .filter((s) => s.length > 0);

    if (termparts.length == 0) {
        results = tags.slice();
    } else if (termparts.length == 1) {
        for (let tag of tags)
            if (
                fuzzyMatch(termparts[0], tag.type) ||
                fuzzyMatch(termparts[0], tag.name)
            )
                results.push(tag);
    } else if (termparts.length == 2) {
        for (let tag of tags)
            if (
                fuzzyMatch(termparts[0], tag.type) &&
                fuzzyMatch(termparts[1], tag.name)
            )
                results.push(tag);
    }

    return results.sort((a, b) => {
        if (a.type < b.type) return -1;
        if (a.type > b.type) return 1;
        if (a.name < b.name) return -1;
        if (a.name > b.name) return 1;
        return 0;
    });
}

export function groupByScope(jobMetrics) {
    let metrics = new Map();
    for (let metric of jobMetrics) {
        if (metrics.has(metric.name)) metrics.get(metric.name).push(metric);
        else metrics.set(metric.name, [metric]);
    }

    return [...metrics.values()].sort((a, b) =>
        a[0].name.localeCompare(b[0].name)
    );
}

const scopeGranularity = {
    node: 10,
    socket: 5,
    memorydomain: 4,
    core: 3,
    hwthread: 2,
    accelerator: 1
};

export function maxScope(scopes) {
    console.assert(
        scopes.length > 0 && scopes.every((x) => scopeGranularity[x] != null)
    );
    let sm = scopes[0],
        gran = scopeGranularity[scopes[0]];
    for (let scope of scopes) {
        let otherGran = scopeGranularity[scope];
        if (otherGran > gran) {
            sm = scope;
            gran = otherGran;
        }
    }
    return sm;
}

export function minScope(scopes) {
    console.assert(
        scopes.length > 0 && scopes.every((x) => scopeGranularity[x] != null)
    );
    let sm = scopes[0],
        gran = scopeGranularity[scopes[0]];
    for (let scope of scopes) {
        let otherGran = scopeGranularity[scope];
        if (otherGran < gran) {
            sm = scope;
            gran = otherGran;
        }
    }
    return sm;
}

export function stickyHeader(datatableHeaderSelector, updatePading) {
    const header = document.querySelector("header > nav.navbar");
    if (!header) return;

    let ticking = false,
        datatableHeader = null;
    const onscroll = (event) => {
        if (ticking) return;

        ticking = true;
        window.requestAnimationFrame(() => {
            ticking = false;
            if (!datatableHeader)
                datatableHeader = document.querySelector(datatableHeaderSelector);

            const top = datatableHeader.getBoundingClientRect().top;
            updatePading(
                top < header.clientHeight ? header.clientHeight - top + 10 : 10
            );
        });
    };

    document.addEventListener("scroll", onscroll);
    onDestroy(() => document.removeEventListener("scroll", onscroll));
}

export function checkMetricDisabled(m, c, s) { // [m]etric, [c]luster, [s]ubcluster
    const metrics = getContext("globalMetrics");
    const result = metrics?.find((gm) => gm.name === m)?.availability?.find((av) => av.cluster === c)?.subClusters?.includes(s)
    return !result
}

export function getStatsItems(presetStats = []) {
    // console.time('stats')
    const globalMetrics = getContext("globalMetrics")
    const result = globalMetrics.map((gm) => {
        if (gm?.footprint) {
            const mc = getMetricConfigDeep(gm.name, null, null)
            if (mc) {
                const presetEntry = presetStats.find((s) => s?.field === (gm.name + '_' + gm.footprint))
                if (presetEntry) {
                    return {
                        field: gm.name + '_' + gm.footprint,
                        text: gm.name + ' (' + gm.footprint + ')',
                        metric: gm.name,
                        from: presetEntry.from,
                        to: presetEntry.to,
                        peak: mc.peak,
                        enabled: true
                    }
                } else {
                    return {
                        field: gm.name + '_' + gm.footprint,
                        text: gm.name + ' (' + gm.footprint + ')',
                        metric: gm.name,
                        from: 0,
                        to: mc.peak,
                        peak: mc.peak,
                        enabled: false
                    }
                }
            }
        }
        return null
    }).filter((r) => r != null)
    // console.timeEnd('stats')
    return [...result];
};

export function getSortItems() {
    //console.time('sort')
    const globalMetrics = getContext("globalMetrics")
    const result = globalMetrics.map((gm) => {
        if (gm?.footprint) {
            return { 
                field: gm.name + '_' + gm.footprint,
                type: 'foot',
                text: gm.name + ' (' + gm.footprint + ')',
                order: 'DESC'
            }
        }
        return null
    }).filter((r) => r != null)
    //console.timeEnd('sort')
    return [...result];
};

function getMetricConfigDeep(metric, cluster, subCluster) {
    const clusters = getContext("clusters");
    if (cluster != null) {
        const c = clusters.find((c) => c.name == cluster);
        if (subCluster != null) {
            const sc = c.subClusters.find((sc) => sc.name == subCluster);
            return sc.metricConfig.find((mc) => mc.name == metric)
        } else {
            let result;
            for (let sc of c.subClusters) {
                const mc = sc.metricConfig.find((mc) => mc.name == metric)
                if (result && mc) { // update result; If lowerIsBetter: Peak is still maximum value, no special case required
                    result.alert = (mc.alert > result.alert) ? mc.alert : result.alert
                    result.caution = (mc.caution > result.caution) ? mc.caution : result.caution
                    result.normal = (mc.normal > result.normal) ? mc.normal : result.normal
                    result.peak = (mc.peak > result.peak) ? mc.peak : result.peak
                } else if (mc) {
                    // start new result
                    result = {...mc};
                }
            }
            return result
        }
    } else {
        let result;
        for (let c of clusters) {
            for (let sc of c.subClusters) {
                const mc = sc.metricConfig.find((mc) => mc.name == metric)
                if (result && mc) { // update result; If lowerIsBetter: Peak is still maximum value, no special case required
                    result.alert = (mc.alert > result.alert) ? mc.alert : result.alert
                    result.caution = (mc.caution > result.caution) ? mc.caution : result.caution
                    result.normal = (mc.normal > result.normal) ? mc.normal : result.normal
                    result.peak = (mc.peak > result.peak) ? mc.peak : result.peak
                } else if (mc) {
                    // Start new result
                    result = {...mc};
                }
            }
        }
        return result
    }
}

export function convert2uplot(canvasData, secondsToMinutes = false, secondsToHours = false) {
    // Prep: Uplot Data Structure
    let uplotData = [[],[]] // [X, Y1, Y2, ...]
    // Iterate if exists
    if (canvasData) {
        canvasData.forEach( cd => {
            if (Object.keys(cd).length == 4) { // MetricHisto Datafromat
                uplotData[0].push(cd?.max ? cd.max : 0)
                uplotData[1].push(cd.count)
            } else { // Default
                if (secondsToHours) {
                    let hours = cd.value / 3600
                    console.log("x seconds to y hours", cd.value, hours)
                    uplotData[0].push(hours)
                } else if (secondsToMinutes) {
                    let minutes = cd.value / 60
                    console.log("x seconds to y minutes", cd.value, minutes)
                    uplotData[0].push(minutes)
                } else {
                    uplotData[0].push(cd.value)
                }
                uplotData[1].push(cd.count)
             }
        })
    }
    return uplotData
}

export function binsFromFootprint(weights, scope, values, numBins) {
    let min = 0, max = 0 //, median = 0
    if (values.length != 0) {
        // Extreme, wrong peak vlaues: Filter here or backend?
        // median = median(values)

        for (let x of values) {
            min = Math.min(min, x)
            max = Math.max(max, x)
        }
        max += 1 // So that we have an exclusive range.
    }

    if (numBins == null || numBins < 3)
        numBins = 3

    let scopeWeights
    switch (scope) {
        case 'core':
            scopeWeights = weights.coreHours
            break
        case 'accelerator':
            scopeWeights = weights.accHours
            break
        default: // every other scope: use 'node'
            scopeWeights = weights.nodeHours
    }

    const rawBins = new Array(numBins).fill(0)
    for (let i = 0; i < values.length; i++)
        rawBins[Math.floor(((values[i] - min) / (max - min)) * numBins)] += scopeWeights ? scopeWeights[i] : 1

    const bins = rawBins.map((count, idx) => ({ 
        value: Math.floor(min + ((idx + 1) / numBins) * (max - min)),
        count: count 
    }))

    return {
        bins: bins
    }
}

export function transformDataForRoofline(flopsAny, memBw) { // Uses Metric Objects: {series:[{},{},...], timestep:60, name:$NAME}
    /* c will contain values from 0 to 1 representing the time */
    let data = null
    const x = [], y = [], c = []

    if (flopsAny && memBw) {
        const nodes = flopsAny.series.length
        const timesteps = flopsAny.series[0].data.length

        for (let i = 0; i < nodes; i++) {
            const flopsData = flopsAny.series[i].data
            const memBwData = memBw.series[i].data
            for (let j = 0; j < timesteps; j++) {
                const f = flopsData[j], m = memBwData[j]
                const intensity = f / m
                if (Number.isNaN(intensity) || !Number.isFinite(intensity))
                    continue

                x.push(intensity)
                y.push(f)
                c.push(j / timesteps)
            }
        }
    } else {
        console.warn("transformData: metrics for 'mem_bw' and/or 'flops_any' missing!")
    }
    if (x.length > 0 && y.length > 0 && c.length > 0) {
        data = [null, [x, y], c] // for dataformat see roofline.svelte
    }
    return data
}

//  Return something to be plotted. The argument shall be the result of the
// `nodeMetrics` GraphQL query.
// Hardcoded metric names required for correct render
export function transformPerNodeDataForRoofline(nodes) {
    let data = null
    const x = [], y = []
    for (let node of nodes) {
        let flopsAny = node.metrics.find(m => m.name == 'flops_any' && m.scope == 'node')?.metric
        let memBw    = node.metrics.find(m => m.name == 'mem_bw'    && m.scope == 'node')?.metric
        if (!flopsAny || !memBw) {
            console.warn("transformPerNodeData: metrics for 'mem_bw' and/or 'flops_any' missing!")
            continue
        }

        let flopsData = flopsAny.series[0].data, memBwData = memBw.series[0].data
        const f = flopsData[flopsData.length - 1], m = memBwData[flopsData.length - 1]
        const intensity = f / m
        if (Number.isNaN(intensity) || !Number.isFinite(intensity))
            continue

        x.push(intensity)
        y.push(f)
    }
    if (x.length > 0 && y.length > 0) {
        data = [null, [x, y], []] // for dataformat see roofline.svelte
    }
    return data
}

export async function fetchJwt(username) {
    const raw = await fetch(`/frontend/jwt/?username=${username}`);

    if (!raw.ok) {
        const message = `An error has occured: ${response.status}`;
        throw new Error(message);
    }

    const res = await raw.text();
    return res;
}

// https://stackoverflow.com/questions/45309447/calculating-median-javascript
// function median(numbers) {
//     const sorted = Array.from(numbers).sort((a, b) => a - b);
//     const middle = Math.floor(sorted.length / 2);
  
//     if (sorted.length % 2 === 0) {
//         return (sorted[middle - 1] + sorted[middle]) / 2;
//     }
  
//     return sorted[middle];
// }