Merge branch 'dev' into ai-review

This commit is contained in:
2025-11-20 08:59:52 +01:00
37 changed files with 1153 additions and 924 deletions

View File

@@ -76,7 +76,7 @@
/* State Init */
let filterComponent = $state(); // see why here: https://stackoverflow.com/questions/58287729/how-can-i-export-a-function-from-a-svelte-component-that-changes-a-value-in-the
let cluster = $state(filterPresets?.cluster);
let cluster = $state({});
let rooflineMaxY = $state(0);
let maxY = $state(-1);
let colWidth1 = $state(0);
@@ -370,7 +370,7 @@
</Col>
</Row>
{:else if $statsQuery.data}
<Row cols={3} class="mb-4">
<Row cols={3} style="margin-bottom: 2rem;">
<Col>
<Table>
<tr>
@@ -419,7 +419,7 @@
{:else}
<Pie
canvasId={`pie-${groupSelection.key}`}
size={colWidth1}
size={colWidth1 / 1.9}
sliceLabel={sortSelection.label}
quantities={$topQuery.data.topList.map(
(t) => t[sortSelection.key],
@@ -490,13 +490,21 @@
{:else if $rooflineQuery.error}
<Card body color="danger">{$rooflineQuery.error.message}</Card>
{:else if $rooflineQuery.data && cluster}
<div class="d-flex justify-content-center align-baseline">
<h5>Job Roofline Heatmap</h5>
<Icon
style="cursor:help; margin-left:0.5rem;"
name="info-circle"
title="Most Populated Areas By Selected Jobs' Average Values"
/>
</div>
<div bind:clientWidth={colWidth2}>
{#key $rooflineQuery.data}
<RooflineHeatmap
width={colWidth2}
height={300}
height={280}
tiles={$rooflineQuery.data.rooflineHeatmap}
subCluster={cluster.subClusters.length == 1
subCluster={cluster.subClusters.length >= 1
? cluster.subClusters[0]
: null}
maxY={rooflineMaxY}

View File

@@ -79,6 +79,11 @@
});
});
$effect(() => {
// Load Metric-Selection for last selected cluster
metrics = selectedCluster ? ccconfig[`metricConfig_jobListMetrics:${selectedCluster}`] : ccconfig.metricConfig_jobListMetrics
});
/* On Mount */
// The filterPresets are handled by the Filters component,
// so we need to wait for it to be ready before we can start a query.

View File

@@ -16,6 +16,7 @@
Row,
Col,
Input,
Button,
InputGroup,
InputGroupText,
Icon,
@@ -63,6 +64,7 @@
query ($cluster: String!, $nodes: [String!], $from: Time!, $to: Time!) {
nodeMetrics(cluster: $cluster, nodes: $nodes, from: $from, to: $to) {
host
state
subCluster
metrics {
name
@@ -97,6 +99,16 @@
}
}
`;
// Node State Colors
const stateColors = {
allocated: 'success',
reserved: 'info',
idle: 'primary',
mixed: 'warning',
down: 'danger',
unknown: 'dark',
notindb: 'secondary'
}
/* State Init */
let from = $state(presetFrom ? presetFrom : new Date(nowEpoch - (4 * 3600 * 1000)));
@@ -123,6 +135,8 @@
})
);
const thisNodeState = $derived($nodeMetricsData?.data?.nodeMetrics[0]?.state ? $nodeMetricsData.data.nodeMetrics[0].state : 'notindb');
/* Effect */
$effect(() => {
loadUnits($initialized);
@@ -138,7 +152,7 @@
}
</script>
<Row cols={{ xs: 2, lg: 4 }}>
<Row cols={{ xs: 2, lg: 5 }}>
{#if $initq.error}
<Card body color="danger">{$initq.error.message}</Card>
{:else if $initq.fetching}
@@ -149,19 +163,18 @@
<InputGroup>
<InputGroupText><Icon name="hdd" /></InputGroupText>
<InputGroupText>Selected Node</InputGroupText>
<Input style="background-color: white;"type="text" value="{hostname} [{cluster} ({$nodeMetricsData?.data ? $nodeMetricsData.data.nodeMetrics[0].subCluster : ''})]" disabled/>
<Input style="background-color: white;" type="text" value="{hostname} [{cluster} {$nodeMetricsData?.data ? `(${$nodeMetricsData.data.nodeMetrics[0].subCluster})` : ''}]" disabled/>
</InputGroup>
</Col>
<!-- Time Col -->
<!-- State Col -->
<Col>
<TimeSelection
presetFrom={from}
presetTo={to}
applyTime={(newFrom, newTo) => {
from = newFrom;
to = newTo;
}}
/>
<InputGroup>
<InputGroupText><Icon name="clipboard2-pulse" /></InputGroupText>
<InputGroupText>Node State</InputGroupText>
<Button class="flex-grow-1 text-center" color={stateColors[thisNodeState]} disabled>
{thisNodeState}
</Button>
</InputGroup>
</Col>
<!-- Concurrent Col -->
<Col class="mt-2 mt-lg-0">
@@ -184,6 +197,17 @@
</InputGroup>
{/if}
</Col>
<!-- Time Col -->
<Col>
<TimeSelection
presetFrom={from}
presetTo={to}
applyTime={(newFrom, newTo) => {
from = newFrom;
to = newTo;
}}
/>
</Col>
<!-- Refresh Col-->
<Col class="mt-2 mt-lg-0">
<Refresher
@@ -217,6 +241,7 @@
cluster={clusters.find((c) => c.name == cluster)}
subCluster={$nodeMetricsData.data.nodeMetrics[0].subCluster}
series={item.metric.series}
enableFlip
forNode
/>
{:else if item.disabled === true && item.metric}

View File

@@ -9,13 +9,17 @@
import {
getContext
} from "svelte"
import {
init,
} from "./generic/utils.js";
import {
Row,
Col,
Card,
CardBody,
TabContent,
TabPane
TabPane,
Spinner
} from "@sveltestrap/sveltestrap";
import StatusDash from "./status/StatusDash.svelte";
@@ -28,8 +32,8 @@
} = $props();
/*Const Init */
const { query: initq } = init();
const useCbColors = getContext("cc-config")?.plotConfiguration_colorblindMode || false
</script>
<!-- Loading indicator & Refresh -->
@@ -40,24 +44,39 @@
</Col>
</Row>
<Card class="overflow-auto" style="height: auto;">
<TabContent>
<TabPane tabId="status-dash" tab="Status" active>
<CardBody>
<StatusDash {presetCluster} {useCbColors} useAltColors></StatusDash>
</CardBody>
</TabPane>
<TabPane tabId="usage-dash" tab="Usage">
<CardBody>
<UsageDash {presetCluster} {useCbColors}></UsageDash>
</CardBody>
</TabPane>
<TabPane tabId="metric-dash" tab="Statistics">
<CardBody>
<StatisticsDash {presetCluster} {useCbColors}></StatisticsDash>
</CardBody>
</TabPane>
</TabContent>
</Card>
{#if $initq.fetching}
<Row cols={1} class="text-center mt-3">
<Col>
<Spinner />
</Col>
</Row>
{:else if $initq.error}
<Row cols={1} class="text-center mt-3">
<Col>
<Card body color="danger">{$initq.error.message}</Card>
</Col>
</Row>
{:else}
<Card class="overflow-auto" style="height: auto;">
<TabContent>
<TabPane tabId="status-dash" tab="Status" active>
<CardBody>
<StatusDash clusters={$initq.data.clusters} {presetCluster} {useCbColors} useAltColors></StatusDash>
</CardBody>
</TabPane>
<TabPane tabId="usage-dash" tab="Usage">
<CardBody>
<UsageDash {presetCluster} {useCbColors}></UsageDash>
</CardBody>
</TabPane>
<TabPane tabId="metric-dash" tab="Statistics">
<CardBody>
<StatisticsDash {presetCluster} {useCbColors}></StatisticsDash>
</CardBody>
</TabPane>
</TabContent>
</Card>
{/if}

View File

@@ -59,6 +59,7 @@
const resampleResolutions = resampleConfig ? [...resampleConfig.resolutions] : [];
const resampleDefault = resampleConfig ? Math.max(...resampleConfig.resolutions) : 0;
const stateOptions = ['all', 'allocated', 'idle', 'reserved', 'mixed', 'down', 'unknown', 'notindb'];
const nowDate = new Date(Date.now());
/* Var Init */
@@ -69,6 +70,7 @@
let from = $state(presetFrom || new Date(nowDate.setHours(nowDate.getHours() - 4)));
let selectedResolution = $state(resampleConfig ? resampleDefault : 0);
let hostnameFilter = $state("");
let hoststateFilter = $state("all");
let pendingHostnameFilter = $state("");
let isMetricsSelectionOpen = $state(false);
@@ -154,7 +156,7 @@
</script>
<!-- ROW1: Tools-->
<Row cols={{ xs: 2, lg: !displayNodeOverview ? (resampleConfig ? 5 : 4) : 4 }} class="mb-3">
<Row cols={{ xs: 2, lg: !displayNodeOverview ? (resampleConfig ? 6 : 5) : 5 }} class="mb-3">
{#if $initq.data}
<!-- List Metric Select Col-->
{#if !displayNodeOverview}
@@ -191,7 +193,7 @@
<Col class="mt-2 mt-lg-0">
<InputGroup>
<InputGroupText><Icon name="hdd" /></InputGroupText>
<InputGroupText>Find Node(s)</InputGroupText>
<InputGroupText>Node(s)</InputGroupText>
<Input
placeholder="Filter hostname ..."
type="text"
@@ -200,6 +202,18 @@
/>
</InputGroup>
</Col>
<!-- State Col-->
<Col class="mt-2 mt-lg-0">
<InputGroup>
<InputGroupText><Icon name="clipboard2-pulse" /></InputGroupText>
<InputGroupText>State</InputGroupText>
<Input type="select" bind:value={hoststateFilter}>
{#each stateOptions as so}
<option value={so}>{so.charAt(0).toUpperCase() + so.slice(1)}</option>
{/each}
</Input>
</InputGroup>
</Col>
<!-- Range Col-->
<Col>
<TimeSelection
@@ -252,10 +266,10 @@
{:else}
{#if displayNodeOverview}
<!-- ROW2-1: Node Overview (Grid Included)-->
<NodeOverview {cluster} {ccconfig} {selectedMetric} {from} {to} {hostnameFilter}/>
<NodeOverview {cluster} {ccconfig} {selectedMetric} {from} {to} {hostnameFilter} {hoststateFilter}/>
{:else}
<!-- ROW2-2: Node List (Grid Included)-->
<NodeList {cluster} {subCluster} {ccconfig} {selectedMetrics} {selectedResolution} {hostnameFilter} {from} {to} {presetSystemUnits}/>
<NodeList {cluster} {subCluster} {ccconfig} {selectedMetrics} {selectedResolution} {hostnameFilter} {hoststateFilter} {from} {to} {presetSystemUnits}/>
{/if}
{/if}

View File

@@ -133,6 +133,11 @@
};
});
$effect(() => {
// Load Metric-Selection for last selected cluster
metrics = selectedCluster ? ccconfig[`metricConfig_jobListMetrics:${selectedCluster}`] : ccconfig.metricConfig_jobListMetrics
});
/* On Mount */
onMount(() => {
filterComponent.updateFilters();
@@ -348,6 +353,7 @@
ylabel="Number of Jobs"
yunit="Jobs"
usesBins
enableFlip
/>
{/snippet}

View File

@@ -27,7 +27,7 @@
function refreshIntervalChanged() {
if (refreshIntervalId != null) clearInterval(refreshIntervalId);
if (refreshInterval == null) return;
refreshIntervalId = setInterval(() => onRefresh(), refreshInterval);
refreshIntervalId = setInterval(() => onRefresh(refreshInterval), refreshInterval);
}
/* Svelte 5 onMount */
@@ -51,7 +51,7 @@
</Input>
<Button
outline
onclick={() => onRefresh()}
onclick={() => onRefresh(refreshInterval)}
disabled={refreshInterval != null}
>
<Icon name="arrow-clockwise" /> Refresh

View File

@@ -39,7 +39,7 @@
yunit = "",
title = "",
forResources = false,
plotSync,
plotSync = null,
} = $props();
/* Const Init */
@@ -204,11 +204,7 @@
live: true,
},
cursor: {
drag: { x: true, y: true },
sync: {
key: plotSync.key,
scales: ["x", null],
}
drag: { x: true, y: true }
}
};
@@ -275,9 +271,12 @@
function update(u) {
const { left, top } = u.cursor;
const width = u?.over?.querySelector(".u-legend")?.offsetWidth ? u.over.querySelector(".u-legend").offsetWidth : 0;
legendEl.style.transform =
"translate(" + (left - width - 15) + "px, " + (top + 15) + "px)";
const internalWidth = u?.over?.querySelector(".u-legend")?.offsetWidth ? u.over.querySelector(".u-legend").offsetWidth : 0;
if (left < (width/2)) {
legendEl.style.transform = "translate(" + (left + 15) + "px, " + (top + 15) + "px)";
} else {
legendEl.style.transform = "translate(" + (left - internalWidth - 15) + "px, " + (top + 15) + "px)";
}
}
return {
@@ -293,6 +292,14 @@
if (!uplot) {
opts.width = ren_width;
opts.height = ren_height;
if (plotSync) {
opts.cursor.sync = {
key: plotSync.key,
scales: ["x", null],
}
}
uplot = new uPlot(opts, data, plotWrapper); // Data is uplot formatted [[X][Ymin][Yavg][Ymax]]
plotSync.sub(uplot)
} else {

View File

@@ -34,6 +34,7 @@
xtime = false,
ylabel = "",
yunit = "",
enableFlip = false,
} = $props();
/* Const Init */
@@ -117,8 +118,12 @@
function update(u) {
const { left, top } = u.cursor;
legendEl.style.transform =
"translate(" + (left + 15) + "px, " + (top + 15) + "px)";
const internalWidth = u?.over?.querySelector(".u-legend")?.offsetWidth ? u.over.querySelector(".u-legend").offsetWidth : 0;
if (enableFlip && (left < (width/2))) {
legendEl.style.transform = "translate(" + (left + 15) + "px, " + (top + 15) + "px)";
} else {
legendEl.style.transform = "translate(" + (left - internalWidth - 15) + "px, " + (top + 15) + "px)";
}
}
return {

View File

@@ -13,7 +13,7 @@
- `statisticsSeries [GraphQL.StatisticsSeries]?`: Min/Max/Median representation of metric data [Default: null]
- `cluster String?`: Cluster name of the parent job / data [Default: ""]
- `subCluster String`: Name of the subCluster of the parent job
- `isShared Bool?`: If this job used shared resources; will adapt threshold indicators accordingly [Default: false]
- `isShared Bool?`: If this job used shared resources; for additional legend display [Default: false]
- `forNode Bool?`: If this plot is used for node data display; will render x-axis as negative time with $now as maximum [Default: false]
- `numhwthreads Number?`: Number of job HWThreads [Default: 0]
- `numaccs Number?`: Number of job Accelerators [Default: 0]
@@ -48,6 +48,8 @@
zoomState = null,
thresholdState = null,
extendedLegendData = null,
plotSync = null,
enableFlip = false,
onZoom
} = $props();
@@ -83,7 +85,6 @@
subClusterTopology,
metricConfig,
scope,
isShared,
numhwthreads,
numaccs
));
@@ -277,7 +278,6 @@
subClusterTopology,
metricConfig,
scope,
isShared,
numhwthreads,
numaccs
) {
@@ -293,32 +293,35 @@
scope = statParts[0]
}
if (
(scope == "node" && isShared == false) ||
metricConfig?.aggregation == "avg"
) {
return {
normal: metricConfig.normal,
caution: metricConfig.caution,
alert: metricConfig.alert,
peak: metricConfig.peak,
};
if (metricConfig?.aggregation == "avg") {
// Return as Configured
return {
normal: metricConfig.normal,
caution: metricConfig.caution,
alert: metricConfig.alert,
peak: metricConfig.peak,
};
}
if (metricConfig?.aggregation == "sum") {
// Scale Thresholds
let fraction;
if (numaccs > 0) fraction = subClusterTopology.accelerators.length / numaccs;
else if (numhwthreads > 0) fraction = subClusterTopology.core.length / numhwthreads;
else fraction = 1; // Fallback
let divisor;
if (isShared == true) { // Shared
if (numaccs > 0) divisor = subClusterTopology.accelerators.length / numaccs;
else if (numhwthreads > 0) divisor = subClusterTopology.core.length / numhwthreads;
}
else if (scope == 'node') divisor = 1; // Use as configured for nodes
else if (scope == 'socket') divisor = subClusterTopology.socket.length;
else if (scope == "memoryDomain") divisor = subClusterTopology.memoryDomain.length;
// Exclusive: Fraction = 1; Shared: Fraction > 1
if (scope == 'node') divisor = fraction;
// Cap divisor at number of available sockets or domains
else if (scope == 'socket') divisor = (fraction < subClusterTopology.socket.length) ? subClusterTopology.socket.length : fraction;
else if (scope == "memoryDomain") divisor = (fraction < subClusterTopology.memoryDomain.length) ? subClusterTopology.socket.length : fraction;
// Use Maximum Division for Smallest Scopes
else if (scope == "core") divisor = subClusterTopology.core.length;
else if (scope == "hwthread") divisor = subClusterTopology.core.length; // alt. name for core
else if (scope == "accelerator") divisor = subClusterTopology.accelerators.length;
else {
console.log('Unknown scope, return default aggregation thresholds ', scope)
console.log('Unknown scope, return default aggregation thresholds for sum', scope)
divisor = 1;
}
@@ -395,9 +398,12 @@
function update(u) {
const { left, top } = u.cursor;
const width = u?.over?.querySelector(".u-legend")?.offsetWidth ? u.over.querySelector(".u-legend").offsetWidth : 0;
legendEl.style.transform =
"translate(" + (left - width - 15) + "px, " + (top + 15) + "px)";
const internalWidth = u?.over?.querySelector(".u-legend")?.offsetWidth ? u.over.querySelector(".u-legend").offsetWidth : 0;
if (enableFlip && (left < (width/2))) {
legendEl.style.transform = "translate(" + (left + 15) + "px, " + (top + 15) + "px)";
} else {
legendEl.style.transform = "translate(" + (left - internalWidth - 15) + "px, " + (top + 15) + "px)";
}
}
if (dataSize <= 12 || useStatsSeries) {
@@ -576,6 +582,14 @@
if (!uplot) {
opts.width = ren_width;
opts.height = ren_height;
if (plotSync) {
opts.cursor.sync = {
key: plotSync.key,
scales: ["x", null],
}
}
if (zoomState && metricConfig?.aggregation == "avg") {
opts.scales = {...zoomState}
} else if (zoomState && metricConfig?.aggregation == "sum") {
@@ -584,6 +598,7 @@
if ((thresholdState === thresholds?.normal)) { opts.scales = {...zoomState} };
} // else: reset scaling to default
}
uplot = new uPlot(opts, plotData, plotWrapper);
} else {
uplot.setSize({ width: ren_width, height: ren_height });

View File

@@ -276,13 +276,13 @@
// Nodes: Color based on Idle vs. Allocated
} else if (nodesData) {
// console.log('In Plot Handler NodesData', nodesData)
if (nodesData[i]?.nodeState == "idle") {
if (nodesData[i]?.schedulerState == "idle") {
//u.ctx.strokeStyle = "rgb(0, 0, 255)";
u.ctx.fillStyle = "rgba(0, 0, 255, 0.5)";
} else if (nodesData[i]?.nodeState == "allocated") {
} else if (nodesData[i]?.schedulerState == "allocated") {
//u.ctx.strokeStyle = "rgb(0, 255, 0)";
u.ctx.fillStyle = "rgba(0, 255, 0, 0.5)";
} else if (nodesData[i]?.nodeState == "notindb") {
} else if (nodesData[i]?.schedulerState == "notindb") {
//u.ctx.strokeStyle = "rgb(0, 0, 0)";
u.ctx.fillStyle = "rgba(0, 0, 0, 0.5)";
} else { // Fallback: All other DEFINED states
@@ -436,11 +436,11 @@
tooltip.style.borderColor = getRGB(u.data[2][i]);
// Nodes: Color based on Idle vs. Allocated
} else if (nodesData) {
if (nodesData[i]?.nodeState == "idle") {
if (nodesData[i]?.schedulerState == "idle") {
tooltip.style.borderColor = "rgb(0, 0, 255)";
} else if (nodesData[i]?.nodeState == "allocated") {
} else if (nodesData[i]?.schedulerState == "allocated") {
tooltip.style.borderColor = "rgb(0, 255, 0)";
} else if (nodesData[i]?.nodeState == "notindb") { // Missing from DB table
} else if (nodesData[i]?.schedulerState == "notindb") { // Missing from DB table
tooltip.style.borderColor = "rgb(0, 0, 0)";
} else { // Fallback: All other DEFINED states
tooltip.style.borderColor = "rgb(255, 0, 0)";
@@ -459,7 +459,7 @@
} else if (nodesData && useColors) {
tooltip.textContent = (
// Tooltip Content as String for Node
`Host: ${getLegendData(u, i).nodeName}\nState: ${getLegendData(u, i).nodeState}\nJobs: ${getLegendData(u, i).numJobs}`
`Host: ${getLegendData(u, i).nodeName}\nState: ${getLegendData(u, i).schedulerState}\nJobs: ${getLegendData(u, i).numJobs}`
);
} else if (nodesData && !useColors) {
tooltip.textContent = (

View File

@@ -34,7 +34,7 @@
const paddingLeft = 40;
const paddingRight = 10;
const paddingTop = 10;
const paddingBottom = 5;
const paddingBottom = 40;
/* Var Init */
let timeoutId = null;
@@ -124,7 +124,7 @@
if (data.xLabel) {
ctx.font = `${labelFontSize}px ${fontFamily}`
let textWidth = ctx.measureText(data.xLabel).width
ctx.fillText(data.xLabel, Math.floor((width / 2) - (textWidth / 2)), height - 20)
ctx.fillText(data.xLabel, Math.floor((width / 2) - (textWidth / 2)), height - paddingBottom + 30)
}
ctx.textAlign = 'center'

View File

@@ -1,34 +1,25 @@
<!--
@component Node State/Health Data Stacked Plot Component, based on uPlot; states by timestamp
Only width/height should change reactively.
Properties:
- `metric String?`: The metric name [Default: ""]
- `width Number?`: The plot width [Default: 0]
- `height Number?`: The plot height [Default: 300]
- `data [Array]`: The data object [Default: null]
- `title String?`: Plot title [Default: ""]
- `xlabel String?`: Plot X axis label [Default: ""]
- `ylabel String?`: Plot Y axis label [Default: ""]
- `yunit String?`: Plot Y axis unit [Default: ""]
- `xticks Array`: Array containing jobIDs [Default: []]
- `xinfo Array`: Array containing job information [Default: []]
- `forResources Bool?`: Render this plot for allocated jobResources [Default: false]
- `plot Sync Object!`: uPlot cursor synchronization key
- `title String?`: Plot title [Default: ""]
- `stateType String?`: Which states to render, affects plot render config [Options: Health, Node; Default: ""]
-->
<script>
import uPlot from "uplot";
import { roundTwoDigits, formatDurationTime, formatUnixTime, formatNumber } from "../units.js";
import { formatUnixTime } from "../units.js";
import { getContext, onMount, onDestroy } from "svelte";
import { Card } from "@sveltestrap/sveltestrap";
// NOTE: Metric Thresholds non-required, Cluster Mixing Allowed
/* Svelte 5 Props */
let {
cluster = "",
width = 0,
height = 300,
data = null,
@@ -36,16 +27,86 @@
ylabel = "",
yunit = "",
title = "",
stateType = "" // Health, Slurm, Both
stateType = "" // Health, Node
} = $props();
/* Const Init */
const clusterCockpitConfig = getContext("cc-config");
const lineWidth = clusterCockpitConfig?.plotConfiguration_lineWidth / window.devicePixelRatio || 2;
const cbmode = clusterCockpitConfig?.plotConfiguration_colorblindMode || false;
const seriesConfig = {
full: {
label: "Full",
scale: "y",
width: lineWidth,
fill: cbmode ? "rgba(0, 110, 0, 0.4)" : "rgba(0, 128, 0, 0.4)",
stroke: cbmode ? "rgb(0, 110, 0)" : "green",
},
partial: {
label: "Partial",
scale: "y",
width: lineWidth,
fill: cbmode ? "rgba(235, 172, 35, 0.4)" : "rgba(255, 215, 0, 0.4)",
stroke: cbmode ? "rgb(235, 172, 35)" : "gold",
},
failed: {
label: "Failed",
scale: "y",
width: lineWidth,
fill: cbmode ? "rgb(181, 29, 20, 0.4)" : "rgba(255, 0, 0, 0.4)",
stroke: cbmode ? "rgb(181, 29, 20)" : "red",
},
idle: {
label: "Idle",
scale: "y",
width: lineWidth,
fill: cbmode ? "rgba(0, 140, 249, 0.4)" : "rgba(0, 0, 255, 0.4)",
stroke: cbmode ? "rgb(0, 140, 249)" : "blue",
},
allocated: {
label: "Allocated",
scale: "y",
width: lineWidth,
fill: cbmode ? "rgba(0, 110, 0, 0.4)" : "rgba(0, 128, 0, 0.4)",
stroke: cbmode ? "rgb(0, 110, 0)" : "green",
},
reserved: {
label: "Reserved",
scale: "y",
width: lineWidth,
fill: cbmode ? "rgba(209, 99, 230, 0.4)" : "rgba(255, 0, 255, 0.4)",
stroke: cbmode ? "rgb(209, 99, 230)" : "magenta",
},
mixed: {
label: "Mixed",
scale: "y",
width: lineWidth,
fill: cbmode ? "rgba(235, 172, 35, 0.4)" : "rgba(255, 215, 0, 0.4)",
stroke: cbmode ? "rgb(235, 172, 35)" : "gold",
},
down: {
label: "Down",
scale: "y",
width: lineWidth,
fill: cbmode ? "rgba(181, 29 ,20, 0.4)" : "rgba(255, 0, 0, 0.4)",
stroke: cbmode ? "rgb(181, 29, 20)" : "red",
},
unknown: {
label: "Unknown",
scale: "y",
width: lineWidth,
fill: "rgba(0, 0, 0, 0.4)",
stroke: "black",
}
};
// Data Prep For uPlot
const sortedData = data.sort((a, b) => a.state.localeCompare(b.state));
const collectLabel = sortedData.map(d => d.state);
// Align Data to Timesteps, Introduces 'undefied' as placeholder, reiterate and set those to 0
const collectData = uPlot.join(sortedData.map(d => [d.times, d.counts])).map(d => d.map(i => i ? i : 0));
// STACKED CHART FUNCTIONS //
function stack(data, omit) {
let data2 = [];
let bands = [];
@@ -74,23 +135,46 @@
};
}
function getOpts(title, series) {
return {
scales: {
x: {
time: false,
function getStackedOpts(title, width, height, series, data) {
let opts = {
width,
height,
title,
plugins: [legendAsTooltipPlugin()],
series,
axes: [
{
scale: "x",
space: 25, // Tick Spacing
rotate: 30,
show: true,
label: xlabel,
values(self, splits) {
return splits.map(s => formatUnixTime(s));
}
},
{
scale: "y",
grid: { show: true },
labelFont: "sans-serif",
label: ylabel + (yunit ? ` (${yunit})` : ''),
// values: (u, vals) => vals.map((v) => formatNumber(v)),
},
],
padding: [5, 10, 0, 0],
scales: {
x: { time: false },
y: { auto: true, distr: 1 },
},
series
legend: {
show: true,
},
cursor: {
drag: { x: true, y: true },
}
};
}
function getStackedOpts(title, series, data, interp) {
let opts = getOpts(title, series);
let interped = interp ? interp(data) : data;
let stacked = stack(interped, i => false);
let stacked = stack(data, i => false);
opts.bands = stacked.bands;
opts.cursor = opts.cursor || {};
@@ -99,7 +183,8 @@
};
opts.series.forEach(s => {
s.value = (u, v, si, i) => data[si][i];
// Format Time Info from Unix TS to LocalTimeString
s.value = (u, v, si, i) => (si === 0) ? formatUnixTime(data[si][i]) : data[si][i];
s.points = s.points || {};
@@ -138,331 +223,7 @@
return {opts, data: stacked.data};
}
function stack2(series) {
// for uplot data
let data = Array(series.length);
let bands = [];
let dataLen = series[0].values.length;
let zeroArr = Array(dataLen).fill(0);
let stackGroups = new Map();
let seriesStackKeys = Array(series.length);
series.forEach((s, si) => {
let vals = s.values.slice();
// apply negY
if (s.negY) {
for (let i = 0; i < vals.length; i++) {
if (vals[i] != null)
vals[i] *= -1;
}
}
if (s.stacking.mode != 'none') {
let hasPos = vals.some(v => v > 0);
// derive stacking key
let stackKey = seriesStackKeys[si] = s.stacking.mode + s.scaleKey + s.stacking.group + (hasPos ? '+' : '-');
let group = stackGroups.get(stackKey);
// initialize stacking group
if (group == null) {
group = {
series: [],
acc: zeroArr.slice(),
dir: hasPos ? -1 : 1,
};
stackGroups.set(stackKey, group);
}
// push for bands gen
group.series.unshift(si);
let stacked = data[si] = Array(dataLen);
let { acc } = group;
for (let i = 0; i < dataLen; i++) {
let v = vals[i];
if (v != null)
stacked[i] = (acc[i] += v);
else
stacked[i] = v; // we may want to coerce to 0 here
}
}
else
data[si] = vals;
});
// re-compute by percent
series.forEach((s, si) => {
if (s.stacking.mode == 'percent') {
let group = stackGroups.get(seriesStackKeys[si]);
let { acc } = group;
// re-negatify percent
let sign = group.dir * -1;
let stacked = data[si];
for (let i = 0; i < dataLen; i++) {
let v = stacked[i];
if (v != null)
stacked[i] = sign * (v / acc[i]);
}
}
});
// generate bands between adjacent group series
stackGroups.forEach(group => {
let { series, dir } = group;
let lastIdx = series.length - 1;
series.forEach((si, i) => {
if (i != lastIdx) {
let nextIdx = series[i + 1];
bands.push({
// since we're not passing x series[0] for stacking, real idxs are actually +1
series: [si + 1, nextIdx + 1],
dir,
});
}
});
});
return {
data,
bands,
};
}
// UPLOT SERIES INIT //
const plotSeries = [
{
label: "Time",
scale: "x",
value: (u, ts, sidx, didx) =>
(didx == null) ? null : formatUnixTime(ts),
}
]
if (stateType === "slurm") {
const resSeries = [
{
label: "Idle",
scale: "y",
width: lineWidth,
stroke: cbmode ? "rgb(136, 204, 238)" : "lightblue",
},
{
label: "Allocated",
scale: "y",
width: lineWidth,
stroke: cbmode ? "rgb(30, 136, 229)" : "green",
},
{
label: "Reserved",
scale: "y",
width: lineWidth,
stroke: cbmode ? "rgb(211, 95, 183)" : "magenta",
},
{
label: "Mixed",
scale: "y",
width: lineWidth,
stroke: cbmode ? "rgb(239, 230, 69)" : "yellow",
},
{
label: "Down",
scale: "y",
width: lineWidth,
stroke: cbmode ? "rgb(225, 86, 44)" : "red",
},
{
label: "Unknown",
scale: "y",
width: lineWidth,
stroke: "black",
}
];
plotSeries.push(...resSeries)
} else if (stateType === "health") {
const resSeries = [
{
label: "Full",
scale: "y",
width: lineWidth,
stroke: cbmode ? "rgb(30, 136, 229)" : "green",
},
{
label: "Partial",
scale: "y",
width: lineWidth,
stroke: cbmode ? "rgb(239, 230, 69)" : "yellow",
},
{
label: "Failed",
scale: "y",
width: lineWidth,
stroke: cbmode ? "rgb(225, 86, 44)" : "red",
}
];
plotSeries.push(...resSeries)
} else {
const resSeries = [
{
label: "Full",
scale: "y",
width: lineWidth,
stroke: cbmode ? "rgb(30, 136, 229)" : "green",
},
{
label: "Partial",
scale: "y",
width: lineWidth,
stroke: cbmode ? "rgb(239, 230, 69)" : "yellow",
},
{
label: "Failed",
scale: "y",
width: lineWidth,
stroke: cbmode ? "rgb(225, 86, 44)" : "red",
},
{
label: "Idle",
scale: "y",
width: lineWidth,
stroke: cbmode ? "rgb(136, 204, 238)" : "lightblue",
},
{
label: "Allocated",
scale: "y",
width: lineWidth,
stroke: cbmode ? "rgb(30, 136, 229)" : "green",
},
{
label: "Reserved",
scale: "y",
width: lineWidth,
stroke: cbmode ? "rgb(211, 95, 183)" : "magenta",
},
{
label: "Mixed",
scale: "y",
width: lineWidth,
stroke: cbmode ? "rgb(239, 230, 69)" : "yellow",
},
{
label: "Down",
scale: "y",
width: lineWidth,
stroke: cbmode ? "rgb(225, 86, 44)" : "red",
},
{
label: "Unknown",
scale: "y",
width: lineWidth,
stroke: "black",
}
];
plotSeries.push(...resSeries)
}
// UPLOT BAND COLORS //
// const plotBands = [
// { series: [5, 4], fill: cbmode ? "rgba(0,0,255,0.1)" : "rgba(0,255,0,0.1)" },
// { series: [4, 3], fill: cbmode ? "rgba(0,255,0,0.1)" : "rgba(255,0,0,0.1)" },
// ];
// UPLOT OPTIONS //
const opts = {
width,
height,
title,
plugins: [legendAsTooltipPlugin()],
series: plotSeries,
axes: [
{
scale: "x",
space: 25, // Tick Spacing
rotate: 30,
show: true,
label: xlabel,
// values(self, splits) {
// return splits.map(s => xticks[s]);
// }
},
{
scale: "y",
grid: { show: true },
labelFont: "sans-serif",
label: ylabel + (yunit ? ` (${yunit})` : ''),
// values: (u, vals) => vals.map((v) => formatNumber(v)),
},
],
// bands: forResources ? [] : plotBands,
padding: [5, 10, 0, 0],
// hooks: {
// draw: [
// (u) => {
// // Draw plot type label:
// let textl = forResources ? "Job Resources by Type" : "Metric Min/Avg/Max for Job Duration";
// let textr = "Earlier <- StartTime -> Later";
// u.ctx.save();
// u.ctx.textAlign = "start";
// u.ctx.fillStyle = "black";
// u.ctx.fillText(textl, u.bbox.left + 10, u.bbox.top + 10);
// u.ctx.textAlign = "end";
// u.ctx.fillStyle = "black";
// u.ctx.fillText(
// textr,
// u.bbox.left + u.bbox.width - 10,
// u.bbox.top + 10,
// );
// u.ctx.restore();
// return;
// },
// ]
// },
scales: {
x: { time: false },
y: {auto: true, distr: 1},
},
legend: {
// Display legend
show: true,
live: true,
},
cursor: {
drag: { x: true, y: true },
// sync: {
// key: plotSync.key,
// scales: ["x", null],
// }
}
};
/* Var Init */
let timeoutId = null;
let uplot = null;
/* State Init */
let plotWrapper = $state(null);
/* Effects */
$effect(() => {
if (plotWrapper) {
onSizeChange(width, height);
}
});
/* Functions */
// UPLOT PLUGIN // converts the legend into a simple tooltip
// UPLOT PLUGIN: Converts the legend into a simple tooltip
function legendAsTooltipPlugin({
className,
style = { backgroundColor: "rgba(255, 249, 196, 0.92)", color: "black" },
@@ -476,7 +237,7 @@
className && legendEl.classList.add(className);
uPlot.assign(legendEl.style, {
minWidth: "100px",
minWidth: "175px",
textAlign: "left",
pointerEvents: "none",
display: "none",
@@ -489,9 +250,9 @@
});
// hide series color markers:
const idents = legendEl.querySelectorAll(".u-marker");
for (let i = 0; i < idents.length; i++)
idents[i].style.display = "none";
// const idents = legendEl.querySelectorAll(".u-marker");
// for (let i = 0; i < idents.length; i++)
// idents[i].style.display = "none";
const overEl = u.over;
overEl.style.overflow = "visible";
@@ -510,9 +271,12 @@
function update(u) {
const { left, top } = u.cursor;
const width = u?.over?.querySelector(".u-legend")?.offsetWidth ? u.over.querySelector(".u-legend").offsetWidth : 0;
legendEl.style.transform =
"translate(" + (left - width - 15) + "px, " + (top + 15) + "px)";
const internalWidth = u?.over?.querySelector(".u-legend")?.offsetWidth ? u.over.querySelector(".u-legend").offsetWidth : 0;
if (left < (width/2)) {
legendEl.style.transform = "translate(" + (left + 15) + "px, " + (top + 15) + "px)";
} else {
legendEl.style.transform = "translate(" + (left - internalWidth - 15) + "px, " + (top + 15) + "px)";
}
}
return {
@@ -523,13 +287,34 @@
};
}
// RENDER HANDLING
// UPLOT SERIES INIT
const plotSeries = [
{
label: "Time",
scale: "x"
},
...collectLabel.map(l => seriesConfig[l])
]
/* Var Init */
let timeoutId = null;
let uplot = null;
/* State Init */
let plotWrapper = $state(null);
/* Effects */
$effect(() => {
if (plotWrapper) {
onSizeChange(width, height);
}
});
/* Functions */
function render(ren_width, ren_height) {
if (!uplot) {
opts.width = ren_width;
opts.height = ren_height;
uplot = new uPlot(opts, data, plotWrapper); // Data is uplot formatted [[X][Ymin][Yavg][Ymax]]
plotSync.sub(uplot)
let { opts, data } = getStackedOpts(title, ren_width, ren_height, plotSeries, collectData);
uplot = new uPlot(opts, data, plotWrapper); // Data is uplot formatted [[X][Y1][Y2]...]
} else {
uplot.setSize({ width: ren_width, height: ren_height });
}
@@ -559,12 +344,12 @@
</script>
<!-- Define $width Wrapper and NoData Card -->
{#if data && data[0].length > 0}
{#if data && collectData[0].length > 0}
<div bind:this={plotWrapper} bind:clientWidth={width}
style="background-color: rgba(255, 255, 255, 1.0);" class="rounded"
></div>
{:else}
<Card body color="warning" class="mx-4 my-2"
>Cannot render plot: No series data returned for <code>{metric?metric:'job resources'}</code></Card
>Cannot render plot: No series data returned for <code>{stateType} State Stacked Chart</code></Card
>
{/if}

View File

@@ -98,7 +98,12 @@
if (!cluster) {
return avail.map((av) => av.cluster).join(', ')
} else {
return avail.find((av) => av.cluster === cluster).subClusters.join(', ')
const subAvail = avail.find((av) => av.cluster === cluster)?.subClusters
if (subAvail) {
return subAvail.join(', ')
} else {
return `Not available for ${cluster}`
}
}
}

View File

@@ -36,7 +36,7 @@
/* Const Init */
const defaultTo = new Date(Date.now());
const defaultFrom = new Date(defaultTo.setHours(defaultTo.getHours() - 4));
const defaultFrom = new Date(new Date(Date.now()).setHours(defaultTo.getHours() - 4));
/* State Init */
let timeType = $state("range");

View File

@@ -51,12 +51,13 @@ export function formatDurationTime(t, forNode = false) {
}
}
export function formatUnixTime(t) {
export function formatUnixTime(t, withDate = false) {
if (t !== null) {
if (isNaN(t)) {
return t;
} else {
return new Date(t * 1000).toLocaleString()
if (withDate) return new Date(t * 1000).toLocaleString();
else return new Date(t * 1000).toLocaleTimeString();
}
}
}

View File

@@ -185,6 +185,7 @@
{isShared}
{zoomState}
{thresholdState}
enableFlip
/>
{:else if statsSeries[selectedScopeIndex] != null && patternMatches}
<MetricPlot
@@ -202,6 +203,7 @@
{thresholdState}
statisticsSeries={statsSeries[selectedScopeIndex]}
useStatsSeries={!!statsSeries[selectedScopeIndex]}
enableFlip
/>
{/if}
{/key}

View File

@@ -21,7 +21,6 @@
getContextClient,
} from "@urql/svelte";
import {
init,
convert2uplot,
} from "../generic/utils.js";
import PlotGrid from "../generic/PlotGrid.svelte";
@@ -35,7 +34,6 @@
} = $props();
/* Const Init */
const { query: initq } = init();
const ccconfig = getContext("cc-config");
const client = getContextClient();
@@ -101,25 +99,18 @@
</Row>
<Row cols={1} class="text-center mt-3">
<Col>
{#if $initq.fetching || $metricStatusQuery.fetching}
<Spinner />
{:else if $initq.error}
<Card body color="danger">{$initq.error.message}</Card>
{:else}
<!-- ... -->
{/if}
</Col>
</Row>
{#if $metricStatusQuery.error}
<Row cols={1}>
{#if $metricStatusQuery.fetching}
<Col>
<Spinner />
</Col>
{:else if $metricStatusQuery.error}
<Col>
<Card body color="danger">{$metricStatusQuery.error.message}</Card>
</Col>
</Row>
{/if}
{/if}
</Row>
{#if $initq.data && $metricStatusQuery.data}
{#if $metricStatusQuery.data}
<!-- Selectable Stats as Histograms : Average Values of Running Jobs -->
{#if selectedHistograms}
<!-- Note: Ignore '#snippet' Error in IDE -->
@@ -132,6 +123,7 @@
ylabel="Number of Jobs"
yunit="Jobs"
usesBins
enableFlip
/>
{/snippet}

View File

@@ -22,34 +22,36 @@
gql,
getContextClient,
} from "@urql/svelte";
import {
init,
} from "../generic/utils.js";
import { scaleNumbers, formatDurationTime } from "../generic/units.js";
import { formatDurationTime } from "../generic/units.js";
import Refresher from "../generic/helper/Refresher.svelte";
import TimeSelection from "../generic/select/TimeSelection.svelte";
import Roofline from "../generic/plots/Roofline.svelte";
import Pie, { colors } from "../generic/plots/Pie.svelte";
import Stacked from "../generic/plots/Stacked.svelte";
/* Svelte 5 Props */
let {
clusters,
presetCluster,
useCbColors = false,
useAltColors = false,
} = $props();
/* Const Init */
const { query: initq } = init();
const client = getContextClient();
/* State Init */
let cluster = $state(presetCluster);
let pieWidth = $state(0);
let stackedWidth = $state(0);
let stackedWidth1 = $state(0);
let stackedWidth2 = $state(0);
let plotWidths = $state([]);
let from = $state(new Date(Date.now() - 5 * 60 * 1000));
let to = $state(new Date(Date.now()));
let stackedFrom = $state(Math.floor(Date.now() / 1000) - 14400);
// Bar Gauges
let allocatedNodes = $state({});
let allocatedCores = $state({});
let allocatedAccs = $state({});
let flopRate = $state({});
let flopRateUnitPrefix = $state({});
@@ -63,46 +65,29 @@
let totalAccs = $state({});
/* Derived */
// Accumulated NodeStates for Piecharts
const nodesStateCounts = $derived(queryStore({
// States for Stacked charts
const statesTimed = $derived(queryStore({
client: client,
query: gql`
query ($filter: [NodeFilter!]) {
nodeStates(filter: $filter) {
query ($filter: [NodeFilter!], $typeNode: String!, $typeHealth: String!) {
nodeStates: nodeStatesTimed(filter: $filter, type: $typeNode) {
state
count
counts
times
}
healthStates: nodeStatesTimed(filter: $filter, type: $typeHealth) {
state
counts
times
}
}
`,
variables: {
filter: { cluster: { eq: cluster }}
},
}));
const refinedStateData = $derived.by(() => {
return $nodesStateCounts?.data?.nodeStates.filter((e) => ['allocated', 'reserved', 'idle', 'mixed','down', 'unknown'].includes(e.state))
});
const refinedHealthData = $derived.by(() => {
return $nodesStateCounts?.data?.nodeStates.filter((e) => ['full', 'partial', 'failed'].includes(e.state))
});
// NodeStates for Stacked charts
const nodesStateTimes = $derived(queryStore({
client: client,
query: gql`
query ($filter: [NodeFilter!]) {
nodeStatesTimed(filter: $filter) {
state
type
count
time
}
}
`,
variables: {
filter: { cluster: { eq: cluster }, timeStart: Date.now() - (24 * 3600 * 1000)} // Add Selector for Timeframe (4h, 12h, 24h)?
filter: { cluster: { eq: cluster }, timeStart: stackedFrom},
typeNode: "node",
typeHealth: "health"
},
requestPolicy: "network-only"
}));
// Note: nodeMetrics are requested on configured $timestep resolution
@@ -177,9 +162,14 @@
hostname
cluster
subCluster
nodeState
schedulerState
}
}
# Get Current States fir Pie Charts
nodeStates(filter: $nodeFilter) {
state
count
}
# totalNodes includes multiples if shared jobs
jobsStatistics(
filter: $jobFilter
@@ -190,6 +180,7 @@
id
totalJobs
totalUsers
totalCores
totalAccs
}
}
@@ -207,10 +198,22 @@
requestPolicy: "network-only"
}));
const refinedStateData = $derived.by(() => {
return $statusQuery?.data?.nodeStates.
filter((e) => ['allocated', 'reserved', 'idle', 'mixed','down', 'unknown'].includes(e.state)).
sort((a, b) => b.count - a.count)
});
const refinedHealthData = $derived.by(() => {
return $statusQuery?.data?.nodeStates.
filter((e) => ['full', 'partial', 'failed'].includes(e.state)).
sort((a, b) => b.count - a.count)
});
/* Effects */
$effect(() => {
if ($initq.data && $statusQuery.data) {
let subClusters = $initq.data.clusters.find(
if ($statusQuery.data) {
let subClusters = clusters.find(
(c) => c.name == cluster,
).subClusters;
for (let subCluster of subClusters) {
@@ -219,6 +222,10 @@
$statusQuery.data.allocatedNodes.find(
({ name }) => name == subCluster.name,
)?.count || 0;
allocatedCores[subCluster.name] =
$statusQuery.data.jobsStatistics.find(
({ id }) => id == subCluster.name,
)?.totalCores || 0;
allocatedAccs[subCluster.name] =
$statusQuery.data.jobsStatistics.find(
({ id }) => id == subCluster.name,
@@ -348,7 +355,7 @@
for (let j = 0; j < subClusterData.length; j++) {
const nodeName = subClusterData[j]?.host ? subClusterData[j].host : "unknown"
const nodeMatch = $statusQuery?.data?.nodes?.items?.find((n) => n.hostname == nodeName && n.subCluster == subClusterData[j].subCluster);
const nodeState = nodeMatch?.nodeState ? nodeMatch.nodeState : "notindb"
const schedulerState = nodeMatch?.schedulerState ? nodeMatch.schedulerState : "notindb"
let numJobs = 0
if ($statusQuery?.data) {
@@ -356,7 +363,7 @@
numJobs = nodeJobs?.length ? nodeJobs.length : 0
}
result.push({nodeName: nodeName, nodeState: nodeState, numJobs: numJobs})
result.push({nodeName: nodeName, schedulerState: schedulerState, numJobs: numJobs})
};
};
return result
@@ -378,14 +385,24 @@
</script>
<!-- Refresher and space for other options -->
<Row class="justify-content-end">
<Row class="justify-content-between">
<Col xs="12" md="5" lg="4" xl="3">
<TimeSelection
customEnabled={false}
applyTime={(newFrom, newTo) => {
stackedFrom = Math.floor(newFrom.getTime() / 1000);
}}
/>
</Col>
<Col xs="12" md="5" lg="4" xl="3">
<Refresher
initially={120}
onRefresh={() => {
console.log('Trigger Refresh StatusTab')
onRefresh={(interval) => {
from = new Date(Date.now() - 5 * 60 * 1000);
to = new Date(Date.now());
if (interval) stackedFrom += Math.floor(interval / 1000);
else stackedFrom += 1 // Workaround: TineSelection not linked, just trigger new data on manual refresh
}}
/>
</Col>
@@ -394,43 +411,40 @@
<hr/>
<!-- Node Stack Charts Dev-->
<!--
{#if $initq.data && $nodesStateTimes.data}
<Row cols={{ lg: 4, md: 2 , sm: 1}} class="mb-3 justify-content-center">
{#if $statesTimed.data}
<Row cols={{ md: 2 , sm: 1}} class="mb-3 justify-content-center">
<Col class="px-3 mt-2 mt-lg-0">
<div bind:clientWidth={stackedWidth}>
{#key $nodesStateTimes.data}
<div bind:clientWidth={stackedWidth1}>
{#key $statesTimed?.data?.nodeStates}
<h4 class="text-center">
{cluster.charAt(0).toUpperCase() + cluster.slice(1)} Node States Over Time
</h4>
<Stacked
{cluster}
data={$nodesStateTimes?.data}
width={stackedWidth * 0.55}
xLabel="Time"
yLabel="Nodes"
data={$statesTimed?.data?.nodeStates}
width={stackedWidth1 * 0.95}
xlabel="Time"
ylabel="Nodes"
yunit = "#Count"
title = "Slurm States"
stateType = "slurm"
title = "Node States"
stateType = "Node"
/>
{/key}
</div>
</Col>
<Col class="px-3 mt-2 mt-lg-0">
<div bind:clientWidth={stackedWidth}>
{#key $nodesStateTimes.data}
<div bind:clientWidth={stackedWidth2}>
{#key $statesTimed?.data?.healthStates}
<h4 class="text-center">
{cluster.charAt(0).toUpperCase() + cluster.slice(1)} Health States Over Time
</h4>
<Stacked
{cluster}
data={$nodesStateTimes?.data}
width={stackedWidth * 0.55}
xLabel="Time"
yLabel="Nodes"
data={$statesTimed?.data?.healthStates}
width={stackedWidth2 * 0.95}
xlabel="Time"
ylabel="Nodes"
yunit = "#Count"
title = "Health States"
stateType = "health"
stateType = "Health"
/>
{/key}
</div>
@@ -439,17 +453,15 @@
{/if}
<hr/>
<hr/>
-->
<!-- Node Health Pis, later Charts -->
{#if $initq.data && $nodesStateCounts.data}
{#if $statusQuery?.data?.nodeStates}
<Row cols={{ lg: 4, md: 2 , sm: 1}} class="mb-3 justify-content-center">
<Col class="px-3 mt-2 mt-lg-0">
<div bind:clientWidth={pieWidth}>
{#key refinedStateData}
<h4 class="text-center">
{cluster.charAt(0).toUpperCase() + cluster.slice(1)} Node States
Current {cluster.charAt(0).toUpperCase() + cluster.slice(1)} Node States
</h4>
<Pie
{useAltColors}
@@ -489,7 +501,7 @@
<div bind:clientWidth={pieWidth}>
{#key refinedHealthData}
<h4 class="text-center">
{cluster.charAt(0).toUpperCase() + cluster.slice(1)} Node Health
Current {cluster.charAt(0).toUpperCase() + cluster.slice(1)} Node Health
</h4>
<Pie
{useAltColors}
@@ -529,8 +541,8 @@
<hr/>
<!-- Gauges & Roofline per Subcluster-->
{#if $initq.data && $statusQuery.data}
{#each $initq.data.clusters.find((c) => c.name == cluster).subClusters as subCluster, i}
{#if $statusQuery.data}
{#each clusters.find((c) => c.name == cluster).subClusters as subCluster, i}
<Row cols={{ lg: 3, md: 1 , sm: 1}} class="mb-3 justify-content-center">
<Col class="px-3">
<Card class="h-auto mt-1">
@@ -579,6 +591,21 @@
Nodes</td
>
</tr>
<tr class="py-2">
<th scope="col">Allocated Cores</th>
<td style="min-width: 100px;"
><div class="col">
<Progress
value={allocatedCores[subCluster.name]}
max={subCluster.socketsPerNode * subCluster.coresPerSocket * subCluster.numberOfNodes}
/>
</div></td
>
<td
>{allocatedCores[subCluster.name]} / {subCluster.socketsPerNode * subCluster.coresPerSocket * subCluster.numberOfNodes}
Cores</td
>
</tr>
{#if totalAccs[subCluster.name] !== null}
<tr class="py-2">
<th scope="col">Allocated Accelerators</th>

View File

@@ -24,7 +24,6 @@
getContextClient,
} from "@urql/svelte";
import {
init,
scramble,
scrambleNames,
convert2uplot,
@@ -41,7 +40,6 @@
} = $props();
/* Const Init */
const { query: initq } = init();
const client = getContextClient();
const durationBinOptions = ["1m","10m","1h","6h","12h"];
@@ -255,6 +253,7 @@
height="275"
usesBins
xtime
enableFlip
/>
{/key}
</Col>
@@ -359,6 +358,7 @@
ylabel="Number of Jobs"
yunit="Jobs"
height="275"
enableFlip
/>
</Col>
<Col xs="6" md="3" lg="2" class="p-2">
@@ -462,6 +462,7 @@
ylabel="Number of Jobs"
yunit="Jobs"
height="275"
enableFlip
/>
</Col>
<Col xs="6" md="3" lg="2" class="p-2">

View File

@@ -8,6 +8,7 @@
- `selectedMetrics [String]`: The array of selected metrics [Default []]
- `selectedResolution Number?`: The selected data resolution [Default: 0]
- `hostnameFilter String?`: The active hostnamefilter [Default: ""]
- `hoststateFilter String?`: The active hoststatefilter [Default: ""]
- `presetSystemUnits Object`: The object of metric units [Default: null]
- `from Date?`: The selected "from" date [Default: null]
- `to Date?`: The selected "to" date [Default: null]
@@ -28,6 +29,7 @@
selectedMetrics = [],
selectedResolution = 0,
hostnameFilter = "",
hoststateFilter = "",
presetSystemUnits = null,
from = null,
to = null
@@ -37,11 +39,14 @@
const client = getContextClient();
const usePaging = ccconfig?.nodeList_usePaging || false;
const nodeListQuery = gql`
query ($cluster: String!, $subCluster: String!, $nodeFilter: String!, $metrics: [String!], $scopes: [MetricScope!]!, $from: Time!, $to: Time!, $paging: PageRequest!, $selectedResolution: Int) {
query ($cluster: String!, $subCluster: String!, $nodeFilter: String!, $stateFilter: String!, $metrics: [String!],
$scopes: [MetricScope!]!, $from: Time!, $to: Time!, $paging: PageRequest!, $selectedResolution: Int
) {
nodeMetricsList(
cluster: $cluster
subCluster: $subCluster
nodeFilter: $nodeFilter
stateFilter: $stateFilter,
scopes: $scopes
metrics: $metrics
from: $from
@@ -51,6 +56,7 @@
) {
items {
host
state
subCluster
metrics {
name
@@ -100,6 +106,7 @@
variables: {
cluster: cluster,
subCluster: subCluster,
stateFilter: hoststateFilter,
nodeFilter: hostnameFilter,
scopes: ["core", "socket", "accelerator"],
metrics: selectedMetrics,
@@ -137,7 +144,7 @@
// Triggers (Except Paging)
from, to
selectedMetrics, selectedResolution
hostnameFilter
hostnameFilter, hoststateFilter
// Continous Scroll: Reset nodes and paging if parameters change: Existing entries will not match new selections
if (!usePaging) {
nodes = [];

View File

@@ -6,6 +6,7 @@
- `cluster String`: The cluster to show status information for
- `selectedMetric String?`: The selectedMetric input [Default: ""]
- `hostnameFilter String?`: The active hostnamefilter [Default: ""]
- `hostnameFilter String?`: The active hoststatefilter [Default: ""]
- `from Date?`: The selected "from" date [Default: null]
- `to Date?`: The selected "to" date [Default: null]
-->
@@ -13,7 +14,7 @@
<script>
import { getContext } from "svelte";
import { queryStore, gql, getContextClient } from "@urql/svelte";
import { Row, Col, Card, Spinner } from "@sveltestrap/sveltestrap";
import { Row, Col, Card, Spinner, Badge } from "@sveltestrap/sveltestrap";
import { checkMetricDisabled } from "../generic/utils.js";
import MetricPlot from "../generic/plots/MetricPlot.svelte";
@@ -23,6 +24,7 @@
cluster = "",
selectedMetric = "",
hostnameFilter = "",
hoststateFilter = "",
from = null,
to = null
} = $props();
@@ -30,6 +32,16 @@
/* Const Init */
const initialized = getContext("initialized");
const client = getContextClient();
// Node State Colors
const stateColors = {
allocated: 'success',
reserved: 'info',
idle: 'primary',
mixed: 'warning',
down: 'danger',
unknown: 'dark',
notindb: 'secondary'
}
/* Derived */
const nodesQuery = $derived(queryStore({
@@ -43,6 +55,7 @@
to: $to
) {
host
state
subCluster
metrics {
name
@@ -75,7 +88,15 @@
}));
const mappedData = $derived(handleQueryData($initialized, $nodesQuery?.data));
const filteredData = $derived(mappedData.filter((h) => h.host.includes(hostnameFilter)));
const filteredData = $derived(mappedData.filter((h) => {
if (hostnameFilter) {
if (hoststateFilter == 'all') return h.host.includes(hostnameFilter)
else return (h.host.includes(hostnameFilter) && h.state == hoststateFilter)
} else {
if (hoststateFilter == 'all') return true
else return h.state == hoststateFilter
}
}));
/* Functions */
function handleQueryData(isInitialized, queryData) {
@@ -94,6 +115,7 @@
if (rawData.length > 0) {
pendingMapped = rawData.map((h) => ({
host: h.host,
state: h?.state? h.state : 'notindb',
subCluster: h.subCluster,
data: h.metrics.filter(
(m) => m?.name == selectedMetric && m.scope == "node",
@@ -125,13 +147,18 @@
{#key selectedMetric}
{#each filteredData as item (item.host)}
<Col class="px-1">
<h4 style="width: 100%; text-align: center;">
<a
style="display: block;padding-top: 15px;"
href="/monitoring/node/{cluster}/{item.host}"
>{item.host} ({item.subCluster})</a
>
</h4>
<div class="d-flex align-items-baseline">
<h4 style="width: 100%; text-align: center;">
<a
style="display: block;padding-top: 15px;"
href="/monitoring/node/{cluster}/{item.host}"
>{item.host} ({item.subCluster})</a
>
</h4>
<span style="margin-right: 0.5rem;">
<Badge color={stateColors[item?.state? item.state : 'notindb']}>{item?.state? item.state : 'notindb'}</Badge>
</span>
</div>
{#if item.disabled === true}
<Card body class="mx-3" color="info"
>Metric disabled for subcluster <code
@@ -149,6 +176,7 @@
{cluster}
subCluster={item.subCluster}
forNode
enableFlip
/>
{/key}
{:else if item.disabled === null}

View File

@@ -13,6 +13,8 @@
import {
Icon,
Button,
Row,
Col,
Card,
CardHeader,
CardBody,
@@ -30,6 +32,7 @@
cluster,
subCluster,
hostname,
hoststate,
dataHealth,
nodeJobsData = null,
} = $props();
@@ -39,6 +42,16 @@
const healthWarn = !dataHealth.includes(true);
// At least one non-returned selected metric: Metric config error?
const metricWarn = dataHealth.includes(false);
// Node State Colors
const stateColors = {
allocated: 'success',
reserved: 'info',
idle: 'primary',
mixed: 'warning',
down: 'danger',
unknown: 'dark',
notindb: 'secondary'
}
/* Derived */
const userList = $derived(nodeJobsData
@@ -68,80 +81,72 @@
</div>
</CardHeader>
<CardBody>
{#if healthWarn}
<InputGroup>
<InputGroupText>
<Icon name="exclamation-circle"/>
</InputGroupText>
<InputGroupText>
Status
</InputGroupText>
<Button color="danger" disabled>
Unhealthy
</Button>
</InputGroup>
{:else if metricWarn}
<InputGroup>
<InputGroupText>
<Icon name="info-circle"/>
</InputGroupText>
<InputGroupText>
Status
</InputGroupText>
<Button color="warning" disabled>
Missing Metric
</Button>
</InputGroup>
{:else if nodeJobsData.jobs.count == 1 && nodeJobsData.jobs.items[0].shared == "none"}
<InputGroup>
<InputGroupText>
<Icon name="circle-fill"/>
</InputGroupText>
<InputGroupText>
Status
</InputGroupText>
<Button color="success" disabled>
Exclusive
</Button>
</InputGroup>
{:else if nodeJobsData.jobs.count >= 1 && !(nodeJobsData.jobs.items[0].shared == "none")}
<InputGroup>
<InputGroupText>
<Icon name="circle-half"/>
</InputGroupText>
<InputGroupText>
Status
</InputGroupText>
<Button color="success" disabled>
Shared
</Button>
</InputGroup>
<!-- Fallback -->
{:else if nodeJobsData.jobs.count >= 1}
<InputGroup>
<InputGroupText>
<Icon name="circle-fill"/>
</InputGroupText>
<InputGroupText>
Status
</InputGroupText>
<Button color="success" disabled>
Allocated Jobs
</Button>
</InputGroup>
{:else}
<InputGroup>
<InputGroupText>
<Icon name="circle"/>
</InputGroupText>
<InputGroupText>
Status
</InputGroupText>
<Button color="secondary" disabled>
Idle
</Button>
</InputGroup>
{/if}
<Row cols={{xs: 1, lg: 2}}>
<Col class="mb-2 mb-lg-0">
<InputGroup size="sm">
{#if healthWarn}
<InputGroupText class="flex-grow-1 flex-lg-grow-0">
<Icon name="exclamation-circle" style="padding-right: 0.5rem;"/>
<span>Jobs</span>
</InputGroupText>
<Button class="flex-grow-1" color="danger" disabled>
No Metrics
</Button>
{:else if metricWarn}
<InputGroupText class="flex-grow-1 flex-lg-grow-0">
<Icon name="info-circle" style="padding-right: 0.5rem;"/>
<span>Jobs</span>
</InputGroupText>
<Button class="flex-grow-1" color="warning" disabled>
Missing Metric
</Button>
{:else if nodeJobsData.jobs.count == 1 && nodeJobsData.jobs.items[0].shared == "none"}
<InputGroupText class="flex-grow-1 flex-lg-grow-0">
<Icon name="circle-fill" style="padding-right: 0.5rem;"/>
<span>Jobs</span>
</InputGroupText>
<Button class="flex-grow-1" color="success" disabled>
Exclusive
</Button>
{:else if nodeJobsData.jobs.count >= 1 && !(nodeJobsData.jobs.items[0].shared == "none")}
<InputGroupText class="flex-grow-1 flex-lg-grow-0">
<Icon name="circle-half" style="padding-right: 0.5rem;"/>
<span>Jobs</span>
</InputGroupText>
<Button class="flex-grow-1" color="success" disabled>
Shared
</Button>
<!-- Fallback -->
{:else if nodeJobsData.jobs.count >= 1}
<InputGroupText class="flex-grow-1 flex-lg-grow-0">
<Icon name="circle-fill" style="padding-right: 0.5rem;"/>
<span>Jobs</span>
</InputGroupText>
<Button class="flex-grow-1" color="success" disabled>
Running
</Button>
{:else}
<InputGroupText class="flex-grow-1 flex-lg-grow-0">
<Icon name="circle" style="padding-right: 0.5rem;"/>
<span>Jobs</span>
</InputGroupText>
<Button class="flex-grow-1" color="secondary" disabled>
None
</Button>
{/if}
</InputGroup>
</Col>
<Col>
<InputGroup size="sm">
<InputGroupText class="flex-grow-1 flex-lg-grow-0">
State
</InputGroupText>
<Button class="flex-grow-1" color={stateColors[hoststate]} disabled>
{hoststate.charAt(0).toUpperCase() + hoststate.slice(1)}
</Button>
</InputGroup>
</Col>
</Row>
<hr class="my-3"/>
<!-- JOBS -->
<InputGroup size="sm" class="justify-content-between mb-3">

View File

@@ -13,6 +13,7 @@
gql,
getContextClient,
} from "@urql/svelte";
import uPlot from "uplot";
import { Card, CardBody, Spinner } from "@sveltestrap/sveltestrap";
import { maxScope, checkMetricDisabled, scramble, scrambleNames } from "../../generic/utils.js";
import MetricPlot from "../../generic/plots/MetricPlot.svelte";
@@ -25,6 +26,9 @@
selectedMetrics,
} = $props();
/* Var Init*/
let plotSync = uPlot.sync(`nodeMetricStack-${nodeData.host}`);
/* Const Init */
const client = getContextClient();
const paging = { itemsPerPage: 50, page: 1 };
@@ -135,7 +139,13 @@
</CardBody>
</Card>
{:else}
<NodeInfo nodeJobsData={$nodeJobsData.data} {cluster} subCluster={nodeData.subCluster} hostname={nodeData.host} {dataHealth}/>
<NodeInfo
{cluster}
{dataHealth}
nodeJobsData={$nodeJobsData.data}
subCluster={nodeData.subCluster}
hostname={nodeData.host}
hoststate={nodeData?.state? nodeData.state: 'notindb'}/>
{/if}
</td>
{#each refinedData as metricData (metricData.data.name)}
@@ -159,6 +169,7 @@
statisticsSeries={metricData.data?.metric.statisticsSeries}
useStatsSeries={!!metricData.data?.metric.statisticsSeries}
height={175}
{plotSync}
forNode
/>
<div class="my-2"></div>
@@ -172,6 +183,7 @@
series={metricData.data.metric.series}
height={175}
{extendedLegendData}
{plotSync}
forNode
/>
{/key}