ClusterCockpit/cc-backend
1
0
Fork 0
mirror of https://github.com/ClusterCockpit/cc-backend synced 2025-02-04 15:25:17 +01:00
Code Issues Packages Releases Activity

initial duration histogram zoom in frontend

Browse Source 
- metric zoom todo
- keeping last zoomState does not work
This commit is contained in:
Christoph Kluge 2025-01-23 12:23:29 +01:00
parent 05bfa9b546
commit 6683a350aa
5 changed files with 200 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
internal/graph/schema.resolvers.go
View File

@ -394,6 +394,12 @@ func (r *queryResolver) JobsStatistics(ctx context.Context, filter []*model.JobF
}
if requireField(ctx, "histDuration") || requireField(ctx, "histNumNodes") || requireField(ctx, "histNumCores") || requireField(ctx, "histNumAccs") {
if numDurationBins == nil {
binCount := 24
numDurationBins = &binCount
}
if groupBy == nil {
stats[0], err = r.Repo.AddHistograms(ctx, filter, stats[0], numDurationBins)
if err != nil {
@ -405,6 +411,12 @@ func (r *queryResolver) JobsStatistics(ctx context.Context, filter []*model.JobF
}
if requireField(ctx, "histMetrics") {
if numMetricBins == nil {
binCount := 10
numMetricBins = &binCount
}
if groupBy == nil {
stats[0], err = r.Repo.AddMetricHistograms(ctx, filter, metrics, stats[0], numMetricBins)
if err != nil {

71
internal/repository/stats.go
View File

@ -454,23 +454,19 @@ func (r *JobRepository) AddHistograms(
// targetBinCount : Frontendargument
// -> Min Bins: 24 -> Min Resolution: By Hour
// -> In Between Bins: 48 -> Resolution by Half Hour
// 96 -> Resolution by Quarter Hour (Skip for Now)
// 96 -> Resolution by Quarter Hour
// 144 -> Resolution by 10 Minutes
// 288 -> Resolution by 5 Minutes
// 720 -> Resolution by 2 Minutes (SKip for Now)
// 720 -> Resolution by 2 Minutes
// -> Max Bins: 1440 -> Max Resolution: By Minute
if targetBinCount == nil {
binCount := 24
targetBinCount = &binCount
}
binSizeSeconds := (86400 / *targetBinCount)
castType := r.getCastType()
var err error
// Return X-Values always as seconds, will be formatted into minutes and hours in frontend
value := fmt.Sprintf(`CAST(ROUND(((CASE WHEN job.job_state = "running" THEN %d - job.start_time ELSE job.duration END) / %d) + 1) as %s) as value`, time.Now().Unix(), binSizeSeconds, castType)
stat.HistDuration, err = r.jobsStatisticsHistogram(ctx, value, filter)
stat.HistDuration, err = r.jobsDurationStatisticsHistogram(ctx, value, filter, binSizeSeconds, targetBinCount)
if err != nil {
log.Warn("Error while loading job statistics histogram: job duration")
return nil, err
@ -508,10 +504,7 @@ func (r *JobRepository) AddMetricHistograms(
) (*model.JobsStatistics, error) {
start := time.Now()
if targetBinCount == nil {
binCount := 10
targetBinCount = &binCount
}
log.Debugf(">>> HELLO ADD HISTO Metrics: Target %d", *targetBinCount)
// Running Jobs Only: First query jobdata from sqlite, then query data and make bins
for _, f := range filter {
@ -563,6 +556,7 @@ func (r *JobRepository) jobsStatisticsHistogram(
}
points := make([]*model.HistoPoint, 0)
// is it possible to introduce zero values here? requires info about bincount
for rows.Next() {
point := model.HistoPoint{}
if err := rows.Scan(&point.Value, &point.Count); err != nil {
@ -576,6 +570,61 @@ func (r *JobRepository) jobsStatisticsHistogram(
return points, nil
}
func (r *JobRepository) jobsDurationStatisticsHistogram(
ctx context.Context,
value string,
filters []*model.JobFilter,
binSizeSeconds int,
targetBinCount *int,
) ([]*model.HistoPoint, error) {
start := time.Now()
query, qerr := SecurityCheck(ctx,
sq.Select(value, "COUNT(job.id) AS count").From("job"))
if qerr != nil {
return nil, qerr
}
// Setup Array
points := make([]*model.HistoPoint, 0)
for i := 1; i <= *targetBinCount; i++ {
point := model.HistoPoint{Value: i * binSizeSeconds, Count: 0}
points = append(points, &point)
}
for _, f := range filters {
query = BuildWhereClause(f, query)
}
rows, err := query.GroupBy("value").RunWith(r.DB).Query()
if err != nil {
log.Error("Error while running query")
return nil, err
}
// Fill Array at matching $Value
for rows.Next() {
point := model.HistoPoint{}
if err := rows.Scan(&point.Value, &point.Count); err != nil {
log.Warn("Error while scanning rows")
return nil, err
}
for _, e := range points {
if e.Value == (point.Value * binSizeSeconds) {
// Note:
// Matching on unmodified integer value (and multiplying point.Value by binSizeSeconds after match)
// causes frontend to loop into highest targetBinCount, due to zoom condition instantly being fullfilled (cause unknown)
e.Count = point.Count
break
}
}
}
log.Debugf("Timer jobsStatisticsHistogram %s", time.Since(start))
return points, nil
}
func (r *JobRepository) jobsMetricStatisticsHistogram(
ctx context.Context,
metric string,

55
web/frontend/src/User.root.svelte
View File

@ -59,6 +59,9 @@
let showFootprint = filterPresets.cluster
? !!ccconfig[`plot_list_showFootprint:${filterPresets.cluster}`]
: !!ccconfig.plot_list_showFootprint;
let numDurationBins;
let numMetricBins;
$: metricsInHistograms = selectedCluster
? ccconfig[`user_view_histogramMetrics:${selectedCluster}`] || []
@ -68,8 +71,8 @@
$: stats = queryStore({
client: client,
query: gql`
query ($jobFilters: [JobFilter!]!, $metricsInHistograms: [String!]) {
jobsStatistics(filter: $jobFilters, metrics: $metricsInHistograms) {
query ($jobFilters: [JobFilter!]!, $metricsInHistograms: [String!], $numDurationBins: Int, $numMetricBins: Int) {
jobsStatistics(filter: $jobFilters, metrics: $metricsInHistograms, numDurationBins: $numDurationBins , numMetricBins: $numMetricBins ) {
totalJobs
shortJobs
totalWalltime
@ -96,9 +99,41 @@
}
}
`,
variables: { jobFilters, metricsInHistograms },
variables: { jobFilters, metricsInHistograms, numDurationBins, numMetricBins },
});
let durationZoomState = null;
let metricZoomState = null;
let pendingDurationBinCount = null;
let pendingMetricBinCount = null;
let pendingZoomState = null;
function handleZoom(detail) {
if ( // States have to differ, causes deathloop if just set
(pendingZoomState?.x?.min !== detail?.lastZoomState?.x?.min) &&
(pendingZoomState?.y?.max !== detail?.lastZoomState?.y?.max)
) {
pendingZoomState = {...detail.lastZoomState};
}
if (detail?.durationBinCount) { // Triggers GQL
pendingDurationBinCount = detail.durationBinCount;
}
if (detail?.metricBinCount) { // Triggers GQL
pendingMetricBinCount = detail.metricBinCount;
}
};
$: if (pendingDurationBinCount !== numDurationBins) {
durationZoomState = {...pendingZoomState};
numDurationBins = pendingDurationBinCount;
}
$: if (pendingMetricBinCount !== numMetricBins) {
metricZoomState = {...pendingZoomState};
numMetricBins = pendingMetricBinCount;
}
onMount(() => filterComponent.updateFilters());
</script>
@ -213,13 +248,17 @@
<Col class="px-1">
{#key $stats.data.jobsStatistics[0].histDuration}
<Histogram
on:zoom={({detail}) => { handleZoom(detail) }}
data={convert2uplot($stats.data.jobsStatistics[0].histDuration)}
title="Duration Distribution"
xlabel="Current Runtimes (Hours)"
xtime={true}
xunit="Hours"
xlabel="Job Runtimes"
xunit="Runtime"
ylabel="Number of Jobs"
yunit="Jobs"
lastBinCount={pendingDurationBinCount}
{durationZoomState}
zoomableHistogram
xtime
/>
{/key}
</Col>
@ -273,6 +312,7 @@
itemsPerRow={3}
>
<Histogram
on:zoom={({detail}) => { handleZoom(detail) }}
data={convert2uplot(item.data)}
usesBins={true}
title="Distribution of '{item.metric} ({item.stat})' footprints"
@ -280,6 +320,9 @@
xunit={item.unit}
ylabel="Number of Jobs"
yunit="Jobs"
lastBinCount={pendingMetricBinCount}
{metricZoomState}
zoomableHistogram
/>
</PlotGrid>
{/key}

84
web/frontend/src/generic/plots/Histogram.svelte
View File

@ -15,8 +15,8 @@
<script>
import uPlot from "uplot";
import { getContext, onMount, onDestroy, createEventDispatcher } from "svelte";
import { formatNumber } from "../units.js";
import { onMount, onDestroy } from "svelte";
import { Card } from "@sveltestrap/sveltestrap";
export let data;
@ -25,18 +25,40 @@
export let height = 250;
export let title = "";
export let xlabel = "";
export let xtime = false;
export let xunit = "";
export let xtime = false;
export let ylabel = "";
export let yunit = "";
export let zoomState = null;
export let lastBinCount = null;
export let zoomableHistogram = false;
const { bars } = uPlot.paths;
const dispatch = createEventDispatcher();
const drawStyles = {
bars: 1,
points: 2,
};
const binCounts = xtime ? [24, 48, 96, 144, 288, 720, 1440] : [10, 20, 50, 100, 200, 500, 1000];
function formatTime(t) {
if (t !== null) {
if (isNaN(t)) {
return t;
} else {
const tAbs = Math.abs(t);
const h = Math.floor(tAbs / 3600);
const m = Math.floor((tAbs % 3600) / 60);
// Re-Add "negativity" to time ticks only as string, so that if-cases work as intended
if (h == 0) return `${m}m`;
else if (m == 0) return `${h}h`;
else return `${h}:${m}h`;
}
}
}
function paths(u, seriesIdx, idx0, idx1, extendGap, buildClip) {
let s = u.series[seriesIdx];
let style = s.drawStyle;
@ -115,6 +137,45 @@
function render() {
let opts = {
hooks: {
init: [
(u) => {
if (zoomableHistogram) {
u.over.addEventListener("dblclick", (e) => {
// console.log('Dispatch Reset')
dispatch('zoom', {
lastZoomState: {
x: { time: false },
y: { auto: true }
}
});
});
}
},
],
setScale: [
(u, key) => {
if (key === 'x') {
if (zoomableHistogram) {
const numX = (u.series[0].idxs[1] - u.series[0].idxs[0])
if (xtime && numX <= 12 && lastBinCount !== 1440) {
console.log("Dispatch for Duration: ", numX, lastBinCount, binCounts[binCounts.indexOf(lastBinCount) + 1])
dispatch('zoom', {
durationBinCount: binCounts[binCounts.indexOf(lastBinCount) + 1],
lastZoomState: u?.scales,
});
} else if (!xtime && numX <= 5 && lastBinCount !== 1000) {
// console.log("Dispatch for Metrics: ", numX, lastBinCount, binCounts[binCounts.indexOf(lastBinCount) + 1])
// dispatch('zoom', {
// metricBinCount: binCounts[binCounts.indexOf(lastBinCount) + 1],
// lastZoomState: u?.scales,
// });
};
}
};
},
]
},
width: width,
height: height,
title: title,
@ -140,7 +201,7 @@
label: xlabel,
labelGap: 10,
size: 25,
incrs: xtime ? [0.25, 0.5, 1, 2, 4, 8, 15, 30, 60, 90, 120, 180, 240] : [1, 2, 5, 10, 25, 50, 100, 250, 500, 1000, 5000, 10000],
incrs: xtime ? [60, 120, 300, 600, 900, 1800, 3600, 7200, 14400] : [1, 2, 5, 10, 20, 50, 100, 200, 500, 1000, 2000, 5000, 10000],
border: {
show: true,
stroke: "#000000",
@ -151,8 +212,12 @@
stroke: "#000000",
},
values: (_, t) => t.map((v) => {
if (!usesBins) console.log("X Scale Val", xlabel, v)
return formatNumber(v)
// if (!usesBins) console.log("X Scale Val", xlabel, v)
if (xtime) {
return formatTime(v);
} else {
return formatNumber(v)
}
}),
},
{
@ -171,7 +236,7 @@
stroke: "#000000",
},
values: (_, t) => t.map((v) => {
if (!usesBins) console.log("Y Scale Val", ylabel, v)
// if (!usesBins) console.log("Y Scale Val", ylabel, v)
return formatNumber(v)
}),
},
@ -184,6 +249,8 @@
const min = u.data[sidx][didx - 1] ? u.data[sidx][didx - 1] : 0;
const max = u.data[sidx][didx];
ts = min + " - " + max; // narrow spaces
} else if (xtime) {
ts = formatTime(ts);
}
return ts;
},
@ -198,6 +265,7 @@
},
{
drawStyle: drawStyles.bars,
width: 1, // 1 / lastBinCount,
lineInterpolation: null,
stroke: "#85abce",
fill: "#85abce", // + "1A", // Transparent Fill
@ -206,6 +274,10 @@
],
};
if (zoomableHistogram && zoomState) {
opts.scales = {...zoomState}
}
uplot = new uPlot(opts, data, plotWrapper);
}

2
web/frontend/src/generic/utils.js
View File

@ -414,7 +414,7 @@ export function convert2uplot(canvasData, secondsToMinutes = false, secondsToHou
if (Object.keys(cd).length == 4) { // MetricHisto Datafromat
uplotData[0].push(cd?.max ? cd.max : 0)
uplotData[1].push(cd.count)
} else { // Default
} else { // Default -> Fill Histodata with zero values on unused value placing -> maybe allows zoom trigger as known
if (secondsToHours) {
let hours = cd.value / 3600
console.log("x seconds to y hours", cd.value, hours)