Drop change on zoom for selector with options

- Up to 7 days worth of runtime
- No zoomState issue and cached results
This commit is contained in:
Christoph Kluge 2025-01-24 14:06:43 +01:00
parent 9e6072fed2
commit 64f60905b4
6 changed files with 93 additions and 135 deletions

View File

@ -236,7 +236,7 @@ type Query {
jobsFootprints(filter: [JobFilter!], metrics: [String!]!): Footprints
jobs(filter: [JobFilter!], page: PageRequest, order: OrderByInput): JobResultList!
jobsStatistics(filter: [JobFilter!], metrics: [String!], page: PageRequest, sortBy: SortByAggregate, groupBy: Aggregate, numDurationBins: Int, numMetricBins: Int): [JobsStatistics!]!
jobsStatistics(filter: [JobFilter!], metrics: [String!], page: PageRequest, sortBy: SortByAggregate, groupBy: Aggregate, numDurationBins: String, numMetricBins: Int): [JobsStatistics!]!
rooflineHeatmap(filter: [JobFilter!]!, rows: Int!, cols: Int!, minX: Float!, minY: Float!, maxX: Float!, maxY: Float!): [[Float!]!]!

View File

@ -257,7 +257,7 @@ type ComplexityRoot struct {
JobMetrics func(childComplexity int, id string, metrics []string, scopes []schema.MetricScope, resolution *int) int
Jobs func(childComplexity int, filter []*model.JobFilter, page *model.PageRequest, order *model.OrderByInput) int
JobsFootprints func(childComplexity int, filter []*model.JobFilter, metrics []string) int
JobsStatistics func(childComplexity int, filter []*model.JobFilter, metrics []string, page *model.PageRequest, sortBy *model.SortByAggregate, groupBy *model.Aggregate, numDurationBins *int, numMetricBins *int) int
JobsStatistics func(childComplexity int, filter []*model.JobFilter, metrics []string, page *model.PageRequest, sortBy *model.SortByAggregate, groupBy *model.Aggregate, numDurationBins *string, numMetricBins *int) int
NodeMetrics func(childComplexity int, cluster string, nodes []string, scopes []schema.MetricScope, metrics []string, from time.Time, to time.Time) int
RooflineHeatmap func(childComplexity int, filter []*model.JobFilter, rows int, cols int, minX float64, minY float64, maxX float64, maxY float64) int
Tags func(childComplexity int) int
@ -382,7 +382,7 @@ type QueryResolver interface {
JobMetrics(ctx context.Context, id string, metrics []string, scopes []schema.MetricScope, resolution *int) ([]*model.JobMetricWithName, error)
JobsFootprints(ctx context.Context, filter []*model.JobFilter, metrics []string) (*model.Footprints, error)
Jobs(ctx context.Context, filter []*model.JobFilter, page *model.PageRequest, order *model.OrderByInput) (*model.JobResultList, error)
JobsStatistics(ctx context.Context, filter []*model.JobFilter, metrics []string, page *model.PageRequest, sortBy *model.SortByAggregate, groupBy *model.Aggregate, numDurationBins *int, numMetricBins *int) ([]*model.JobsStatistics, error)
JobsStatistics(ctx context.Context, filter []*model.JobFilter, metrics []string, page *model.PageRequest, sortBy *model.SortByAggregate, groupBy *model.Aggregate, numDurationBins *string, numMetricBins *int) ([]*model.JobsStatistics, error)
RooflineHeatmap(ctx context.Context, filter []*model.JobFilter, rows int, cols int, minX float64, minY float64, maxX float64, maxY float64) ([][]float64, error)
NodeMetrics(ctx context.Context, cluster string, nodes []string, scopes []schema.MetricScope, metrics []string, from time.Time, to time.Time) ([]*model.NodeMetrics, error)
}
@ -1372,7 +1372,7 @@ func (e *executableSchema) Complexity(typeName, field string, childComplexity in
return 0, false
}
return e.complexity.Query.JobsStatistics(childComplexity, args["filter"].([]*model.JobFilter), args["metrics"].([]string), args["page"].(*model.PageRequest), args["sortBy"].(*model.SortByAggregate), args["groupBy"].(*model.Aggregate), args["numDurationBins"].(*int), args["numMetricBins"].(*int)), true
return e.complexity.Query.JobsStatistics(childComplexity, args["filter"].([]*model.JobFilter), args["metrics"].([]string), args["page"].(*model.PageRequest), args["sortBy"].(*model.SortByAggregate), args["groupBy"].(*model.Aggregate), args["numDurationBins"].(*string), args["numMetricBins"].(*int)), true
case "Query.nodeMetrics":
if e.complexity.Query.NodeMetrics == nil {
@ -2132,7 +2132,7 @@ type Query {
jobsFootprints(filter: [JobFilter!], metrics: [String!]!): Footprints
jobs(filter: [JobFilter!], page: PageRequest, order: OrderByInput): JobResultList!
jobsStatistics(filter: [JobFilter!], metrics: [String!], page: PageRequest, sortBy: SortByAggregate, groupBy: Aggregate, numDurationBins: Int, numMetricBins: Int): [JobsStatistics!]!
jobsStatistics(filter: [JobFilter!], metrics: [String!], page: PageRequest, sortBy: SortByAggregate, groupBy: Aggregate, numDurationBins: String, numMetricBins: Int): [JobsStatistics!]!
rooflineHeatmap(filter: [JobFilter!]!, rows: Int!, cols: Int!, minX: Float!, minY: Float!, maxX: Float!, maxY: Float!): [[Float!]!]!
@ -2985,22 +2985,22 @@ func (ec *executionContext) field_Query_jobsStatistics_argsGroupBy(
func (ec *executionContext) field_Query_jobsStatistics_argsNumDurationBins(
ctx context.Context,
rawArgs map[string]interface{},
) (*int, error) {
) (*string, error) {
// We won't call the directive if the argument is null.
// Set call_argument_directives_with_null to true to call directives
// even if the argument is null.
_, ok := rawArgs["numDurationBins"]
if !ok {
var zeroVal *int
var zeroVal *string
return zeroVal, nil
}
ctx = graphql.WithPathContext(ctx, graphql.NewPathWithField("numDurationBins"))
if tmp, ok := rawArgs["numDurationBins"]; ok {
return ec.unmarshalOInt2ᚖint(ctx, tmp)
return ec.unmarshalOString2ᚖstring(ctx, tmp)
}
var zeroVal *int
var zeroVal *string
return zeroVal, nil
}
@ -9829,7 +9829,7 @@ func (ec *executionContext) _Query_jobsStatistics(ctx context.Context, field gra
}()
resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (interface{}, error) {
ctx = rctx // use context from middleware stack in children
return ec.resolvers.Query().JobsStatistics(rctx, fc.Args["filter"].([]*model.JobFilter), fc.Args["metrics"].([]string), fc.Args["page"].(*model.PageRequest), fc.Args["sortBy"].(*model.SortByAggregate), fc.Args["groupBy"].(*model.Aggregate), fc.Args["numDurationBins"].(*int), fc.Args["numMetricBins"].(*int))
return ec.resolvers.Query().JobsStatistics(rctx, fc.Args["filter"].([]*model.JobFilter), fc.Args["metrics"].([]string), fc.Args["page"].(*model.PageRequest), fc.Args["sortBy"].(*model.SortByAggregate), fc.Args["groupBy"].(*model.Aggregate), fc.Args["numDurationBins"].(*string), fc.Args["numMetricBins"].(*int))
})
if err != nil {
ec.Error(ctx, err)

View File

@ -357,12 +357,12 @@ func (r *queryResolver) Jobs(ctx context.Context, filter []*model.JobFilter, pag
}
// JobsStatistics is the resolver for the jobsStatistics field.
func (r *queryResolver) JobsStatistics(ctx context.Context, filter []*model.JobFilter, metrics []string, page *model.PageRequest, sortBy *model.SortByAggregate, groupBy *model.Aggregate, numDurationBins *int, numMetricBins *int) ([]*model.JobsStatistics, error) {
func (r *queryResolver) JobsStatistics(ctx context.Context, filter []*model.JobFilter, metrics []string, page *model.PageRequest, sortBy *model.SortByAggregate, groupBy *model.Aggregate, numDurationBins *string, numMetricBins *int) ([]*model.JobsStatistics, error) {
var err error
var stats []*model.JobsStatistics
// Top Level Defaults
var defaultDurationBins int = 25
var defaultDurationBins string = "1h"
var defaultMetricBins int = 10
if requireField(ctx, "totalJobs") || requireField(ctx, "totalWalltime") || requireField(ctx, "totalNodes") || requireField(ctx, "totalCores") ||

View File

@ -446,28 +446,38 @@ func (r *JobRepository) AddHistograms(
ctx context.Context,
filter []*model.JobFilter,
stat *model.JobsStatistics,
targetBinCount *int,
durationBins *string,
) (*model.JobsStatistics, error) {
start := time.Now()
// targetBinCount : Frontendargument
// -> Min Bins: 25 -> Min Resolution: By Hour
// -> In Between Bins: 50 -> Resolution by Half Hour
// 100 -> Resolution by Quarter Hour
// 150 -> Resolution by 10 Minutes
// 300 -> Resolution by 5 Minutes
// 750 -> Resolution by 2 Minutes
// -> Max Bins: 1500 -> Max Resolution: By Minute
binSizeSeconds := (90000 / *targetBinCount)
// Important Note: Fixed to 25h max display range -> Too site specific! Configurable or Extend? -> Start view with "classic" by hour histogram, zoom mostly required for "small" runtimes
var targetBinCount int
var targetBinSize int
switch {
case *durationBins == "1m": // 1 Minute Bins + Max 60 Bins -> Max 60 Minutes
targetBinCount = 60
targetBinSize = 60
case *durationBins == "10m": // 10 Minute Bins + Max 72 Bins -> Max 12 Hours
targetBinCount = 72
targetBinSize = 600
case *durationBins == "1h": // 1 Hour Bins + Max 48 Bins -> Max 48 Hours
targetBinCount = 48
targetBinSize = 3600
case *durationBins == "6h": // 6 Hour Bins + Max 12 Bins -> Max 3 Days
targetBinCount = 12
targetBinSize = 21600
case *durationBins == "12h": // 12 hour Bins + Max 14 Bins -> Max 7 Days
targetBinCount = 14
targetBinSize = 43200
default: // 24h
targetBinCount = 24
targetBinSize = 3600
}
castType := r.getCastType()
var err error
// Return X-Values always as seconds, will be formatted into minutes and hours in frontend
value := fmt.Sprintf(`CAST(ROUND(((CASE WHEN job.job_state = "running" THEN %d - job.start_time ELSE job.duration END) / %d) + 1) as %s) as value`, time.Now().Unix(), binSizeSeconds, castType)
stat.HistDuration, err = r.jobsDurationStatisticsHistogram(ctx, value, filter, binSizeSeconds, targetBinCount)
value := fmt.Sprintf(`CAST(ROUND(((CASE WHEN job.job_state = "running" THEN %d - job.start_time ELSE job.duration END) / %d) + 1) as %s) as value`, time.Now().Unix(), targetBinSize, castType)
stat.HistDuration, err = r.jobsDurationStatisticsHistogram(ctx, value, filter, targetBinSize, &targetBinCount)
if err != nil {
log.Warn("Error while loading job statistics histogram: job duration")
return nil, err

View File

@ -17,6 +17,9 @@
Icon,
Card,
Spinner,
Input,
InputGroup,
InputGroupText
} from "@sveltestrap/sveltestrap";
import {
queryStore,
@ -59,9 +62,11 @@
let showFootprint = filterPresets.cluster
? !!ccconfig[`plot_list_showFootprint:${filterPresets.cluster}`]
: !!ccconfig.plot_list_showFootprint;
let numDurationBins;
let numMetricBins;
let numDurationBins = "1h";
let numMetricBins = 10;
let durationBinOptions = ["1m","10m","1h","6h","12h"];
let metricBinOptions = [10, 20, 50, 100];
$: metricsInHistograms = selectedCluster
? ccconfig[`user_view_histogramMetrics:${selectedCluster}`] || []
@ -71,7 +76,7 @@
$: stats = queryStore({
client: client,
query: gql`
query ($jobFilters: [JobFilter!]!, $metricsInHistograms: [String!], $numDurationBins: Int, $numMetricBins: Int) {
query ($jobFilters: [JobFilter!]!, $metricsInHistograms: [String!], $numDurationBins: String, $numMetricBins: Int) {
jobsStatistics(filter: $jobFilters, metrics: $metricsInHistograms, numDurationBins: $numDurationBins , numMetricBins: $numMetricBins ) {
totalJobs
shortJobs
@ -102,38 +107,6 @@
variables: { jobFilters, metricsInHistograms, numDurationBins, numMetricBins },
});
let durationZoomState = null;
let metricZoomState = null;
let pendingDurationBinCount = null;
let pendingMetricBinCount = null;
let pendingZoomState = null;
function handleZoom(detail) {
if ( // States have to differ, causes deathloop if just set
(pendingZoomState?.x?.min !== detail?.lastZoomState?.x?.min) &&
(pendingZoomState?.y?.max !== detail?.lastZoomState?.y?.max)
) {
pendingZoomState = {...detail.lastZoomState};
}
if (detail?.durationBinCount) { // Triggers GQL
pendingDurationBinCount = detail.durationBinCount;
}
if (detail?.metricBinCount) { // Triggers GQL
pendingMetricBinCount = detail.metricBinCount;
}
};
$: if (pendingDurationBinCount !== numDurationBins) {
durationZoomState = {...pendingZoomState};
numDurationBins = pendingDurationBinCount;
}
$: if (pendingMetricBinCount !== numMetricBins) {
metricZoomState = {...pendingZoomState};
numMetricBins = pendingMetricBinCount;
}
onMount(() => filterComponent.updateFilters());
</script>
@ -153,8 +126,8 @@
{/if}
<!-- ROW2: Tools-->
<Row cols={{ xs: 1, md: 2, lg: 4}} class="mb-3">
<Col lg="2" class="mb-2 mb-lg-0">
<Row cols={{ xs: 1, md: 2, lg: 6}} class="mb-3">
<Col class="mb-2 mb-lg-0">
<ButtonGroup class="w-100">
<Button outline color="primary" on:click={() => (isSortingOpen = true)}>
<Icon name="sort-up" /> Sorting
@ -168,7 +141,7 @@
</Button>
</ButtonGroup>
</Col>
<Col lg="4" xl="6" class="mb-1 mb-lg-0">
<Col lg="4" class="mb-1 mb-lg-0">
<Filters
{filterPresets}
{matchedJobs}
@ -183,12 +156,27 @@
}}
/>
</Col>
<Col lg="3" xl="2" class="mb-2 mb-lg-0">
<Col class="mb-2 mb-lg-0">
<InputGroup>
<InputGroupText>
<Icon name="bar-chart-line-fill" />
</InputGroupText>
<InputGroupText>
Duration Bin Size
</InputGroupText>
<Input type="select" bind:value={numDurationBins} style="max-width: 120px;">
{#each durationBinOptions as dbin}
<option value={dbin}>{dbin}</option>
{/each}
</Input>
</InputGroup>
</Col>
<Col class="mb-2 mb-lg-0">
<TextFilter
on:set-filter={({ detail }) => filterComponent.updateFilters(detail)}
/>
</Col>
<Col lg="3" xl="2" class="mb-1 mb-lg-0">
<Col class="mb-1 mb-lg-0">
<Refresher on:refresh={() => {
jobList.refreshJobs()
jobList.refreshAllMetrics()
@ -248,16 +236,13 @@
<Col class="px-1">
{#key $stats.data.jobsStatistics[0].histDuration}
<Histogram
on:zoom={({detail}) => { handleZoom(detail) }}
data={convert2uplot($stats.data.jobsStatistics[0].histDuration)}
title="Duration Distribution"
xlabel="Job Runtimes"
xunit="Runtime"
ylabel="Number of Jobs"
yunit="Jobs"
lastBinCount={pendingDurationBinCount}
{durationZoomState}
zoomableHistogram
usesBins
xtime
/>
{/key}
@ -278,16 +263,32 @@
</Row>
<!-- ROW4+5: Selectable Histograms -->
<Row cols={{ xs: 1, md: 5}}>
<Col>
<Row>
<Col xs="12" md="3" lg="2" class="mb-2 mb-md-0">
<Button
outline
color="secondary"
class="w-100"
on:click={() => (isHistogramSelectionOpen = true)}
>
<Icon name="bar-chart-line" /> Select Histograms
</Button>
</Col>
<Col xs="12" md="9" lg="10" class="mb-2 mb-md-0">
<InputGroup>
<InputGroupText>
<Icon name="bar-chart-line-fill" />
</InputGroupText>
<InputGroupText>
Metric Bins
</InputGroupText>
<Input type="select" bind:value={numMetricBins} style="max-width: 120px;">
{#each metricBinOptions as mbin}
<option value={mbin}>{mbin}</option>
{/each}
</Input>
</InputGroup>
</Col>
</Row>
{#if metricsInHistograms?.length > 0}
{#if $stats.error}
@ -312,17 +313,13 @@
itemsPerRow={3}
>
<Histogram
on:zoom={({detail}) => { handleZoom(detail) }}
data={convert2uplot(item.data)}
usesBins={true}
title="Distribution of '{item.metric} ({item.stat})' footprints"
xlabel={`${item.metric} bin maximum ${item?.unit ? `[${item.unit}]` : ``}`}
xunit={item.unit}
ylabel="Number of Jobs"
yunit="Jobs"
lastBinCount={pendingMetricBinCount}
{metricZoomState}
zoomableHistogram
usesBins
/>
</PlotGrid>
{/key}

View File

@ -15,7 +15,7 @@
<script>
import uPlot from "uplot";
import { getContext, onMount, onDestroy, createEventDispatcher } from "svelte";
import { onMount, onDestroy } from "svelte";
import { formatNumber } from "../units.js";
import { Card } from "@sveltestrap/sveltestrap";
@ -29,21 +29,13 @@
export let xtime = false;
export let ylabel = "";
export let yunit = "";
export let zoomState = null;
export let lastBinCount = null;
export let zoomableHistogram = false;
const { bars } = uPlot.paths;
const dispatch = createEventDispatcher();
const drawStyles = {
bars: 1,
points: 2,
};
// TimeBins: Include Hour "24-25"
const binCounts = xtime ? [25, 50, 100, 150, 300, 750, 1500] : [10, 20, 50, 100, 200]; // , 500, 1000
function formatTime(t) {
if (t !== null) {
if (isNaN(t)) {
@ -52,7 +44,6 @@
const tAbs = Math.abs(t);
const h = Math.floor(tAbs / 3600);
const m = Math.floor((tAbs % 3600) / 60);
// Re-Add "negativity" to time ticks only as string, so that if-cases work as intended
if (h == 0) return `${m}m`;
else if (m == 0) return `${h}h`;
else return `${h}:${m}h`;
@ -138,45 +129,6 @@
function render() {
let opts = {
hooks: {
init: [
(u) => {
if (zoomableHistogram) {
u.over.addEventListener("dblclick", (e) => {
console.log('Dispatch Reset')
dispatch('zoom', {
lastZoomState: {
x: { time: false },
y: { auto: true }
}
});
});
}
},
],
setScale: [
(u, key) => {
if (key === 'x') {
if (zoomableHistogram) {
const numX = (u.series[0].idxs[1] - u.series[0].idxs[0])
if (xtime && numX <= 12 && lastBinCount !== 1500) {
// console.log("Dispatch for Duration: ", numX, lastBinCount, binCounts[binCounts.indexOf(lastBinCount) + 1])
dispatch('zoom', {
durationBinCount: binCounts[binCounts.indexOf(lastBinCount) + 1],
lastZoomState: u?.scales,
});
} else if (!xtime && numX <= 6 && lastBinCount !== 200) {
// console.log("Dispatch for Metrics: ", numX, lastBinCount, binCounts[binCounts.indexOf(lastBinCount) + 1])
dispatch('zoom', {
metricBinCount: binCounts[binCounts.indexOf(lastBinCount) + 1],
lastZoomState: u?.scales,
});
};
}
};
},
]
},
width: width,
height: height,
title: title,
@ -202,7 +154,7 @@
label: xlabel,
labelGap: 10,
size: 25,
incrs: xtime ? [60, 120, 300, 600, 900, 1800, 3600, 7200, 14400, 18000] : [1, 2, 5, 10, 20, 50, 100, 200, 500, 1000, 2000, 5000, 10000],
incrs: xtime ? [60, 120, 300, 600, 1800, 3600, 7200, 14400, 18000, 21600, 43200, 86400] : [1, 2, 5, 10, 20, 50, 100, 200, 500, 1000, 2000, 5000, 10000],
border: {
show: true,
stroke: "#000000",
@ -244,7 +196,11 @@
{
label: xunit !== "" ? xunit : null,
value: (u, ts, sidx, didx) => {
if (usesBins) {
if (usesBins && xtime) {
const min = u.data[sidx][didx - 1] ? formatTime(u.data[sidx][didx - 1]) : 0;
const max = formatTime(u.data[sidx][didx]);
ts = min + "-" + max; // narrow spaces
} else if (usesBins) {
const min = u.data[sidx][didx - 1] ? u.data[sidx][didx - 1] : 0;
const max = u.data[sidx][didx];
ts = min + "-" + max; // narrow spaces
@ -273,11 +229,6 @@
],
};
if (zoomableHistogram && zoomState) {
console.log("Apply ZoomState ...", zoomState)
opts.scales = {...zoomState}
}
uplot = new uPlot(opts, data, plotWrapper);
}