From 6683a350aa4a837ad0d4541a5326a011518e4615 Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Thu, 23 Jan 2025 12:23:29 +0100 Subject: [PATCH] initial duration histogram zoom in frontend - metric zoom todo - keeping last zoomState does not work --- internal/graph/schema.resolvers.go | 12 +++ internal/repository/stats.go | 71 +++++++++++++--- web/frontend/src/User.root.svelte | 55 ++++++++++-- .../src/generic/plots/Histogram.svelte | 84 +++++++++++++++++-- web/frontend/src/generic/utils.js | 2 +- 5 files changed, 200 insertions(+), 24 deletions(-) diff --git a/internal/graph/schema.resolvers.go b/internal/graph/schema.resolvers.go index aae5812..98f9a83 100644 --- a/internal/graph/schema.resolvers.go +++ b/internal/graph/schema.resolvers.go @@ -394,6 +394,12 @@ func (r *queryResolver) JobsStatistics(ctx context.Context, filter []*model.JobF } if requireField(ctx, "histDuration") || requireField(ctx, "histNumNodes") || requireField(ctx, "histNumCores") || requireField(ctx, "histNumAccs") { + + if numDurationBins == nil { + binCount := 24 + numDurationBins = &binCount + } + if groupBy == nil { stats[0], err = r.Repo.AddHistograms(ctx, filter, stats[0], numDurationBins) if err != nil { @@ -405,6 +411,12 @@ func (r *queryResolver) JobsStatistics(ctx context.Context, filter []*model.JobF } if requireField(ctx, "histMetrics") { + + if numMetricBins == nil { + binCount := 10 + numMetricBins = &binCount + } + if groupBy == nil { stats[0], err = r.Repo.AddMetricHistograms(ctx, filter, metrics, stats[0], numMetricBins) if err != nil { diff --git a/internal/repository/stats.go b/internal/repository/stats.go index 5a055a4..12caa6c 100644 --- a/internal/repository/stats.go +++ b/internal/repository/stats.go @@ -454,23 +454,19 @@ func (r *JobRepository) AddHistograms( // targetBinCount : Frontendargument // -> Min Bins: 24 -> Min Resolution: By Hour // -> In Between Bins: 48 -> Resolution by Half Hour - // 96 -> Resolution by Quarter Hour (Skip for Now) + // 96 -> Resolution by Quarter Hour // 144 -> Resolution by 10 Minutes // 288 -> Resolution by 5 Minutes - // 720 -> Resolution by 2 Minutes (SKip for Now) + // 720 -> Resolution by 2 Minutes // -> Max Bins: 1440 -> Max Resolution: By Minute - if targetBinCount == nil { - binCount := 24 - targetBinCount = &binCount - } binSizeSeconds := (86400 / *targetBinCount) castType := r.getCastType() var err error // Return X-Values always as seconds, will be formatted into minutes and hours in frontend value := fmt.Sprintf(`CAST(ROUND(((CASE WHEN job.job_state = "running" THEN %d - job.start_time ELSE job.duration END) / %d) + 1) as %s) as value`, time.Now().Unix(), binSizeSeconds, castType) - stat.HistDuration, err = r.jobsStatisticsHistogram(ctx, value, filter) + stat.HistDuration, err = r.jobsDurationStatisticsHistogram(ctx, value, filter, binSizeSeconds, targetBinCount) if err != nil { log.Warn("Error while loading job statistics histogram: job duration") return nil, err @@ -508,10 +504,7 @@ func (r *JobRepository) AddMetricHistograms( ) (*model.JobsStatistics, error) { start := time.Now() - if targetBinCount == nil { - binCount := 10 - targetBinCount = &binCount - } + log.Debugf(">>> HELLO ADD HISTO Metrics: Target %d", *targetBinCount) // Running Jobs Only: First query jobdata from sqlite, then query data and make bins for _, f := range filter { @@ -563,6 +556,7 @@ func (r *JobRepository) jobsStatisticsHistogram( } points := make([]*model.HistoPoint, 0) + // is it possible to introduce zero values here? requires info about bincount for rows.Next() { point := model.HistoPoint{} if err := rows.Scan(&point.Value, &point.Count); err != nil { @@ -576,6 +570,61 @@ func (r *JobRepository) jobsStatisticsHistogram( return points, nil } +func (r *JobRepository) jobsDurationStatisticsHistogram( + ctx context.Context, + value string, + filters []*model.JobFilter, + binSizeSeconds int, + targetBinCount *int, +) ([]*model.HistoPoint, error) { + start := time.Now() + query, qerr := SecurityCheck(ctx, + sq.Select(value, "COUNT(job.id) AS count").From("job")) + + if qerr != nil { + return nil, qerr + } + + // Setup Array + points := make([]*model.HistoPoint, 0) + for i := 1; i <= *targetBinCount; i++ { + point := model.HistoPoint{Value: i * binSizeSeconds, Count: 0} + points = append(points, &point) + } + + for _, f := range filters { + query = BuildWhereClause(f, query) + } + + rows, err := query.GroupBy("value").RunWith(r.DB).Query() + if err != nil { + log.Error("Error while running query") + return nil, err + } + + // Fill Array at matching $Value + for rows.Next() { + point := model.HistoPoint{} + if err := rows.Scan(&point.Value, &point.Count); err != nil { + log.Warn("Error while scanning rows") + return nil, err + } + + for _, e := range points { + if e.Value == (point.Value * binSizeSeconds) { + // Note: + // Matching on unmodified integer value (and multiplying point.Value by binSizeSeconds after match) + // causes frontend to loop into highest targetBinCount, due to zoom condition instantly being fullfilled (cause unknown) + e.Count = point.Count + break + } + } + } + + log.Debugf("Timer jobsStatisticsHistogram %s", time.Since(start)) + return points, nil +} + func (r *JobRepository) jobsMetricStatisticsHistogram( ctx context.Context, metric string, diff --git a/web/frontend/src/User.root.svelte b/web/frontend/src/User.root.svelte index 4c958a9..853a89b 100644 --- a/web/frontend/src/User.root.svelte +++ b/web/frontend/src/User.root.svelte @@ -59,6 +59,9 @@ let showFootprint = filterPresets.cluster ? !!ccconfig[`plot_list_showFootprint:${filterPresets.cluster}`] : !!ccconfig.plot_list_showFootprint; + let numDurationBins; + let numMetricBins; + $: metricsInHistograms = selectedCluster ? ccconfig[`user_view_histogramMetrics:${selectedCluster}`] || [] @@ -68,8 +71,8 @@ $: stats = queryStore({ client: client, query: gql` - query ($jobFilters: [JobFilter!]!, $metricsInHistograms: [String!]) { - jobsStatistics(filter: $jobFilters, metrics: $metricsInHistograms) { + query ($jobFilters: [JobFilter!]!, $metricsInHistograms: [String!], $numDurationBins: Int, $numMetricBins: Int) { + jobsStatistics(filter: $jobFilters, metrics: $metricsInHistograms, numDurationBins: $numDurationBins , numMetricBins: $numMetricBins ) { totalJobs shortJobs totalWalltime @@ -96,9 +99,41 @@ } } `, - variables: { jobFilters, metricsInHistograms }, + variables: { jobFilters, metricsInHistograms, numDurationBins, numMetricBins }, }); + let durationZoomState = null; + let metricZoomState = null; + let pendingDurationBinCount = null; + let pendingMetricBinCount = null; + let pendingZoomState = null; + function handleZoom(detail) { + if ( // States have to differ, causes deathloop if just set + (pendingZoomState?.x?.min !== detail?.lastZoomState?.x?.min) && + (pendingZoomState?.y?.max !== detail?.lastZoomState?.y?.max) + ) { + pendingZoomState = {...detail.lastZoomState}; + } + + if (detail?.durationBinCount) { // Triggers GQL + pendingDurationBinCount = detail.durationBinCount; + } + + if (detail?.metricBinCount) { // Triggers GQL + pendingMetricBinCount = detail.metricBinCount; + } + }; + + $: if (pendingDurationBinCount !== numDurationBins) { + durationZoomState = {...pendingZoomState}; + numDurationBins = pendingDurationBinCount; + } + + $: if (pendingMetricBinCount !== numMetricBins) { + metricZoomState = {...pendingZoomState}; + numMetricBins = pendingMetricBinCount; + } + onMount(() => filterComponent.updateFilters()); @@ -213,13 +248,17 @@ {#key $stats.data.jobsStatistics[0].histDuration} { handleZoom(detail) }} data={convert2uplot($stats.data.jobsStatistics[0].histDuration)} title="Duration Distribution" - xlabel="Current Runtimes (Hours)" - xtime={true} - xunit="Hours" + xlabel="Job Runtimes" + xunit="Runtime" ylabel="Number of Jobs" yunit="Jobs" + lastBinCount={pendingDurationBinCount} + {durationZoomState} + zoomableHistogram + xtime /> {/key} @@ -273,6 +312,7 @@ itemsPerRow={3} > { handleZoom(detail) }} data={convert2uplot(item.data)} usesBins={true} title="Distribution of '{item.metric} ({item.stat})' footprints" @@ -280,6 +320,9 @@ xunit={item.unit} ylabel="Number of Jobs" yunit="Jobs" + lastBinCount={pendingMetricBinCount} + {metricZoomState} + zoomableHistogram /> {/key} diff --git a/web/frontend/src/generic/plots/Histogram.svelte b/web/frontend/src/generic/plots/Histogram.svelte index fbc8f14..51ca543 100644 --- a/web/frontend/src/generic/plots/Histogram.svelte +++ b/web/frontend/src/generic/plots/Histogram.svelte @@ -15,8 +15,8 @@