From f933cad87fb29775c45b61043bb9e39a48b399ae Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Tue, 29 Aug 2023 14:01:01 +0200 Subject: [PATCH] feat: add select to status view pie charts - 'Jobs' as generic default value for top lists - Prepare histograms for cores and accs in schema --- api/schema.graphqls | 4 +- internal/graph/generated/generated.go | 134 +++++++++++++++++++++++- internal/graph/model/models_gen.go | 6 +- internal/repository/stats.go | 35 ++++--- web/frontend/src/Status.root.svelte | 142 ++++++++++++++++++-------- web/frontend/src/plots/Pie.svelte | 4 +- 6 files changed, 267 insertions(+), 58 deletions(-) diff --git a/api/schema.graphqls b/api/schema.graphqls index 303ede5..729c712 100644 --- a/api/schema.graphqls +++ b/api/schema.graphqls @@ -167,7 +167,7 @@ type TimeWeights { } enum Aggregate { USER, PROJECT, CLUSTER } -enum SortByAggregate { WALLTIME, TOTALNODES, NODEHOURS, TOTALCORES, COREHOURS, TOTALACCS, ACCHOURS } +enum SortByAggregate { WALLTIME, TOTALJOBS, TOTALNODES, NODEHOURS, TOTALCORES, COREHOURS, TOTALACCS, ACCHOURS } type NodeMetrics { host: String! @@ -301,6 +301,8 @@ type JobsStatistics { totalAccHours: Int! # Sum of the gpu hours of all matched jobs histDuration: [HistoPoint!]! # value: hour, count: number of jobs with a rounded duration of value histNumNodes: [HistoPoint!]! # value: number of nodes, count: number of jobs with that number of nodes + histNumCores: [HistoPoint!]! # value: number of cores, count: number of jobs with that number of cores + histNumAccs: [HistoPoint!]! # value: number of accs, count: number of jobs with that number of accs } input PageRequest { diff --git a/internal/graph/generated/generated.go b/internal/graph/generated/generated.go index f06698c..efbfae9 100644 --- a/internal/graph/generated/generated.go +++ b/internal/graph/generated/generated.go @@ -141,6 +141,8 @@ type ComplexityRoot struct { JobsStatistics struct { HistDuration func(childComplexity int) int + HistNumAccs func(childComplexity int) int + HistNumCores func(childComplexity int) int HistNumNodes func(childComplexity int) int ID func(childComplexity int) int Name func(childComplexity int) int @@ -728,6 +730,20 @@ func (e *executableSchema) Complexity(typeName, field string, childComplexity in return e.complexity.JobsStatistics.HistDuration(childComplexity), true + case "JobsStatistics.histNumAccs": + if e.complexity.JobsStatistics.HistNumAccs == nil { + break + } + + return e.complexity.JobsStatistics.HistNumAccs(childComplexity), true + + case "JobsStatistics.histNumCores": + if e.complexity.JobsStatistics.HistNumCores == nil { + break + } + + return e.complexity.JobsStatistics.HistNumCores(childComplexity), true + case "JobsStatistics.histNumNodes": if e.complexity.JobsStatistics.HistNumNodes == nil { break @@ -1751,7 +1767,7 @@ type TimeWeights { } enum Aggregate { USER, PROJECT, CLUSTER } -enum SortByAggregate { WALLTIME, TOTALNODES, NODEHOURS, TOTALCORES, COREHOURS, TOTALACCS, ACCHOURS } +enum SortByAggregate { WALLTIME, TOTALJOBS, TOTALNODES, NODEHOURS, TOTALCORES, COREHOURS, TOTALACCS, ACCHOURS } type NodeMetrics { host: String! @@ -1885,6 +1901,8 @@ type JobsStatistics { totalAccHours: Int! # Sum of the gpu hours of all matched jobs histDuration: [HistoPoint!]! # value: hour, count: number of jobs with a rounded duration of value histNumNodes: [HistoPoint!]! # value: number of nodes, count: number of jobs with that number of nodes + histNumCores: [HistoPoint!]! # value: number of cores, count: number of jobs with that number of cores + histNumAccs: [HistoPoint!]! # value: number of accs, count: number of jobs with that number of accs } input PageRequest { @@ -5522,6 +5540,106 @@ func (ec *executionContext) fieldContext_JobsStatistics_histNumNodes(ctx context return fc, nil } +func (ec *executionContext) _JobsStatistics_histNumCores(ctx context.Context, field graphql.CollectedField, obj *model.JobsStatistics) (ret graphql.Marshaler) { + fc, err := ec.fieldContext_JobsStatistics_histNumCores(ctx, field) + if err != nil { + return graphql.Null + } + ctx = graphql.WithFieldContext(ctx, fc) + defer func() { + if r := recover(); r != nil { + ec.Error(ctx, ec.Recover(ctx, r)) + ret = graphql.Null + } + }() + resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (interface{}, error) { + ctx = rctx // use context from middleware stack in children + return obj.HistNumCores, nil + }) + if err != nil { + ec.Error(ctx, err) + return graphql.Null + } + if resTmp == nil { + if !graphql.HasFieldError(ctx, fc) { + ec.Errorf(ctx, "must not be null") + } + return graphql.Null + } + res := resTmp.([]*model.HistoPoint) + fc.Result = res + return ec.marshalNHistoPoint2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐHistoPointᚄ(ctx, field.Selections, res) +} + +func (ec *executionContext) fieldContext_JobsStatistics_histNumCores(ctx context.Context, field graphql.CollectedField) (fc *graphql.FieldContext, err error) { + fc = &graphql.FieldContext{ + Object: "JobsStatistics", + Field: field, + IsMethod: false, + IsResolver: false, + Child: func(ctx context.Context, field graphql.CollectedField) (*graphql.FieldContext, error) { + switch field.Name { + case "count": + return ec.fieldContext_HistoPoint_count(ctx, field) + case "value": + return ec.fieldContext_HistoPoint_value(ctx, field) + } + return nil, fmt.Errorf("no field named %q was found under type HistoPoint", field.Name) + }, + } + return fc, nil +} + +func (ec *executionContext) _JobsStatistics_histNumAccs(ctx context.Context, field graphql.CollectedField, obj *model.JobsStatistics) (ret graphql.Marshaler) { + fc, err := ec.fieldContext_JobsStatistics_histNumAccs(ctx, field) + if err != nil { + return graphql.Null + } + ctx = graphql.WithFieldContext(ctx, fc) + defer func() { + if r := recover(); r != nil { + ec.Error(ctx, ec.Recover(ctx, r)) + ret = graphql.Null + } + }() + resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (interface{}, error) { + ctx = rctx // use context from middleware stack in children + return obj.HistNumAccs, nil + }) + if err != nil { + ec.Error(ctx, err) + return graphql.Null + } + if resTmp == nil { + if !graphql.HasFieldError(ctx, fc) { + ec.Errorf(ctx, "must not be null") + } + return graphql.Null + } + res := resTmp.([]*model.HistoPoint) + fc.Result = res + return ec.marshalNHistoPoint2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐHistoPointᚄ(ctx, field.Selections, res) +} + +func (ec *executionContext) fieldContext_JobsStatistics_histNumAccs(ctx context.Context, field graphql.CollectedField) (fc *graphql.FieldContext, err error) { + fc = &graphql.FieldContext{ + Object: "JobsStatistics", + Field: field, + IsMethod: false, + IsResolver: false, + Child: func(ctx context.Context, field graphql.CollectedField) (*graphql.FieldContext, error) { + switch field.Name { + case "count": + return ec.fieldContext_HistoPoint_count(ctx, field) + case "value": + return ec.fieldContext_HistoPoint_value(ctx, field) + } + return nil, fmt.Errorf("no field named %q was found under type HistoPoint", field.Name) + }, + } + return fc, nil +} + func (ec *executionContext) _MetricConfig_name(ctx context.Context, field graphql.CollectedField, obj *schema.MetricConfig) (ret graphql.Marshaler) { fc, err := ec.fieldContext_MetricConfig_name(ctx, field) if err != nil { @@ -7309,6 +7427,10 @@ func (ec *executionContext) fieldContext_Query_jobsStatistics(ctx context.Contex return ec.fieldContext_JobsStatistics_histDuration(ctx, field) case "histNumNodes": return ec.fieldContext_JobsStatistics_histNumNodes(ctx, field) + case "histNumCores": + return ec.fieldContext_JobsStatistics_histNumCores(ctx, field) + case "histNumAccs": + return ec.fieldContext_JobsStatistics_histNumAccs(ctx, field) } return nil, fmt.Errorf("no field named %q was found under type JobsStatistics", field.Name) }, @@ -12778,6 +12900,16 @@ func (ec *executionContext) _JobsStatistics(ctx context.Context, sel ast.Selecti if out.Values[i] == graphql.Null { out.Invalids++ } + case "histNumCores": + out.Values[i] = ec._JobsStatistics_histNumCores(ctx, field, obj) + if out.Values[i] == graphql.Null { + out.Invalids++ + } + case "histNumAccs": + out.Values[i] = ec._JobsStatistics_histNumAccs(ctx, field, obj) + if out.Values[i] == graphql.Null { + out.Invalids++ + } default: panic("unknown field " + strconv.Quote(field.Name)) } diff --git a/internal/graph/model/models_gen.go b/internal/graph/model/models_gen.go index 3997b2d..99fc96f 100644 --- a/internal/graph/model/models_gen.go +++ b/internal/graph/model/models_gen.go @@ -99,6 +99,8 @@ type JobsStatistics struct { TotalAccHours int `json:"totalAccHours"` HistDuration []*HistoPoint `json:"histDuration"` HistNumNodes []*HistoPoint `json:"histNumNodes"` + HistNumCores []*HistoPoint `json:"histNumCores"` + HistNumAccs []*HistoPoint `json:"histNumAccs"` } type MetricFootprints struct { @@ -195,6 +197,7 @@ type SortByAggregate string const ( SortByAggregateWalltime SortByAggregate = "WALLTIME" + SortByAggregateTotaljobs SortByAggregate = "TOTALJOBS" SortByAggregateTotalnodes SortByAggregate = "TOTALNODES" SortByAggregateNodehours SortByAggregate = "NODEHOURS" SortByAggregateTotalcores SortByAggregate = "TOTALCORES" @@ -205,6 +208,7 @@ const ( var AllSortByAggregate = []SortByAggregate{ SortByAggregateWalltime, + SortByAggregateTotaljobs, SortByAggregateTotalnodes, SortByAggregateNodehours, SortByAggregateTotalcores, @@ -215,7 +219,7 @@ var AllSortByAggregate = []SortByAggregate{ func (e SortByAggregate) IsValid() bool { switch e { - case SortByAggregateWalltime, SortByAggregateTotalnodes, SortByAggregateNodehours, SortByAggregateTotalcores, SortByAggregateCorehours, SortByAggregateTotalaccs, SortByAggregateAcchours: + case SortByAggregateWalltime, SortByAggregateTotaljobs, SortByAggregateTotalnodes, SortByAggregateNodehours, SortByAggregateTotalcores, SortByAggregateCorehours, SortByAggregateTotalaccs, SortByAggregateAcchours: return true } return false diff --git a/internal/repository/stats.go b/internal/repository/stats.go index c766238..4585f44 100644 --- a/internal/repository/stats.go +++ b/internal/repository/stats.go @@ -24,6 +24,7 @@ var groupBy2column = map[model.Aggregate]string{ } var sortBy2column = map[model.SortByAggregate]string{ + model.SortByAggregateTotaljobs: "totalJobs", model.SortByAggregateWalltime: "totalWalltime", model.SortByAggregateTotalnodes: "totalNodes", model.SortByAggregateNodehours: "totalNodeHours", @@ -71,7 +72,7 @@ func (r *JobRepository) buildStatsQuery( if col != "" { // Scan columns: id, totalJobs, totalWalltime, totalNodes, totalNodeHours, totalCores, totalCoreHours, totalAccs, totalAccHours - query = sq.Select(col, "COUNT(job.id)", + query = sq.Select(col, "COUNT(job.id) as totalJobs", fmt.Sprintf("CAST(ROUND(SUM(job.duration) / 3600) as %s) as totalWalltime", castType), fmt.Sprintf("CAST(SUM(job.num_nodes) as %s) as totalNodes", castType), fmt.Sprintf("CAST(ROUND(SUM(job.duration * job.num_nodes) / 3600) as %s) as totalNodeHours", castType), @@ -168,8 +169,15 @@ func (r *JobRepository) JobsStatsGrouped( } if id.Valid { - var totalCores, totalCoreHours, totalAccs, totalAccHours int + var totalJobs, totalNodes, totalNodeHours, totalCores, totalCoreHours, totalAccs, totalAccHours int + if jobs.Valid { + totalJobs = int(jobs.Int64) + } + + if nodes.Valid { + totalNodes = int(nodes.Int64) + } if cores.Valid { totalCores = int(cores.Int64) } @@ -177,6 +185,9 @@ func (r *JobRepository) JobsStatsGrouped( totalAccs = int(accs.Int64) } + if nodeHours.Valid { + totalNodeHours = int(nodeHours.Int64) + } if coreHours.Valid { totalCoreHours = int(coreHours.Int64) } @@ -190,8 +201,10 @@ func (r *JobRepository) JobsStatsGrouped( &model.JobsStatistics{ ID: id.String, Name: name, - TotalJobs: int(jobs.Int64), + TotalJobs: totalJobs, TotalWalltime: int(walltime.Int64), + TotalNodes: totalNodes, + TotalNodeHours: totalNodeHours, TotalCores: totalCores, TotalCoreHours: totalCoreHours, TotalAccs: totalAccs, @@ -202,6 +215,8 @@ func (r *JobRepository) JobsStatsGrouped( ID: id.String, TotalJobs: int(jobs.Int64), TotalWalltime: int(walltime.Int64), + TotalNodes: totalNodes, + TotalNodeHours: totalNodeHours, TotalCores: totalCores, TotalCoreHours: totalCoreHours, TotalAccs: totalAccs, @@ -228,16 +243,11 @@ func (r *JobRepository) jobsStats( } if jobs.Valid { - var totalCoreHours, totalAccHours int - // var totalCores, totalAccs int - - // if cores.Valid { - // totalCores = int(cores.Int64) - // } - // if accs.Valid { - // totalAccs = int(accs.Int64) - // } + var totalNodeHours, totalCoreHours, totalAccHours int + if nodeHours.Valid { + totalNodeHours = int(nodeHours.Int64) + } if coreHours.Valid { totalCoreHours = int(coreHours.Int64) } @@ -248,6 +258,7 @@ func (r *JobRepository) jobsStats( &model.JobsStatistics{ TotalJobs: int(jobs.Int64), TotalWalltime: int(walltime.Int64), + TotalNodeHours: totalNodeHours, TotalCoreHours: totalCoreHours, TotalAccHours: totalAccHours}) } diff --git a/web/frontend/src/Status.root.svelte b/web/frontend/src/Status.root.svelte index 06a1893..4c9e8d5 100644 --- a/web/frontend/src/Status.root.svelte +++ b/web/frontend/src/Status.root.svelte @@ -14,6 +14,14 @@ let plotWidths = [], colWidth1 = 0, colWidth2 let from = new Date(Date.now() - 5 * 60 * 1000), to = new Date(Date.now()) + const topOptions = [ + {key: 'totalJobs', label: 'Jobs'}, + {key: 'totalNodes', label: 'Nodes'}, + {key: 'totalCores', label: 'Cores'}, + {key: 'totalAccs', label: 'Accelerators'}, + ] + let topProjectSelection = topOptions[0] // Default: Jobs + let topUserSelection = topOptions[0] // Default: Jobs const client = getContextClient(); $: mainQuery = queryStore({ @@ -51,29 +59,33 @@ $: topUserQuery = queryStore({ client: client, query: gql` - query($filter: [JobFilter!]!, $paging: PageRequest!) { - topUser: jobsStatistics(filter: $filter, page: $paging, sortBy: TOTALCORES, groupBy: USER) { + query($filter: [JobFilter!]!, $paging: PageRequest!, $sortBy: SortByAggregate!) { + topUser: jobsStatistics(filter: $filter, page: $paging, sortBy: $sortBy, groupBy: USER) { id + totalJobs totalNodes totalCores + totalAccs } } `, - variables: { filter: [{ state: ['running'] }, { cluster: { eq: cluster } }], paging } + variables: { filter: [{ state: ['running'] }, { cluster: { eq: cluster } }], paging, sortBy: topUserSelection.key.toUpperCase() } }) $: topProjectQuery = queryStore({ client: client, query: gql` - query($filter: [JobFilter!]!, $paging: PageRequest!) { - topProjects: jobsStatistics(filter: $filter, page: $paging, sortBy: TOTALCORES, groupBy: PROJECT) { + query($filter: [JobFilter!]!, $paging: PageRequest!, $sortBy: SortByAggregate!) { + topProjects: jobsStatistics(filter: $filter, page: $paging, sortBy: $sortBy, groupBy: PROJECT) { id + totalJobs totalNodes totalCores + totalAccs } } `, - variables: { filter: [{ state: ['running'] }, { cluster: { eq: cluster } }], paging } + variables: { filter: [{ state: ['running'] }, { cluster: { eq: cluster } }], paging, sortBy: topProjectSelection.key.toUpperCase() } }) const sumUp = (data, subcluster, metric) => data.reduce((sum, node) => node.subCluster == subcluster @@ -188,51 +200,99 @@
-

Top Users

- {#key $topUserQuery.data} +

Top Users on {cluster.charAt(0).toUpperCase() + cluster.slice(1)}

+ {#if $topUserQuery.fetching} + + {:else if $topUserQuery.error} + {$topUserQuery.error.message} + {:else} tu.totalCores)} + sliceLabel={topUserSelection.label} + quantities={$topUserQuery.data.topUser.map((tu) => tu[topUserSelection.key])} entities={$topUserQuery.data.topUser.map((tu) => tu.id)} /> - {/key} + {/if}
- - - {#each $topUserQuery.data.topUser as { id, totalCores, totalNodes }, i} - - - - - - {/each} -
LegendUser NameNumber of Cores
{id}{totalCores}
- - -

Top Projects

- {#key $topProjectQuery.data} - tp.totalCores)} - entities={$topProjectQuery.data.topProjects.map((tp) => tp.id)} - /> + {#key $topUserQuery.data} + {#if $topUserQuery.fetching} + + {:else if $topUserQuery.error} + {$topUserQuery.error.message} + {:else} + + + + + + + {#each $topUserQuery.data.topUser as tu, i} + + + + + + {/each} +
LegendUser NameNumber of + +
{tu.id}{tu[topUserSelection.key]}
+ {/if} {/key} + +

Top Projects on {cluster.charAt(0).toUpperCase() + cluster.slice(1)}

+ {#if $topProjectQuery.fetching} + + {:else if $topProjectQuery.error} + {$topProjectQuery.error.message} + {:else} + tp[topProjectSelection.key])} + entities={$topProjectQuery.data.topProjects.map((tp) => tp.id)} + /> + {/if} + - - - {#each $topProjectQuery.data.topProjects as { id, totalCores, totalNodes }, i} - - - - - - {/each} -
LegendProject CodeNumber of Cores
{id}{totalCores}
+ {#key $topProjectQuery.data} + {#if $topProjectQuery.fetching} + + {:else if $topProjectQuery.error} + {$topProjectQuery.error.message} + {:else} + + + + + + + {#each $topProjectQuery.data.topProjects as tp, i} + + + + + + {/each} +
LegendProject CodeNumber of + +
{tp.id}{tp[topProjectSelection.key]}
+ {/if} + {/key}
diff --git a/web/frontend/src/plots/Pie.svelte b/web/frontend/src/plots/Pie.svelte index 6355f09..11dc2c9 100644 --- a/web/frontend/src/plots/Pie.svelte +++ b/web/frontend/src/plots/Pie.svelte @@ -43,14 +43,14 @@ export let entities export let displayLegend = false - const data = { + $: data = { labels: entities, datasets: [ { label: sliceLabel, data: quantities, fill: 1, - backgroundColor: colors.slice(0, quantities.length), + backgroundColor: colors.slice(0, quantities.length) } ] }