From f7571211fdc46fa601c2cc65a0ee17827598b7ef Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Fri, 21 Jul 2023 16:33:53 +0200 Subject: [PATCH 01/19] initial branch commit --- api/schema.graphqls | 2 +- internal/graph/generated/generated.go | 28 +++++++++++++------------- internal/graph/model/models_gen.go | 4 ++-- internal/graph/util.go | 17 ++++++++++++++-- internal/metricdata/cc-metric-store.go | 2 +- internal/metricdata/metricdata.go | 4 ++-- web/frontend/src/Analysis.root.svelte | 6 +++--- 7 files changed, 38 insertions(+), 25 deletions(-) diff --git a/api/schema.graphqls b/api/schema.graphqls index 82681c0..4802117 100644 --- a/api/schema.graphqls +++ b/api/schema.graphqls @@ -156,7 +156,7 @@ type MetricFootprints { } type Footprints { - nodehours: [NullableFloat!]! + timeweights: [NullableFloat!]! metrics: [MetricFootprints!]! } diff --git a/internal/graph/generated/generated.go b/internal/graph/generated/generated.go index 229c6b5..1f3c349 100644 --- a/internal/graph/generated/generated.go +++ b/internal/graph/generated/generated.go @@ -68,8 +68,8 @@ type ComplexityRoot struct { } Footprints struct { - Metrics func(childComplexity int) int - Nodehours func(childComplexity int) int + Metrics func(childComplexity int) int + Timeweights func(childComplexity int) int } HistoPoint struct { @@ -406,12 +406,12 @@ func (e *executableSchema) Complexity(typeName, field string, childComplexity in return e.complexity.Footprints.Metrics(childComplexity), true - case "Footprints.nodehours": - if e.complexity.Footprints.Nodehours == nil { + case "Footprints.timeweights": + if e.complexity.Footprints.Timeweights == nil { break } - return e.complexity.Footprints.Nodehours(childComplexity), true + return e.complexity.Footprints.Timeweights(childComplexity), true case "HistoPoint.count": if e.complexity.HistoPoint.Count == nil { @@ -1666,7 +1666,7 @@ type MetricFootprints { } type Footprints { - nodehours: [NullableFloat!]! + timeweights: [NullableFloat!]! metrics: [MetricFootprints!]! } @@ -2753,8 +2753,8 @@ func (ec *executionContext) fieldContext_Count_count(ctx context.Context, field return fc, nil } -func (ec *executionContext) _Footprints_nodehours(ctx context.Context, field graphql.CollectedField, obj *model.Footprints) (ret graphql.Marshaler) { - fc, err := ec.fieldContext_Footprints_nodehours(ctx, field) +func (ec *executionContext) _Footprints_timeweights(ctx context.Context, field graphql.CollectedField, obj *model.Footprints) (ret graphql.Marshaler) { + fc, err := ec.fieldContext_Footprints_timeweights(ctx, field) if err != nil { return graphql.Null } @@ -2767,7 +2767,7 @@ func (ec *executionContext) _Footprints_nodehours(ctx context.Context, field gra }() resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (interface{}, error) { ctx = rctx // use context from middleware stack in children - return obj.Nodehours, nil + return obj.Timeweights, nil }) if err != nil { ec.Error(ctx, err) @@ -2784,7 +2784,7 @@ func (ec *executionContext) _Footprints_nodehours(ctx context.Context, field gra return ec.marshalNNullableFloat2ᚕgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋpkgᚋschemaᚐFloatᚄ(ctx, field.Selections, res) } -func (ec *executionContext) fieldContext_Footprints_nodehours(ctx context.Context, field graphql.CollectedField) (fc *graphql.FieldContext, err error) { +func (ec *executionContext) fieldContext_Footprints_timeweights(ctx context.Context, field graphql.CollectedField) (fc *graphql.FieldContext, err error) { fc = &graphql.FieldContext{ Object: "Footprints", Field: field, @@ -6945,8 +6945,8 @@ func (ec *executionContext) fieldContext_Query_jobsFootprints(ctx context.Contex IsResolver: true, Child: func(ctx context.Context, field graphql.CollectedField) (*graphql.FieldContext, error) { switch field.Name { - case "nodehours": - return ec.fieldContext_Footprints_nodehours(ctx, field) + case "timeweights": + return ec.fieldContext_Footprints_timeweights(ctx, field) case "metrics": return ec.fieldContext_Footprints_metrics(ctx, field) } @@ -11715,9 +11715,9 @@ func (ec *executionContext) _Footprints(ctx context.Context, sel ast.SelectionSe switch field.Name { case "__typename": out.Values[i] = graphql.MarshalString("Footprints") - case "nodehours": + case "timeweights": - out.Values[i] = ec._Footprints_nodehours(ctx, field, obj) + out.Values[i] = ec._Footprints_timeweights(ctx, field, obj) if out.Values[i] == graphql.Null { invalids++ diff --git a/internal/graph/model/models_gen.go b/internal/graph/model/models_gen.go index 8284051..7dacdf2 100644 --- a/internal/graph/model/models_gen.go +++ b/internal/graph/model/models_gen.go @@ -22,8 +22,8 @@ type FloatRange struct { } type Footprints struct { - Nodehours []schema.Float `json:"nodehours"` - Metrics []*MetricFootprints `json:"metrics"` + Timeweights []schema.Float `json:"timeweights"` + Metrics []*MetricFootprints `json:"metrics"` } type HistoPoint struct { diff --git a/internal/graph/util.go b/internal/graph/util.go index c9423e1..64676c8 100644 --- a/internal/graph/util.go +++ b/internal/graph/util.go @@ -107,6 +107,8 @@ func (r *queryResolver) jobsFootprints(ctx context.Context, filter []*model.JobF } nodehours := make([]schema.Float, 0, len(jobs)) + acchours := make([]schema.Float, 0, len(jobs)) + hwthours := make([]schema.Float, 0, len(jobs)) for _, job := range jobs { if job.MonitoringStatus == schema.MonitoringStatusDisabled || job.MonitoringStatus == schema.MonitoringStatusArchivingFailed { continue @@ -117,7 +119,18 @@ func (r *queryResolver) jobsFootprints(ctx context.Context, filter []*model.JobF return nil, err } + // #166 collect arrays: Null values or no null values? nodehours = append(nodehours, schema.Float(float64(job.Duration)/60.0*float64(job.NumNodes))) + if job.NumAcc > 0 { + acchours = append(acchours, schema.Float(float64(job.Duration)/60.0*float64(job.NumAcc))) + } else { + acchours = append(acchours, schema.Float(0.0)) + } + if job.NumHWThreads > 0 { + hwthours = append(hwthours, schema.Float(float64(job.Duration)/60.0*float64(job.NumHWThreads))) + } else { + hwthours = append(hwthours, schema.Float(0.0)) + } } res := make([]*model.MetricFootprints, len(avgs)) @@ -129,8 +142,8 @@ func (r *queryResolver) jobsFootprints(ctx context.Context, filter []*model.JobF } return &model.Footprints{ - Nodehours: nodehours, - Metrics: res, + Timeweights: nodehours, + Metrics: res, }, nil } diff --git a/internal/metricdata/cc-metric-store.go b/internal/metricdata/cc-metric-store.go index 6b3153f..cfaa6fd 100644 --- a/internal/metricdata/cc-metric-store.go +++ b/internal/metricdata/cc-metric-store.go @@ -506,7 +506,7 @@ func (ccms *CCMetricStore) LoadStats( metrics []string, ctx context.Context) (map[string]map[string]schema.MetricStatistics, error) { - queries, _, err := ccms.buildQueries(job, metrics, []schema.MetricScope{schema.MetricScopeNode}) + queries, _, err := ccms.buildQueries(job, metrics, []schema.MetricScope{schema.MetricScopeNode}) // #166 Add scope shere for analysis view accelerator normalization? if err != nil { log.Warn("Error while building query") return nil, err diff --git a/internal/metricdata/metricdata.go b/internal/metricdata/metricdata.go index 08898bd..fc91e7d 100644 --- a/internal/metricdata/metricdata.go +++ b/internal/metricdata/metricdata.go @@ -182,7 +182,7 @@ func LoadAverages( ctx context.Context) error { if job.State != schema.JobStateRunning && useArchive { - return archive.LoadAveragesFromArchive(job, metrics, data) + return archive.LoadAveragesFromArchive(job, metrics, data) // #166 change also here } repo, ok := metricDataRepos[job.Cluster] @@ -190,7 +190,7 @@ func LoadAverages( return fmt.Errorf("METRICDATA/METRICDATA > no metric data repository configured for '%s'", job.Cluster) } - stats, err := repo.LoadStats(job, metrics, ctx) + stats, err := repo.LoadStats(job, metrics, ctx) // #166 how to handle stats for acc normalizazion? if err != nil { log.Errorf("Error while loading statistics for job %v (User %v, Project %v)", job.JobID, job.User, job.Project) return err diff --git a/web/frontend/src/Analysis.root.svelte b/web/frontend/src/Analysis.root.svelte index 2e6f5b5..2ecf9db 100644 --- a/web/frontend/src/Analysis.root.svelte +++ b/web/frontend/src/Analysis.root.svelte @@ -76,7 +76,7 @@ query: gql` query($jobFilters: [JobFilter!]!, $metrics: [String!]!) { footprints: jobsFootprints(filter: $jobFilters, metrics: $metrics) { - nodehours, + timeweights, metrics { metric, data } } }`, @@ -229,7 +229,7 @@ let:width renderFor="analysis" items={metricsInHistograms.map(metric => ({ metric, ...binsFromFootprint( - $footprintsQuery.data.footprints.nodehours, + $footprintsQuery.data.footprints.timeweights, $footprintsQuery.data.footprints.metrics.find(f => f.metric == metric).data, numBins) }))} itemsPerRow={ccconfig.plot_view_plotsPerRow}> @@ -271,7 +271,7 @@ (metricConfig(cluster.name, item.m1)?.unit?.base ? metricConfig(cluster.name, item.m1)?.unit?.base : '')}]`} yLabel={`${item.m2} [${(metricConfig(cluster.name, item.m2)?.unit?.prefix ? metricConfig(cluster.name, item.m2)?.unit?.prefix : '') + (metricConfig(cluster.name, item.m2)?.unit?.base ? metricConfig(cluster.name, item.m2)?.unit?.base : '')}]`} - X={item.f1} Y={item.f2} S={$footprintsQuery.data.footprints.nodehours} /> + X={item.f1} Y={item.f2} S={$footprintsQuery.data.footprints.timeweights} /> From 6a1e35107ff57a116b8ad3986fd96b4b2518b8d8 Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Thu, 24 Aug 2023 11:52:36 +0200 Subject: [PATCH 02/19] fix: analysis metric histogram normalized by scope - native acc metrics normalized by accHours - native core metrics normalized by coreHours --- api/schema.graphqls | 10 +- internal/graph/generated/generated.go | 270 ++++++++++++++++++++++++-- internal/graph/model/models_gen.go | 14 +- internal/graph/util.go | 44 ++++- internal/metricdata/metricdata.go | 2 +- web/frontend/src/Analysis.root.svelte | 18 +- web/frontend/src/utils.js | 28 ++- 7 files changed, 336 insertions(+), 50 deletions(-) diff --git a/api/schema.graphqls b/api/schema.graphqls index 4802117..bd5d7db 100644 --- a/api/schema.graphqls +++ b/api/schema.graphqls @@ -156,12 +156,18 @@ type MetricFootprints { } type Footprints { - timeweights: [NullableFloat!]! + timeWeights: TimeWeights! metrics: [MetricFootprints!]! } +type TimeWeights { + nodeHours: [NullableFloat!]! + accHours: [NullableFloat!]! + coreHours: [NullableFloat!]! +} + enum Aggregate { USER, PROJECT, CLUSTER } -enum Weights { NODE_COUNT, NODE_HOURS } +enum Weights { NODE_COUNT, NODE_HOURS, CORE_COUNT, CORE_HOURS } type NodeMetrics { host: String! diff --git a/internal/graph/generated/generated.go b/internal/graph/generated/generated.go index 5f5bacc..278e8f2 100644 --- a/internal/graph/generated/generated.go +++ b/internal/graph/generated/generated.go @@ -69,7 +69,7 @@ type ComplexityRoot struct { Footprints struct { Metrics func(childComplexity int) int - Timeweights func(childComplexity int) int + TimeWeights func(childComplexity int) int } HistoPoint struct { @@ -265,6 +265,12 @@ type ComplexityRoot struct { To func(childComplexity int) int } + TimeWeights struct { + AccHours func(childComplexity int) int + CoreHours func(childComplexity int) int + NodeHours func(childComplexity int) int + } + Topology struct { Accelerators func(childComplexity int) int Core func(childComplexity int) int @@ -406,12 +412,12 @@ func (e *executableSchema) Complexity(typeName, field string, childComplexity in return e.complexity.Footprints.Metrics(childComplexity), true - case "Footprints.timeweights": - if e.complexity.Footprints.Timeweights == nil { + case "Footprints.timeWeights": + if e.complexity.Footprints.TimeWeights == nil { break } - return e.complexity.Footprints.Timeweights(childComplexity), true + return e.complexity.Footprints.TimeWeights(childComplexity), true case "HistoPoint.count": if e.complexity.HistoPoint.Count == nil { @@ -1356,6 +1362,27 @@ func (e *executableSchema) Complexity(typeName, field string, childComplexity in return e.complexity.TimeRangeOutput.To(childComplexity), true + case "TimeWeights.accHours": + if e.complexity.TimeWeights.AccHours == nil { + break + } + + return e.complexity.TimeWeights.AccHours(childComplexity), true + + case "TimeWeights.coreHours": + if e.complexity.TimeWeights.CoreHours == nil { + break + } + + return e.complexity.TimeWeights.CoreHours(childComplexity), true + + case "TimeWeights.nodeHours": + if e.complexity.TimeWeights.NodeHours == nil { + break + } + + return e.complexity.TimeWeights.NodeHours(childComplexity), true + case "Topology.accelerators": if e.complexity.Topology.Accelerators == nil { break @@ -1703,12 +1730,18 @@ type MetricFootprints { } type Footprints { - timeweights: [NullableFloat!]! + timeWeights: TimeWeights! metrics: [MetricFootprints!]! } +type TimeWeights { + nodeHours: [NullableFloat!]! + accHours: [NullableFloat!]! + coreHours: [NullableFloat!]! +} + enum Aggregate { USER, PROJECT, CLUSTER } -enum Weights { NODE_COUNT, NODE_HOURS } +enum Weights { NODE_COUNT, NODE_HOURS, CORE_COUNT, CORE_HOURS } type NodeMetrics { host: String! @@ -1836,7 +1869,7 @@ type JobsStatistics { shortJobs: Int! # Number of jobs with a duration of less than duration totalWalltime: Int! # Sum of the duration of all matched jobs in hours totalNodeHours: Int! # Sum of the node hours of all matched jobs - totalCoreHours: Int! # Sum of the core hours of all matched jobs + totalCoreHours: Int! # Sum of the core hours of all matched jobs <-- Das nehmen statt totaljobs in hsitograms mit totaljobs + bei analysis metric histos weighted totalAccHours: Int! # Sum of the gpu hours of all matched jobs histDuration: [HistoPoint!]! # value: hour, count: number of jobs with a rounded duration of value histNumNodes: [HistoPoint!]! # value: number of nodes, count: number of jobs with that number of nodes @@ -2790,8 +2823,8 @@ func (ec *executionContext) fieldContext_Count_count(ctx context.Context, field return fc, nil } -func (ec *executionContext) _Footprints_timeweights(ctx context.Context, field graphql.CollectedField, obj *model.Footprints) (ret graphql.Marshaler) { - fc, err := ec.fieldContext_Footprints_timeweights(ctx, field) +func (ec *executionContext) _Footprints_timeWeights(ctx context.Context, field graphql.CollectedField, obj *model.Footprints) (ret graphql.Marshaler) { + fc, err := ec.fieldContext_Footprints_timeWeights(ctx, field) if err != nil { return graphql.Null } @@ -2804,7 +2837,7 @@ func (ec *executionContext) _Footprints_timeweights(ctx context.Context, field g }() resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (interface{}, error) { ctx = rctx // use context from middleware stack in children - return obj.Timeweights, nil + return obj.TimeWeights, nil }) if err != nil { ec.Error(ctx, err) @@ -2816,19 +2849,27 @@ func (ec *executionContext) _Footprints_timeweights(ctx context.Context, field g } return graphql.Null } - res := resTmp.([]schema.Float) + res := resTmp.(*model.TimeWeights) fc.Result = res - return ec.marshalNNullableFloat2ᚕgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋpkgᚋschemaᚐFloatᚄ(ctx, field.Selections, res) + return ec.marshalNTimeWeights2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐTimeWeights(ctx, field.Selections, res) } -func (ec *executionContext) fieldContext_Footprints_timeweights(ctx context.Context, field graphql.CollectedField) (fc *graphql.FieldContext, err error) { +func (ec *executionContext) fieldContext_Footprints_timeWeights(ctx context.Context, field graphql.CollectedField) (fc *graphql.FieldContext, err error) { fc = &graphql.FieldContext{ Object: "Footprints", Field: field, IsMethod: false, IsResolver: false, Child: func(ctx context.Context, field graphql.CollectedField) (*graphql.FieldContext, error) { - return nil, errors.New("field of type NullableFloat does not have child fields") + switch field.Name { + case "nodeHours": + return ec.fieldContext_TimeWeights_nodeHours(ctx, field) + case "accHours": + return ec.fieldContext_TimeWeights_accHours(ctx, field) + case "coreHours": + return ec.fieldContext_TimeWeights_coreHours(ctx, field) + } + return nil, fmt.Errorf("no field named %q was found under type TimeWeights", field.Name) }, } return fc, nil @@ -6994,8 +7035,8 @@ func (ec *executionContext) fieldContext_Query_jobsFootprints(ctx context.Contex IsResolver: true, Child: func(ctx context.Context, field graphql.CollectedField) (*graphql.FieldContext, error) { switch field.Name { - case "timeweights": - return ec.fieldContext_Footprints_timeweights(ctx, field) + case "timeWeights": + return ec.fieldContext_Footprints_timeWeights(ctx, field) case "metrics": return ec.fieldContext_Footprints_metrics(ctx, field) } @@ -8930,6 +8971,138 @@ func (ec *executionContext) fieldContext_TimeRangeOutput_to(ctx context.Context, return fc, nil } +func (ec *executionContext) _TimeWeights_nodeHours(ctx context.Context, field graphql.CollectedField, obj *model.TimeWeights) (ret graphql.Marshaler) { + fc, err := ec.fieldContext_TimeWeights_nodeHours(ctx, field) + if err != nil { + return graphql.Null + } + ctx = graphql.WithFieldContext(ctx, fc) + defer func() { + if r := recover(); r != nil { + ec.Error(ctx, ec.Recover(ctx, r)) + ret = graphql.Null + } + }() + resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (interface{}, error) { + ctx = rctx // use context from middleware stack in children + return obj.NodeHours, nil + }) + if err != nil { + ec.Error(ctx, err) + return graphql.Null + } + if resTmp == nil { + if !graphql.HasFieldError(ctx, fc) { + ec.Errorf(ctx, "must not be null") + } + return graphql.Null + } + res := resTmp.([]schema.Float) + fc.Result = res + return ec.marshalNNullableFloat2ᚕgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋpkgᚋschemaᚐFloatᚄ(ctx, field.Selections, res) +} + +func (ec *executionContext) fieldContext_TimeWeights_nodeHours(ctx context.Context, field graphql.CollectedField) (fc *graphql.FieldContext, err error) { + fc = &graphql.FieldContext{ + Object: "TimeWeights", + Field: field, + IsMethod: false, + IsResolver: false, + Child: func(ctx context.Context, field graphql.CollectedField) (*graphql.FieldContext, error) { + return nil, errors.New("field of type NullableFloat does not have child fields") + }, + } + return fc, nil +} + +func (ec *executionContext) _TimeWeights_accHours(ctx context.Context, field graphql.CollectedField, obj *model.TimeWeights) (ret graphql.Marshaler) { + fc, err := ec.fieldContext_TimeWeights_accHours(ctx, field) + if err != nil { + return graphql.Null + } + ctx = graphql.WithFieldContext(ctx, fc) + defer func() { + if r := recover(); r != nil { + ec.Error(ctx, ec.Recover(ctx, r)) + ret = graphql.Null + } + }() + resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (interface{}, error) { + ctx = rctx // use context from middleware stack in children + return obj.AccHours, nil + }) + if err != nil { + ec.Error(ctx, err) + return graphql.Null + } + if resTmp == nil { + if !graphql.HasFieldError(ctx, fc) { + ec.Errorf(ctx, "must not be null") + } + return graphql.Null + } + res := resTmp.([]schema.Float) + fc.Result = res + return ec.marshalNNullableFloat2ᚕgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋpkgᚋschemaᚐFloatᚄ(ctx, field.Selections, res) +} + +func (ec *executionContext) fieldContext_TimeWeights_accHours(ctx context.Context, field graphql.CollectedField) (fc *graphql.FieldContext, err error) { + fc = &graphql.FieldContext{ + Object: "TimeWeights", + Field: field, + IsMethod: false, + IsResolver: false, + Child: func(ctx context.Context, field graphql.CollectedField) (*graphql.FieldContext, error) { + return nil, errors.New("field of type NullableFloat does not have child fields") + }, + } + return fc, nil +} + +func (ec *executionContext) _TimeWeights_coreHours(ctx context.Context, field graphql.CollectedField, obj *model.TimeWeights) (ret graphql.Marshaler) { + fc, err := ec.fieldContext_TimeWeights_coreHours(ctx, field) + if err != nil { + return graphql.Null + } + ctx = graphql.WithFieldContext(ctx, fc) + defer func() { + if r := recover(); r != nil { + ec.Error(ctx, ec.Recover(ctx, r)) + ret = graphql.Null + } + }() + resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (interface{}, error) { + ctx = rctx // use context from middleware stack in children + return obj.CoreHours, nil + }) + if err != nil { + ec.Error(ctx, err) + return graphql.Null + } + if resTmp == nil { + if !graphql.HasFieldError(ctx, fc) { + ec.Errorf(ctx, "must not be null") + } + return graphql.Null + } + res := resTmp.([]schema.Float) + fc.Result = res + return ec.marshalNNullableFloat2ᚕgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋpkgᚋschemaᚐFloatᚄ(ctx, field.Selections, res) +} + +func (ec *executionContext) fieldContext_TimeWeights_coreHours(ctx context.Context, field graphql.CollectedField) (fc *graphql.FieldContext, err error) { + fc = &graphql.FieldContext{ + Object: "TimeWeights", + Field: field, + IsMethod: false, + IsResolver: false, + Child: func(ctx context.Context, field graphql.CollectedField) (*graphql.FieldContext, error) { + return nil, errors.New("field of type NullableFloat does not have child fields") + }, + } + return fc, nil +} + func (ec *executionContext) _Topology_node(ctx context.Context, field graphql.CollectedField, obj *schema.Topology) (ret graphql.Marshaler) { fc, err := ec.fieldContext_Topology_node(ctx, field) if err != nil { @@ -11848,10 +12021,8 @@ func (ec *executionContext) _Footprints(ctx context.Context, sel ast.SelectionSe switch field.Name { case "__typename": out.Values[i] = graphql.MarshalString("Footprints") - case "timeweights": - - out.Values[i] = ec._Footprints_timeweights(ctx, field, obj) - + case "timeWeights": + out.Values[i] = ec._Footprints_timeWeights(ctx, field, obj) if out.Values[i] == graphql.Null { out.Invalids++ } @@ -13600,6 +13771,55 @@ func (ec *executionContext) _TimeRangeOutput(ctx context.Context, sel ast.Select return out } +var timeWeightsImplementors = []string{"TimeWeights"} + +func (ec *executionContext) _TimeWeights(ctx context.Context, sel ast.SelectionSet, obj *model.TimeWeights) graphql.Marshaler { + fields := graphql.CollectFields(ec.OperationContext, sel, timeWeightsImplementors) + + out := graphql.NewFieldSet(fields) + deferred := make(map[string]*graphql.FieldSet) + for i, field := range fields { + switch field.Name { + case "__typename": + out.Values[i] = graphql.MarshalString("TimeWeights") + case "nodeHours": + out.Values[i] = ec._TimeWeights_nodeHours(ctx, field, obj) + if out.Values[i] == graphql.Null { + out.Invalids++ + } + case "accHours": + out.Values[i] = ec._TimeWeights_accHours(ctx, field, obj) + if out.Values[i] == graphql.Null { + out.Invalids++ + } + case "coreHours": + out.Values[i] = ec._TimeWeights_coreHours(ctx, field, obj) + if out.Values[i] == graphql.Null { + out.Invalids++ + } + default: + panic("unknown field " + strconv.Quote(field.Name)) + } + } + out.Dispatch(ctx) + if out.Invalids > 0 { + return graphql.Null + } + + atomic.AddInt32(&ec.deferred, int32(len(deferred))) + + for label, dfs := range deferred { + ec.processDeferredGroup(graphql.DeferredGroup{ + Label: label, + Path: graphql.GetPath(ctx), + FieldSet: dfs, + Context: ctx, + }) + } + + return out +} + var topologyImplementors = []string{"Topology"} func (ec *executionContext) _Topology(ctx context.Context, sel ast.SelectionSet, obj *schema.Topology) graphql.Marshaler { @@ -15333,6 +15553,16 @@ func (ec *executionContext) marshalNTime2timeᚐTime(ctx context.Context, sel as return res } +func (ec *executionContext) marshalNTimeWeights2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐTimeWeights(ctx context.Context, sel ast.SelectionSet, v *model.TimeWeights) graphql.Marshaler { + if v == nil { + if !graphql.HasFieldError(ctx, graphql.GetFieldContext(ctx)) { + ec.Errorf(ctx, "the requested element is null which the schema does not allow") + } + return graphql.Null + } + return ec._TimeWeights(ctx, sel, v) +} + func (ec *executionContext) marshalNTopology2githubᚗcomᚋClusterCockpitᚋccᚑbackendᚋpkgᚋschemaᚐTopology(ctx context.Context, sel ast.SelectionSet, v schema.Topology) graphql.Marshaler { return ec._Topology(ctx, sel, &v) } diff --git a/internal/graph/model/models_gen.go b/internal/graph/model/models_gen.go index 3f0cca5..faffae7 100644 --- a/internal/graph/model/models_gen.go +++ b/internal/graph/model/models_gen.go @@ -22,7 +22,7 @@ type FloatRange struct { } type Footprints struct { - Timeweights []schema.Float `json:"timeweights"` + TimeWeights *TimeWeights `json:"timeWeights"` Metrics []*MetricFootprints `json:"metrics"` } @@ -133,6 +133,12 @@ type TimeRangeOutput struct { To time.Time `json:"to"` } +type TimeWeights struct { + NodeHours []schema.Float `json:"nodeHours"` + AccHours []schema.Float `json:"accHours"` + CoreHours []schema.Float `json:"coreHours"` +} + type User struct { Username string `json:"username"` Name string `json:"name"` @@ -228,16 +234,20 @@ type Weights string const ( WeightsNodeCount Weights = "NODE_COUNT" WeightsNodeHours Weights = "NODE_HOURS" + WeightsCoreCount Weights = "CORE_COUNT" + WeightsCoreHours Weights = "CORE_HOURS" ) var AllWeights = []Weights{ WeightsNodeCount, WeightsNodeHours, + WeightsCoreCount, + WeightsCoreHours, } func (e Weights) IsValid() bool { switch e { - case WeightsNodeCount, WeightsNodeHours: + case WeightsNodeCount, WeightsNodeHours, WeightsCoreCount, WeightsCoreHours: return true } return false diff --git a/internal/graph/util.go b/internal/graph/util.go index 64676c8..b3a0080 100644 --- a/internal/graph/util.go +++ b/internal/graph/util.go @@ -13,6 +13,7 @@ import ( "github.com/99designs/gqlgen/graphql" "github.com/ClusterCockpit/cc-backend/internal/graph/model" "github.com/ClusterCockpit/cc-backend/internal/metricdata" + "github.com/ClusterCockpit/cc-backend/pkg/archive" "github.com/ClusterCockpit/cc-backend/pkg/log" "github.com/ClusterCockpit/cc-backend/pkg/schema" ) @@ -106,9 +107,11 @@ func (r *queryResolver) jobsFootprints(ctx context.Context, filter []*model.JobF avgs[i] = make([]schema.Float, 0, len(jobs)) } - nodehours := make([]schema.Float, 0, len(jobs)) - acchours := make([]schema.Float, 0, len(jobs)) - hwthours := make([]schema.Float, 0, len(jobs)) + timeweights := new(model.TimeWeights) + timeweights.NodeHours = make([]schema.Float, 0, len(jobs)) + timeweights.AccHours = make([]schema.Float, 0, len(jobs)) + timeweights.CoreHours = make([]schema.Float, 0, len(jobs)) + for _, job := range jobs { if job.MonitoringStatus == schema.MonitoringStatusDisabled || job.MonitoringStatus == schema.MonitoringStatusArchivingFailed { continue @@ -120,16 +123,16 @@ func (r *queryResolver) jobsFootprints(ctx context.Context, filter []*model.JobF } // #166 collect arrays: Null values or no null values? - nodehours = append(nodehours, schema.Float(float64(job.Duration)/60.0*float64(job.NumNodes))) + timeweights.NodeHours = append(timeweights.NodeHours, schema.Float(float64(job.Duration)/60.0*float64(job.NumNodes))) if job.NumAcc > 0 { - acchours = append(acchours, schema.Float(float64(job.Duration)/60.0*float64(job.NumAcc))) + timeweights.AccHours = append(timeweights.AccHours, schema.Float(float64(job.Duration)/60.0*float64(job.NumAcc))) } else { - acchours = append(acchours, schema.Float(0.0)) + timeweights.AccHours = append(timeweights.AccHours, schema.Float(1.0)) } if job.NumHWThreads > 0 { - hwthours = append(hwthours, schema.Float(float64(job.Duration)/60.0*float64(job.NumHWThreads))) + timeweights.CoreHours = append(timeweights.CoreHours, schema.Float(float64(job.Duration)/60.0*float64(numCoresForJob(job)))) } else { - hwthours = append(hwthours, schema.Float(0.0)) + timeweights.CoreHours = append(timeweights.CoreHours, schema.Float(1.0)) } } @@ -142,11 +145,34 @@ func (r *queryResolver) jobsFootprints(ctx context.Context, filter []*model.JobF } return &model.Footprints{ - Timeweights: nodehours, + TimeWeights: timeweights, Metrics: res, }, nil } +func numCoresForJob(job *schema.Job) (numCores int) { + + subcluster, scerr := archive.GetSubCluster(job.Cluster, job.SubCluster) + if scerr != nil { + return 1 + } + + totalJobCores := 0 + topology := subcluster.Topology + + for _, host := range job.Resources { + hwthreads := host.HWThreads + if hwthreads == nil { + hwthreads = topology.Node + } + + hostCores, _ := topology.GetCoresFromHWThreads(hwthreads) + totalJobCores += len(hostCores) + } + + return totalJobCores +} + func requireField(ctx context.Context, name string) bool { fields := graphql.CollectAllFields(ctx) diff --git a/internal/metricdata/metricdata.go b/internal/metricdata/metricdata.go index fc91e7d..3117f8c 100644 --- a/internal/metricdata/metricdata.go +++ b/internal/metricdata/metricdata.go @@ -182,7 +182,7 @@ func LoadAverages( ctx context.Context) error { if job.State != schema.JobStateRunning && useArchive { - return archive.LoadAveragesFromArchive(job, metrics, data) // #166 change also here + return archive.LoadAveragesFromArchive(job, metrics, data) // #166 change also here? } repo, ok := metricDataRepos[job.Cluster] diff --git a/web/frontend/src/Analysis.root.svelte b/web/frontend/src/Analysis.root.svelte index 9df1282..4022826 100644 --- a/web/frontend/src/Analysis.root.svelte +++ b/web/frontend/src/Analysis.root.svelte @@ -78,7 +78,7 @@ query: gql` query($jobFilters: [JobFilter!]!, $metrics: [String!]!) { footprints: jobsFootprints(filter: $jobFilters, metrics: $metrics) { - timeweights, + timeWeights { nodeHours, accHours, coreHours }, metrics { metric, data } } }`, @@ -244,8 +244,9 @@ - These histograms show the distribution of the averages of all jobs matching the filters. Each job/average is weighted by its node hours. - Note that some metrics could be disabled for specific subclusters as per metriConfig and thus could affect shown average values. + These histograms show the distribution of the averages of all jobs matching the filters. Each job/average is weighted by its node hours by default + (Accelerator hours for native accelerator scope metrics, coreHours for native core scope metrics). + Note that some metrics could be disabled for specific subclusters as per metricConfig and thus could affect shown average values.
@@ -257,7 +258,8 @@ let:width renderFor="analysis" items={metricsInHistograms.map(metric => ({ metric, ...binsFromFootprint( - $footprintsQuery.data.footprints.timeweights, + $footprintsQuery.data.footprints.timeWeights, + metricConfig(cluster.name, metric)?.scope, $footprintsQuery.data.footprints.metrics.find(f => f.metric == metric).data, numBins) }))} itemsPerRow={ccconfig.plot_view_plotsPerRow}> @@ -265,11 +267,11 @@ data={convert2uplot(item.bins)} width={width} height={250} title="Average Distribution of '{item.metric}'" - xlabel={`${item.metric} average [${(metricConfig(cluster.name, item.metric)?.unit?.prefix ? metricConfig(cluster.name, item.metric)?.unit?.prefix : '') + + xlabel={`${item.metric} bin maximum [${(metricConfig(cluster.name, item.metric)?.unit?.prefix ? metricConfig(cluster.name, item.metric)?.unit?.prefix : '') + (metricConfig(cluster.name, item.metric)?.unit?.base ? metricConfig(cluster.name, item.metric)?.unit?.base : '')}]`} xunit={`${(metricConfig(cluster.name, item.metric)?.unit?.prefix ? metricConfig(cluster.name, item.metric)?.unit?.prefix : '') + (metricConfig(cluster.name, item.metric)?.unit?.base ? metricConfig(cluster.name, item.metric)?.unit?.base : '')}`} - ylabel="Node Hours" + ylabel="Normalized Hours" yunit="Hours"/> @@ -279,7 +281,7 @@ Each circle represents one job. The size of a circle is proportional to its node hours. Darker circles mean multiple jobs have the same averages for the respective metrics. - Note that some metrics could be disabled for specific subclusters as per metriConfig and thus could affect shown average values. + Note that some metrics could be disabled for specific subclusters as per metricConfig and thus could affect shown average values.
@@ -301,7 +303,7 @@ (metricConfig(cluster.name, item.m1)?.unit?.base ? metricConfig(cluster.name, item.m1)?.unit?.base : '')}]`} yLabel={`${item.m2} [${(metricConfig(cluster.name, item.m2)?.unit?.prefix ? metricConfig(cluster.name, item.m2)?.unit?.prefix : '') + (metricConfig(cluster.name, item.m2)?.unit?.base ? metricConfig(cluster.name, item.m2)?.unit?.base : '')}]`} - X={item.f1} Y={item.f2} S={$footprintsQuery.data.footprints.timeweights} /> + X={item.f1} Y={item.f2} S={$footprintsQuery.data.footprints.timeWeights.nodeHours} /> diff --git a/web/frontend/src/utils.js b/web/frontend/src/utils.js index f68fec4..5e9cdae 100644 --- a/web/frontend/src/utils.js +++ b/web/frontend/src/utils.js @@ -325,7 +325,7 @@ export function convert2uplot(canvasData) { return uplotData } -export function binsFromFootprint(weights, values, numBins) { +export function binsFromFootprint(weights, scope, values, numBins) { let min = 0, max = 0 if (values.length != 0) { for (let x of values) { @@ -338,10 +338,23 @@ export function binsFromFootprint(weights, values, numBins) { if (numBins == null || numBins < 3) numBins = 3 + let scopeWeights + switch (scope) { + case 'core': + scopeWeights = weights.coreHours + break + case 'accelerator': + scopeWeights = weights.accHours + break + default: // every other scope: use 'node' + scopeWeights = weights.nodeHours + } + const bins = new Array(numBins).fill(0) for (let i = 0; i < values.length; i++) - bins[Math.floor(((values[i] - min) / (max - min)) * numBins)] += weights ? weights[i] : 1 + bins[Math.floor(((values[i] - min) / (max - min)) * numBins)] += scopeWeights ? scopeWeights[i] : 1 + // Manual Canvas Original // return { // label: idx => { // let start = min + (idx / numBins) * (max - min) @@ -355,14 +368,13 @@ export function binsFromFootprint(weights, values, numBins) { return { bins: bins.map((count, idx) => ({ - value: idx => { // Get rounded down next integer to bins' Start-Stop Mean Value - let start = min + (idx / numBins) * (max - min) + value: idx => { // Use bins' max value instead of mean + // let start = min + (idx / numBins) * (max - min) let stop = min + ((idx + 1) / numBins) * (max - min) - return `${formatNumber(Math.floor((start+stop)/2))}` + // return `${formatNumber(Math.floor((start+stop)/2))}` + return Math.floor(stop) }, count: count - })), - min: min, - max: max + })) } } From 4eceab4dc7c909f6cf1dbb3c95b870211eb2a028 Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Thu, 24 Aug 2023 12:51:55 +0200 Subject: [PATCH 03/19] fix: change analysis top users to core hours --- internal/repository/job.go | 6 ++++++ web/frontend/src/Analysis.root.svelte | 4 ++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/internal/repository/job.go b/internal/repository/job.go index 449c267..b14fd53 100644 --- a/internal/repository/job.go +++ b/internal/repository/job.go @@ -478,6 +478,12 @@ func (r *JobRepository) CountGroupedJobs( now := time.Now().Unix() count = fmt.Sprintf(`sum(job.num_nodes * (CASE WHEN job.job_state = "running" THEN %d - job.start_time ELSE job.duration END)) as count`, now) runner = r.DB + case model.WeightsCoreCount: + count = "sum(job.num_hwthreads) as count" + case model.WeightsCoreHours: + now := time.Now().Unix() + count = fmt.Sprintf(`sum(job.num_hwthreads * (CASE WHEN job.job_state = "running" THEN %d - job.start_time ELSE job.duration END)) as count`, now) + runner = r.DB default: log.Debugf("CountGroupedJobs() Weight %v unknown.", *weight) } diff --git a/web/frontend/src/Analysis.root.svelte b/web/frontend/src/Analysis.root.svelte index 4022826..c6e4575 100644 --- a/web/frontend/src/Analysis.root.svelte +++ b/web/frontend/src/Analysis.root.svelte @@ -67,7 +67,7 @@ histNumNodes { count, value } } - topUsers: jobsCount(filter: $jobFilters, groupBy: USER, weight: NODE_HOURS, limit: 5) { name, count } + topUsers: jobsCount(filter: $jobFilters, groupBy: USER, weight: CORE_HOURS, limit: 5) { name, count } } `, variables: { jobFilters } @@ -172,7 +172,7 @@ - + {#each $statsQuery.data.topUsers.sort((a, b) => b.count - a.count) as { name, count }, i} From 28609a33722a4ce02e8e7d511bd411d4adea0176 Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Thu, 24 Aug 2023 12:56:35 +0200 Subject: [PATCH 04/19] adapt core timeweight to sqlite name logic --- internal/graph/util.go | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/internal/graph/util.go b/internal/graph/util.go index b3a0080..b61bcc7 100644 --- a/internal/graph/util.go +++ b/internal/graph/util.go @@ -13,9 +13,9 @@ import ( "github.com/99designs/gqlgen/graphql" "github.com/ClusterCockpit/cc-backend/internal/graph/model" "github.com/ClusterCockpit/cc-backend/internal/metricdata" - "github.com/ClusterCockpit/cc-backend/pkg/archive" "github.com/ClusterCockpit/cc-backend/pkg/log" "github.com/ClusterCockpit/cc-backend/pkg/schema" + // "github.com/ClusterCockpit/cc-backend/pkg/archive" ) const MAX_JOBS_FOR_ANALYSIS = 500 @@ -130,7 +130,7 @@ func (r *queryResolver) jobsFootprints(ctx context.Context, filter []*model.JobF timeweights.AccHours = append(timeweights.AccHours, schema.Float(1.0)) } if job.NumHWThreads > 0 { - timeweights.CoreHours = append(timeweights.CoreHours, schema.Float(float64(job.Duration)/60.0*float64(numCoresForJob(job)))) + timeweights.CoreHours = append(timeweights.CoreHours, schema.Float(float64(job.Duration)/60.0*float64(job.NumHWThreads))) // SQLite HWThreads == Cores; numCoresForJob(job) } else { timeweights.CoreHours = append(timeweights.CoreHours, schema.Float(1.0)) } @@ -150,28 +150,28 @@ func (r *queryResolver) jobsFootprints(ctx context.Context, filter []*model.JobF }, nil } -func numCoresForJob(job *schema.Job) (numCores int) { +// func numCoresForJob(job *schema.Job) (numCores int) { - subcluster, scerr := archive.GetSubCluster(job.Cluster, job.SubCluster) - if scerr != nil { - return 1 - } +// subcluster, scerr := archive.GetSubCluster(job.Cluster, job.SubCluster) +// if scerr != nil { +// return 1 +// } - totalJobCores := 0 - topology := subcluster.Topology +// totalJobCores := 0 +// topology := subcluster.Topology - for _, host := range job.Resources { - hwthreads := host.HWThreads - if hwthreads == nil { - hwthreads = topology.Node - } +// for _, host := range job.Resources { +// hwthreads := host.HWThreads +// if hwthreads == nil { +// hwthreads = topology.Node +// } - hostCores, _ := topology.GetCoresFromHWThreads(hwthreads) - totalJobCores += len(hostCores) - } +// hostCores, _ := topology.GetCoresFromHWThreads(hwthreads) +// totalJobCores += len(hostCores) +// } - return totalJobCores -} +// return totalJobCores +// } func requireField(ctx context.Context, name string) bool { fields := graphql.CollectAllFields(ctx) From 77677a9f1b741e81b1c9a81cb350a38ed907c433 Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Thu, 24 Aug 2023 13:11:46 +0200 Subject: [PATCH 05/19] specify label --- web/frontend/src/Analysis.root.svelte | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/frontend/src/Analysis.root.svelte b/web/frontend/src/Analysis.root.svelte index c6e4575..f60b2b9 100644 --- a/web/frontend/src/Analysis.root.svelte +++ b/web/frontend/src/Analysis.root.svelte @@ -163,7 +163,7 @@ {#key $statsQuery.data.topUsers} b.count - a.count).map((tu) => tu.count)} entities={$statsQuery.data.topUsers.sort((a, b) => b.count - a.count).map((tu) => tu.name)} /> From a7dd3fbc0b531ce73311d2fe937321672e79d333 Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Thu, 24 Aug 2023 14:26:23 +0200 Subject: [PATCH 06/19] fix bug in stats AddJobCount --- internal/repository/stats.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/internal/repository/stats.go b/internal/repository/stats.go index 8a74008..ee48ee1 100644 --- a/internal/repository/stats.go +++ b/internal/repository/stats.go @@ -321,7 +321,7 @@ func (r *JobRepository) AddJobCount( return nil, err } - counts := make(map[string]int) + var count int for rows.Next() { var cnt sql.NullInt64 @@ -329,20 +329,22 @@ func (r *JobRepository) AddJobCount( log.Warn("Error while scanning rows") return nil, err } + + count = int(cnt.Int64) } switch kind { case "running": for _, s := range stats { - s.RunningJobs = counts[s.ID] + s.RunningJobs = count } case "short": for _, s := range stats { - s.ShortJobs = counts[s.ID] + s.ShortJobs = count } } - log.Debugf("Timer JobJobCount %s", time.Since(start)) + log.Debugf("Timer AddJobCount %s", time.Since(start)) return stats, nil } From 3b8bcf7b32374857568be7128629001ef192ee76 Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Thu, 24 Aug 2023 14:51:26 +0200 Subject: [PATCH 07/19] Remove obsolete jobsCount resolver --- api/schema.graphqls | 1 - internal/graph/generated/generated.go | 193 +------------------------- internal/graph/schema.resolvers.go | 18 --- internal/repository/job.go | 69 --------- 4 files changed, 1 insertion(+), 280 deletions(-) diff --git a/api/schema.graphqls b/api/schema.graphqls index bd5d7db..7f427af 100644 --- a/api/schema.graphqls +++ b/api/schema.graphqls @@ -199,7 +199,6 @@ type Query { jobs(filter: [JobFilter!], page: PageRequest, order: OrderByInput): JobResultList! jobsStatistics(filter: [JobFilter!], groupBy: Aggregate): [JobsStatistics!]! - jobsCount(filter: [JobFilter]!, groupBy: Aggregate!, weight: Weights, limit: Int): [Count!]! rooflineHeatmap(filter: [JobFilter!]!, rows: Int!, cols: Int!, minX: Float!, minY: Float!, maxX: Float!, maxY: Float!): [[Float!]!]! diff --git a/internal/graph/generated/generated.go b/internal/graph/generated/generated.go index 278e8f2..7544148 100644 --- a/internal/graph/generated/generated.go +++ b/internal/graph/generated/generated.go @@ -202,7 +202,6 @@ type ComplexityRoot struct { Job func(childComplexity int, id string) int JobMetrics func(childComplexity int, id string, metrics []string, scopes []schema.MetricScope) int Jobs func(childComplexity int, filter []*model.JobFilter, page *model.PageRequest, order *model.OrderByInput) int - JobsCount func(childComplexity int, filter []*model.JobFilter, groupBy model.Aggregate, weight *model.Weights, limit *int) int JobsFootprints func(childComplexity int, filter []*model.JobFilter, metrics []string) int JobsStatistics func(childComplexity int, filter []*model.JobFilter, groupBy *model.Aggregate) int NodeMetrics func(childComplexity int, cluster string, nodes []string, scopes []schema.MetricScope, metrics []string, from time.Time, to time.Time) int @@ -319,7 +318,6 @@ type QueryResolver interface { JobsFootprints(ctx context.Context, filter []*model.JobFilter, metrics []string) (*model.Footprints, error) Jobs(ctx context.Context, filter []*model.JobFilter, page *model.PageRequest, order *model.OrderByInput) (*model.JobResultList, error) JobsStatistics(ctx context.Context, filter []*model.JobFilter, groupBy *model.Aggregate) ([]*model.JobsStatistics, error) - JobsCount(ctx context.Context, filter []*model.JobFilter, groupBy model.Aggregate, weight *model.Weights, limit *int) ([]*model.Count, error) RooflineHeatmap(ctx context.Context, filter []*model.JobFilter, rows int, cols int, minX float64, minY float64, maxX float64, maxY float64) ([][]float64, error) NodeMetrics(ctx context.Context, cluster string, nodes []string, scopes []schema.MetricScope, metrics []string, from time.Time, to time.Time) ([]*model.NodeMetrics, error) } @@ -1052,18 +1050,6 @@ func (e *executableSchema) Complexity(typeName, field string, childComplexity in return e.complexity.Query.Jobs(childComplexity, args["filter"].([]*model.JobFilter), args["page"].(*model.PageRequest), args["order"].(*model.OrderByInput)), true - case "Query.jobsCount": - if e.complexity.Query.JobsCount == nil { - break - } - - args, err := ec.field_Query_jobsCount_args(context.TODO(), rawArgs) - if err != nil { - return 0, false - } - - return e.complexity.Query.JobsCount(childComplexity, args["filter"].([]*model.JobFilter), args["groupBy"].(model.Aggregate), args["weight"].(*model.Weights), args["limit"].(*int)), true - case "Query.jobsFootprints": if e.complexity.Query.JobsFootprints == nil { break @@ -1773,7 +1759,6 @@ type Query { jobs(filter: [JobFilter!], page: PageRequest, order: OrderByInput): JobResultList! jobsStatistics(filter: [JobFilter!], groupBy: Aggregate): [JobsStatistics!]! - jobsCount(filter: [JobFilter]!, groupBy: Aggregate!, weight: Weights, limit: Int): [Count!]! rooflineHeatmap(filter: [JobFilter!]!, rows: Int!, cols: Int!, minX: Float!, minY: Float!, maxX: Float!, maxY: Float!): [[Float!]!]! @@ -1869,7 +1854,7 @@ type JobsStatistics { shortJobs: Int! # Number of jobs with a duration of less than duration totalWalltime: Int! # Sum of the duration of all matched jobs in hours totalNodeHours: Int! # Sum of the node hours of all matched jobs - totalCoreHours: Int! # Sum of the core hours of all matched jobs <-- Das nehmen statt totaljobs in hsitograms mit totaljobs + bei analysis metric histos weighted + totalCoreHours: Int! # Sum of the core hours of all matched jobs totalAccHours: Int! # Sum of the gpu hours of all matched jobs histDuration: [HistoPoint!]! # value: hour, count: number of jobs with a rounded duration of value histNumNodes: [HistoPoint!]! # value: number of nodes, count: number of jobs with that number of nodes @@ -2076,48 +2061,6 @@ func (ec *executionContext) field_Query_job_args(ctx context.Context, rawArgs ma return args, nil } -func (ec *executionContext) field_Query_jobsCount_args(ctx context.Context, rawArgs map[string]interface{}) (map[string]interface{}, error) { - var err error - args := map[string]interface{}{} - var arg0 []*model.JobFilter - if tmp, ok := rawArgs["filter"]; ok { - ctx := graphql.WithPathContext(ctx, graphql.NewPathWithField("filter")) - arg0, err = ec.unmarshalNJobFilter2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐJobFilter(ctx, tmp) - if err != nil { - return nil, err - } - } - args["filter"] = arg0 - var arg1 model.Aggregate - if tmp, ok := rawArgs["groupBy"]; ok { - ctx := graphql.WithPathContext(ctx, graphql.NewPathWithField("groupBy")) - arg1, err = ec.unmarshalNAggregate2githubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐAggregate(ctx, tmp) - if err != nil { - return nil, err - } - } - args["groupBy"] = arg1 - var arg2 *model.Weights - if tmp, ok := rawArgs["weight"]; ok { - ctx := graphql.WithPathContext(ctx, graphql.NewPathWithField("weight")) - arg2, err = ec.unmarshalOWeights2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐWeights(ctx, tmp) - if err != nil { - return nil, err - } - } - args["weight"] = arg2 - var arg3 *int - if tmp, ok := rawArgs["limit"]; ok { - ctx := graphql.WithPathContext(ctx, graphql.NewPathWithField("limit")) - arg3, err = ec.unmarshalOInt2ᚖint(ctx, tmp) - if err != nil { - return nil, err - } - } - args["limit"] = arg3 - return args, nil -} - func (ec *executionContext) field_Query_jobsFootprints_args(ctx context.Context, rawArgs map[string]interface{}) (map[string]interface{}, error) { var err error args := map[string]interface{}{} @@ -7201,67 +7144,6 @@ func (ec *executionContext) fieldContext_Query_jobsStatistics(ctx context.Contex return fc, nil } -func (ec *executionContext) _Query_jobsCount(ctx context.Context, field graphql.CollectedField) (ret graphql.Marshaler) { - fc, err := ec.fieldContext_Query_jobsCount(ctx, field) - if err != nil { - return graphql.Null - } - ctx = graphql.WithFieldContext(ctx, fc) - defer func() { - if r := recover(); r != nil { - ec.Error(ctx, ec.Recover(ctx, r)) - ret = graphql.Null - } - }() - resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (interface{}, error) { - ctx = rctx // use context from middleware stack in children - return ec.resolvers.Query().JobsCount(rctx, fc.Args["filter"].([]*model.JobFilter), fc.Args["groupBy"].(model.Aggregate), fc.Args["weight"].(*model.Weights), fc.Args["limit"].(*int)) - }) - if err != nil { - ec.Error(ctx, err) - return graphql.Null - } - if resTmp == nil { - if !graphql.HasFieldError(ctx, fc) { - ec.Errorf(ctx, "must not be null") - } - return graphql.Null - } - res := resTmp.([]*model.Count) - fc.Result = res - return ec.marshalNCount2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐCountᚄ(ctx, field.Selections, res) -} - -func (ec *executionContext) fieldContext_Query_jobsCount(ctx context.Context, field graphql.CollectedField) (fc *graphql.FieldContext, err error) { - fc = &graphql.FieldContext{ - Object: "Query", - Field: field, - IsMethod: true, - IsResolver: true, - Child: func(ctx context.Context, field graphql.CollectedField) (*graphql.FieldContext, error) { - switch field.Name { - case "name": - return ec.fieldContext_Count_name(ctx, field) - case "count": - return ec.fieldContext_Count_count(ctx, field) - } - return nil, fmt.Errorf("no field named %q was found under type Count", field.Name) - }, - } - defer func() { - if r := recover(); r != nil { - err = ec.Recover(ctx, r) - ec.Error(ctx, err) - } - }() - ctx = graphql.WithFieldContext(ctx, fc) - if fc.Args, err = ec.field_Query_jobsCount_args(ctx, field.ArgumentMap(ec.Variables)); err != nil { - ec.Error(ctx, err) - return fc, err - } - return fc, nil -} - func (ec *executionContext) _Query_rooflineHeatmap(ctx context.Context, field graphql.CollectedField) (ret graphql.Marshaler) { fc, err := ec.fieldContext_Query_rooflineHeatmap(ctx, field) if err != nil { @@ -13269,28 +13151,6 @@ func (ec *executionContext) _Query(ctx context.Context, sel ast.SelectionSet) gr func(ctx context.Context) graphql.Marshaler { return innerFunc(ctx, out) }) } - out.Concurrently(i, func(ctx context.Context) graphql.Marshaler { return rrm(innerCtx) }) - case "jobsCount": - field := field - - innerFunc := func(ctx context.Context, fs *graphql.FieldSet) (res graphql.Marshaler) { - defer func() { - if r := recover(); r != nil { - ec.Error(ctx, ec.Recover(ctx, r)) - } - }() - res = ec._Query_jobsCount(ctx, field) - if res == graphql.Null { - atomic.AddUint32(&fs.Invalids, 1) - } - return res - } - - rrm := func(ctx context.Context) graphql.Marshaler { - return ec.OperationContext.RootResolverMiddleware(ctx, - func(ctx context.Context) graphql.Marshaler { return innerFunc(ctx, out) }) - } - out.Concurrently(i, func(ctx context.Context) graphql.Marshaler { return rrm(innerCtx) }) case "rooflineHeatmap": field := field @@ -14292,16 +14152,6 @@ func (ec *executionContext) marshalNAccelerator2ᚖgithubᚗcomᚋClusterCockpit return ec._Accelerator(ctx, sel, v) } -func (ec *executionContext) unmarshalNAggregate2githubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐAggregate(ctx context.Context, v interface{}) (model.Aggregate, error) { - var res model.Aggregate - err := res.UnmarshalGQL(v) - return res, graphql.ErrorOnPath(ctx, err) -} - -func (ec *executionContext) marshalNAggregate2githubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐAggregate(ctx context.Context, sel ast.SelectionSet, v model.Aggregate) graphql.Marshaler { - return v -} - func (ec *executionContext) unmarshalNBoolean2bool(ctx context.Context, v interface{}) (bool, error) { res, err := graphql.UnmarshalBoolean(v) return res, graphql.ErrorOnPath(ctx, err) @@ -14804,23 +14654,6 @@ func (ec *executionContext) marshalNJob2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑ return ec._Job(ctx, sel, v) } -func (ec *executionContext) unmarshalNJobFilter2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐJobFilter(ctx context.Context, v interface{}) ([]*model.JobFilter, error) { - var vSlice []interface{} - if v != nil { - vSlice = graphql.CoerceList(v) - } - var err error - res := make([]*model.JobFilter, len(vSlice)) - for i := range vSlice { - ctx := graphql.WithPathContext(ctx, graphql.NewPathWithIndex(i)) - res[i], err = ec.unmarshalOJobFilter2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐJobFilter(ctx, vSlice[i]) - if err != nil { - return nil, err - } - } - return res, nil -} - func (ec *executionContext) unmarshalNJobFilter2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐJobFilterᚄ(ctx context.Context, v interface{}) ([]*model.JobFilter, error) { var vSlice []interface{} if v != nil { @@ -16157,14 +15990,6 @@ func (ec *executionContext) unmarshalOJobFilter2ᚕᚖgithubᚗcomᚋClusterCock return res, nil } -func (ec *executionContext) unmarshalOJobFilter2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐJobFilter(ctx context.Context, v interface{}) (*model.JobFilter, error) { - if v == nil { - return nil, nil - } - res, err := ec.unmarshalInputJobFilter(ctx, v) - return &res, graphql.ErrorOnPath(ctx, err) -} - func (ec *executionContext) marshalOJobLinkResultList2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐJobLinkResultList(ctx context.Context, sel ast.SelectionSet, v *model.JobLinkResultList) graphql.Marshaler { if v == nil { return graphql.Null @@ -16429,22 +16254,6 @@ func (ec *executionContext) marshalOUser2ᚖgithubᚗcomᚋClusterCockpitᚋcc return ec._User(ctx, sel, v) } -func (ec *executionContext) unmarshalOWeights2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐWeights(ctx context.Context, v interface{}) (*model.Weights, error) { - if v == nil { - return nil, nil - } - var res = new(model.Weights) - err := res.UnmarshalGQL(v) - return res, graphql.ErrorOnPath(ctx, err) -} - -func (ec *executionContext) marshalOWeights2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐWeights(ctx context.Context, sel ast.SelectionSet, v *model.Weights) graphql.Marshaler { - if v == nil { - return graphql.Null - } - return v -} - func (ec *executionContext) marshalO__EnumValue2ᚕgithubᚗcomᚋ99designsᚋgqlgenᚋgraphqlᚋintrospectionᚐEnumValueᚄ(ctx context.Context, sel ast.SelectionSet, v []introspection.EnumValue) graphql.Marshaler { if v == nil { return graphql.Null diff --git a/internal/graph/schema.resolvers.go b/internal/graph/schema.resolvers.go index 8d34fb3..a19ed8b 100644 --- a/internal/graph/schema.resolvers.go +++ b/internal/graph/schema.resolvers.go @@ -294,24 +294,6 @@ func (r *queryResolver) JobsStatistics(ctx context.Context, filter []*model.JobF return stats, nil } -// JobsCount is the resolver for the jobsCount field. -func (r *queryResolver) JobsCount(ctx context.Context, filter []*model.JobFilter, groupBy model.Aggregate, weight *model.Weights, limit *int) ([]*model.Count, error) { - counts, err := r.Repo.CountGroupedJobs(ctx, groupBy, filter, weight, limit) - if err != nil { - log.Warn("Error while counting grouped jobs") - return nil, err - } - - res := make([]*model.Count, 0, len(counts)) - for name, count := range counts { - res = append(res, &model.Count{ - Name: name, - Count: count, - }) - } - return res, nil -} - // RooflineHeatmap is the resolver for the rooflineHeatmap field. func (r *queryResolver) RooflineHeatmap(ctx context.Context, filter []*model.JobFilter, rows int, cols int, minX float64, minY float64, maxX float64, maxY float64) ([][]float64, error) { return r.rooflineHeatmap(ctx, filter, rows, cols, minX, minY, maxX, maxY) diff --git a/internal/repository/job.go b/internal/repository/job.go index b14fd53..76834d1 100644 --- a/internal/repository/job.go +++ b/internal/repository/job.go @@ -455,75 +455,6 @@ func (r *JobRepository) DeleteJobById(id int64) error { return err } -// TODO: Use node hours instead: SELECT job.user, sum(job.num_nodes * (CASE WHEN job.job_state = "running" THEN CAST(strftime('%s', 'now') AS INTEGER) - job.start_time ELSE job.duration END)) as x FROM job GROUP BY user ORDER BY x DESC; -func (r *JobRepository) CountGroupedJobs( - ctx context.Context, - aggreg model.Aggregate, - filters []*model.JobFilter, - weight *model.Weights, - limit *int) (map[string]int, error) { - - start := time.Now() - if !aggreg.IsValid() { - return nil, errors.New("invalid aggregate") - } - - runner := (sq.BaseRunner)(r.stmtCache) - count := "count(*) as count" - if weight != nil { - switch *weight { - case model.WeightsNodeCount: - count = "sum(job.num_nodes) as count" - case model.WeightsNodeHours: - now := time.Now().Unix() - count = fmt.Sprintf(`sum(job.num_nodes * (CASE WHEN job.job_state = "running" THEN %d - job.start_time ELSE job.duration END)) as count`, now) - runner = r.DB - case model.WeightsCoreCount: - count = "sum(job.num_hwthreads) as count" - case model.WeightsCoreHours: - now := time.Now().Unix() - count = fmt.Sprintf(`sum(job.num_hwthreads * (CASE WHEN job.job_state = "running" THEN %d - job.start_time ELSE job.duration END)) as count`, now) - runner = r.DB - default: - log.Debugf("CountGroupedJobs() Weight %v unknown.", *weight) - } - } - - q, qerr := SecurityCheck(ctx, sq.Select("job."+string(aggreg), count).From("job").GroupBy("job."+string(aggreg)).OrderBy("count DESC")) - - if qerr != nil { - return nil, qerr - } - - for _, f := range filters { - q = BuildWhereClause(f, q) - } - if limit != nil { - q = q.Limit(uint64(*limit)) - } - - counts := map[string]int{} - rows, err := q.RunWith(runner).Query() - if err != nil { - log.Error("Error while running query") - return nil, err - } - - for rows.Next() { - var group string - var count int - if err := rows.Scan(&group, &count); err != nil { - log.Warn("Error while scanning rows") - return nil, err - } - - counts[group] = count - } - - log.Debugf("Timer CountGroupedJobs %s", time.Since(start)) - return counts, nil -} - func (r *JobRepository) UpdateMonitoringStatus(job int64, monitoringStatus int32) (err error) { stmt := sq.Update("job"). Set("monitoring_status", monitoringStatus). From 13d99a6ae0706a1493f9eca947249d87075d1cc0 Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Thu, 24 Aug 2023 14:55:49 +0200 Subject: [PATCH 08/19] Fix typo in Jobstats resolver --- internal/graph/schema.resolvers.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/internal/graph/schema.resolvers.go b/internal/graph/schema.resolvers.go index a19ed8b..f2a3c65 100644 --- a/internal/graph/schema.resolvers.go +++ b/internal/graph/schema.resolvers.go @@ -256,22 +256,21 @@ func (r *queryResolver) JobsStatistics(ctx context.Context, filter []*model.JobF } } else { stats = make([]*model.JobsStatistics, 0, 1) - stats = append(stats, - &model.JobsStatistics{}) + stats = append(stats, &model.JobsStatistics{}) } if groupBy != nil { if requireField(ctx, "shortJobs") { stats, err = r.Repo.AddJobCountGrouped(ctx, filter, groupBy, stats, "short") } - if requireField(ctx, "RunningJobs") { + if requireField(ctx, "runningJobs") { stats, err = r.Repo.AddJobCountGrouped(ctx, filter, groupBy, stats, "running") } } else { if requireField(ctx, "shortJobs") { stats, err = r.Repo.AddJobCount(ctx, filter, stats, "short") } - if requireField(ctx, "RunningJobs") { + if requireField(ctx, "runningJobs") { stats, err = r.Repo.AddJobCount(ctx, filter, stats, "running") } } From ba42f4efc0b1871630fb7b82fb85efdbe3f9ded8 Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Thu, 24 Aug 2023 16:20:52 +0200 Subject: [PATCH 09/19] del obsolete enum --- api/schema.graphqls | 1 - 1 file changed, 1 deletion(-) diff --git a/api/schema.graphqls b/api/schema.graphqls index 7f427af..081cab6 100644 --- a/api/schema.graphqls +++ b/api/schema.graphqls @@ -167,7 +167,6 @@ type TimeWeights { } enum Aggregate { USER, PROJECT, CLUSTER } -enum Weights { NODE_COUNT, NODE_HOURS, CORE_COUNT, CORE_HOURS } type NodeMetrics { host: String! From d7117f3d49bb456ff5c53685ebe9bd364b2a7679 Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Fri, 25 Aug 2023 13:14:34 +0200 Subject: [PATCH 10/19] Add sorting and paging to JobStatsGrouped --- api/schema.graphqls | 4 +- internal/graph/generated/generated.go | 56 +++++++++++++---- internal/graph/model/models_gen.go | 90 +++++++++++++-------------- internal/graph/schema.resolvers.go | 4 +- internal/repository/stats.go | 62 +++++++++++++----- internal/repository/stats_test.go | 16 +++++ 6 files changed, 158 insertions(+), 74 deletions(-) diff --git a/api/schema.graphqls b/api/schema.graphqls index 7f427af..eb3e270 100644 --- a/api/schema.graphqls +++ b/api/schema.graphqls @@ -167,7 +167,7 @@ type TimeWeights { } enum Aggregate { USER, PROJECT, CLUSTER } -enum Weights { NODE_COUNT, NODE_HOURS, CORE_COUNT, CORE_HOURS } +enum SortByAggregate { WALLTIME, NODEHOURS, COREHOURS, ACCHOURS } type NodeMetrics { host: String! @@ -198,7 +198,7 @@ type Query { jobsFootprints(filter: [JobFilter!], metrics: [String!]!): Footprints jobs(filter: [JobFilter!], page: PageRequest, order: OrderByInput): JobResultList! - jobsStatistics(filter: [JobFilter!], groupBy: Aggregate): [JobsStatistics!]! + jobsStatistics(filter: [JobFilter!], page: PageRequest, sortBy: SortByAggregate, groupBy: Aggregate): [JobsStatistics!]! rooflineHeatmap(filter: [JobFilter!]!, rows: Int!, cols: Int!, minX: Float!, minY: Float!, maxX: Float!, maxY: Float!): [[Float!]!]! diff --git a/internal/graph/generated/generated.go b/internal/graph/generated/generated.go index 7544148..355b25a 100644 --- a/internal/graph/generated/generated.go +++ b/internal/graph/generated/generated.go @@ -203,7 +203,7 @@ type ComplexityRoot struct { JobMetrics func(childComplexity int, id string, metrics []string, scopes []schema.MetricScope) int Jobs func(childComplexity int, filter []*model.JobFilter, page *model.PageRequest, order *model.OrderByInput) int JobsFootprints func(childComplexity int, filter []*model.JobFilter, metrics []string) int - JobsStatistics func(childComplexity int, filter []*model.JobFilter, groupBy *model.Aggregate) int + JobsStatistics func(childComplexity int, filter []*model.JobFilter, page *model.PageRequest, sortBy *model.SortByAggregate, groupBy *model.Aggregate) int NodeMetrics func(childComplexity int, cluster string, nodes []string, scopes []schema.MetricScope, metrics []string, from time.Time, to time.Time) int RooflineHeatmap func(childComplexity int, filter []*model.JobFilter, rows int, cols int, minX float64, minY float64, maxX float64, maxY float64) int Tags func(childComplexity int) int @@ -317,7 +317,7 @@ type QueryResolver interface { JobMetrics(ctx context.Context, id string, metrics []string, scopes []schema.MetricScope) ([]*model.JobMetricWithName, error) JobsFootprints(ctx context.Context, filter []*model.JobFilter, metrics []string) (*model.Footprints, error) Jobs(ctx context.Context, filter []*model.JobFilter, page *model.PageRequest, order *model.OrderByInput) (*model.JobResultList, error) - JobsStatistics(ctx context.Context, filter []*model.JobFilter, groupBy *model.Aggregate) ([]*model.JobsStatistics, error) + JobsStatistics(ctx context.Context, filter []*model.JobFilter, page *model.PageRequest, sortBy *model.SortByAggregate, groupBy *model.Aggregate) ([]*model.JobsStatistics, error) RooflineHeatmap(ctx context.Context, filter []*model.JobFilter, rows int, cols int, minX float64, minY float64, maxX float64, maxY float64) ([][]float64, error) NodeMetrics(ctx context.Context, cluster string, nodes []string, scopes []schema.MetricScope, metrics []string, from time.Time, to time.Time) ([]*model.NodeMetrics, error) } @@ -1072,7 +1072,7 @@ func (e *executableSchema) Complexity(typeName, field string, childComplexity in return 0, false } - return e.complexity.Query.JobsStatistics(childComplexity, args["filter"].([]*model.JobFilter), args["groupBy"].(*model.Aggregate)), true + return e.complexity.Query.JobsStatistics(childComplexity, args["filter"].([]*model.JobFilter), args["page"].(*model.PageRequest), args["sortBy"].(*model.SortByAggregate), args["groupBy"].(*model.Aggregate)), true case "Query.nodeMetrics": if e.complexity.Query.NodeMetrics == nil { @@ -1727,7 +1727,7 @@ type TimeWeights { } enum Aggregate { USER, PROJECT, CLUSTER } -enum Weights { NODE_COUNT, NODE_HOURS, CORE_COUNT, CORE_HOURS } +enum SortByAggregate { WALLTIME, NODEHOURS, COREHOURS, ACCHOURS } type NodeMetrics { host: String! @@ -1758,7 +1758,7 @@ type Query { jobsFootprints(filter: [JobFilter!], metrics: [String!]!): Footprints jobs(filter: [JobFilter!], page: PageRequest, order: OrderByInput): JobResultList! - jobsStatistics(filter: [JobFilter!], groupBy: Aggregate): [JobsStatistics!]! + jobsStatistics(filter: [JobFilter!], page: PageRequest, sortBy: SortByAggregate, groupBy: Aggregate): [JobsStatistics!]! rooflineHeatmap(filter: [JobFilter!]!, rows: Int!, cols: Int!, minX: Float!, minY: Float!, maxX: Float!, maxY: Float!): [[Float!]!]! @@ -2097,15 +2097,33 @@ func (ec *executionContext) field_Query_jobsStatistics_args(ctx context.Context, } } args["filter"] = arg0 - var arg1 *model.Aggregate - if tmp, ok := rawArgs["groupBy"]; ok { - ctx := graphql.WithPathContext(ctx, graphql.NewPathWithField("groupBy")) - arg1, err = ec.unmarshalOAggregate2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐAggregate(ctx, tmp) + var arg1 *model.PageRequest + if tmp, ok := rawArgs["page"]; ok { + ctx := graphql.WithPathContext(ctx, graphql.NewPathWithField("page")) + arg1, err = ec.unmarshalOPageRequest2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐPageRequest(ctx, tmp) if err != nil { return nil, err } } - args["groupBy"] = arg1 + args["page"] = arg1 + var arg2 *model.SortByAggregate + if tmp, ok := rawArgs["sortBy"]; ok { + ctx := graphql.WithPathContext(ctx, graphql.NewPathWithField("sortBy")) + arg2, err = ec.unmarshalOSortByAggregate2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐSortByAggregate(ctx, tmp) + if err != nil { + return nil, err + } + } + args["sortBy"] = arg2 + var arg3 *model.Aggregate + if tmp, ok := rawArgs["groupBy"]; ok { + ctx := graphql.WithPathContext(ctx, graphql.NewPathWithField("groupBy")) + arg3, err = ec.unmarshalOAggregate2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐAggregate(ctx, tmp) + if err != nil { + return nil, err + } + } + args["groupBy"] = arg3 return args, nil } @@ -7079,7 +7097,7 @@ func (ec *executionContext) _Query_jobsStatistics(ctx context.Context, field gra }() resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (interface{}, error) { ctx = rctx // use context from middleware stack in children - return ec.resolvers.Query().JobsStatistics(rctx, fc.Args["filter"].([]*model.JobFilter), fc.Args["groupBy"].(*model.Aggregate)) + return ec.resolvers.Query().JobsStatistics(rctx, fc.Args["filter"].([]*model.JobFilter), fc.Args["page"].(*model.PageRequest), fc.Args["sortBy"].(*model.SortByAggregate), fc.Args["groupBy"].(*model.Aggregate)) }) if err != nil { ec.Error(ctx, err) @@ -16140,6 +16158,22 @@ func (ec *executionContext) marshalOSeries2ᚕgithubᚗcomᚋClusterCockpitᚋcc return ret } +func (ec *executionContext) unmarshalOSortByAggregate2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐSortByAggregate(ctx context.Context, v interface{}) (*model.SortByAggregate, error) { + if v == nil { + return nil, nil + } + var res = new(model.SortByAggregate) + err := res.UnmarshalGQL(v) + return res, graphql.ErrorOnPath(ctx, err) +} + +func (ec *executionContext) marshalOSortByAggregate2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐSortByAggregate(ctx context.Context, sel ast.SelectionSet, v *model.SortByAggregate) graphql.Marshaler { + if v == nil { + return graphql.Null + } + return v +} + func (ec *executionContext) marshalOStatsSeries2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋpkgᚋschemaᚐStatsSeries(ctx context.Context, sel ast.SelectionSet, v *schema.StatsSeries) graphql.Marshaler { if v == nil { return graphql.Null diff --git a/internal/graph/model/models_gen.go b/internal/graph/model/models_gen.go index faffae7..609e6a4 100644 --- a/internal/graph/model/models_gen.go +++ b/internal/graph/model/models_gen.go @@ -188,6 +188,51 @@ func (e Aggregate) MarshalGQL(w io.Writer) { fmt.Fprint(w, strconv.Quote(e.String())) } +type SortByAggregate string + +const ( + SortByAggregateWalltime SortByAggregate = "WALLTIME" + SortByAggregateNodehours SortByAggregate = "NODEHOURS" + SortByAggregateCorehours SortByAggregate = "COREHOURS" + SortByAggregateAcchours SortByAggregate = "ACCHOURS" +) + +var AllSortByAggregate = []SortByAggregate{ + SortByAggregateWalltime, + SortByAggregateNodehours, + SortByAggregateCorehours, + SortByAggregateAcchours, +} + +func (e SortByAggregate) IsValid() bool { + switch e { + case SortByAggregateWalltime, SortByAggregateNodehours, SortByAggregateCorehours, SortByAggregateAcchours: + return true + } + return false +} + +func (e SortByAggregate) String() string { + return string(e) +} + +func (e *SortByAggregate) UnmarshalGQL(v interface{}) error { + str, ok := v.(string) + if !ok { + return fmt.Errorf("enums must be strings") + } + + *e = SortByAggregate(str) + if !e.IsValid() { + return fmt.Errorf("%s is not a valid SortByAggregate", str) + } + return nil +} + +func (e SortByAggregate) MarshalGQL(w io.Writer) { + fmt.Fprint(w, strconv.Quote(e.String())) +} + type SortDirectionEnum string const ( @@ -228,48 +273,3 @@ func (e *SortDirectionEnum) UnmarshalGQL(v interface{}) error { func (e SortDirectionEnum) MarshalGQL(w io.Writer) { fmt.Fprint(w, strconv.Quote(e.String())) } - -type Weights string - -const ( - WeightsNodeCount Weights = "NODE_COUNT" - WeightsNodeHours Weights = "NODE_HOURS" - WeightsCoreCount Weights = "CORE_COUNT" - WeightsCoreHours Weights = "CORE_HOURS" -) - -var AllWeights = []Weights{ - WeightsNodeCount, - WeightsNodeHours, - WeightsCoreCount, - WeightsCoreHours, -} - -func (e Weights) IsValid() bool { - switch e { - case WeightsNodeCount, WeightsNodeHours, WeightsCoreCount, WeightsCoreHours: - return true - } - return false -} - -func (e Weights) String() string { - return string(e) -} - -func (e *Weights) UnmarshalGQL(v interface{}) error { - str, ok := v.(string) - if !ok { - return fmt.Errorf("enums must be strings") - } - - *e = Weights(str) - if !e.IsValid() { - return fmt.Errorf("%s is not a valid Weights", str) - } - return nil -} - -func (e Weights) MarshalGQL(w io.Writer) { - fmt.Fprint(w, strconv.Quote(e.String())) -} diff --git a/internal/graph/schema.resolvers.go b/internal/graph/schema.resolvers.go index f2a3c65..83aec04 100644 --- a/internal/graph/schema.resolvers.go +++ b/internal/graph/schema.resolvers.go @@ -244,7 +244,7 @@ func (r *queryResolver) Jobs(ctx context.Context, filter []*model.JobFilter, pag } // JobsStatistics is the resolver for the jobsStatistics field. -func (r *queryResolver) JobsStatistics(ctx context.Context, filter []*model.JobFilter, groupBy *model.Aggregate) ([]*model.JobsStatistics, error) { +func (r *queryResolver) JobsStatistics(ctx context.Context, filter []*model.JobFilter, page *model.PageRequest, sortBy *model.SortByAggregate, groupBy *model.Aggregate) ([]*model.JobsStatistics, error) { var err error var stats []*model.JobsStatistics @@ -252,7 +252,7 @@ func (r *queryResolver) JobsStatistics(ctx context.Context, filter []*model.JobF if groupBy == nil { stats, err = r.Repo.JobsStats(ctx, filter) } else { - stats, err = r.Repo.JobsStatsGrouped(ctx, filter, groupBy) + stats, err = r.Repo.JobsStatsGrouped(ctx, filter, page, sortBy, groupBy) } } else { stats = make([]*model.JobsStatistics, 0, 1) diff --git a/internal/repository/stats.go b/internal/repository/stats.go index ee48ee1..18d495b 100644 --- a/internal/repository/stats.go +++ b/internal/repository/stats.go @@ -23,6 +23,13 @@ var groupBy2column = map[model.Aggregate]string{ model.AggregateCluster: "job.cluster", } +var sortBy2column = map[model.SortByAggregate]string{ + model.SortByAggregateWalltime: "totalWalltime", + model.SortByAggregateNodehours: "totalNodeHours", + model.SortByAggregateCorehours: "totalCoreHours", + model.SortByAggregateAcchours: "totalAccHours", +} + func (r *JobRepository) buildCountQuery( filter []*model.JobFilter, kind string, @@ -62,11 +69,12 @@ func (r *JobRepository) buildStatsQuery( if col != "" { // Scan columns: id, totalJobs, totalWalltime, totalNodeHours, totalCoreHours, totalAccHours query = sq.Select(col, "COUNT(job.id)", - fmt.Sprintf("CAST(ROUND(SUM(job.duration) / 3600) as %s)", castType), - fmt.Sprintf("CAST(ROUND(SUM(job.duration * job.num_nodes) / 3600) as %s)", castType), - fmt.Sprintf("CAST(ROUND(SUM(job.duration * job.num_hwthreads) / 3600) as %s)", castType), - fmt.Sprintf("CAST(ROUND(SUM(job.duration * job.num_acc) / 3600) as %s)", castType), + fmt.Sprintf("CAST(ROUND(SUM(job.duration) / 3600) as %s) as totalWalltime", castType), + fmt.Sprintf("CAST(ROUND(SUM(job.duration * job.num_nodes) / 3600) as %s) as totalNodeHours", castType), + fmt.Sprintf("CAST(ROUND(SUM(job.duration * job.num_hwthreads) / 3600) as %s) as totalCoreHours", castType), + fmt.Sprintf("CAST(ROUND(SUM(job.duration * job.num_acc) / 3600) as %s) as totalAccHours", castType), ).From("job").GroupBy(col) + } else { // Scan columns: totalJobs, totalWalltime, totalNodeHours, totalCoreHours, totalAccHours query = sq.Select("COUNT(job.id)", @@ -112,16 +120,28 @@ func (r *JobRepository) getCastType() string { func (r *JobRepository) JobsStatsGrouped( ctx context.Context, filter []*model.JobFilter, + page *model.PageRequest, + sortBy *model.SortByAggregate, groupBy *model.Aggregate) ([]*model.JobsStatistics, error) { start := time.Now() col := groupBy2column[*groupBy] query := r.buildStatsQuery(filter, col) + query, err := SecurityCheck(ctx, query) if err != nil { return nil, err } + if sortBy != nil { + sortBy := sortBy2column[*sortBy] + query = query.OrderBy(fmt.Sprintf("%s DESC", sortBy)) + } + if page != nil && page.ItemsPerPage != -1 { + limit := uint64(page.ItemsPerPage) + query = query.Offset((uint64(page.Page) - 1) * limit).Limit(limit) + } + rows, err := query.RunWith(r.DB).Query() if err != nil { log.Warn("Error while querying DB for job statistics") @@ -174,17 +194,10 @@ func (r *JobRepository) JobsStatsGrouped( return stats, nil } -func (r *JobRepository) JobsStats( - ctx context.Context, +func (r *JobRepository) jobsStats( + query sq.SelectBuilder, filter []*model.JobFilter) ([]*model.JobsStatistics, error) { - start := time.Now() - query := r.buildStatsQuery(filter, "") - query, err := SecurityCheck(ctx, query) - if err != nil { - return nil, err - } - row := query.RunWith(r.DB).QueryRow() stats := make([]*model.JobsStatistics, 0, 1) @@ -211,10 +224,31 @@ func (r *JobRepository) JobsStats( TotalAccHours: totalAccHours}) } - log.Debugf("Timer JobStats %s", time.Since(start)) return stats, nil } +func (r *JobRepository) testJobsStats( + filter []*model.JobFilter) ([]*model.JobsStatistics, error) { + + query := r.buildStatsQuery(filter, "") + return r.jobsStats(query, filter) +} + +func (r *JobRepository) JobsStats( + ctx context.Context, + filter []*model.JobFilter) ([]*model.JobsStatistics, error) { + + start := time.Now() + query := r.buildStatsQuery(filter, "") + query, err := SecurityCheck(ctx, query) + if err != nil { + return nil, err + } + + log.Debugf("Timer JobStats %s", time.Since(start)) + return r.jobsStats(query, filter) +} + func (r *JobRepository) JobCountGrouped( ctx context.Context, filter []*model.JobFilter, diff --git a/internal/repository/stats_test.go b/internal/repository/stats_test.go index b1a815e..2672b3f 100644 --- a/internal/repository/stats_test.go +++ b/internal/repository/stats_test.go @@ -7,6 +7,8 @@ package repository import ( "fmt" "testing" + + "github.com/ClusterCockpit/cc-backend/internal/graph/model" ) func TestBuildJobStatsQuery(t *testing.T) { @@ -19,3 +21,17 @@ func TestBuildJobStatsQuery(t *testing.T) { fmt.Printf("SQL: %s\n", sql) } + +func TestJobStats(t *testing.T) { + r := setup(t) + + filter := &model.JobFilter{} + var err error + var stats []*model.JobsStatistics + stats, err = r.testJobsStats([]*model.JobFilter{filter}) + noErr(t, err) + + if stats[0].TotalJobs != 98 { + t.Fatalf("Want 98, Got %d", stats[0].TotalJobs) + } +} From ce758610b6e9c94528c25047a03343a7a90fec63 Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Fri, 25 Aug 2023 17:38:25 +0200 Subject: [PATCH 11/19] change: implement topX query in jobsStatistics --- api/schema.graphqls | 5 +- internal/graph/generated/generated.go | 182 +++++++++++++++++++++++++- internal/graph/model/models_gen.go | 19 ++- internal/repository/stats.go | 38 ++++-- web/frontend/src/Analysis.root.svelte | 27 +++- web/frontend/src/Status.root.svelte | 63 ++++++--- 6 files changed, 293 insertions(+), 41 deletions(-) diff --git a/api/schema.graphqls b/api/schema.graphqls index eb3e270..303ede5 100644 --- a/api/schema.graphqls +++ b/api/schema.graphqls @@ -167,7 +167,7 @@ type TimeWeights { } enum Aggregate { USER, PROJECT, CLUSTER } -enum SortByAggregate { WALLTIME, NODEHOURS, COREHOURS, ACCHOURS } +enum SortByAggregate { WALLTIME, TOTALNODES, NODEHOURS, TOTALCORES, COREHOURS, TOTALACCS, ACCHOURS } type NodeMetrics { host: String! @@ -293,8 +293,11 @@ type JobsStatistics { runningJobs: Int! # Number of running jobs shortJobs: Int! # Number of jobs with a duration of less than duration totalWalltime: Int! # Sum of the duration of all matched jobs in hours + totalNodes: Int! # Sum of the nodes of all matched jobs totalNodeHours: Int! # Sum of the node hours of all matched jobs + totalCores: Int! # Sum of the cores of all matched jobs totalCoreHours: Int! # Sum of the core hours of all matched jobs + totalAccs: Int! # Sum of the accs of all matched jobs totalAccHours: Int! # Sum of the gpu hours of all matched jobs histDuration: [HistoPoint!]! # value: hour, count: number of jobs with a rounded duration of value histNumNodes: [HistoPoint!]! # value: number of nodes, count: number of jobs with that number of nodes diff --git a/internal/graph/generated/generated.go b/internal/graph/generated/generated.go index 355b25a..f06698c 100644 --- a/internal/graph/generated/generated.go +++ b/internal/graph/generated/generated.go @@ -147,9 +147,12 @@ type ComplexityRoot struct { RunningJobs func(childComplexity int) int ShortJobs func(childComplexity int) int TotalAccHours func(childComplexity int) int + TotalAccs func(childComplexity int) int TotalCoreHours func(childComplexity int) int + TotalCores func(childComplexity int) int TotalJobs func(childComplexity int) int TotalNodeHours func(childComplexity int) int + TotalNodes func(childComplexity int) int TotalWalltime func(childComplexity int) int } @@ -767,6 +770,13 @@ func (e *executableSchema) Complexity(typeName, field string, childComplexity in return e.complexity.JobsStatistics.TotalAccHours(childComplexity), true + case "JobsStatistics.totalAccs": + if e.complexity.JobsStatistics.TotalAccs == nil { + break + } + + return e.complexity.JobsStatistics.TotalAccs(childComplexity), true + case "JobsStatistics.totalCoreHours": if e.complexity.JobsStatistics.TotalCoreHours == nil { break @@ -774,6 +784,13 @@ func (e *executableSchema) Complexity(typeName, field string, childComplexity in return e.complexity.JobsStatistics.TotalCoreHours(childComplexity), true + case "JobsStatistics.totalCores": + if e.complexity.JobsStatistics.TotalCores == nil { + break + } + + return e.complexity.JobsStatistics.TotalCores(childComplexity), true + case "JobsStatistics.totalJobs": if e.complexity.JobsStatistics.TotalJobs == nil { break @@ -788,6 +805,13 @@ func (e *executableSchema) Complexity(typeName, field string, childComplexity in return e.complexity.JobsStatistics.TotalNodeHours(childComplexity), true + case "JobsStatistics.totalNodes": + if e.complexity.JobsStatistics.TotalNodes == nil { + break + } + + return e.complexity.JobsStatistics.TotalNodes(childComplexity), true + case "JobsStatistics.totalWalltime": if e.complexity.JobsStatistics.TotalWalltime == nil { break @@ -1727,7 +1751,7 @@ type TimeWeights { } enum Aggregate { USER, PROJECT, CLUSTER } -enum SortByAggregate { WALLTIME, NODEHOURS, COREHOURS, ACCHOURS } +enum SortByAggregate { WALLTIME, TOTALNODES, NODEHOURS, TOTALCORES, COREHOURS, TOTALACCS, ACCHOURS } type NodeMetrics { host: String! @@ -1853,8 +1877,11 @@ type JobsStatistics { runningJobs: Int! # Number of running jobs shortJobs: Int! # Number of jobs with a duration of less than duration totalWalltime: Int! # Sum of the duration of all matched jobs in hours + totalNodes: Int! # Sum of the nodes of all matched jobs totalNodeHours: Int! # Sum of the node hours of all matched jobs + totalCores: Int! # Sum of the cores of all matched jobs totalCoreHours: Int! # Sum of the core hours of all matched jobs + totalAccs: Int! # Sum of the accs of all matched jobs totalAccHours: Int! # Sum of the gpu hours of all matched jobs histDuration: [HistoPoint!]! # value: hour, count: number of jobs with a rounded duration of value histNumNodes: [HistoPoint!]! # value: number of nodes, count: number of jobs with that number of nodes @@ -5131,6 +5158,50 @@ func (ec *executionContext) fieldContext_JobsStatistics_totalWalltime(ctx contex return fc, nil } +func (ec *executionContext) _JobsStatistics_totalNodes(ctx context.Context, field graphql.CollectedField, obj *model.JobsStatistics) (ret graphql.Marshaler) { + fc, err := ec.fieldContext_JobsStatistics_totalNodes(ctx, field) + if err != nil { + return graphql.Null + } + ctx = graphql.WithFieldContext(ctx, fc) + defer func() { + if r := recover(); r != nil { + ec.Error(ctx, ec.Recover(ctx, r)) + ret = graphql.Null + } + }() + resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (interface{}, error) { + ctx = rctx // use context from middleware stack in children + return obj.TotalNodes, nil + }) + if err != nil { + ec.Error(ctx, err) + return graphql.Null + } + if resTmp == nil { + if !graphql.HasFieldError(ctx, fc) { + ec.Errorf(ctx, "must not be null") + } + return graphql.Null + } + res := resTmp.(int) + fc.Result = res + return ec.marshalNInt2int(ctx, field.Selections, res) +} + +func (ec *executionContext) fieldContext_JobsStatistics_totalNodes(ctx context.Context, field graphql.CollectedField) (fc *graphql.FieldContext, err error) { + fc = &graphql.FieldContext{ + Object: "JobsStatistics", + Field: field, + IsMethod: false, + IsResolver: false, + Child: func(ctx context.Context, field graphql.CollectedField) (*graphql.FieldContext, error) { + return nil, errors.New("field of type Int does not have child fields") + }, + } + return fc, nil +} + func (ec *executionContext) _JobsStatistics_totalNodeHours(ctx context.Context, field graphql.CollectedField, obj *model.JobsStatistics) (ret graphql.Marshaler) { fc, err := ec.fieldContext_JobsStatistics_totalNodeHours(ctx, field) if err != nil { @@ -5175,6 +5246,50 @@ func (ec *executionContext) fieldContext_JobsStatistics_totalNodeHours(ctx conte return fc, nil } +func (ec *executionContext) _JobsStatistics_totalCores(ctx context.Context, field graphql.CollectedField, obj *model.JobsStatistics) (ret graphql.Marshaler) { + fc, err := ec.fieldContext_JobsStatistics_totalCores(ctx, field) + if err != nil { + return graphql.Null + } + ctx = graphql.WithFieldContext(ctx, fc) + defer func() { + if r := recover(); r != nil { + ec.Error(ctx, ec.Recover(ctx, r)) + ret = graphql.Null + } + }() + resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (interface{}, error) { + ctx = rctx // use context from middleware stack in children + return obj.TotalCores, nil + }) + if err != nil { + ec.Error(ctx, err) + return graphql.Null + } + if resTmp == nil { + if !graphql.HasFieldError(ctx, fc) { + ec.Errorf(ctx, "must not be null") + } + return graphql.Null + } + res := resTmp.(int) + fc.Result = res + return ec.marshalNInt2int(ctx, field.Selections, res) +} + +func (ec *executionContext) fieldContext_JobsStatistics_totalCores(ctx context.Context, field graphql.CollectedField) (fc *graphql.FieldContext, err error) { + fc = &graphql.FieldContext{ + Object: "JobsStatistics", + Field: field, + IsMethod: false, + IsResolver: false, + Child: func(ctx context.Context, field graphql.CollectedField) (*graphql.FieldContext, error) { + return nil, errors.New("field of type Int does not have child fields") + }, + } + return fc, nil +} + func (ec *executionContext) _JobsStatistics_totalCoreHours(ctx context.Context, field graphql.CollectedField, obj *model.JobsStatistics) (ret graphql.Marshaler) { fc, err := ec.fieldContext_JobsStatistics_totalCoreHours(ctx, field) if err != nil { @@ -5219,6 +5334,50 @@ func (ec *executionContext) fieldContext_JobsStatistics_totalCoreHours(ctx conte return fc, nil } +func (ec *executionContext) _JobsStatistics_totalAccs(ctx context.Context, field graphql.CollectedField, obj *model.JobsStatistics) (ret graphql.Marshaler) { + fc, err := ec.fieldContext_JobsStatistics_totalAccs(ctx, field) + if err != nil { + return graphql.Null + } + ctx = graphql.WithFieldContext(ctx, fc) + defer func() { + if r := recover(); r != nil { + ec.Error(ctx, ec.Recover(ctx, r)) + ret = graphql.Null + } + }() + resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (interface{}, error) { + ctx = rctx // use context from middleware stack in children + return obj.TotalAccs, nil + }) + if err != nil { + ec.Error(ctx, err) + return graphql.Null + } + if resTmp == nil { + if !graphql.HasFieldError(ctx, fc) { + ec.Errorf(ctx, "must not be null") + } + return graphql.Null + } + res := resTmp.(int) + fc.Result = res + return ec.marshalNInt2int(ctx, field.Selections, res) +} + +func (ec *executionContext) fieldContext_JobsStatistics_totalAccs(ctx context.Context, field graphql.CollectedField) (fc *graphql.FieldContext, err error) { + fc = &graphql.FieldContext{ + Object: "JobsStatistics", + Field: field, + IsMethod: false, + IsResolver: false, + Child: func(ctx context.Context, field graphql.CollectedField) (*graphql.FieldContext, error) { + return nil, errors.New("field of type Int does not have child fields") + }, + } + return fc, nil +} + func (ec *executionContext) _JobsStatistics_totalAccHours(ctx context.Context, field graphql.CollectedField, obj *model.JobsStatistics) (ret graphql.Marshaler) { fc, err := ec.fieldContext_JobsStatistics_totalAccHours(ctx, field) if err != nil { @@ -7134,10 +7293,16 @@ func (ec *executionContext) fieldContext_Query_jobsStatistics(ctx context.Contex return ec.fieldContext_JobsStatistics_shortJobs(ctx, field) case "totalWalltime": return ec.fieldContext_JobsStatistics_totalWalltime(ctx, field) + case "totalNodes": + return ec.fieldContext_JobsStatistics_totalNodes(ctx, field) case "totalNodeHours": return ec.fieldContext_JobsStatistics_totalNodeHours(ctx, field) + case "totalCores": + return ec.fieldContext_JobsStatistics_totalCores(ctx, field) case "totalCoreHours": return ec.fieldContext_JobsStatistics_totalCoreHours(ctx, field) + case "totalAccs": + return ec.fieldContext_JobsStatistics_totalAccs(ctx, field) case "totalAccHours": return ec.fieldContext_JobsStatistics_totalAccHours(ctx, field) case "histDuration": @@ -12573,16 +12738,31 @@ func (ec *executionContext) _JobsStatistics(ctx context.Context, sel ast.Selecti if out.Values[i] == graphql.Null { out.Invalids++ } + case "totalNodes": + out.Values[i] = ec._JobsStatistics_totalNodes(ctx, field, obj) + if out.Values[i] == graphql.Null { + out.Invalids++ + } case "totalNodeHours": out.Values[i] = ec._JobsStatistics_totalNodeHours(ctx, field, obj) if out.Values[i] == graphql.Null { out.Invalids++ } + case "totalCores": + out.Values[i] = ec._JobsStatistics_totalCores(ctx, field, obj) + if out.Values[i] == graphql.Null { + out.Invalids++ + } case "totalCoreHours": out.Values[i] = ec._JobsStatistics_totalCoreHours(ctx, field, obj) if out.Values[i] == graphql.Null { out.Invalids++ } + case "totalAccs": + out.Values[i] = ec._JobsStatistics_totalAccs(ctx, field, obj) + if out.Values[i] == graphql.Null { + out.Invalids++ + } case "totalAccHours": out.Values[i] = ec._JobsStatistics_totalAccHours(ctx, field, obj) if out.Values[i] == graphql.Null { diff --git a/internal/graph/model/models_gen.go b/internal/graph/model/models_gen.go index 609e6a4..3997b2d 100644 --- a/internal/graph/model/models_gen.go +++ b/internal/graph/model/models_gen.go @@ -91,8 +91,11 @@ type JobsStatistics struct { RunningJobs int `json:"runningJobs"` ShortJobs int `json:"shortJobs"` TotalWalltime int `json:"totalWalltime"` + TotalNodes int `json:"totalNodes"` TotalNodeHours int `json:"totalNodeHours"` + TotalCores int `json:"totalCores"` TotalCoreHours int `json:"totalCoreHours"` + TotalAccs int `json:"totalAccs"` TotalAccHours int `json:"totalAccHours"` HistDuration []*HistoPoint `json:"histDuration"` HistNumNodes []*HistoPoint `json:"histNumNodes"` @@ -191,22 +194,28 @@ func (e Aggregate) MarshalGQL(w io.Writer) { type SortByAggregate string const ( - SortByAggregateWalltime SortByAggregate = "WALLTIME" - SortByAggregateNodehours SortByAggregate = "NODEHOURS" - SortByAggregateCorehours SortByAggregate = "COREHOURS" - SortByAggregateAcchours SortByAggregate = "ACCHOURS" + SortByAggregateWalltime SortByAggregate = "WALLTIME" + SortByAggregateTotalnodes SortByAggregate = "TOTALNODES" + SortByAggregateNodehours SortByAggregate = "NODEHOURS" + SortByAggregateTotalcores SortByAggregate = "TOTALCORES" + SortByAggregateCorehours SortByAggregate = "COREHOURS" + SortByAggregateTotalaccs SortByAggregate = "TOTALACCS" + SortByAggregateAcchours SortByAggregate = "ACCHOURS" ) var AllSortByAggregate = []SortByAggregate{ SortByAggregateWalltime, + SortByAggregateTotalnodes, SortByAggregateNodehours, + SortByAggregateTotalcores, SortByAggregateCorehours, + SortByAggregateTotalaccs, SortByAggregateAcchours, } func (e SortByAggregate) IsValid() bool { switch e { - case SortByAggregateWalltime, SortByAggregateNodehours, SortByAggregateCorehours, SortByAggregateAcchours: + case SortByAggregateWalltime, SortByAggregateTotalnodes, SortByAggregateNodehours, SortByAggregateTotalcores, SortByAggregateCorehours, SortByAggregateTotalaccs, SortByAggregateAcchours: return true } return false diff --git a/internal/repository/stats.go b/internal/repository/stats.go index 18d495b..905d74b 100644 --- a/internal/repository/stats.go +++ b/internal/repository/stats.go @@ -24,10 +24,13 @@ var groupBy2column = map[model.Aggregate]string{ } var sortBy2column = map[model.SortByAggregate]string{ - model.SortByAggregateWalltime: "totalWalltime", - model.SortByAggregateNodehours: "totalNodeHours", - model.SortByAggregateCorehours: "totalCoreHours", - model.SortByAggregateAcchours: "totalAccHours", + model.SortByAggregateWalltime: "totalWalltime", + model.SortByAggregateTotalnodes: "totalNodes", + model.SortByAggregateNodehours: "totalNodeHours", + model.SortByAggregateTotalcores: "totalCores", + model.SortByAggregateCorehours: "totalCoreHours", + model.SortByAggregateTotalaccs: "totalAccs", + model.SortByAggregateAcchours: "totalAccHours", } func (r *JobRepository) buildCountQuery( @@ -67,20 +70,26 @@ func (r *JobRepository) buildStatsQuery( castType := r.getCastType() if col != "" { - // Scan columns: id, totalJobs, totalWalltime, totalNodeHours, totalCoreHours, totalAccHours + // Scan columns: id, totalJobs, totalWalltime, totalNodes, totalNodeHours, totalCores, totalCoreHours, totalAccs, totalAccHours query = sq.Select(col, "COUNT(job.id)", fmt.Sprintf("CAST(ROUND(SUM(job.duration) / 3600) as %s) as totalWalltime", castType), + fmt.Sprintf("CAST(SUM(job.num_nodes) as %s) as totalNodes", castType), fmt.Sprintf("CAST(ROUND(SUM(job.duration * job.num_nodes) / 3600) as %s) as totalNodeHours", castType), + fmt.Sprintf("CAST(SUM(job.num_hwthreads) as %s) as totalCores", castType), fmt.Sprintf("CAST(ROUND(SUM(job.duration * job.num_hwthreads) / 3600) as %s) as totalCoreHours", castType), + fmt.Sprintf("CAST(SUM(job.num_acc) as %s) as totalAccs", castType), fmt.Sprintf("CAST(ROUND(SUM(job.duration * job.num_acc) / 3600) as %s) as totalAccHours", castType), ).From("job").GroupBy(col) } else { - // Scan columns: totalJobs, totalWalltime, totalNodeHours, totalCoreHours, totalAccHours + // Scan columns: totalJobs, totalWalltime, totalNodes, totalNodeHours, totalCores, totalCoreHours, totalAccs, totalAccHours query = sq.Select("COUNT(job.id)", fmt.Sprintf("CAST(ROUND(SUM(job.duration) / 3600) as %s)", castType), + fmt.Sprintf("CAST(SUM(job.num_nodes) as %s)", castType), fmt.Sprintf("CAST(ROUND(SUM(job.duration * job.num_nodes) / 3600) as %s)", castType), + fmt.Sprintf("CAST(SUM(job.num_hwthreads) as %s)", castType), fmt.Sprintf("CAST(ROUND(SUM(job.duration * job.num_hwthreads) / 3600) as %s)", castType), + fmt.Sprintf("CAST(SUM(job.num_acc) as %s)", castType), fmt.Sprintf("CAST(ROUND(SUM(job.duration * job.num_acc) / 3600) as %s)", castType), ).From("job") } @@ -152,14 +161,21 @@ func (r *JobRepository) JobsStatsGrouped( for rows.Next() { var id sql.NullString - var jobs, walltime, nodeHours, coreHours, accHours sql.NullInt64 - if err := rows.Scan(&id, &jobs, &walltime, &nodeHours, &coreHours, &accHours); err != nil { + var jobs, walltime, nodes, nodeHours, cores, coreHours, accs, accHours sql.NullInt64 + if err := rows.Scan(&id, &jobs, &walltime, &nodes, &nodeHours, &cores, &coreHours, &accs, &accHours); err != nil { log.Warn("Error while scanning rows") return nil, err } if id.Valid { - var totalCoreHours, totalAccHours int + var totalCores, totalCoreHours, totalAccs, totalAccHours int + + if cores.Valid { + totalCores = int(cores.Int64) + } + if accs.Valid { + totalAccs = int(accs.Int64) + } if coreHours.Valid { totalCoreHours = int(coreHours.Int64) @@ -176,7 +192,9 @@ func (r *JobRepository) JobsStatsGrouped( Name: name, TotalJobs: int(jobs.Int64), TotalWalltime: int(walltime.Int64), + TotalCores: totalCores, TotalCoreHours: totalCoreHours, + TotalAccs: totalAccs, TotalAccHours: totalAccHours}) } else { stats = append(stats, @@ -184,7 +202,9 @@ func (r *JobRepository) JobsStatsGrouped( ID: id.String, TotalJobs: int(jobs.Int64), TotalWalltime: int(walltime.Int64), + TotalCores: totalCores, TotalCoreHours: totalCoreHours, + TotalAccs: totalAccs, TotalAccHours: totalAccHours}) } } diff --git a/web/frontend/src/Analysis.root.svelte b/web/frontend/src/Analysis.root.svelte index f60b2b9..4a87600 100644 --- a/web/frontend/src/Analysis.root.svelte +++ b/web/frontend/src/Analysis.root.svelte @@ -66,13 +66,26 @@ histDuration { count, value } histNumNodes { count, value } } - - topUsers: jobsCount(filter: $jobFilters, groupBy: USER, weight: CORE_HOURS, limit: 5) { name, count } } `, variables: { jobFilters } }) + const paging = { itemsPerPage: 5, page: 1 }; // Top 5 + // const sorting = { field: "totalCoreHours", order: "DESC" }; + $: topQuery = queryStore({ + client: client, + query: gql` + query($jobFilters: [JobFilter!]!, $paging: PageRequest!) { + jobsStatistics(filter: $jobFilters, page: $paging, sortBy: TOTALCOREHOURS, groupBy: USER) { + id + totalCoreHours + } + } + `, + variables: { jobFilters, paging } + }) + $: footprintsQuery = queryStore({ client: client, query: gql` @@ -164,8 +177,8 @@ b.count - a.count).map((tu) => tu.count)} - entities={$statsQuery.data.topUsers.sort((a, b) => b.count - a.count).map((tu) => tu.name)} + quantities={$topQuery.data.jobsStatistics.map((tu) => tu.totalCoreHours)} + entities={$topQuery.data.jobsStatistics.map((tu) => tu.id)} /> {/key} @@ -173,11 +186,11 @@
LegendUser NameNode Hours
LegendUser NameCore Hours
- {#each $statsQuery.data.topUsers.sort((a, b) => b.count - a.count) as { name, count }, i} + {#each $topQuery.data.jobsStatistics as { id, totalCoreHours }, i} - - + + {/each}
LegendUser NameCore Hours
{name}{count}{id}{totalCoreHours}
diff --git a/web/frontend/src/Status.root.svelte b/web/frontend/src/Status.root.svelte index 244862c..06a1893 100644 --- a/web/frontend/src/Status.root.svelte +++ b/web/frontend/src/Status.root.svelte @@ -38,9 +38,7 @@ histNumNodes { count, value } } - allocatedNodes(cluster: $cluster) { name, count } - topUsers: jobsCount(filter: $filter, groupBy: USER, weight: NODE_COUNT, limit: 10) { name, count } - topProjects: jobsCount(filter: $filter, groupBy: PROJECT, weight: NODE_COUNT, limit: 10) { name, count } + allocatedNodes(cluster: $cluster) { name, count } }`, variables: { cluster: cluster, metrics: ['flops_any', 'mem_bw'], from: from.toISOString(), to: to.toISOString(), @@ -48,6 +46,36 @@ } }) + const paging = { itemsPerPage: 10, page: 1 }; // Top 10 + // const sorting = { field: "totalCores", order: "DESC" }; + $: topUserQuery = queryStore({ + client: client, + query: gql` + query($filter: [JobFilter!]!, $paging: PageRequest!) { + topUser: jobsStatistics(filter: $filter, page: $paging, sortBy: TOTALCORES, groupBy: USER) { + id + totalNodes + totalCores + } + } + `, + variables: { filter: [{ state: ['running'] }, { cluster: { eq: cluster } }], paging } + }) + + $: topProjectQuery = queryStore({ + client: client, + query: gql` + query($filter: [JobFilter!]!, $paging: PageRequest!) { + topProjects: jobsStatistics(filter: $filter, page: $paging, sortBy: TOTALCORES, groupBy: PROJECT) { + id + totalNodes + totalCores + } + } + `, + variables: { filter: [{ state: ['running'] }, { cluster: { eq: cluster } }], paging } + }) + const sumUp = (data, subcluster, metric) => data.reduce((sum, node) => node.subCluster == subcluster ? sum + (node.metrics.find(m => m.name == metric)?.metric.series.reduce((sum, series) => sum + series.data[series.data.length - 1], 0) || 0) : sum, 0) @@ -161,48 +189,47 @@

Top Users

- {#key $mainQuery.data} + {#key $topUserQuery.data} b.count - a.count).map((tu) => tu.count)} - entities={$mainQuery.data.topUsers.sort((a, b) => b.count - a.count).map((tu) => tu.name)} - + quantities={$topUserQuery.data.topUser.map((tu) => tu.totalCores)} + entities={$topUserQuery.data.topUser.map((tu) => tu.id)} /> {/key}
- - {#each $mainQuery.data.topUsers.sort((a, b) => b.count - a.count) as { name, count }, i} + + {#each $topUserQuery.data.topUser as { id, totalCores, totalNodes }, i} - - + + {/each}
LegendUser NameNumber of Nodes
LegendUser NameNumber of Cores
{name}{count}{id}{totalCores}

Top Projects

- {#key $mainQuery.data} + {#key $topProjectQuery.data} b.count - a.count).map((tp) => tp.count)} - entities={$mainQuery.data.topProjects.sort((a, b) => b.count - a.count).map((tp) => tp.name)} + quantities={$topProjectQuery.data.topProjects.map((tp) => tp.totalCores)} + entities={$topProjectQuery.data.topProjects.map((tp) => tp.id)} /> {/key} - - {#each $mainQuery.data.topProjects.sort((a, b) => b.count - a.count) as { name, count }, i} + + {#each $topProjectQuery.data.topProjects as { id, totalCores, totalNodes }, i} - - + + {/each}
LegendProject CodeNumber of Nodes
LegendProject CodeNumber of Cores
{name}{count}{id}{totalCores}
From c84b81921257da407c04492a65f60b1f9b1df7ea Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Mon, 28 Aug 2023 10:00:20 +0200 Subject: [PATCH 12/19] Fix frontend errors - todo: debug backend handling --- internal/repository/stats.go | 12 ++++++++++-- web/frontend/src/Analysis.root.svelte | 2 +- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/internal/repository/stats.go b/internal/repository/stats.go index 905d74b..c766238 100644 --- a/internal/repository/stats.go +++ b/internal/repository/stats.go @@ -221,14 +221,22 @@ func (r *JobRepository) jobsStats( row := query.RunWith(r.DB).QueryRow() stats := make([]*model.JobsStatistics, 0, 1) - var jobs, walltime, nodeHours, coreHours, accHours sql.NullInt64 - if err := row.Scan(&jobs, &walltime, &nodeHours, &coreHours, &accHours); err != nil { + var jobs, walltime, nodes, nodeHours, cores, coreHours, accs, accHours sql.NullInt64 + if err := row.Scan(&jobs, &walltime, &nodes, &nodeHours, &cores, &coreHours, &accs, &accHours); err != nil { log.Warn("Error while scanning rows") return nil, err } if jobs.Valid { var totalCoreHours, totalAccHours int + // var totalCores, totalAccs int + + // if cores.Valid { + // totalCores = int(cores.Int64) + // } + // if accs.Valid { + // totalAccs = int(accs.Int64) + // } if coreHours.Valid { totalCoreHours = int(coreHours.Int64) diff --git a/web/frontend/src/Analysis.root.svelte b/web/frontend/src/Analysis.root.svelte index 4a87600..76e1813 100644 --- a/web/frontend/src/Analysis.root.svelte +++ b/web/frontend/src/Analysis.root.svelte @@ -77,7 +77,7 @@ client: client, query: gql` query($jobFilters: [JobFilter!]!, $paging: PageRequest!) { - jobsStatistics(filter: $jobFilters, page: $paging, sortBy: TOTALCOREHOURS, groupBy: USER) { + jobsStatistics(filter: $jobFilters, page: $paging, sortBy: COREHOURS, groupBy: USER) { id totalCoreHours } From 69519ec0401ce134a4b98e44ffeb6842683eba70 Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Mon, 28 Aug 2023 10:19:26 +0200 Subject: [PATCH 13/19] Add requireField cases to resolver --- internal/graph/schema.resolvers.go | 3 ++- web/frontend/src/Analysis.root.svelte | 8 ++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/internal/graph/schema.resolvers.go b/internal/graph/schema.resolvers.go index 83aec04..684783a 100644 --- a/internal/graph/schema.resolvers.go +++ b/internal/graph/schema.resolvers.go @@ -248,7 +248,8 @@ func (r *queryResolver) JobsStatistics(ctx context.Context, filter []*model.JobF var err error var stats []*model.JobsStatistics - if requireField(ctx, "totalJobs") { + if requireField(ctx, "totalJobs") || requireField(ctx, "totalNodes") || requireField(ctx, "totalCores") || requireField(ctx, "totalAccs") || + requireField(ctx, "totalNodeHours") || requireField(ctx, "totalCoreHours") || requireField(ctx, "totalAccHours") { if groupBy == nil { stats, err = r.Repo.JobsStats(ctx, filter) } else { diff --git a/web/frontend/src/Analysis.root.svelte b/web/frontend/src/Analysis.root.svelte index 76e1813..88c3a32 100644 --- a/web/frontend/src/Analysis.root.svelte +++ b/web/frontend/src/Analysis.root.svelte @@ -77,7 +77,7 @@ client: client, query: gql` query($jobFilters: [JobFilter!]!, $paging: PageRequest!) { - jobsStatistics(filter: $jobFilters, page: $paging, sortBy: COREHOURS, groupBy: USER) { + topUser: jobsStatistics(filter: $jobFilters, page: $paging, sortBy: COREHOURS, groupBy: USER) { id totalCoreHours } @@ -177,8 +177,8 @@ tu.totalCoreHours)} - entities={$topQuery.data.jobsStatistics.map((tu) => tu.id)} + quantities={$topQuery.data.topUser.map((tu) => tu.totalCoreHours)} + entities={$topQuery.data.topUser.map((tu) => tu.id)} /> {/key} @@ -186,7 +186,7 @@ - {#each $topQuery.data.jobsStatistics as { id, totalCoreHours }, i} + {#each $topQuery.data.topUser as { id, totalCoreHours }, i} From f933cad87fb29775c45b61043bb9e39a48b399ae Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Tue, 29 Aug 2023 14:01:01 +0200 Subject: [PATCH 14/19] feat: add select to status view pie charts - 'Jobs' as generic default value for top lists - Prepare histograms for cores and accs in schema --- api/schema.graphqls | 4 +- internal/graph/generated/generated.go | 134 +++++++++++++++++++++++- internal/graph/model/models_gen.go | 6 +- internal/repository/stats.go | 35 ++++--- web/frontend/src/Status.root.svelte | 142 ++++++++++++++++++-------- web/frontend/src/plots/Pie.svelte | 4 +- 6 files changed, 267 insertions(+), 58 deletions(-) diff --git a/api/schema.graphqls b/api/schema.graphqls index 303ede5..729c712 100644 --- a/api/schema.graphqls +++ b/api/schema.graphqls @@ -167,7 +167,7 @@ type TimeWeights { } enum Aggregate { USER, PROJECT, CLUSTER } -enum SortByAggregate { WALLTIME, TOTALNODES, NODEHOURS, TOTALCORES, COREHOURS, TOTALACCS, ACCHOURS } +enum SortByAggregate { WALLTIME, TOTALJOBS, TOTALNODES, NODEHOURS, TOTALCORES, COREHOURS, TOTALACCS, ACCHOURS } type NodeMetrics { host: String! @@ -301,6 +301,8 @@ type JobsStatistics { totalAccHours: Int! # Sum of the gpu hours of all matched jobs histDuration: [HistoPoint!]! # value: hour, count: number of jobs with a rounded duration of value histNumNodes: [HistoPoint!]! # value: number of nodes, count: number of jobs with that number of nodes + histNumCores: [HistoPoint!]! # value: number of cores, count: number of jobs with that number of cores + histNumAccs: [HistoPoint!]! # value: number of accs, count: number of jobs with that number of accs } input PageRequest { diff --git a/internal/graph/generated/generated.go b/internal/graph/generated/generated.go index f06698c..efbfae9 100644 --- a/internal/graph/generated/generated.go +++ b/internal/graph/generated/generated.go @@ -141,6 +141,8 @@ type ComplexityRoot struct { JobsStatistics struct { HistDuration func(childComplexity int) int + HistNumAccs func(childComplexity int) int + HistNumCores func(childComplexity int) int HistNumNodes func(childComplexity int) int ID func(childComplexity int) int Name func(childComplexity int) int @@ -728,6 +730,20 @@ func (e *executableSchema) Complexity(typeName, field string, childComplexity in return e.complexity.JobsStatistics.HistDuration(childComplexity), true + case "JobsStatistics.histNumAccs": + if e.complexity.JobsStatistics.HistNumAccs == nil { + break + } + + return e.complexity.JobsStatistics.HistNumAccs(childComplexity), true + + case "JobsStatistics.histNumCores": + if e.complexity.JobsStatistics.HistNumCores == nil { + break + } + + return e.complexity.JobsStatistics.HistNumCores(childComplexity), true + case "JobsStatistics.histNumNodes": if e.complexity.JobsStatistics.HistNumNodes == nil { break @@ -1751,7 +1767,7 @@ type TimeWeights { } enum Aggregate { USER, PROJECT, CLUSTER } -enum SortByAggregate { WALLTIME, TOTALNODES, NODEHOURS, TOTALCORES, COREHOURS, TOTALACCS, ACCHOURS } +enum SortByAggregate { WALLTIME, TOTALJOBS, TOTALNODES, NODEHOURS, TOTALCORES, COREHOURS, TOTALACCS, ACCHOURS } type NodeMetrics { host: String! @@ -1885,6 +1901,8 @@ type JobsStatistics { totalAccHours: Int! # Sum of the gpu hours of all matched jobs histDuration: [HistoPoint!]! # value: hour, count: number of jobs with a rounded duration of value histNumNodes: [HistoPoint!]! # value: number of nodes, count: number of jobs with that number of nodes + histNumCores: [HistoPoint!]! # value: number of cores, count: number of jobs with that number of cores + histNumAccs: [HistoPoint!]! # value: number of accs, count: number of jobs with that number of accs } input PageRequest { @@ -5522,6 +5540,106 @@ func (ec *executionContext) fieldContext_JobsStatistics_histNumNodes(ctx context return fc, nil } +func (ec *executionContext) _JobsStatistics_histNumCores(ctx context.Context, field graphql.CollectedField, obj *model.JobsStatistics) (ret graphql.Marshaler) { + fc, err := ec.fieldContext_JobsStatistics_histNumCores(ctx, field) + if err != nil { + return graphql.Null + } + ctx = graphql.WithFieldContext(ctx, fc) + defer func() { + if r := recover(); r != nil { + ec.Error(ctx, ec.Recover(ctx, r)) + ret = graphql.Null + } + }() + resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (interface{}, error) { + ctx = rctx // use context from middleware stack in children + return obj.HistNumCores, nil + }) + if err != nil { + ec.Error(ctx, err) + return graphql.Null + } + if resTmp == nil { + if !graphql.HasFieldError(ctx, fc) { + ec.Errorf(ctx, "must not be null") + } + return graphql.Null + } + res := resTmp.([]*model.HistoPoint) + fc.Result = res + return ec.marshalNHistoPoint2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐHistoPointᚄ(ctx, field.Selections, res) +} + +func (ec *executionContext) fieldContext_JobsStatistics_histNumCores(ctx context.Context, field graphql.CollectedField) (fc *graphql.FieldContext, err error) { + fc = &graphql.FieldContext{ + Object: "JobsStatistics", + Field: field, + IsMethod: false, + IsResolver: false, + Child: func(ctx context.Context, field graphql.CollectedField) (*graphql.FieldContext, error) { + switch field.Name { + case "count": + return ec.fieldContext_HistoPoint_count(ctx, field) + case "value": + return ec.fieldContext_HistoPoint_value(ctx, field) + } + return nil, fmt.Errorf("no field named %q was found under type HistoPoint", field.Name) + }, + } + return fc, nil +} + +func (ec *executionContext) _JobsStatistics_histNumAccs(ctx context.Context, field graphql.CollectedField, obj *model.JobsStatistics) (ret graphql.Marshaler) { + fc, err := ec.fieldContext_JobsStatistics_histNumAccs(ctx, field) + if err != nil { + return graphql.Null + } + ctx = graphql.WithFieldContext(ctx, fc) + defer func() { + if r := recover(); r != nil { + ec.Error(ctx, ec.Recover(ctx, r)) + ret = graphql.Null + } + }() + resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (interface{}, error) { + ctx = rctx // use context from middleware stack in children + return obj.HistNumAccs, nil + }) + if err != nil { + ec.Error(ctx, err) + return graphql.Null + } + if resTmp == nil { + if !graphql.HasFieldError(ctx, fc) { + ec.Errorf(ctx, "must not be null") + } + return graphql.Null + } + res := resTmp.([]*model.HistoPoint) + fc.Result = res + return ec.marshalNHistoPoint2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐHistoPointᚄ(ctx, field.Selections, res) +} + +func (ec *executionContext) fieldContext_JobsStatistics_histNumAccs(ctx context.Context, field graphql.CollectedField) (fc *graphql.FieldContext, err error) { + fc = &graphql.FieldContext{ + Object: "JobsStatistics", + Field: field, + IsMethod: false, + IsResolver: false, + Child: func(ctx context.Context, field graphql.CollectedField) (*graphql.FieldContext, error) { + switch field.Name { + case "count": + return ec.fieldContext_HistoPoint_count(ctx, field) + case "value": + return ec.fieldContext_HistoPoint_value(ctx, field) + } + return nil, fmt.Errorf("no field named %q was found under type HistoPoint", field.Name) + }, + } + return fc, nil +} + func (ec *executionContext) _MetricConfig_name(ctx context.Context, field graphql.CollectedField, obj *schema.MetricConfig) (ret graphql.Marshaler) { fc, err := ec.fieldContext_MetricConfig_name(ctx, field) if err != nil { @@ -7309,6 +7427,10 @@ func (ec *executionContext) fieldContext_Query_jobsStatistics(ctx context.Contex return ec.fieldContext_JobsStatistics_histDuration(ctx, field) case "histNumNodes": return ec.fieldContext_JobsStatistics_histNumNodes(ctx, field) + case "histNumCores": + return ec.fieldContext_JobsStatistics_histNumCores(ctx, field) + case "histNumAccs": + return ec.fieldContext_JobsStatistics_histNumAccs(ctx, field) } return nil, fmt.Errorf("no field named %q was found under type JobsStatistics", field.Name) }, @@ -12778,6 +12900,16 @@ func (ec *executionContext) _JobsStatistics(ctx context.Context, sel ast.Selecti if out.Values[i] == graphql.Null { out.Invalids++ } + case "histNumCores": + out.Values[i] = ec._JobsStatistics_histNumCores(ctx, field, obj) + if out.Values[i] == graphql.Null { + out.Invalids++ + } + case "histNumAccs": + out.Values[i] = ec._JobsStatistics_histNumAccs(ctx, field, obj) + if out.Values[i] == graphql.Null { + out.Invalids++ + } default: panic("unknown field " + strconv.Quote(field.Name)) } diff --git a/internal/graph/model/models_gen.go b/internal/graph/model/models_gen.go index 3997b2d..99fc96f 100644 --- a/internal/graph/model/models_gen.go +++ b/internal/graph/model/models_gen.go @@ -99,6 +99,8 @@ type JobsStatistics struct { TotalAccHours int `json:"totalAccHours"` HistDuration []*HistoPoint `json:"histDuration"` HistNumNodes []*HistoPoint `json:"histNumNodes"` + HistNumCores []*HistoPoint `json:"histNumCores"` + HistNumAccs []*HistoPoint `json:"histNumAccs"` } type MetricFootprints struct { @@ -195,6 +197,7 @@ type SortByAggregate string const ( SortByAggregateWalltime SortByAggregate = "WALLTIME" + SortByAggregateTotaljobs SortByAggregate = "TOTALJOBS" SortByAggregateTotalnodes SortByAggregate = "TOTALNODES" SortByAggregateNodehours SortByAggregate = "NODEHOURS" SortByAggregateTotalcores SortByAggregate = "TOTALCORES" @@ -205,6 +208,7 @@ const ( var AllSortByAggregate = []SortByAggregate{ SortByAggregateWalltime, + SortByAggregateTotaljobs, SortByAggregateTotalnodes, SortByAggregateNodehours, SortByAggregateTotalcores, @@ -215,7 +219,7 @@ var AllSortByAggregate = []SortByAggregate{ func (e SortByAggregate) IsValid() bool { switch e { - case SortByAggregateWalltime, SortByAggregateTotalnodes, SortByAggregateNodehours, SortByAggregateTotalcores, SortByAggregateCorehours, SortByAggregateTotalaccs, SortByAggregateAcchours: + case SortByAggregateWalltime, SortByAggregateTotaljobs, SortByAggregateTotalnodes, SortByAggregateNodehours, SortByAggregateTotalcores, SortByAggregateCorehours, SortByAggregateTotalaccs, SortByAggregateAcchours: return true } return false diff --git a/internal/repository/stats.go b/internal/repository/stats.go index c766238..4585f44 100644 --- a/internal/repository/stats.go +++ b/internal/repository/stats.go @@ -24,6 +24,7 @@ var groupBy2column = map[model.Aggregate]string{ } var sortBy2column = map[model.SortByAggregate]string{ + model.SortByAggregateTotaljobs: "totalJobs", model.SortByAggregateWalltime: "totalWalltime", model.SortByAggregateTotalnodes: "totalNodes", model.SortByAggregateNodehours: "totalNodeHours", @@ -71,7 +72,7 @@ func (r *JobRepository) buildStatsQuery( if col != "" { // Scan columns: id, totalJobs, totalWalltime, totalNodes, totalNodeHours, totalCores, totalCoreHours, totalAccs, totalAccHours - query = sq.Select(col, "COUNT(job.id)", + query = sq.Select(col, "COUNT(job.id) as totalJobs", fmt.Sprintf("CAST(ROUND(SUM(job.duration) / 3600) as %s) as totalWalltime", castType), fmt.Sprintf("CAST(SUM(job.num_nodes) as %s) as totalNodes", castType), fmt.Sprintf("CAST(ROUND(SUM(job.duration * job.num_nodes) / 3600) as %s) as totalNodeHours", castType), @@ -168,8 +169,15 @@ func (r *JobRepository) JobsStatsGrouped( } if id.Valid { - var totalCores, totalCoreHours, totalAccs, totalAccHours int + var totalJobs, totalNodes, totalNodeHours, totalCores, totalCoreHours, totalAccs, totalAccHours int + if jobs.Valid { + totalJobs = int(jobs.Int64) + } + + if nodes.Valid { + totalNodes = int(nodes.Int64) + } if cores.Valid { totalCores = int(cores.Int64) } @@ -177,6 +185,9 @@ func (r *JobRepository) JobsStatsGrouped( totalAccs = int(accs.Int64) } + if nodeHours.Valid { + totalNodeHours = int(nodeHours.Int64) + } if coreHours.Valid { totalCoreHours = int(coreHours.Int64) } @@ -190,8 +201,10 @@ func (r *JobRepository) JobsStatsGrouped( &model.JobsStatistics{ ID: id.String, Name: name, - TotalJobs: int(jobs.Int64), + TotalJobs: totalJobs, TotalWalltime: int(walltime.Int64), + TotalNodes: totalNodes, + TotalNodeHours: totalNodeHours, TotalCores: totalCores, TotalCoreHours: totalCoreHours, TotalAccs: totalAccs, @@ -202,6 +215,8 @@ func (r *JobRepository) JobsStatsGrouped( ID: id.String, TotalJobs: int(jobs.Int64), TotalWalltime: int(walltime.Int64), + TotalNodes: totalNodes, + TotalNodeHours: totalNodeHours, TotalCores: totalCores, TotalCoreHours: totalCoreHours, TotalAccs: totalAccs, @@ -228,16 +243,11 @@ func (r *JobRepository) jobsStats( } if jobs.Valid { - var totalCoreHours, totalAccHours int - // var totalCores, totalAccs int - - // if cores.Valid { - // totalCores = int(cores.Int64) - // } - // if accs.Valid { - // totalAccs = int(accs.Int64) - // } + var totalNodeHours, totalCoreHours, totalAccHours int + if nodeHours.Valid { + totalNodeHours = int(nodeHours.Int64) + } if coreHours.Valid { totalCoreHours = int(coreHours.Int64) } @@ -248,6 +258,7 @@ func (r *JobRepository) jobsStats( &model.JobsStatistics{ TotalJobs: int(jobs.Int64), TotalWalltime: int(walltime.Int64), + TotalNodeHours: totalNodeHours, TotalCoreHours: totalCoreHours, TotalAccHours: totalAccHours}) } diff --git a/web/frontend/src/Status.root.svelte b/web/frontend/src/Status.root.svelte index 06a1893..4c9e8d5 100644 --- a/web/frontend/src/Status.root.svelte +++ b/web/frontend/src/Status.root.svelte @@ -14,6 +14,14 @@ let plotWidths = [], colWidth1 = 0, colWidth2 let from = new Date(Date.now() - 5 * 60 * 1000), to = new Date(Date.now()) + const topOptions = [ + {key: 'totalJobs', label: 'Jobs'}, + {key: 'totalNodes', label: 'Nodes'}, + {key: 'totalCores', label: 'Cores'}, + {key: 'totalAccs', label: 'Accelerators'}, + ] + let topProjectSelection = topOptions[0] // Default: Jobs + let topUserSelection = topOptions[0] // Default: Jobs const client = getContextClient(); $: mainQuery = queryStore({ @@ -51,29 +59,33 @@ $: topUserQuery = queryStore({ client: client, query: gql` - query($filter: [JobFilter!]!, $paging: PageRequest!) { - topUser: jobsStatistics(filter: $filter, page: $paging, sortBy: TOTALCORES, groupBy: USER) { + query($filter: [JobFilter!]!, $paging: PageRequest!, $sortBy: SortByAggregate!) { + topUser: jobsStatistics(filter: $filter, page: $paging, sortBy: $sortBy, groupBy: USER) { id + totalJobs totalNodes totalCores + totalAccs } } `, - variables: { filter: [{ state: ['running'] }, { cluster: { eq: cluster } }], paging } + variables: { filter: [{ state: ['running'] }, { cluster: { eq: cluster } }], paging, sortBy: topUserSelection.key.toUpperCase() } }) $: topProjectQuery = queryStore({ client: client, query: gql` - query($filter: [JobFilter!]!, $paging: PageRequest!) { - topProjects: jobsStatistics(filter: $filter, page: $paging, sortBy: TOTALCORES, groupBy: PROJECT) { + query($filter: [JobFilter!]!, $paging: PageRequest!, $sortBy: SortByAggregate!) { + topProjects: jobsStatistics(filter: $filter, page: $paging, sortBy: $sortBy, groupBy: PROJECT) { id + totalJobs totalNodes totalCores + totalAccs } } `, - variables: { filter: [{ state: ['running'] }, { cluster: { eq: cluster } }], paging } + variables: { filter: [{ state: ['running'] }, { cluster: { eq: cluster } }], paging, sortBy: topProjectSelection.key.toUpperCase() } }) const sumUp = (data, subcluster, metric) => data.reduce((sum, node) => node.subCluster == subcluster @@ -188,51 +200,99 @@
-

Top Users

- {#key $topUserQuery.data} +

Top Users on {cluster.charAt(0).toUpperCase() + cluster.slice(1)}

+ {#if $topUserQuery.fetching} + + {:else if $topUserQuery.error} + {$topUserQuery.error.message} + {:else} tu.totalCores)} + sliceLabel={topUserSelection.label} + quantities={$topUserQuery.data.topUser.map((tu) => tu[topUserSelection.key])} entities={$topUserQuery.data.topUser.map((tu) => tu.id)} /> - {/key} + {/if}
-
LegendUser NameCore Hours
{id}
- - {#each $topUserQuery.data.topUser as { id, totalCores, totalNodes }, i} - - - - - - {/each} -
LegendUser NameNumber of Cores
{id}{totalCores}
- - -

Top Projects

- {#key $topProjectQuery.data} - tp.totalCores)} - entities={$topProjectQuery.data.topProjects.map((tp) => tp.id)} - /> + {#key $topUserQuery.data} + {#if $topUserQuery.fetching} + + {:else if $topUserQuery.error} + {$topUserQuery.error.message} + {:else} + + + + + + + {#each $topUserQuery.data.topUser as tu, i} + + + + + + {/each} +
LegendUser NameNumber of + +
{tu.id}{tu[topUserSelection.key]}
+ {/if} {/key} + +

Top Projects on {cluster.charAt(0).toUpperCase() + cluster.slice(1)}

+ {#if $topProjectQuery.fetching} + + {:else if $topProjectQuery.error} + {$topProjectQuery.error.message} + {:else} + tp[topProjectSelection.key])} + entities={$topProjectQuery.data.topProjects.map((tp) => tp.id)} + /> + {/if} + - - - {#each $topProjectQuery.data.topProjects as { id, totalCores, totalNodes }, i} - - - - - - {/each} -
LegendProject CodeNumber of Cores
{id}{totalCores}
+ {#key $topProjectQuery.data} + {#if $topProjectQuery.fetching} + + {:else if $topProjectQuery.error} + {$topProjectQuery.error.message} + {:else} + + + + + + + {#each $topProjectQuery.data.topProjects as tp, i} + + + + + + {/each} +
LegendProject CodeNumber of + +
{tp.id}{tp[topProjectSelection.key]}
+ {/if} + {/key}
diff --git a/web/frontend/src/plots/Pie.svelte b/web/frontend/src/plots/Pie.svelte index 6355f09..11dc2c9 100644 --- a/web/frontend/src/plots/Pie.svelte +++ b/web/frontend/src/plots/Pie.svelte @@ -43,14 +43,14 @@ export let entities export let displayLegend = false - const data = { + $: data = { labels: entities, datasets: [ { label: sliceLabel, data: quantities, fill: 1, - backgroundColor: colors.slice(0, quantities.length), + backgroundColor: colors.slice(0, quantities.length) } ] } From 3014f59cc2cfa92ffaef1f02088dc0bb7636b0e3 Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Tue, 29 Aug 2023 14:02:23 +0200 Subject: [PATCH 15/19] feat: add new distribution plots to status view - numCores and numAccs --- internal/graph/schema.resolvers.go | 2 +- internal/repository/stats.go | 12 ++++++++++++ web/frontend/src/Status.root.svelte | 30 +++++++++++++++++++++++++++++ 3 files changed, 43 insertions(+), 1 deletion(-) diff --git a/internal/graph/schema.resolvers.go b/internal/graph/schema.resolvers.go index 684783a..e3cc1ea 100644 --- a/internal/graph/schema.resolvers.go +++ b/internal/graph/schema.resolvers.go @@ -280,7 +280,7 @@ func (r *queryResolver) JobsStatistics(ctx context.Context, filter []*model.JobF return nil, err } - if requireField(ctx, "histDuration") || requireField(ctx, "histNumNodes") { + if requireField(ctx, "histDuration") || requireField(ctx, "histNumNodes") || requireField(ctx, "histNumCores") || requireField(ctx, "histNumAccs") { if groupBy == nil { stats[0], err = r.Repo.AddHistograms(ctx, filter, stats[0]) if err != nil { diff --git a/internal/repository/stats.go b/internal/repository/stats.go index 4585f44..485a5cd 100644 --- a/internal/repository/stats.go +++ b/internal/repository/stats.go @@ -442,6 +442,18 @@ func (r *JobRepository) AddHistograms( return nil, err } + stat.HistNumCores, err = r.jobsStatisticsHistogram(ctx, "job.num_hwthreads as value", filter) + if err != nil { + log.Warn("Error while loading job statistics histogram: num hwthreads") + return nil, err + } + + stat.HistNumAccs, err = r.jobsStatisticsHistogram(ctx, "job.num_acc as value", filter) + if err != nil { + log.Warn("Error while loading job statistics histogram: num acc") + return nil, err + } + log.Debugf("Timer AddHistograms %s", time.Since(start)) return stat, nil } diff --git a/web/frontend/src/Status.root.svelte b/web/frontend/src/Status.root.svelte index 4c9e8d5..3192dbf 100644 --- a/web/frontend/src/Status.root.svelte +++ b/web/frontend/src/Status.root.svelte @@ -44,6 +44,8 @@ stats: jobsStatistics(filter: $filter) { histDuration { count, value } histNumNodes { count, value } + histNumCores { count, value } + histNumAccs { count, value } } allocatedNodes(cluster: $cluster) { name, count } @@ -324,4 +326,32 @@ {/key} + + +
+ {#key $mainQuery.data.stats} + + {/key} +
+ + + {#key $mainQuery.data.stats} + + {/key} + + {/if} \ No newline at end of file From 1771883754e6415dcff3bbbd1735a3792d6f86e2 Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Tue, 29 Aug 2023 15:58:25 +0200 Subject: [PATCH 16/19] fix missing key tag on piecharts --- web/frontend/src/Status.root.svelte | 53 +++++++++++++++-------------- 1 file changed, 28 insertions(+), 25 deletions(-) diff --git a/web/frontend/src/Status.root.svelte b/web/frontend/src/Status.root.svelte index 3192dbf..4e7269a 100644 --- a/web/frontend/src/Status.root.svelte +++ b/web/frontend/src/Status.root.svelte @@ -57,7 +57,6 @@ }) const paging = { itemsPerPage: 10, page: 1 }; // Top 10 - // const sorting = { field: "totalCores", order: "DESC" }; $: topUserQuery = queryStore({ client: client, query: gql` @@ -203,18 +202,20 @@

Top Users on {cluster.charAt(0).toUpperCase() + cluster.slice(1)}

- {#if $topUserQuery.fetching} - - {:else if $topUserQuery.error} - {$topUserQuery.error.message} - {:else} - tu[topUserSelection.key])} - entities={$topUserQuery.data.topUser.map((tu) => tu.id)} - /> - {/if} + {#key $topUserQuery.data} + {#if $topUserQuery.fetching} + + {:else if $topUserQuery.error} + {$topUserQuery.error.message} + {:else} + tu[topUserSelection.key])} + entities={$topUserQuery.data.topUser.map((tu) => tu.id)} + /> + {/if} + {/key}
@@ -251,18 +252,20 @@

Top Projects on {cluster.charAt(0).toUpperCase() + cluster.slice(1)}

- {#if $topProjectQuery.fetching} - - {:else if $topProjectQuery.error} - {$topProjectQuery.error.message} - {:else} - tp[topProjectSelection.key])} - entities={$topProjectQuery.data.topProjects.map((tp) => tp.id)} - /> - {/if} + {#key $topProjectQuery.data} + {#if $topProjectQuery.fetching} + + {:else if $topProjectQuery.error} + {$topProjectQuery.error.message} + {:else} + tp[topProjectSelection.key])} + entities={$topProjectQuery.data.topProjects.map((tp) => tp.id)} + /> + {/if} + {/key} {#key $topProjectQuery.data} From 59c749a16424cedb67573b561870f00b40eb823e Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Tue, 29 Aug 2023 17:38:17 +0200 Subject: [PATCH 17/19] feat: add select to analysis view pie chart - 'Walltime' as generic default value for top list - Change from nodes distribution to cores distribution --- api/schema.graphqls | 2 +- internal/graph/generated/generated.go | 2 +- internal/graph/model/models_gen.go | 26 +++--- internal/graph/schema.resolvers.go | 4 +- internal/repository/stats.go | 24 +++--- web/frontend/src/Analysis.root.svelte | 120 +++++++++++++++++++------- 6 files changed, 122 insertions(+), 56 deletions(-) diff --git a/api/schema.graphqls b/api/schema.graphqls index 729c712..69e32e2 100644 --- a/api/schema.graphqls +++ b/api/schema.graphqls @@ -167,7 +167,7 @@ type TimeWeights { } enum Aggregate { USER, PROJECT, CLUSTER } -enum SortByAggregate { WALLTIME, TOTALJOBS, TOTALNODES, NODEHOURS, TOTALCORES, COREHOURS, TOTALACCS, ACCHOURS } +enum SortByAggregate { TOTALWALLTIME, TOTALJOBS, TOTALNODES, TOTALNODEHOURS, TOTALCORES, TOTALCOREHOURS, TOTALACCS, TOTALACCHOURS } type NodeMetrics { host: String! diff --git a/internal/graph/generated/generated.go b/internal/graph/generated/generated.go index efbfae9..f29e2a0 100644 --- a/internal/graph/generated/generated.go +++ b/internal/graph/generated/generated.go @@ -1767,7 +1767,7 @@ type TimeWeights { } enum Aggregate { USER, PROJECT, CLUSTER } -enum SortByAggregate { WALLTIME, TOTALJOBS, TOTALNODES, NODEHOURS, TOTALCORES, COREHOURS, TOTALACCS, ACCHOURS } +enum SortByAggregate { TOTALWALLTIME, TOTALJOBS, TOTALNODES, TOTALNODEHOURS, TOTALCORES, TOTALCOREHOURS, TOTALACCS, TOTALACCHOURS } type NodeMetrics { host: String! diff --git a/internal/graph/model/models_gen.go b/internal/graph/model/models_gen.go index 99fc96f..050784b 100644 --- a/internal/graph/model/models_gen.go +++ b/internal/graph/model/models_gen.go @@ -196,30 +196,30 @@ func (e Aggregate) MarshalGQL(w io.Writer) { type SortByAggregate string const ( - SortByAggregateWalltime SortByAggregate = "WALLTIME" - SortByAggregateTotaljobs SortByAggregate = "TOTALJOBS" - SortByAggregateTotalnodes SortByAggregate = "TOTALNODES" - SortByAggregateNodehours SortByAggregate = "NODEHOURS" - SortByAggregateTotalcores SortByAggregate = "TOTALCORES" - SortByAggregateCorehours SortByAggregate = "COREHOURS" - SortByAggregateTotalaccs SortByAggregate = "TOTALACCS" - SortByAggregateAcchours SortByAggregate = "ACCHOURS" + SortByAggregateTotalwalltime SortByAggregate = "TOTALWALLTIME" + SortByAggregateTotaljobs SortByAggregate = "TOTALJOBS" + SortByAggregateTotalnodes SortByAggregate = "TOTALNODES" + SortByAggregateTotalnodehours SortByAggregate = "TOTALNODEHOURS" + SortByAggregateTotalcores SortByAggregate = "TOTALCORES" + SortByAggregateTotalcorehours SortByAggregate = "TOTALCOREHOURS" + SortByAggregateTotalaccs SortByAggregate = "TOTALACCS" + SortByAggregateTotalacchours SortByAggregate = "TOTALACCHOURS" ) var AllSortByAggregate = []SortByAggregate{ - SortByAggregateWalltime, + SortByAggregateTotalwalltime, SortByAggregateTotaljobs, SortByAggregateTotalnodes, - SortByAggregateNodehours, + SortByAggregateTotalnodehours, SortByAggregateTotalcores, - SortByAggregateCorehours, + SortByAggregateTotalcorehours, SortByAggregateTotalaccs, - SortByAggregateAcchours, + SortByAggregateTotalacchours, } func (e SortByAggregate) IsValid() bool { switch e { - case SortByAggregateWalltime, SortByAggregateTotaljobs, SortByAggregateTotalnodes, SortByAggregateNodehours, SortByAggregateTotalcores, SortByAggregateCorehours, SortByAggregateTotalaccs, SortByAggregateAcchours: + case SortByAggregateTotalwalltime, SortByAggregateTotaljobs, SortByAggregateTotalnodes, SortByAggregateTotalnodehours, SortByAggregateTotalcores, SortByAggregateTotalcorehours, SortByAggregateTotalaccs, SortByAggregateTotalacchours: return true } return false diff --git a/internal/graph/schema.resolvers.go b/internal/graph/schema.resolvers.go index e3cc1ea..9e5e111 100644 --- a/internal/graph/schema.resolvers.go +++ b/internal/graph/schema.resolvers.go @@ -248,8 +248,8 @@ func (r *queryResolver) JobsStatistics(ctx context.Context, filter []*model.JobF var err error var stats []*model.JobsStatistics - if requireField(ctx, "totalJobs") || requireField(ctx, "totalNodes") || requireField(ctx, "totalCores") || requireField(ctx, "totalAccs") || - requireField(ctx, "totalNodeHours") || requireField(ctx, "totalCoreHours") || requireField(ctx, "totalAccHours") { + if requireField(ctx, "totalJobs") || requireField(ctx, "totalWalltime") || requireField(ctx, "totalNodes") || requireField(ctx, "totalCores") || + requireField(ctx, "totalAccs") || requireField(ctx, "totalNodeHours") || requireField(ctx, "totalCoreHours") || requireField(ctx, "totalAccHours") { if groupBy == nil { stats, err = r.Repo.JobsStats(ctx, filter) } else { diff --git a/internal/repository/stats.go b/internal/repository/stats.go index 485a5cd..193eb06 100644 --- a/internal/repository/stats.go +++ b/internal/repository/stats.go @@ -24,14 +24,14 @@ var groupBy2column = map[model.Aggregate]string{ } var sortBy2column = map[model.SortByAggregate]string{ - model.SortByAggregateTotaljobs: "totalJobs", - model.SortByAggregateWalltime: "totalWalltime", - model.SortByAggregateTotalnodes: "totalNodes", - model.SortByAggregateNodehours: "totalNodeHours", - model.SortByAggregateTotalcores: "totalCores", - model.SortByAggregateCorehours: "totalCoreHours", - model.SortByAggregateTotalaccs: "totalAccs", - model.SortByAggregateAcchours: "totalAccHours", + model.SortByAggregateTotaljobs: "totalJobs", + model.SortByAggregateTotalwalltime: "totalWalltime", + model.SortByAggregateTotalnodes: "totalNodes", + model.SortByAggregateTotalnodehours: "totalNodeHours", + model.SortByAggregateTotalcores: "totalCores", + model.SortByAggregateTotalcorehours: "totalCoreHours", + model.SortByAggregateTotalaccs: "totalAccs", + model.SortByAggregateTotalacchours: "totalAccHours", } func (r *JobRepository) buildCountQuery( @@ -169,12 +169,16 @@ func (r *JobRepository) JobsStatsGrouped( } if id.Valid { - var totalJobs, totalNodes, totalNodeHours, totalCores, totalCoreHours, totalAccs, totalAccHours int + var totalJobs, totalWalltime, totalNodes, totalNodeHours, totalCores, totalCoreHours, totalAccs, totalAccHours int if jobs.Valid { totalJobs = int(jobs.Int64) } + if walltime.Valid { + totalWalltime = int(walltime.Int64) + } + if nodes.Valid { totalNodes = int(nodes.Int64) } @@ -202,7 +206,7 @@ func (r *JobRepository) JobsStatsGrouped( ID: id.String, Name: name, TotalJobs: totalJobs, - TotalWalltime: int(walltime.Int64), + TotalWalltime: totalWalltime, TotalNodes: totalNodes, TotalNodeHours: totalNodeHours, TotalCores: totalCores, diff --git a/web/frontend/src/Analysis.root.svelte b/web/frontend/src/Analysis.root.svelte index 88c3a32..57ebfc5 100644 --- a/web/frontend/src/Analysis.root.svelte +++ b/web/frontend/src/Analysis.root.svelte @@ -42,6 +42,19 @@ $: metrics = [...new Set([...metricsInHistograms, ...metricsInScatterplots.flat()])] + const sortOptions = [ + {key: 'totalWalltime', label: 'Walltime'}, + {key: 'totalNodeHours', label: 'Node Hours'}, + {key: 'totalCoreHours', label: 'Core Hours'}, + {key: 'totalAccHours', label: 'Accelerator Hours'} + ] + const groupOptions = [ + {key: 'User', label: 'User Name'}, + {key: 'Project', label: 'Project ID'} + ] + let sortSelection = sortOptions[0] // Default: Walltime + let groupSelection = groupOptions[0] // Default: Users + getContext('on-init')(({ data }) => { if (data != null) { cluster = data.clusters.find(c => c.name == filterPresets.cluster) @@ -62,28 +75,31 @@ totalJobs shortJobs totalWalltime + totalNodeHours totalCoreHours + totalAccHours histDuration { count, value } - histNumNodes { count, value } + histNumCores { count, value } } } `, variables: { jobFilters } }) - const paging = { itemsPerPage: 5, page: 1 }; // Top 5 - // const sorting = { field: "totalCoreHours", order: "DESC" }; $: topQuery = queryStore({ client: client, query: gql` - query($jobFilters: [JobFilter!]!, $paging: PageRequest!) { - topUser: jobsStatistics(filter: $jobFilters, page: $paging, sortBy: COREHOURS, groupBy: USER) { + query($jobFilters: [JobFilter!]!, $paging: PageRequest!, $sortBy: SortByAggregate!, $groupBy: Aggregate!) { + topList: jobsStatistics(filter: $jobFilters, page: $paging, sortBy: $sortBy, groupBy: $groupBy) { id + totalWalltime + totalNodeHours totalCoreHours + totalAccHours } } `, - variables: { jobFilters, paging } + variables: { jobFilters, paging: { itemsPerPage: 10, page: 1 }, sortBy: sortSelection.key.toUpperCase(), groupBy: groupSelection.key.toUpperCase() } }) $: footprintsQuery = queryStore({ @@ -164,36 +180,82 @@ Total Walltime {$statsQuery.data.stats[0].totalWalltime} + + Total Node Hours + {$statsQuery.data.stats[0].totalNodeHours} + Total Core Hours {$statsQuery.data.stats[0].totalCoreHours} + + Total Accelerator Hours + {$statsQuery.data.stats[0].totalAccHours} +
-
Top Users
- {#key $statsQuery.data.topUsers} - tu.totalCoreHours)} - entities={$topQuery.data.topUser.map((tu) => tu.id)} - /> +
Top + +
+ {#key $topQuery.data} + {#if $topQuery.fetching} + + {:else if $topQuery.error} + {$topQuery.error.message} + {:else} + t[sortSelection.key])} + entities={$topQuery.data.topList.map((t) => t.id)} + /> + {/if} {/key}
- - - {#each $topQuery.data.topUser as { id, totalCoreHours }, i} - - - - - - {/each} -
LegendUser NameCore Hours
{id}{totalCoreHours}
+ {#key $topQuery.data} + {#if $topQuery.fetching} + + {:else if $topQuery.error} + {$topQuery.error.message} + {:else} + + + + + + + {#each $topQuery.data.topList as te, i} + + + {#if groupSelection.key == 'User'} + + {:else} + + {/if} + + + {/each} +
Legend{groupSelection.label} + +
{te.id}{te.id}{te[sortSelection.key]}
+ {/if} + {/key} @@ -230,13 +292,13 @@
- {#key $statsQuery.data.stats[0].histNumNodes} + {#key $statsQuery.data.stats[0].histNumCores} {/key} From 9533f06eaf20696e9506431c2ad9c39c4b5298db Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Wed, 30 Aug 2023 15:04:50 +0200 Subject: [PATCH 18/19] Refactor repository tests Add context to tests. Remove special test routines --- internal/repository/query.go | 62 ++++++-------------------- internal/repository/repository_test.go | 25 +++++++++-- internal/repository/stats.go | 34 +++++--------- internal/repository/stats_test.go | 6 +-- 4 files changed, 47 insertions(+), 80 deletions(-) diff --git a/internal/repository/query.go b/internal/repository/query.go index 0501fe1..84b8048 100644 --- a/internal/repository/query.go +++ b/internal/repository/query.go @@ -18,13 +18,17 @@ import ( sq "github.com/Masterminds/squirrel" ) -// SecurityCheck-less, private: Returns a list of jobs matching the provided filters. page and order are optional- -func (r *JobRepository) queryJobs( - query sq.SelectBuilder, +func (r *JobRepository) QueryJobs( + ctx context.Context, filters []*model.JobFilter, page *model.PageRequest, order *model.OrderByInput) ([]*schema.Job, error) { + query, qerr := SecurityCheck(ctx, sq.Select(jobColumns...).From("job")) + if qerr != nil { + return nil, qerr + } + if order != nil { field := toSnakeCase(order.Field) @@ -67,34 +71,15 @@ func (r *JobRepository) queryJobs( return jobs, nil } -// testFunction for queryJobs -func (r *JobRepository) testQueryJobs( - filters []*model.JobFilter, - page *model.PageRequest, - order *model.OrderByInput) ([]*schema.Job, error) { - - return r.queryJobs(sq.Select(jobColumns...).From("job"), filters, page, order) -} - -// Public function with added securityCheck, calls private queryJobs function above -func (r *JobRepository) QueryJobs( +func (r *JobRepository) CountJobs( ctx context.Context, - filters []*model.JobFilter, - page *model.PageRequest, - order *model.OrderByInput) ([]*schema.Job, error) { - - query, qerr := SecurityCheck(ctx, sq.Select(jobColumns...).From("job")) - if qerr != nil { - return nil, qerr - } - - return r.queryJobs(query, filters, page, order) -} - -// SecurityCheck-less, private: Returns the number of jobs matching the filters -func (r *JobRepository) countJobs(query sq.SelectBuilder, filters []*model.JobFilter) (int, error) { + query, qerr := SecurityCheck(ctx, sq.Select("count(*)").From("job")) + if qerr != nil { + return 0, qerr + } + for _, f := range filters { query = BuildWhereClause(f, query) } @@ -107,27 +92,6 @@ func (r *JobRepository) countJobs(query sq.SelectBuilder, return count, nil } -// testFunction for countJobs -func (r *JobRepository) testCountJobs( - filters []*model.JobFilter) (int, error) { - - return r.countJobs(sq.Select("count(*)").From("job"), filters) -} - -// Public function with added securityCheck, calls private countJobs function above -func (r *JobRepository) CountJobs( - ctx context.Context, - filters []*model.JobFilter) (int, error) { - - query, qerr := SecurityCheck(ctx, sq.Select("count(*)").From("job")) - - if qerr != nil { - return 0, qerr - } - - return r.countJobs(query, filters) -} - func SecurityCheck(ctx context.Context, query sq.SelectBuilder) (sq.SelectBuilder, error) { user := GetUserFromContext(ctx) if user == nil { diff --git a/internal/repository/repository_test.go b/internal/repository/repository_test.go index efb5395..48b692f 100644 --- a/internal/repository/repository_test.go +++ b/internal/repository/repository_test.go @@ -5,10 +5,12 @@ package repository import ( + "context" "testing" "github.com/ClusterCockpit/cc-backend/internal/graph/model" "github.com/ClusterCockpit/cc-backend/pkg/log" + "github.com/ClusterCockpit/cc-backend/pkg/schema" _ "github.com/mattn/go-sqlite3" ) @@ -94,7 +96,7 @@ func BenchmarkDB_CountJobs(b *testing.B) { b.RunParallel(func(pb *testing.PB) { for pb.Next() { - _, err := db.testCountJobs([]*model.JobFilter{filter}) + _, err := db.CountJobs(getContext(b), []*model.JobFilter{filter}) noErr(b, err) } }) @@ -118,20 +120,37 @@ func BenchmarkDB_QueryJobs(b *testing.B) { b.RunParallel(func(pb *testing.PB) { for pb.Next() { - _, err := db.testQueryJobs([]*model.JobFilter{filter}, page, order) + _, err := db.QueryJobs(getContext(b), []*model.JobFilter{filter}, page, order) noErr(b, err) } }) }) } +func getContext(tb testing.TB) context.Context { + tb.Helper() + + var roles []string + roles = append(roles, schema.GetRoleString(schema.RoleAdmin)) + projects := make([]string, 0) + + user := &schema.User{ + Username: "demo", + Name: "The man", + Roles: roles, + Projects: projects, + AuthSource: schema.AuthViaLDAP, + } + ctx := context.Background() + return context.WithValue(ctx, ContextUserKey, user) +} + func setup(tb testing.TB) *JobRepository { tb.Helper() log.Init("warn", true) dbfile := "testdata/job.db" err := MigrateDB("sqlite3", dbfile) noErr(tb, err) - Connect("sqlite3", dbfile) return GetJobRepository() } diff --git a/internal/repository/stats.go b/internal/repository/stats.go index 193eb06..3ac3ffd 100644 --- a/internal/repository/stats.go +++ b/internal/repository/stats.go @@ -233,10 +233,17 @@ func (r *JobRepository) JobsStatsGrouped( return stats, nil } -func (r *JobRepository) jobsStats( - query sq.SelectBuilder, +func (r *JobRepository) JobsStats( + ctx context.Context, filter []*model.JobFilter) ([]*model.JobsStatistics, error) { + start := time.Now() + query := r.buildStatsQuery(filter, "") + query, err := SecurityCheck(ctx, query) + if err != nil { + return nil, err + } + row := query.RunWith(r.DB).QueryRow() stats := make([]*model.JobsStatistics, 0, 1) @@ -267,29 +274,8 @@ func (r *JobRepository) jobsStats( TotalAccHours: totalAccHours}) } - return stats, nil -} - -func (r *JobRepository) testJobsStats( - filter []*model.JobFilter) ([]*model.JobsStatistics, error) { - - query := r.buildStatsQuery(filter, "") - return r.jobsStats(query, filter) -} - -func (r *JobRepository) JobsStats( - ctx context.Context, - filter []*model.JobFilter) ([]*model.JobsStatistics, error) { - - start := time.Now() - query := r.buildStatsQuery(filter, "") - query, err := SecurityCheck(ctx, query) - if err != nil { - return nil, err - } - log.Debugf("Timer JobStats %s", time.Since(start)) - return r.jobsStats(query, filter) + return stats, nil } func (r *JobRepository) JobCountGrouped( diff --git a/internal/repository/stats_test.go b/internal/repository/stats_test.go index 2672b3f..6ed3f72 100644 --- a/internal/repository/stats_test.go +++ b/internal/repository/stats_test.go @@ -26,12 +26,10 @@ func TestJobStats(t *testing.T) { r := setup(t) filter := &model.JobFilter{} - var err error - var stats []*model.JobsStatistics - stats, err = r.testJobsStats([]*model.JobFilter{filter}) + stats, err := r.JobsStats(getContext(t), []*model.JobFilter{filter}) noErr(t, err) - if stats[0].TotalJobs != 98 { + if stats[0].TotalJobs != 6 { t.Fatalf("Want 98, Got %d", stats[0].TotalJobs) } } From b6230927212a79937abf608006050a54b7aa0274 Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Wed, 30 Aug 2023 15:15:53 +0200 Subject: [PATCH 19/19] feat: persist analysis and status pie selections --- internal/config/config.go | 34 ++++++++------- web/frontend/src/Analysis.root.svelte | 60 ++++++++++++++++++++++++--- web/frontend/src/Status.root.svelte | 54 ++++++++++++++++++++++-- 3 files changed, 124 insertions(+), 24 deletions(-) diff --git a/internal/config/config.go b/internal/config/config.go index 08d01c6..253951c 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -26,21 +26,25 @@ var Keys schema.ProgramConfig = schema.ProgramConfig{ StopJobsExceedingWalltime: 0, ShortRunningJobsDuration: 5 * 60, UiDefaults: map[string]interface{}{ - "analysis_view_histogramMetrics": []string{"flops_any", "mem_bw", "mem_used"}, - "analysis_view_scatterPlotMetrics": [][]string{{"flops_any", "mem_bw"}, {"flops_any", "cpu_load"}, {"cpu_load", "mem_bw"}}, - "job_view_nodestats_selectedMetrics": []string{"flops_any", "mem_bw", "mem_used"}, - "job_view_polarPlotMetrics": []string{"flops_any", "mem_bw", "mem_used"}, - "job_view_selectedMetrics": []string{"flops_any", "mem_bw", "mem_used"}, - "plot_general_colorBackground": true, - "plot_general_colorscheme": []string{"#00bfff", "#0000ff", "#ff00ff", "#ff0000", "#ff8000", "#ffff00", "#80ff00"}, - "plot_general_lineWidth": 3, - "plot_list_jobsPerPage": 50, - "plot_list_selectedMetrics": []string{"cpu_load", "mem_used", "flops_any", "mem_bw"}, - "plot_view_plotsPerRow": 3, - "plot_view_showPolarplot": true, - "plot_view_showRoofline": true, - "plot_view_showStatTable": true, - "system_view_selectedMetric": "cpu_load", + "analysis_view_histogramMetrics": []string{"flops_any", "mem_bw", "mem_used"}, + "analysis_view_scatterPlotMetrics": [][]string{{"flops_any", "mem_bw"}, {"flops_any", "cpu_load"}, {"cpu_load", "mem_bw"}}, + "job_view_nodestats_selectedMetrics": []string{"flops_any", "mem_bw", "mem_used"}, + "job_view_polarPlotMetrics": []string{"flops_any", "mem_bw", "mem_used"}, + "job_view_selectedMetrics": []string{"flops_any", "mem_bw", "mem_used"}, + "plot_general_colorBackground": true, + "plot_general_colorscheme": []string{"#00bfff", "#0000ff", "#ff00ff", "#ff0000", "#ff8000", "#ffff00", "#80ff00"}, + "plot_general_lineWidth": 3, + "plot_list_jobsPerPage": 50, + "plot_list_selectedMetrics": []string{"cpu_load", "mem_used", "flops_any", "mem_bw"}, + "plot_view_plotsPerRow": 3, + "plot_view_showPolarplot": true, + "plot_view_showRoofline": true, + "plot_view_showStatTable": true, + "system_view_selectedMetric": "cpu_load", + "analysis_view_selectedTopEntity": "user", + "analysis_view_selectedTopCategory": "totalWalltime", + "status_view_selectedTopUserCategory": "totalJobs", + "status_view_selectedTopProjectCategory": "totalJobs", }, } diff --git a/web/frontend/src/Analysis.root.svelte b/web/frontend/src/Analysis.root.svelte index 57ebfc5..67cc652 100644 --- a/web/frontend/src/Analysis.root.svelte +++ b/web/frontend/src/Analysis.root.svelte @@ -1,7 +1,7 @@ @@ -200,7 +248,7 @@ diff --git a/web/frontend/src/Status.root.svelte b/web/frontend/src/Status.root.svelte index 4e7269a..c0a67ad 100644 --- a/web/frontend/src/Status.root.svelte +++ b/web/frontend/src/Status.root.svelte @@ -1,4 +1,5 @@