Merge branch 'master' into import-data-sanitation

This commit is contained in:
Jan Eitzinger
2023-04-07 08:57:42 +02:00
110 changed files with 8191 additions and 1464 deletions

View File

@@ -89,6 +89,7 @@ type ComplexityRoot struct {
Exclusive func(childComplexity int) int
ID func(childComplexity int) int
JobID func(childComplexity int) int
JobName func(childComplexity int) int
MetaData func(childComplexity int) int
MonitoringStatus func(childComplexity int) int
NumAcc func(childComplexity int) int
@@ -131,6 +132,7 @@ type ComplexityRoot struct {
HistDuration func(childComplexity int) int
HistNumNodes func(childComplexity int) int
ID func(childComplexity int) int
Name func(childComplexity int) int
ShortJobs func(childComplexity int) int
TotalCoreHours func(childComplexity int) int
TotalJobs func(childComplexity int) int
@@ -274,6 +276,8 @@ type ClusterResolver interface {
Partitions(ctx context.Context, obj *schema.Cluster) ([]string, error)
}
type JobResolver interface {
JobName(ctx context.Context, obj *schema.Job) (*string, error)
Tags(ctx context.Context, obj *schema.Job) ([]*schema.Tag, error)
MetaData(ctx context.Context, obj *schema.Job) (interface{}, error)
@@ -466,6 +470,13 @@ func (e *executableSchema) Complexity(typeName, field string, childComplexity in
return e.complexity.Job.JobID(childComplexity), true
case "Job.jobName":
if e.complexity.Job.JobName == nil {
break
}
return e.complexity.Job.JobName(childComplexity), true
case "Job.metaData":
if e.complexity.Job.MetaData == nil {
break
@@ -676,6 +687,13 @@ func (e *executableSchema) Complexity(typeName, field string, childComplexity in
return e.complexity.JobsStatistics.ID(childComplexity), true
case "JobsStatistics.name":
if e.complexity.JobsStatistics.Name == nil {
break
}
return e.complexity.JobsStatistics.Name(childComplexity), true
case "JobsStatistics.shortJobs":
if e.complexity.JobsStatistics.ShortJobs == nil {
break
@@ -1433,6 +1451,7 @@ type Job {
jobId: Int!
user: String!
project: String!
jobName: String
cluster: String!
subCluster: String!
startTime: Time!
@@ -1635,6 +1654,7 @@ input JobFilter {
arrayJobId: Int
user: StringInput
project: StringInput
jobName: StringInput
cluster: StringInput
partition: StringInput
duration: IntRange
@@ -1668,6 +1688,7 @@ input StringInput {
contains: String
startsWith: String
endsWith: String
in: [String!]
}
input IntRange { from: Int!, to: Int! }
@@ -1688,6 +1709,7 @@ type HistoPoint {
type JobsStatistics {
id: ID! # If ` + "`" + `groupBy` + "`" + ` was used, ID of the user/project/cluster
name: String # if User-Statistics: Given Name of Account (ID) Owner
totalJobs: Int! # Number of jobs that matched
shortJobs: Int! # Number of jobs with a duration of less than 2 minutes
totalWalltime: Int! # Sum of the duration of all matched jobs in hours
@@ -3090,6 +3112,47 @@ func (ec *executionContext) fieldContext_Job_project(ctx context.Context, field
return fc, nil
}
func (ec *executionContext) _Job_jobName(ctx context.Context, field graphql.CollectedField, obj *schema.Job) (ret graphql.Marshaler) {
fc, err := ec.fieldContext_Job_jobName(ctx, field)
if err != nil {
return graphql.Null
}
ctx = graphql.WithFieldContext(ctx, fc)
defer func() {
if r := recover(); r != nil {
ec.Error(ctx, ec.Recover(ctx, r))
ret = graphql.Null
}
}()
resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (interface{}, error) {
ctx = rctx // use context from middleware stack in children
return ec.resolvers.Job().JobName(rctx, obj)
})
if err != nil {
ec.Error(ctx, err)
return graphql.Null
}
if resTmp == nil {
return graphql.Null
}
res := resTmp.(*string)
fc.Result = res
return ec.marshalOString2ᚖstring(ctx, field.Selections, res)
}
func (ec *executionContext) fieldContext_Job_jobName(ctx context.Context, field graphql.CollectedField) (fc *graphql.FieldContext, err error) {
fc = &graphql.FieldContext{
Object: "Job",
Field: field,
IsMethod: true,
IsResolver: true,
Child: func(ctx context.Context, field graphql.CollectedField) (*graphql.FieldContext, error) {
return nil, errors.New("field of type String does not have child fields")
},
}
return fc, nil
}
func (ec *executionContext) _Job_cluster(ctx context.Context, field graphql.CollectedField, obj *schema.Job) (ret graphql.Marshaler) {
fc, err := ec.fieldContext_Job_cluster(ctx, field)
if err != nil {
@@ -4282,6 +4345,8 @@ func (ec *executionContext) fieldContext_JobResultList_items(ctx context.Context
return ec.fieldContext_Job_user(ctx, field)
case "project":
return ec.fieldContext_Job_project(ctx, field)
case "jobName":
return ec.fieldContext_Job_jobName(ctx, field)
case "cluster":
return ec.fieldContext_Job_cluster(ctx, field)
case "subCluster":
@@ -4492,6 +4557,47 @@ func (ec *executionContext) fieldContext_JobsStatistics_id(ctx context.Context,
return fc, nil
}
func (ec *executionContext) _JobsStatistics_name(ctx context.Context, field graphql.CollectedField, obj *model.JobsStatistics) (ret graphql.Marshaler) {
fc, err := ec.fieldContext_JobsStatistics_name(ctx, field)
if err != nil {
return graphql.Null
}
ctx = graphql.WithFieldContext(ctx, fc)
defer func() {
if r := recover(); r != nil {
ec.Error(ctx, ec.Recover(ctx, r))
ret = graphql.Null
}
}()
resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (interface{}, error) {
ctx = rctx // use context from middleware stack in children
return obj.Name, nil
})
if err != nil {
ec.Error(ctx, err)
return graphql.Null
}
if resTmp == nil {
return graphql.Null
}
res := resTmp.(*string)
fc.Result = res
return ec.marshalOString2ᚖstring(ctx, field.Selections, res)
}
func (ec *executionContext) fieldContext_JobsStatistics_name(ctx context.Context, field graphql.CollectedField) (fc *graphql.FieldContext, err error) {
fc = &graphql.FieldContext{
Object: "JobsStatistics",
Field: field,
IsMethod: false,
IsResolver: false,
Child: func(ctx context.Context, field graphql.CollectedField) (*graphql.FieldContext, error) {
return nil, errors.New("field of type String does not have child fields")
},
}
return fc, nil
}
func (ec *executionContext) _JobsStatistics_totalJobs(ctx context.Context, field graphql.CollectedField, obj *model.JobsStatistics) (ret graphql.Marshaler) {
fc, err := ec.fieldContext_JobsStatistics_totalJobs(ctx, field)
if err != nil {
@@ -5557,7 +5663,6 @@ func (ec *executionContext) _Mutation_createTag(ctx context.Context, field graph
})
if err != nil {
ec.Error(ctx, err)
return graphql.Null
}
if resTmp == nil {
if !graphql.HasFieldError(ctx, fc) {
@@ -5620,7 +5725,6 @@ func (ec *executionContext) _Mutation_deleteTag(ctx context.Context, field graph
})
if err != nil {
ec.Error(ctx, err)
return graphql.Null
}
if resTmp == nil {
if !graphql.HasFieldError(ctx, fc) {
@@ -5675,7 +5779,6 @@ func (ec *executionContext) _Mutation_addTagsToJob(ctx context.Context, field gr
})
if err != nil {
ec.Error(ctx, err)
return graphql.Null
}
if resTmp == nil {
if !graphql.HasFieldError(ctx, fc) {
@@ -5738,7 +5841,6 @@ func (ec *executionContext) _Mutation_removeTagsFromJob(ctx context.Context, fie
})
if err != nil {
ec.Error(ctx, err)
return graphql.Null
}
if resTmp == nil {
if !graphql.HasFieldError(ctx, fc) {
@@ -5801,7 +5903,6 @@ func (ec *executionContext) _Mutation_updateConfiguration(ctx context.Context, f
})
if err != nil {
ec.Error(ctx, err)
return graphql.Null
}
if resTmp == nil {
return graphql.Null
@@ -5993,7 +6094,6 @@ func (ec *executionContext) _Query_clusters(ctx context.Context, field graphql.C
})
if err != nil {
ec.Error(ctx, err)
return graphql.Null
}
if resTmp == nil {
if !graphql.HasFieldError(ctx, fc) {
@@ -6047,7 +6147,6 @@ func (ec *executionContext) _Query_tags(ctx context.Context, field graphql.Colle
})
if err != nil {
ec.Error(ctx, err)
return graphql.Null
}
if resTmp == nil {
if !graphql.HasFieldError(ctx, fc) {
@@ -6099,7 +6198,6 @@ func (ec *executionContext) _Query_user(ctx context.Context, field graphql.Colle
})
if err != nil {
ec.Error(ctx, err)
return graphql.Null
}
if resTmp == nil {
return graphql.Null
@@ -6159,7 +6257,6 @@ func (ec *executionContext) _Query_allocatedNodes(ctx context.Context, field gra
})
if err != nil {
ec.Error(ctx, err)
return graphql.Null
}
if resTmp == nil {
if !graphql.HasFieldError(ctx, fc) {
@@ -6220,7 +6317,6 @@ func (ec *executionContext) _Query_job(ctx context.Context, field graphql.Collec
})
if err != nil {
ec.Error(ctx, err)
return graphql.Null
}
if resTmp == nil {
return graphql.Null
@@ -6246,6 +6342,8 @@ func (ec *executionContext) fieldContext_Query_job(ctx context.Context, field gr
return ec.fieldContext_Job_user(ctx, field)
case "project":
return ec.fieldContext_Job_project(ctx, field)
case "jobName":
return ec.fieldContext_Job_jobName(ctx, field)
case "cluster":
return ec.fieldContext_Job_cluster(ctx, field)
case "subCluster":
@@ -6318,7 +6416,6 @@ func (ec *executionContext) _Query_jobMetrics(ctx context.Context, field graphql
})
if err != nil {
ec.Error(ctx, err)
return graphql.Null
}
if resTmp == nil {
if !graphql.HasFieldError(ctx, fc) {
@@ -6381,7 +6478,6 @@ func (ec *executionContext) _Query_jobsFootprints(ctx context.Context, field gra
})
if err != nil {
ec.Error(ctx, err)
return graphql.Null
}
if resTmp == nil {
return graphql.Null
@@ -6439,7 +6535,6 @@ func (ec *executionContext) _Query_jobs(ctx context.Context, field graphql.Colle
})
if err != nil {
ec.Error(ctx, err)
return graphql.Null
}
if resTmp == nil {
if !graphql.HasFieldError(ctx, fc) {
@@ -6504,7 +6599,6 @@ func (ec *executionContext) _Query_jobsStatistics(ctx context.Context, field gra
})
if err != nil {
ec.Error(ctx, err)
return graphql.Null
}
if resTmp == nil {
if !graphql.HasFieldError(ctx, fc) {
@@ -6527,6 +6621,8 @@ func (ec *executionContext) fieldContext_Query_jobsStatistics(ctx context.Contex
switch field.Name {
case "id":
return ec.fieldContext_JobsStatistics_id(ctx, field)
case "name":
return ec.fieldContext_JobsStatistics_name(ctx, field)
case "totalJobs":
return ec.fieldContext_JobsStatistics_totalJobs(ctx, field)
case "shortJobs":
@@ -6575,7 +6671,6 @@ func (ec *executionContext) _Query_jobsCount(ctx context.Context, field graphql.
})
if err != nil {
ec.Error(ctx, err)
return graphql.Null
}
if resTmp == nil {
if !graphql.HasFieldError(ctx, fc) {
@@ -6636,7 +6731,6 @@ func (ec *executionContext) _Query_rooflineHeatmap(ctx context.Context, field gr
})
if err != nil {
ec.Error(ctx, err)
return graphql.Null
}
if resTmp == nil {
if !graphql.HasFieldError(ctx, fc) {
@@ -6691,7 +6785,6 @@ func (ec *executionContext) _Query_nodeMetrics(ctx context.Context, field graphq
})
if err != nil {
ec.Error(ctx, err)
return graphql.Null
}
if resTmp == nil {
if !graphql.HasFieldError(ctx, fc) {
@@ -6754,7 +6847,6 @@ func (ec *executionContext) _Query___type(ctx context.Context, field graphql.Col
})
if err != nil {
ec.Error(ctx, err)
return graphql.Null
}
if resTmp == nil {
return graphql.Null
@@ -6828,7 +6920,6 @@ func (ec *executionContext) _Query___schema(ctx context.Context, field graphql.C
})
if err != nil {
ec.Error(ctx, err)
return graphql.Null
}
if resTmp == nil {
return graphql.Null
@@ -10650,7 +10741,7 @@ func (ec *executionContext) unmarshalInputJobFilter(ctx context.Context, obj int
asMap[k] = v
}
fieldsInOrder := [...]string{"tags", "jobId", "arrayJobId", "user", "project", "cluster", "partition", "duration", "minRunningFor", "numNodes", "numAccelerators", "numHWThreads", "startTime", "state", "flopsAnyAvg", "memBwAvg", "loadAvg", "memUsedMax"}
fieldsInOrder := [...]string{"tags", "jobId", "arrayJobId", "user", "project", "jobName", "cluster", "partition", "duration", "minRunningFor", "numNodes", "numAccelerators", "numHWThreads", "startTime", "state", "flopsAnyAvg", "memBwAvg", "loadAvg", "memUsedMax"}
for _, k := range fieldsInOrder {
v, ok := asMap[k]
if !ok {
@@ -10697,6 +10788,14 @@ func (ec *executionContext) unmarshalInputJobFilter(ctx context.Context, obj int
if err != nil {
return it, err
}
case "jobName":
var err error
ctx := graphql.WithPathContext(ctx, graphql.NewPathWithField("jobName"))
it.JobName, err = ec.unmarshalOStringInput2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐStringInput(ctx, v)
if err != nil {
return it, err
}
case "cluster":
var err error
@@ -10890,7 +10989,7 @@ func (ec *executionContext) unmarshalInputStringInput(ctx context.Context, obj i
asMap[k] = v
}
fieldsInOrder := [...]string{"eq", "contains", "startsWith", "endsWith"}
fieldsInOrder := [...]string{"eq", "contains", "startsWith", "endsWith", "in"}
for _, k := range fieldsInOrder {
v, ok := asMap[k]
if !ok {
@@ -10929,6 +11028,14 @@ func (ec *executionContext) unmarshalInputStringInput(ctx context.Context, obj i
if err != nil {
return it, err
}
case "in":
var err error
ctx := graphql.WithPathContext(ctx, graphql.NewPathWithField("in"))
it.In, err = ec.unmarshalOString2ᚕstringᚄ(ctx, v)
if err != nil {
return it, err
}
}
}
@@ -11261,6 +11368,23 @@ func (ec *executionContext) _Job(ctx context.Context, sel ast.SelectionSet, obj
if out.Values[i] == graphql.Null {
atomic.AddUint32(&invalids, 1)
}
case "jobName":
field := field
innerFunc := func(ctx context.Context) (res graphql.Marshaler) {
defer func() {
if r := recover(); r != nil {
ec.Error(ctx, ec.Recover(ctx, r))
}
}()
res = ec._Job_jobName(ctx, field, obj)
return res
}
out.Concurrently(i, func() graphql.Marshaler {
return innerFunc(ctx)
})
case "cluster":
out.Values[i] = ec._Job_cluster(ctx, field, obj)
@@ -11570,6 +11694,10 @@ func (ec *executionContext) _JobsStatistics(ctx context.Context, sel ast.Selecti
if out.Values[i] == graphql.Null {
invalids++
}
case "name":
out.Values[i] = ec._JobsStatistics_name(ctx, field, obj)
case "totalJobs":
out.Values[i] = ec._JobsStatistics_totalJobs(ctx, field, obj)
@@ -11832,7 +11960,6 @@ func (ec *executionContext) _Mutation(ctx context.Context, sel ast.SelectionSet)
})
out := graphql.NewFieldSet(fields)
var invalids uint32
for i, field := range fields {
innerCtx := graphql.WithRootFieldContext(ctx, &graphql.RootFieldContext{
Object: field.Name,
@@ -11848,36 +11975,24 @@ func (ec *executionContext) _Mutation(ctx context.Context, sel ast.SelectionSet)
return ec._Mutation_createTag(ctx, field)
})
if out.Values[i] == graphql.Null {
invalids++
}
case "deleteTag":
out.Values[i] = ec.OperationContext.RootResolverMiddleware(innerCtx, func(ctx context.Context) (res graphql.Marshaler) {
return ec._Mutation_deleteTag(ctx, field)
})
if out.Values[i] == graphql.Null {
invalids++
}
case "addTagsToJob":
out.Values[i] = ec.OperationContext.RootResolverMiddleware(innerCtx, func(ctx context.Context) (res graphql.Marshaler) {
return ec._Mutation_addTagsToJob(ctx, field)
})
if out.Values[i] == graphql.Null {
invalids++
}
case "removeTagsFromJob":
out.Values[i] = ec.OperationContext.RootResolverMiddleware(innerCtx, func(ctx context.Context) (res graphql.Marshaler) {
return ec._Mutation_removeTagsFromJob(ctx, field)
})
if out.Values[i] == graphql.Null {
invalids++
}
case "updateConfiguration":
out.Values[i] = ec.OperationContext.RootResolverMiddleware(innerCtx, func(ctx context.Context) (res graphql.Marshaler) {
@@ -11889,9 +12004,6 @@ func (ec *executionContext) _Mutation(ctx context.Context, sel ast.SelectionSet)
}
}
out.Dispatch()
if invalids > 0 {
return graphql.Null
}
return out
}
@@ -11946,7 +12058,6 @@ func (ec *executionContext) _Query(ctx context.Context, sel ast.SelectionSet) gr
})
out := graphql.NewFieldSet(fields)
var invalids uint32
for i, field := range fields {
innerCtx := graphql.WithRootFieldContext(ctx, &graphql.RootFieldContext{
Object: field.Name,
@@ -11966,9 +12077,6 @@ func (ec *executionContext) _Query(ctx context.Context, sel ast.SelectionSet) gr
}
}()
res = ec._Query_clusters(ctx, field)
if res == graphql.Null {
atomic.AddUint32(&invalids, 1)
}
return res
}
@@ -11989,9 +12097,6 @@ func (ec *executionContext) _Query(ctx context.Context, sel ast.SelectionSet) gr
}
}()
res = ec._Query_tags(ctx, field)
if res == graphql.Null {
atomic.AddUint32(&invalids, 1)
}
return res
}
@@ -12032,9 +12137,6 @@ func (ec *executionContext) _Query(ctx context.Context, sel ast.SelectionSet) gr
}
}()
res = ec._Query_allocatedNodes(ctx, field)
if res == graphql.Null {
atomic.AddUint32(&invalids, 1)
}
return res
}
@@ -12075,9 +12177,6 @@ func (ec *executionContext) _Query(ctx context.Context, sel ast.SelectionSet) gr
}
}()
res = ec._Query_jobMetrics(ctx, field)
if res == graphql.Null {
atomic.AddUint32(&invalids, 1)
}
return res
}
@@ -12118,9 +12217,6 @@ func (ec *executionContext) _Query(ctx context.Context, sel ast.SelectionSet) gr
}
}()
res = ec._Query_jobs(ctx, field)
if res == graphql.Null {
atomic.AddUint32(&invalids, 1)
}
return res
}
@@ -12141,9 +12237,6 @@ func (ec *executionContext) _Query(ctx context.Context, sel ast.SelectionSet) gr
}
}()
res = ec._Query_jobsStatistics(ctx, field)
if res == graphql.Null {
atomic.AddUint32(&invalids, 1)
}
return res
}
@@ -12164,9 +12257,6 @@ func (ec *executionContext) _Query(ctx context.Context, sel ast.SelectionSet) gr
}
}()
res = ec._Query_jobsCount(ctx, field)
if res == graphql.Null {
atomic.AddUint32(&invalids, 1)
}
return res
}
@@ -12187,9 +12277,6 @@ func (ec *executionContext) _Query(ctx context.Context, sel ast.SelectionSet) gr
}
}()
res = ec._Query_rooflineHeatmap(ctx, field)
if res == graphql.Null {
atomic.AddUint32(&invalids, 1)
}
return res
}
@@ -12210,9 +12297,6 @@ func (ec *executionContext) _Query(ctx context.Context, sel ast.SelectionSet) gr
}
}()
res = ec._Query_nodeMetrics(ctx, field)
if res == graphql.Null {
atomic.AddUint32(&invalids, 1)
}
return res
}
@@ -12240,9 +12324,6 @@ func (ec *executionContext) _Query(ctx context.Context, sel ast.SelectionSet) gr
}
}
out.Dispatch()
if invalids > 0 {
return graphql.Null
}
return out
}

View File

@@ -42,6 +42,7 @@ type JobFilter struct {
ArrayJobID *int `json:"arrayJobId"`
User *StringInput `json:"user"`
Project *StringInput `json:"project"`
JobName *StringInput `json:"jobName"`
Cluster *StringInput `json:"cluster"`
Partition *StringInput `json:"partition"`
Duration *schema.IntRange `json:"duration"`
@@ -72,6 +73,7 @@ type JobResultList struct {
type JobsStatistics struct {
ID string `json:"id"`
Name *string `json:"name"`
TotalJobs int `json:"totalJobs"`
ShortJobs int `json:"shortJobs"`
TotalWalltime int `json:"totalWalltime"`
@@ -102,10 +104,11 @@ type PageRequest struct {
}
type StringInput struct {
Eq *string `json:"eq"`
Contains *string `json:"contains"`
StartsWith *string `json:"startsWith"`
EndsWith *string `json:"endsWith"`
Eq *string `json:"eq"`
Contains *string `json:"contains"`
StartsWith *string `json:"startsWith"`
EndsWith *string `json:"endsWith"`
In []string `json:"in"`
}
type TimeRangeOutput struct {

View File

@@ -2,6 +2,7 @@ package graph
// This file will be automatically regenerated based on the schema, any resolver implementations
// will be copied through when generating and any unknown code will be moved to the end.
// Code generated by github.com/99designs/gqlgen version v0.17.24
import (
"context"
@@ -16,6 +17,7 @@ import (
"github.com/ClusterCockpit/cc-backend/internal/metricdata"
"github.com/ClusterCockpit/cc-backend/internal/repository"
"github.com/ClusterCockpit/cc-backend/pkg/archive"
"github.com/ClusterCockpit/cc-backend/pkg/log"
"github.com/ClusterCockpit/cc-backend/pkg/schema"
)
@@ -24,6 +26,11 @@ func (r *clusterResolver) Partitions(ctx context.Context, obj *schema.Cluster) (
return r.Repo.Partitions(obj.Name)
}
// JobName is the resolver for the jobName field.
func (r *jobResolver) JobName(ctx context.Context, obj *schema.Job) (*string, error) {
return r.Repo.FetchJobName(obj)
}
// Tags is the resolver for the tags field.
func (r *jobResolver) Tags(ctx context.Context, obj *schema.Job) ([]*schema.Tag, error) {
return r.Repo.GetTags(&obj.ID)
@@ -43,6 +50,7 @@ func (r *jobResolver) UserData(ctx context.Context, obj *schema.Job) (*model.Use
func (r *mutationResolver) CreateTag(ctx context.Context, typeArg string, name string) (*schema.Tag, error) {
id, err := r.Repo.CreateTag(typeArg, name)
if err != nil {
log.Warn("Error while creating tag")
return nil, err
}
@@ -58,6 +66,7 @@ func (r *mutationResolver) DeleteTag(ctx context.Context, id string) (string, er
func (r *mutationResolver) AddTagsToJob(ctx context.Context, job string, tagIds []string) ([]*schema.Tag, error) {
jid, err := strconv.ParseInt(job, 10, 64)
if err != nil {
log.Warn("Error while adding tag to job")
return nil, err
}
@@ -65,10 +74,12 @@ func (r *mutationResolver) AddTagsToJob(ctx context.Context, job string, tagIds
for _, tagId := range tagIds {
tid, err := strconv.ParseInt(tagId, 10, 64)
if err != nil {
log.Warn("Error while parsing tag id")
return nil, err
}
if tags, err = r.Repo.AddTag(jid, tid); err != nil {
log.Warn("Error while adding tag")
return nil, err
}
}
@@ -80,6 +91,7 @@ func (r *mutationResolver) AddTagsToJob(ctx context.Context, job string, tagIds
func (r *mutationResolver) RemoveTagsFromJob(ctx context.Context, job string, tagIds []string) ([]*schema.Tag, error) {
jid, err := strconv.ParseInt(job, 10, 64)
if err != nil {
log.Warn("Error while parsing job id")
return nil, err
}
@@ -87,10 +99,12 @@ func (r *mutationResolver) RemoveTagsFromJob(ctx context.Context, job string, ta
for _, tagId := range tagIds {
tid, err := strconv.ParseInt(tagId, 10, 64)
if err != nil {
log.Warn("Error while parsing tag id")
return nil, err
}
if tags, err = r.Repo.RemoveTag(jid, tid); err != nil {
log.Warn("Error while removing tag")
return nil, err
}
}
@@ -101,6 +115,7 @@ func (r *mutationResolver) RemoveTagsFromJob(ctx context.Context, job string, ta
// UpdateConfiguration is the resolver for the updateConfiguration field.
func (r *mutationResolver) UpdateConfiguration(ctx context.Context, name string, value string) (*string, error) {
if err := repository.GetUserCfgRepo().UpdateConfig(name, value, auth.GetUser(ctx)); err != nil {
log.Warn("Error while updating user config")
return nil, err
}
@@ -126,6 +141,7 @@ func (r *queryResolver) User(ctx context.Context, username string) (*model.User,
func (r *queryResolver) AllocatedNodes(ctx context.Context, cluster string) ([]*model.Count, error) {
data, err := r.Repo.AllocatedNodes(cluster)
if err != nil {
log.Warn("Error while fetching allocated nodes")
return nil, err
}
@@ -144,15 +160,17 @@ func (r *queryResolver) AllocatedNodes(ctx context.Context, cluster string) ([]*
func (r *queryResolver) Job(ctx context.Context, id string) (*schema.Job, error) {
numericId, err := strconv.ParseInt(id, 10, 64)
if err != nil {
log.Warn("Error while parsing job id")
return nil, err
}
job, err := r.Repo.FindById(numericId)
if err != nil {
log.Warn("Error while finding job by id")
return nil, err
}
if user := auth.GetUser(ctx); user != nil && !user.HasRole(auth.RoleAdmin) && !user.HasRole(auth.RoleSupport) && job.User != user.Username {
if user := auth.GetUser(ctx); user != nil && job.User != user.Username && user.HasNotRoles([]auth.Role{auth.RoleAdmin, auth.RoleSupport, auth.RoleManager}) {
return nil, errors.New("you are not allowed to see this job")
}
@@ -163,17 +181,23 @@ func (r *queryResolver) Job(ctx context.Context, id string) (*schema.Job, error)
func (r *queryResolver) JobMetrics(ctx context.Context, id string, metrics []string, scopes []schema.MetricScope) ([]*model.JobMetricWithName, error) {
job, err := r.Query().Job(ctx, id)
if err != nil {
log.Warn("Error while querying job for metrics")
return nil, err
}
data, err := metricdata.LoadData(job, metrics, scopes, ctx)
if err != nil {
log.Warn("Error while loading job data")
return nil, err
}
res := []*model.JobMetricWithName{}
for name, md := range data {
for scope, metric := range md {
if metric.Scope != schema.MetricScope(scope) {
log.Panic("metric.Scope != schema.MetricScope(scope) : Should not happen!")
}
res = append(res, &model.JobMetricWithName{
Name: name,
Scope: scope,
@@ -201,11 +225,13 @@ func (r *queryResolver) Jobs(ctx context.Context, filter []*model.JobFilter, pag
jobs, err := r.Repo.QueryJobs(ctx, filter, page, order)
if err != nil {
log.Warn("Error while querying jobs")
return nil, err
}
count, err := r.Repo.CountJobs(ctx, filter)
if err != nil {
log.Warn("Error while counting jobs")
return nil, err
}
@@ -214,13 +240,14 @@ func (r *queryResolver) Jobs(ctx context.Context, filter []*model.JobFilter, pag
// JobsStatistics is the resolver for the jobsStatistics field.
func (r *queryResolver) JobsStatistics(ctx context.Context, filter []*model.JobFilter, groupBy *model.Aggregate) ([]*model.JobsStatistics, error) {
return r.jobsStatistics(ctx, filter, groupBy)
return r.Repo.JobsStatistics(ctx, filter, groupBy)
}
// JobsCount is the resolver for the jobsCount field.
func (r *queryResolver) JobsCount(ctx context.Context, filter []*model.JobFilter, groupBy model.Aggregate, weight *model.Weights, limit *int) ([]*model.Count, error) {
counts, err := r.Repo.CountGroupedJobs(ctx, groupBy, filter, weight, limit)
if err != nil {
log.Warn("Error while counting grouped jobs")
return nil, err
}
@@ -254,6 +281,7 @@ func (r *queryResolver) NodeMetrics(ctx context.Context, cluster string, nodes [
data, err := metricdata.LoadNodeData(cluster, metrics, nodes, scopes, from, to, ctx)
if err != nil {
log.Warn("Error while loading node data")
return nil, err
}

View File

@@ -6,197 +6,16 @@ package graph
import (
"context"
"database/sql"
"errors"
"fmt"
"math"
"time"
"github.com/99designs/gqlgen/graphql"
"github.com/ClusterCockpit/cc-backend/internal/graph/model"
"github.com/ClusterCockpit/cc-backend/internal/metricdata"
"github.com/ClusterCockpit/cc-backend/internal/repository"
"github.com/ClusterCockpit/cc-backend/pkg/archive"
"github.com/ClusterCockpit/cc-backend/pkg/log"
"github.com/ClusterCockpit/cc-backend/pkg/schema"
sq "github.com/Masterminds/squirrel"
)
// GraphQL validation should make sure that no unkown values can be specified.
var groupBy2column = map[model.Aggregate]string{
model.AggregateUser: "job.user",
model.AggregateProject: "job.project",
model.AggregateCluster: "job.cluster",
}
const ShortJobDuration int = 5 * 60
// Helper function for the jobsStatistics GraphQL query placed here so that schema.resolvers.go is not too full.
func (r *queryResolver) jobsStatistics(ctx context.Context, filter []*model.JobFilter, groupBy *model.Aggregate) ([]*model.JobsStatistics, error) {
// In case `groupBy` is nil (not used), the model.JobsStatistics used is at the key '' (empty string)
stats := map[string]*model.JobsStatistics{}
// `socketsPerNode` and `coresPerSocket` can differ from cluster to cluster, so we need to explicitly loop over those.
for _, cluster := range archive.Clusters {
for _, subcluster := range cluster.SubClusters {
corehoursCol := fmt.Sprintf("CAST(ROUND(SUM(job.duration * job.num_nodes * %d * %d) / 3600) as int)", subcluster.SocketsPerNode, subcluster.CoresPerSocket)
var query sq.SelectBuilder
if groupBy == nil {
query = sq.Select(
"''",
"COUNT(job.id)",
"CAST(ROUND(SUM(job.duration) / 3600) as int)",
corehoursCol,
).From("job")
} else {
col := groupBy2column[*groupBy]
query = sq.Select(
col,
"COUNT(job.id)",
"CAST(ROUND(SUM(job.duration) / 3600) as int)",
corehoursCol,
).From("job").GroupBy(col)
}
query = query.
Where("job.cluster = ?", cluster.Name).
Where("job.subcluster = ?", subcluster.Name)
query = repository.SecurityCheck(ctx, query)
for _, f := range filter {
query = repository.BuildWhereClause(f, query)
}
rows, err := query.RunWith(r.DB).Query()
if err != nil {
return nil, err
}
for rows.Next() {
var id sql.NullString
var jobs, walltime, corehours sql.NullInt64
if err := rows.Scan(&id, &jobs, &walltime, &corehours); err != nil {
return nil, err
}
if id.Valid {
if s, ok := stats[id.String]; ok {
s.TotalJobs += int(jobs.Int64)
s.TotalWalltime += int(walltime.Int64)
s.TotalCoreHours += int(corehours.Int64)
} else {
stats[id.String] = &model.JobsStatistics{
ID: id.String,
TotalJobs: int(jobs.Int64),
TotalWalltime: int(walltime.Int64),
TotalCoreHours: int(corehours.Int64),
}
}
}
}
}
}
if groupBy == nil {
query := sq.Select("COUNT(job.id)").From("job").Where("job.duration < ?", ShortJobDuration)
query = repository.SecurityCheck(ctx, query)
for _, f := range filter {
query = repository.BuildWhereClause(f, query)
}
if err := query.RunWith(r.DB).QueryRow().Scan(&(stats[""].ShortJobs)); err != nil {
return nil, err
}
} else {
col := groupBy2column[*groupBy]
query := sq.Select(col, "COUNT(job.id)").From("job").Where("job.duration < ?", ShortJobDuration)
query = repository.SecurityCheck(ctx, query)
for _, f := range filter {
query = repository.BuildWhereClause(f, query)
}
rows, err := query.RunWith(r.DB).Query()
if err != nil {
return nil, err
}
for rows.Next() {
var id sql.NullString
var shortJobs sql.NullInt64
if err := rows.Scan(&id, &shortJobs); err != nil {
return nil, err
}
if id.Valid {
stats[id.String].ShortJobs = int(shortJobs.Int64)
}
}
}
// Calculating the histogram data is expensive, so only do it if needed.
// An explicit resolver can not be used because we need to know the filters.
histogramsNeeded := false
fields := graphql.CollectFieldsCtx(ctx, nil)
for _, col := range fields {
if col.Name == "histDuration" || col.Name == "histNumNodes" {
histogramsNeeded = true
}
}
res := make([]*model.JobsStatistics, 0, len(stats))
for _, stat := range stats {
res = append(res, stat)
id, col := "", ""
if groupBy != nil {
id = stat.ID
col = groupBy2column[*groupBy]
}
if histogramsNeeded {
var err error
value := fmt.Sprintf(`CAST(ROUND((CASE WHEN job.job_state = "running" THEN %d - job.start_time ELSE job.duration END) / 3600) as int) as value`, time.Now().Unix())
stat.HistDuration, err = r.jobsStatisticsHistogram(ctx, value, filter, id, col)
if err != nil {
return nil, err
}
stat.HistNumNodes, err = r.jobsStatisticsHistogram(ctx, "job.num_nodes as value", filter, id, col)
if err != nil {
return nil, err
}
}
}
return res, nil
}
// `value` must be the column grouped by, but renamed to "value". `id` and `col` can optionally be used
// to add a condition to the query of the kind "<col> = <id>".
func (r *queryResolver) jobsStatisticsHistogram(ctx context.Context, value string, filters []*model.JobFilter, id, col string) ([]*model.HistoPoint, error) {
query := sq.Select(value, "COUNT(job.id) AS count").From("job")
query = repository.SecurityCheck(ctx, query)
for _, f := range filters {
query = repository.BuildWhereClause(f, query)
}
if len(id) != 0 && len(col) != 0 {
query = query.Where(col+" = ?", id)
}
rows, err := query.GroupBy("value").RunWith(r.DB).Query()
if err != nil {
return nil, err
}
points := make([]*model.HistoPoint, 0)
for rows.Next() {
point := model.HistoPoint{}
if err := rows.Scan(&point.Value, &point.Count); err != nil {
return nil, err
}
points = append(points, &point)
}
return points, nil
}
const MAX_JOBS_FOR_ANALYSIS = 500
// Helper function for the rooflineHeatmap GraphQL query placed here so that schema.resolvers.go is not too full.
@@ -208,10 +27,11 @@ func (r *queryResolver) rooflineHeatmap(
jobs, err := r.Repo.QueryJobs(ctx, filter, &model.PageRequest{Page: 1, ItemsPerPage: MAX_JOBS_FOR_ANALYSIS + 1}, nil)
if err != nil {
log.Error("Error while querying jobs for roofline")
return nil, err
}
if len(jobs) > MAX_JOBS_FOR_ANALYSIS {
return nil, fmt.Errorf("too many jobs matched (max: %d)", MAX_JOBS_FOR_ANALYSIS)
return nil, fmt.Errorf("GRAPH/STATS > too many jobs matched (max: %d)", MAX_JOBS_FOR_ANALYSIS)
}
fcols, frows := float64(cols), float64(rows)
@@ -228,19 +48,20 @@ func (r *queryResolver) rooflineHeatmap(
jobdata, err := metricdata.LoadData(job, []string{"flops_any", "mem_bw"}, []schema.MetricScope{schema.MetricScopeNode}, ctx)
if err != nil {
log.Error("Error while loading metrics for roofline")
return nil, err
}
flops_, membw_ := jobdata["flops_any"], jobdata["mem_bw"]
if flops_ == nil && membw_ == nil {
return nil, fmt.Errorf("'flops_any' or 'mem_bw' missing for job %d", job.ID)
return nil, fmt.Errorf("GRAPH/STATS > 'flops_any' or 'mem_bw' missing for job %d", job.ID)
}
flops, ok1 := flops_["node"]
membw, ok2 := membw_["node"]
if !ok1 || !ok2 {
// TODO/FIXME:
return nil, errors.New("todo: rooflineHeatmap() query not implemented for where flops_any or mem_bw not available at 'node' level")
return nil, errors.New("GRAPH/STATS > todo: rooflineHeatmap() query not implemented for where flops_any or mem_bw not available at 'node' level")
}
for n := 0; n < len(flops.Series); n++ {
@@ -272,10 +93,11 @@ func (r *queryResolver) rooflineHeatmap(
func (r *queryResolver) jobsFootprints(ctx context.Context, filter []*model.JobFilter, metrics []string) (*model.Footprints, error) {
jobs, err := r.Repo.QueryJobs(ctx, filter, &model.PageRequest{Page: 1, ItemsPerPage: MAX_JOBS_FOR_ANALYSIS + 1}, nil)
if err != nil {
log.Error("Error while querying jobs for footprint")
return nil, err
}
if len(jobs) > MAX_JOBS_FOR_ANALYSIS {
return nil, fmt.Errorf("too many jobs matched (max: %d)", MAX_JOBS_FOR_ANALYSIS)
return nil, fmt.Errorf("GRAPH/STATS > too many jobs matched (max: %d)", MAX_JOBS_FOR_ANALYSIS)
}
avgs := make([][]schema.Float, len(metrics))
@@ -290,6 +112,7 @@ func (r *queryResolver) jobsFootprints(ctx context.Context, filter []*model.JobF
}
if err := metricdata.LoadAverages(job, metrics, avgs, ctx); err != nil {
log.Error("Error while loading averages for footprint")
return nil, err
}