From 836c1aea3ac3bcd77bc5528b53ea3a1effd74be8 Mon Sep 17 00:00:00 2001 From: Lou Knauer Date: Fri, 30 Apr 2021 10:26:13 +0200 Subject: [PATCH] change tags filter api, add load_avg, use camelCase in API --- graph/generated/generated.go | 166 ++++++++++++++++++++++++----------- graph/model/models.go | 37 ++++---- graph/model/models_gen.go | 3 +- graph/resolver.go | 47 +++++----- graph/schema.graphqls | 5 +- 5 files changed, 160 insertions(+), 98 deletions(-) diff --git a/graph/generated/generated.go b/graph/generated/generated.go index 94289a7..e439a3c 100644 --- a/graph/generated/generated.go +++ b/graph/generated/generated.go @@ -62,21 +62,21 @@ type ComplexityRoot struct { } Job struct { - ClusterID func(childComplexity int) int - Duration func(childComplexity int) int - FileBw_avg func(childComplexity int) int - FlopsAny_avg func(childComplexity int) int - HasProfile func(childComplexity int) int - ID func(childComplexity int) int - JobID func(childComplexity int) int - MemBw_avg func(childComplexity int) int - MemUsed_max func(childComplexity int) int - NetBw_avg func(childComplexity int) int - NumNodes func(childComplexity int) int - ProjectID func(childComplexity int) int - StartTime func(childComplexity int) int - Tags func(childComplexity int) int - UserID func(childComplexity int) int + ClusterID func(childComplexity int) int + Duration func(childComplexity int) int + FileBwAvg func(childComplexity int) int + FlopsAnyAvg func(childComplexity int) int + HasProfile func(childComplexity int) int + ID func(childComplexity int) int + JobID func(childComplexity int) int + MemBwAvg func(childComplexity int) int + MemUsedMax func(childComplexity int) int + NetBwAvg func(childComplexity int) int + NumNodes func(childComplexity int) int + ProjectID func(childComplexity int) int + StartTime func(childComplexity int) int + Tags func(childComplexity int) int + UserID func(childComplexity int) int } JobMetric struct { @@ -138,7 +138,7 @@ type ComplexityRoot struct { Query struct { Clusters func(childComplexity int) int JobByID func(childComplexity int, jobID string) int - JobMetrics func(childComplexity int, jobID string, metrics []*string) int + JobMetrics func(childComplexity int, jobID string, clusterID *string, startTime *time.Time, metrics []*string) int Jobs func(childComplexity int, filter *model.JobFilterList, page *model.PageRequest, order *model.OrderByInput) int JobsStatistics func(childComplexity int, filter *model.JobFilterList) int Tags func(childComplexity int, jobID *string) int @@ -153,7 +153,7 @@ type QueryResolver interface { JobByID(ctx context.Context, jobID string) (*model.Job, error) Jobs(ctx context.Context, filter *model.JobFilterList, page *model.PageRequest, order *model.OrderByInput) (*model.JobResultList, error) JobsStatistics(ctx context.Context, filter *model.JobFilterList) (*model.JobsStatistics, error) - JobMetrics(ctx context.Context, jobID string, metrics []*string) ([]*model.JobMetricWithName, error) + JobMetrics(ctx context.Context, jobID string, clusterID *string, startTime *time.Time, metrics []*string) ([]*model.JobMetricWithName, error) Tags(ctx context.Context, jobID *string) ([]*model.JobTag, error) } @@ -264,18 +264,18 @@ func (e *executableSchema) Complexity(typeName, field string, childComplexity in return e.complexity.Job.Duration(childComplexity), true case "Job.fileBw_avg": - if e.complexity.Job.FileBw_avg == nil { + if e.complexity.Job.FileBwAvg == nil { break } - return e.complexity.Job.FileBw_avg(childComplexity), true + return e.complexity.Job.FileBwAvg(childComplexity), true case "Job.flopsAny_avg": - if e.complexity.Job.FlopsAny_avg == nil { + if e.complexity.Job.FlopsAnyAvg == nil { break } - return e.complexity.Job.FlopsAny_avg(childComplexity), true + return e.complexity.Job.FlopsAnyAvg(childComplexity), true case "Job.hasProfile": if e.complexity.Job.HasProfile == nil { @@ -299,25 +299,25 @@ func (e *executableSchema) Complexity(typeName, field string, childComplexity in return e.complexity.Job.JobID(childComplexity), true case "Job.memBw_avg": - if e.complexity.Job.MemBw_avg == nil { + if e.complexity.Job.MemBwAvg == nil { break } - return e.complexity.Job.MemBw_avg(childComplexity), true + return e.complexity.Job.MemBwAvg(childComplexity), true case "Job.memUsed_max": - if e.complexity.Job.MemUsed_max == nil { + if e.complexity.Job.MemUsedMax == nil { break } - return e.complexity.Job.MemUsed_max(childComplexity), true + return e.complexity.Job.MemUsedMax(childComplexity), true case "Job.netBw_avg": - if e.complexity.Job.NetBw_avg == nil { + if e.complexity.Job.NetBwAvg == nil { break } - return e.complexity.Job.NetBw_avg(childComplexity), true + return e.complexity.Job.NetBwAvg(childComplexity), true case "Job.numNodes": if e.complexity.Job.NumNodes == nil { @@ -607,7 +607,7 @@ func (e *executableSchema) Complexity(typeName, field string, childComplexity in return 0, false } - return e.complexity.Query.JobMetrics(childComplexity, args["jobId"].(string), args["metrics"].([]*string)), true + return e.complexity.Query.JobMetrics(childComplexity, args["jobId"].(string), args["clusterId"].(*string), args["startTime"].(*time.Time), args["metrics"].([]*string)), true case "Query.jobs": if e.complexity.Query.Jobs == nil { @@ -775,7 +775,7 @@ type Query { jobById(jobId: String!): Job jobs(filter: JobFilterList, page: PageRequest, order: OrderByInput): JobResultList! jobsStatistics(filter: JobFilterList): JobsStatistics! - jobMetrics(jobId: String!, metrics: [String]): [JobMetricWithName]! + jobMetrics(jobId: String!, clusterId: String, startTime: Time, metrics: [String]): [JobMetricWithName]! # Return all known tags or, if jobId is specified, only tags from this job tags(jobId: String): [JobTag!]! @@ -810,8 +810,7 @@ input JobFilterList { } input JobFilter { - tagName: String - tagType: String + tags: [ID!] jobId: StringInput userId: StringInput projectId: StringInput @@ -936,15 +935,33 @@ func (ec *executionContext) field_Query_jobMetrics_args(ctx context.Context, raw } } args["jobId"] = arg0 - var arg1 []*string - if tmp, ok := rawArgs["metrics"]; ok { - ctx := graphql.WithPathContext(ctx, graphql.NewPathWithField("metrics")) - arg1, err = ec.unmarshalOString2ᚕᚖstring(ctx, tmp) + var arg1 *string + if tmp, ok := rawArgs["clusterId"]; ok { + ctx := graphql.WithPathContext(ctx, graphql.NewPathWithField("clusterId")) + arg1, err = ec.unmarshalOString2ᚖstring(ctx, tmp) if err != nil { return nil, err } } - args["metrics"] = arg1 + args["clusterId"] = arg1 + var arg2 *time.Time + if tmp, ok := rawArgs["startTime"]; ok { + ctx := graphql.WithPathContext(ctx, graphql.NewPathWithField("startTime")) + arg2, err = ec.unmarshalOTime2ᚖtimeᚐTime(ctx, tmp) + if err != nil { + return nil, err + } + } + args["startTime"] = arg2 + var arg3 []*string + if tmp, ok := rawArgs["metrics"]; ok { + ctx := graphql.WithPathContext(ctx, graphql.NewPathWithField("metrics")) + arg3, err = ec.unmarshalOString2ᚕᚖstring(ctx, tmp) + if err != nil { + return nil, err + } + } + args["metrics"] = arg3 return args, nil } @@ -1767,7 +1784,7 @@ func (ec *executionContext) _Job_memUsed_max(ctx context.Context, field graphql. ctx = graphql.WithFieldContext(ctx, fc) resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (interface{}, error) { ctx = rctx // use context from middleware stack in children - return obj.MemUsed_max, nil + return obj.MemUsedMax, nil }) if err != nil { ec.Error(ctx, err) @@ -1799,7 +1816,7 @@ func (ec *executionContext) _Job_flopsAny_avg(ctx context.Context, field graphql ctx = graphql.WithFieldContext(ctx, fc) resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (interface{}, error) { ctx = rctx // use context from middleware stack in children - return obj.FlopsAny_avg, nil + return obj.FlopsAnyAvg, nil }) if err != nil { ec.Error(ctx, err) @@ -1831,7 +1848,7 @@ func (ec *executionContext) _Job_memBw_avg(ctx context.Context, field graphql.Co ctx = graphql.WithFieldContext(ctx, fc) resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (interface{}, error) { ctx = rctx // use context from middleware stack in children - return obj.MemBw_avg, nil + return obj.MemBwAvg, nil }) if err != nil { ec.Error(ctx, err) @@ -1863,7 +1880,7 @@ func (ec *executionContext) _Job_netBw_avg(ctx context.Context, field graphql.Co ctx = graphql.WithFieldContext(ctx, fc) resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (interface{}, error) { ctx = rctx // use context from middleware stack in children - return obj.NetBw_avg, nil + return obj.NetBwAvg, nil }) if err != nil { ec.Error(ctx, err) @@ -1895,7 +1912,7 @@ func (ec *executionContext) _Job_fileBw_avg(ctx context.Context, field graphql.C ctx = graphql.WithFieldContext(ctx, fc) resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (interface{}, error) { ctx = rctx // use context from middleware stack in children - return obj.FileBw_avg, nil + return obj.FileBwAvg, nil }) if err != nil { ec.Error(ctx, err) @@ -3232,7 +3249,7 @@ func (ec *executionContext) _Query_jobMetrics(ctx context.Context, field graphql fc.Args = args resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (interface{}, error) { ctx = rctx // use context from middleware stack in children - return ec.resolvers.Query().JobMetrics(rctx, args["jobId"].(string), args["metrics"].([]*string)) + return ec.resolvers.Query().JobMetrics(rctx, args["jobId"].(string), args["clusterId"].(*string), args["startTime"].(*time.Time), args["metrics"].([]*string)) }) if err != nil { ec.Error(ctx, err) @@ -4579,19 +4596,11 @@ func (ec *executionContext) unmarshalInputJobFilter(ctx context.Context, obj int for k, v := range asMap { switch k { - case "tagName": + case "tags": var err error - ctx := graphql.WithPathContext(ctx, graphql.NewPathWithField("tagName")) - it.TagName, err = ec.unmarshalOString2ᚖstring(ctx, v) - if err != nil { - return it, err - } - case "tagType": - var err error - - ctx := graphql.WithPathContext(ctx, graphql.NewPathWithField("tagType")) - it.TagType, err = ec.unmarshalOString2ᚖstring(ctx, v) + ctx := graphql.WithPathContext(ctx, graphql.NewPathWithField("tags")) + it.Tags, err = ec.unmarshalOID2ᚕstringᚄ(ctx, v) if err != nil { return it, err } @@ -6498,6 +6507,42 @@ func (ec *executionContext) marshalOHistoPoint2ᚖgithubᚗcomᚋClusterCockpit return ec._HistoPoint(ctx, sel, v) } +func (ec *executionContext) unmarshalOID2ᚕstringᚄ(ctx context.Context, v interface{}) ([]string, error) { + if v == nil { + return nil, nil + } + var vSlice []interface{} + if v != nil { + if tmp1, ok := v.([]interface{}); ok { + vSlice = tmp1 + } else { + vSlice = []interface{}{v} + } + } + var err error + res := make([]string, len(vSlice)) + for i := range vSlice { + ctx := graphql.WithPathContext(ctx, graphql.NewPathWithIndex(i)) + res[i], err = ec.unmarshalNID2string(ctx, vSlice[i]) + if err != nil { + return nil, err + } + } + return res, nil +} + +func (ec *executionContext) marshalOID2ᚕstringᚄ(ctx context.Context, sel ast.SelectionSet, v []string) graphql.Marshaler { + if v == nil { + return graphql.Null + } + ret := make(graphql.Array, len(v)) + for i := range v { + ret[i] = ec.marshalNID2string(ctx, sel, v[i]) + } + + return ret +} + func (ec *executionContext) unmarshalOInt2ᚖint(ctx context.Context, v interface{}) (*int, error) { if v == nil { return nil, nil @@ -6729,6 +6774,21 @@ func (ec *executionContext) unmarshalOStringInput2ᚖgithubᚗcomᚋClusterCockp return &res, graphql.ErrorOnPath(ctx, err) } +func (ec *executionContext) unmarshalOTime2ᚖtimeᚐTime(ctx context.Context, v interface{}) (*time.Time, error) { + if v == nil { + return nil, nil + } + res, err := graphql.UnmarshalTime(v) + return &res, graphql.ErrorOnPath(ctx, err) +} + +func (ec *executionContext) marshalOTime2ᚖtimeᚐTime(ctx context.Context, sel ast.SelectionSet, v *time.Time) graphql.Marshaler { + if v == nil { + return graphql.Null + } + return graphql.MarshalTime(*v) +} + func (ec *executionContext) unmarshalOTimeRange2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑjobarchiveᚋgraphᚋmodelᚐTimeRange(ctx context.Context, v interface{}) (*model.TimeRange, error) { if v == nil { return nil, nil diff --git a/graph/model/models.go b/graph/model/models.go index 4aa5419..085e6c7 100644 --- a/graph/model/models.go +++ b/graph/model/models.go @@ -5,24 +5,25 @@ import ( ) type Job struct { - ID string `json:"id"` - JobID string `json:"jobId" db:"job_id"` - UserID string `json:"userId" db:"user_id"` - ProjectID string `json:"projectId" db:"project_id"` - ClusterID string `json:"clusterId" db:"cluster_id"` - StartTime time.Time `json:"startTime" db:"start_time"` - Duration int `json:"duration" db:"duration"` - Walltime *int `json:"walltime" db:"walltime"` - Jobstate *string `json:"jobstate" db:"job_state"` - NumNodes int `json:"numNodes" db:"num_nodes"` - NodeList string `json:"nodelist" db:"node_list"` - HasProfile bool `json:"hasProfile" db:"has_profile"` - MemUsed_max *float64 `json:"memUsedMax" db:"mem_used_max"` - FlopsAny_avg *float64 `json:"flopsAnyAvg" db:"flops_any_avg"` - MemBw_avg *float64 `json:"memBwAvg" db:"mem_bw_avg"` - NetBw_avg *float64 `json:"netBwAvg" db:"net_bw_avg"` - FileBw_avg *float64 `json:"fileBwAvg" db:"file_bw_avg"` - Tags []JobTag `json:"tags"` + ID string `json:"id"` + JobID string `json:"jobId" db:"job_id"` + UserID string `json:"userId" db:"user_id"` + ProjectID string `json:"projectId" db:"project_id"` + ClusterID string `json:"clusterId" db:"cluster_id"` + StartTime time.Time `json:"startTime" db:"start_time"` + Duration int `json:"duration" db:"duration"` + Walltime *int `json:"walltime" db:"walltime"` + Jobstate *string `json:"jobstate" db:"job_state"` + NumNodes int `json:"numNodes" db:"num_nodes"` + NodeList string `json:"nodelist" db:"node_list"` + HasProfile bool `json:"hasProfile" db:"has_profile"` + MemUsedMax *float64 `json:"memUsedMax" db:"mem_used_max"` + FlopsAnyAvg *float64 `json:"flopsAnyAvg" db:"flops_any_avg"` + MemBwAvg *float64 `json:"memBwAvg" db:"mem_bw_avg"` + NetBwAvg *float64 `json:"netBwAvg" db:"net_bw_avg"` + FileBwAvg *float64 `json:"fileBwAvg" db:"file_bw_avg"` + LoadAvg *float64 `json:"loadAvg" db:"load_avg"` + Tags []JobTag `json:"tags"` } type JobTag struct { diff --git a/graph/model/models_gen.go b/graph/model/models_gen.go index 36708eb..1ca355a 100644 --- a/graph/model/models_gen.go +++ b/graph/model/models_gen.go @@ -35,8 +35,7 @@ type IntRange struct { } type JobFilter struct { - TagName *string `json:"tagName"` - TagType *string `json:"tagType"` + Tags []string `json:"tags"` JobID *StringInput `json:"jobId"` UserID *StringInput `json:"userId"` ProjectID *StringInput `json:"projectId"` diff --git a/graph/resolver.go b/graph/resolver.go index a7a6b65..53352a7 100644 --- a/graph/resolver.go +++ b/graph/resolver.go @@ -9,6 +9,8 @@ import ( "os" "strconv" "encoding/json" + "time" + "regexp" "github.com/ClusterCockpit/cc-jobarchive/graph/generated" "github.com/ClusterCockpit/cc-jobarchive/graph/model" @@ -61,19 +63,10 @@ func buildQueryConditions(filterList *model.JobFilterList) (string, string) { var conditions []string var join string - joinJobtags := ` - JOIN jobtag ON jobtag.job_id = job.id - JOIN tag ON tag.id = jobtag.tag_id - ` - for _, condition := range filterList.List { - if condition.TagName != nil { - conditions = append(conditions, fmt.Sprintf("tag.tag_name = '%s'", *condition.TagName)) - join = joinJobtags - } - if condition.TagType != nil { - conditions = append(conditions, fmt.Sprintf("tag.tag_type = '%s'", *condition.TagType)) - join = joinJobtags + if condition.Tags != nil && len(condition.Tags) > 0 { + conditions = append(conditions, "jobtag.tag_id IN ('" + strings.Join(condition.Tags, "', '") + "')") + join = ` JOIN jobtag ON jobtag.job_id = job.id ` } if condition.JobID != nil { conditions = addStringCondition(conditions, `job.job_id`, condition.JobID) @@ -101,8 +94,7 @@ func buildQueryConditions(filterList *model.JobFilterList) (string, string) { return strings.Join(conditions, " AND "), join } -func readJobDataFile(jobId string) ([]byte, error) { - // TODO: Use suffix as cluster-id! +func readJobDataFile(jobId string, clusterId *string, startTime *time.Time) ([]byte, error) { jobId = strings.Split(jobId, ".")[0] id, err := strconv.Atoi(jobId) if err != nil { @@ -110,7 +102,13 @@ func readJobDataFile(jobId string) ([]byte, error) { } lvl1, lvl2 := id / 1000, id % 1000 - filepath := fmt.Sprintf("./job-data/%d/%03d/data.json", lvl1, lvl2) + var filepath string + if clusterId == nil { + filepath = fmt.Sprintf("./job-data/%d/%03d/data.json", lvl1, lvl2) + } else { + filepath = fmt.Sprintf("./job-data/%s/%d/%03d/data.json", *clusterId, lvl1, lvl2) + } + f, err := os.ReadFile(filepath) if err != nil { return nil, err @@ -128,6 +126,14 @@ func contains(s []*string, e string) bool { return false } +func toSnakeCase(str string) string { + matchFirstCap := regexp.MustCompile("(.)([A-Z][a-z]+)") + matchAllCap := regexp.MustCompile("([a-z0-9])([A-Z])") + snake := matchFirstCap.ReplaceAllString(str, "${1}_${2}") + snake = matchAllCap.ReplaceAllString(snake, "${1}_${2}") + return strings.ToLower(snake) +} + // Queries func (r *queryResolver) JobByID( @@ -177,7 +183,8 @@ func (r *queryResolver) Jobs( } if orderBy != nil { - ob = fmt.Sprintf("ORDER BY %s %s", orderBy.Field, *orderBy.Order) + ob = fmt.Sprintf("ORDER BY %s %s", + toSnakeCase(orderBy.Field), *orderBy.Order) } qstr := `SELECT job.* ` @@ -322,18 +329,14 @@ func (r *queryResolver) Clusters(ctx context.Context) ([]*model.Cluster, error) func (r *queryResolver) JobMetrics( ctx context.Context, jobId string, + clusterId *string, startTime *time.Time, metrics []*string) ([]*model.JobMetricWithName, error) { - f, err := readJobDataFile(jobId) + f, err := readJobDataFile(jobId, clusterId, startTime) if err != nil { return nil, err } - /* - * GraphQL has no Map-Type, so - * this is the best i could come up with. - * This is only for testing anyways? - */ var list []*model.JobMetricWithName var metricMap map[string]*model.JobMetric diff --git a/graph/schema.graphqls b/graph/schema.graphqls index 0efe609..21ab748 100644 --- a/graph/schema.graphqls +++ b/graph/schema.graphqls @@ -78,7 +78,7 @@ type Query { jobById(jobId: String!): Job jobs(filter: JobFilterList, page: PageRequest, order: OrderByInput): JobResultList! jobsStatistics(filter: JobFilterList): JobsStatistics! - jobMetrics(jobId: String!, metrics: [String]): [JobMetricWithName]! + jobMetrics(jobId: String!, clusterId: String, startTime: Time, metrics: [String]): [JobMetricWithName]! # Return all known tags or, if jobId is specified, only tags from this job tags(jobId: String): [JobTag!]! @@ -113,8 +113,7 @@ input JobFilterList { } input JobFilter { - tagName: String - tagType: String + tags: [ID!] jobId: StringInput userId: StringInput projectId: StringInput