From 017f9b214095fe1a82ee2a97721aac123cf55640 Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Thu, 1 Aug 2024 18:59:24 +0200 Subject: [PATCH 01/25] feat: Add tag scopes to front and backend, initial commit --- api/schema.graphqls | 3 +- internal/api/rest.go | 24 +-- internal/graph/generated/generated.go | 93 ++++++++- internal/graph/schema.resolvers.go | 16 +- internal/importer/handleImport.go | 4 +- internal/repository/tags.go | 209 +++++++++++++++++---- internal/routerConfig/routes.go | 4 +- pkg/archive/archive.go | 5 +- pkg/schema/job.go | 7 +- web/frontend/src/Job.root.svelte | 9 +- web/frontend/src/Node.root.svelte | 2 +- web/frontend/src/generic/JobList.svelte | 1 + web/frontend/src/generic/helper/Tag.svelte | 8 +- web/frontend/src/generic/utils.js | 2 +- web/frontend/src/job.entrypoint.js | 1 + web/frontend/src/job/TagManagement.svelte | 58 ++++-- web/templates/monitoring/job.tmpl | 1 + web/templates/monitoring/taglist.tmpl | 12 +- 18 files changed, 361 insertions(+), 98 deletions(-) diff --git a/api/schema.graphqls b/api/schema.graphqls index 568c15d..c38a4a1 100644 --- a/api/schema.graphqls +++ b/api/schema.graphqls @@ -113,6 +113,7 @@ type Tag { id: ID! type: String! name: String! + scope: String! } type Resource { @@ -235,7 +236,7 @@ type Query { } type Mutation { - createTag(type: String!, name: String!): Tag! + createTag(type: String!, name: String!, scope: String!): Tag! deleteTag(id: ID!): ID! addTagsToJob(job: ID!, tagIds: [ID!]!): [Tag!]! removeTagsFromJob(job: ID!, tagIds: [ID!]!): [Tag!]! diff --git a/internal/api/rest.go b/internal/api/rest.go index c8f4e7a..4efdd4a 100644 --- a/internal/api/rest.go +++ b/internal/api/rest.go @@ -176,8 +176,9 @@ type ErrorResponse struct { // ApiTag model type ApiTag struct { // Tag Type - Type string `json:"type" example:"Debug"` - Name string `json:"name" example:"Testjob"` // Tag Name + Type string `json:"type" example:"Debug"` + Name string `json:"name" example:"Testjob"` // Tag Name + Scope string `json:"scope" example:"global"` // Tag Scope for Frontend Display } // ApiMeta model @@ -419,7 +420,7 @@ func (api *RestApi) getJobs(rw http.ResponseWriter, r *http.Request) { StartTime: job.StartTime.Unix(), } - res.Tags, err = api.JobRepository.GetTags(&job.ID) + res.Tags, err = api.JobRepository.GetTags(r.Context(), &job.ID) if err != nil { handleError(err, http.StatusInternalServerError, rw) return @@ -492,7 +493,7 @@ func (api *RestApi) getCompleteJobById(rw http.ResponseWriter, r *http.Request) return } - job.Tags, err = api.JobRepository.GetTags(&job.ID) + job.Tags, err = api.JobRepository.GetTags(r.Context(), &job.ID) if err != nil { handleError(err, http.StatusInternalServerError, rw) return @@ -578,7 +579,7 @@ func (api *RestApi) getJobById(rw http.ResponseWriter, r *http.Request) { return } - job.Tags, err = api.JobRepository.GetTags(&job.ID) + job.Tags, err = api.JobRepository.GetTags(r.Context(), &job.ID) if err != nil { handleError(err, http.StatusInternalServerError, rw) return @@ -711,7 +712,7 @@ func (api *RestApi) tagJob(rw http.ResponseWriter, r *http.Request) { return } - job.Tags, err = api.JobRepository.GetTags(&job.ID) + job.Tags, err = api.JobRepository.GetTags(r.Context(), &job.ID) if err != nil { http.Error(rw, err.Error(), http.StatusInternalServerError) return @@ -724,16 +725,17 @@ func (api *RestApi) tagJob(rw http.ResponseWriter, r *http.Request) { } for _, tag := range req { - tagId, err := api.JobRepository.AddTagOrCreate(job.ID, tag.Type, tag.Name) + tagId, err := api.JobRepository.AddTagOrCreate(r.Context(), job.ID, tag.Type, tag.Name, tag.Scope) if err != nil { http.Error(rw, err.Error(), http.StatusInternalServerError) return } job.Tags = append(job.Tags, &schema.Tag{ - ID: tagId, - Type: tag.Type, - Name: tag.Name, + ID: tagId, + Type: tag.Type, + Name: tag.Name, + Scope: tag.Scope, }) } @@ -801,7 +803,7 @@ func (api *RestApi) startJob(rw http.ResponseWriter, r *http.Request) { unlockOnce.Do(api.RepositoryMutex.Unlock) for _, tag := range req.Tags { - if _, err := api.JobRepository.AddTagOrCreate(id, tag.Type, tag.Name); err != nil { + if _, err := api.JobRepository.AddTagOrCreate(r.Context(), id, tag.Type, tag.Name, tag.Scope); err != nil { http.Error(rw, err.Error(), http.StatusInternalServerError) handleError(fmt.Errorf("adding tag to new job %d failed: %w", id, err), http.StatusInternalServerError, rw) return diff --git a/internal/graph/generated/generated.go b/internal/graph/generated/generated.go index 9ca0a60..54c8e70 100644 --- a/internal/graph/generated/generated.go +++ b/internal/graph/generated/generated.go @@ -229,7 +229,7 @@ type ComplexityRoot struct { Mutation struct { AddTagsToJob func(childComplexity int, job string, tagIds []string) int - CreateTag func(childComplexity int, typeArg string, name string) int + CreateTag func(childComplexity int, typeArg string, name string, scope string) int DeleteTag func(childComplexity int, id string) int RemoveTagsFromJob func(childComplexity int, job string, tagIds []string) int UpdateConfiguration func(childComplexity int, name string, value string) int @@ -303,9 +303,10 @@ type ComplexityRoot struct { } Tag struct { - ID func(childComplexity int) int - Name func(childComplexity int) int - Type func(childComplexity int) int + ID func(childComplexity int) int + Name func(childComplexity int) int + Scope func(childComplexity int) int + Type func(childComplexity int) int } TimeRangeOutput struct { @@ -355,7 +356,7 @@ type MetricValueResolver interface { Name(ctx context.Context, obj *schema.MetricValue) (*string, error) } type MutationResolver interface { - CreateTag(ctx context.Context, typeArg string, name string) (*schema.Tag, error) + CreateTag(ctx context.Context, typeArg string, name string, scope string) (*schema.Tag, error) DeleteTag(ctx context.Context, id string) (string, error) AddTagsToJob(ctx context.Context, job string, tagIds []string) ([]*schema.Tag, error) RemoveTagsFromJob(ctx context.Context, job string, tagIds []string) ([]*schema.Tag, error) @@ -1183,7 +1184,7 @@ func (e *executableSchema) Complexity(typeName, field string, childComplexity in return 0, false } - return e.complexity.Mutation.CreateTag(childComplexity, args["type"].(string), args["name"].(string)), true + return e.complexity.Mutation.CreateTag(childComplexity, args["type"].(string), args["name"].(string), args["scope"].(string)), true case "Mutation.deleteTag": if e.complexity.Mutation.DeleteTag == nil { @@ -1602,6 +1603,13 @@ func (e *executableSchema) Complexity(typeName, field string, childComplexity in return e.complexity.Tag.Name(childComplexity), true + case "Tag.scope": + if e.complexity.Tag.Scope == nil { + break + } + + return e.complexity.Tag.Scope(childComplexity), true + case "Tag.type": if e.complexity.Tag.Type == nil { break @@ -1949,6 +1957,7 @@ type Tag { id: ID! type: String! name: String! + scope: String! } type Resource { @@ -2071,7 +2080,7 @@ type Query { } type Mutation { - createTag(type: String!, name: String!): Tag! + createTag(type: String!, name: String!, scope: String!): Tag! deleteTag(id: ID!): ID! addTagsToJob(job: ID!, tagIds: [ID!]!): [Tag!]! removeTagsFromJob(job: ID!, tagIds: [ID!]!): [Tag!]! @@ -2244,6 +2253,15 @@ func (ec *executionContext) field_Mutation_createTag_args(ctx context.Context, r } } args["name"] = arg1 + var arg2 string + if tmp, ok := rawArgs["scope"]; ok { + ctx := graphql.WithPathContext(ctx, graphql.NewPathWithField("scope")) + arg2, err = ec.unmarshalNString2string(ctx, tmp) + if err != nil { + return nil, err + } + } + args["scope"] = arg2 return args, nil } @@ -4622,6 +4640,8 @@ func (ec *executionContext) fieldContext_Job_tags(_ context.Context, field graph return ec.fieldContext_Tag_type(ctx, field) case "name": return ec.fieldContext_Tag_name(ctx, field) + case "scope": + return ec.fieldContext_Tag_scope(ctx, field) } return nil, fmt.Errorf("no field named %q was found under type Tag", field.Name) }, @@ -7680,7 +7700,7 @@ func (ec *executionContext) _Mutation_createTag(ctx context.Context, field graph }() resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (interface{}, error) { ctx = rctx // use context from middleware stack in children - return ec.resolvers.Mutation().CreateTag(rctx, fc.Args["type"].(string), fc.Args["name"].(string)) + return ec.resolvers.Mutation().CreateTag(rctx, fc.Args["type"].(string), fc.Args["name"].(string), fc.Args["scope"].(string)) }) if err != nil { ec.Error(ctx, err) @@ -7711,6 +7731,8 @@ func (ec *executionContext) fieldContext_Mutation_createTag(ctx context.Context, return ec.fieldContext_Tag_type(ctx, field) case "name": return ec.fieldContext_Tag_name(ctx, field) + case "scope": + return ec.fieldContext_Tag_scope(ctx, field) } return nil, fmt.Errorf("no field named %q was found under type Tag", field.Name) }, @@ -7829,6 +7851,8 @@ func (ec *executionContext) fieldContext_Mutation_addTagsToJob(ctx context.Conte return ec.fieldContext_Tag_type(ctx, field) case "name": return ec.fieldContext_Tag_name(ctx, field) + case "scope": + return ec.fieldContext_Tag_scope(ctx, field) } return nil, fmt.Errorf("no field named %q was found under type Tag", field.Name) }, @@ -7892,6 +7916,8 @@ func (ec *executionContext) fieldContext_Mutation_removeTagsFromJob(ctx context. return ec.fieldContext_Tag_type(ctx, field) case "name": return ec.fieldContext_Tag_name(ctx, field) + case "scope": + return ec.fieldContext_Tag_scope(ctx, field) } return nil, fmt.Errorf("no field named %q was found under type Tag", field.Name) }, @@ -8199,6 +8225,8 @@ func (ec *executionContext) fieldContext_Query_tags(_ context.Context, field gra return ec.fieldContext_Tag_type(ctx, field) case "name": return ec.fieldContext_Tag_name(ctx, field) + case "scope": + return ec.fieldContext_Tag_scope(ctx, field) } return nil, fmt.Errorf("no field named %q was found under type Tag", field.Name) }, @@ -10547,6 +10575,50 @@ func (ec *executionContext) fieldContext_Tag_name(_ context.Context, field graph return fc, nil } +func (ec *executionContext) _Tag_scope(ctx context.Context, field graphql.CollectedField, obj *schema.Tag) (ret graphql.Marshaler) { + fc, err := ec.fieldContext_Tag_scope(ctx, field) + if err != nil { + return graphql.Null + } + ctx = graphql.WithFieldContext(ctx, fc) + defer func() { + if r := recover(); r != nil { + ec.Error(ctx, ec.Recover(ctx, r)) + ret = graphql.Null + } + }() + resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (interface{}, error) { + ctx = rctx // use context from middleware stack in children + return obj.Scope, nil + }) + if err != nil { + ec.Error(ctx, err) + return graphql.Null + } + if resTmp == nil { + if !graphql.HasFieldError(ctx, fc) { + ec.Errorf(ctx, "must not be null") + } + return graphql.Null + } + res := resTmp.(string) + fc.Result = res + return ec.marshalNString2string(ctx, field.Selections, res) +} + +func (ec *executionContext) fieldContext_Tag_scope(_ context.Context, field graphql.CollectedField) (fc *graphql.FieldContext, err error) { + fc = &graphql.FieldContext{ + Object: "Tag", + Field: field, + IsMethod: false, + IsResolver: false, + Child: func(ctx context.Context, field graphql.CollectedField) (*graphql.FieldContext, error) { + return nil, errors.New("field of type String does not have child fields") + }, + } + return fc, nil +} + func (ec *executionContext) _TimeRangeOutput_from(ctx context.Context, field graphql.CollectedField, obj *model.TimeRangeOutput) (ret graphql.Marshaler) { fc, err := ec.fieldContext_TimeRangeOutput_from(ctx, field) if err != nil { @@ -15666,6 +15738,11 @@ func (ec *executionContext) _Tag(ctx context.Context, sel ast.SelectionSet, obj if out.Values[i] == graphql.Null { out.Invalids++ } + case "scope": + out.Values[i] = ec._Tag_scope(ctx, field, obj) + if out.Values[i] == graphql.Null { + out.Invalids++ + } default: panic("unknown field " + strconv.Quote(field.Name)) } diff --git a/internal/graph/schema.resolvers.go b/internal/graph/schema.resolvers.go index f36e25a..4673354 100644 --- a/internal/graph/schema.resolvers.go +++ b/internal/graph/schema.resolvers.go @@ -29,7 +29,7 @@ func (r *clusterResolver) Partitions(ctx context.Context, obj *schema.Cluster) ( // Tags is the resolver for the tags field. func (r *jobResolver) Tags(ctx context.Context, obj *schema.Job) ([]*schema.Tag, error) { - return r.Repo.GetTags(&obj.ID) + return r.Repo.GetTags(ctx, &obj.ID) } // ConcurrentJobs is the resolver for the concurrentJobs field. @@ -86,14 +86,14 @@ func (r *metricValueResolver) Name(ctx context.Context, obj *schema.MetricValue) } // CreateTag is the resolver for the createTag field. -func (r *mutationResolver) CreateTag(ctx context.Context, typeArg string, name string) (*schema.Tag, error) { - id, err := r.Repo.CreateTag(typeArg, name) +func (r *mutationResolver) CreateTag(ctx context.Context, typeArg string, name string, scope string) (*schema.Tag, error) { + id, err := r.Repo.CreateTag(typeArg, name, scope) if err != nil { log.Warn("Error while creating tag") return nil, err } - return &schema.Tag{ID: id, Type: typeArg, Name: name}, nil + return &schema.Tag{ID: id, Type: typeArg, Name: name, Scope: scope}, nil } // DeleteTag is the resolver for the deleteTag field. @@ -103,6 +103,7 @@ func (r *mutationResolver) DeleteTag(ctx context.Context, id string) (string, er // AddTagsToJob is the resolver for the addTagsToJob field. func (r *mutationResolver) AddTagsToJob(ctx context.Context, job string, tagIds []string) ([]*schema.Tag, error) { + // Selectable Tags Pre-Filtered by Scope in Frontend: No backend check required jid, err := strconv.ParseInt(job, 10, 64) if err != nil { log.Warn("Error while adding tag to job") @@ -117,7 +118,7 @@ func (r *mutationResolver) AddTagsToJob(ctx context.Context, job string, tagIds return nil, err } - if tags, err = r.Repo.AddTag(jid, tid); err != nil { + if tags, err = r.Repo.AddTag(ctx, jid, tid); err != nil { log.Warn("Error while adding tag") return nil, err } @@ -128,6 +129,7 @@ func (r *mutationResolver) AddTagsToJob(ctx context.Context, job string, tagIds // RemoveTagsFromJob is the resolver for the removeTagsFromJob field. func (r *mutationResolver) RemoveTagsFromJob(ctx context.Context, job string, tagIds []string) ([]*schema.Tag, error) { + // Removable Tags Pre-Filtered by Scope in Frontend: No backend check required jid, err := strconv.ParseInt(job, 10, 64) if err != nil { log.Warn("Error while parsing job id") @@ -142,7 +144,7 @@ func (r *mutationResolver) RemoveTagsFromJob(ctx context.Context, job string, ta return nil, err } - if tags, err = r.Repo.RemoveTag(jid, tid); err != nil { + if tags, err = r.Repo.RemoveTag(ctx, jid, tid); err != nil { log.Warn("Error while removing tag") return nil, err } @@ -168,7 +170,7 @@ func (r *queryResolver) Clusters(ctx context.Context) ([]*schema.Cluster, error) // Tags is the resolver for the tags field. func (r *queryResolver) Tags(ctx context.Context) ([]*schema.Tag, error) { - return r.Repo.GetTags(nil) + return r.Repo.GetTags(ctx, nil) } // GlobalMetrics is the resolver for the globalMetrics field. diff --git a/internal/importer/handleImport.go b/internal/importer/handleImport.go index c4d55ab..54158c4 100644 --- a/internal/importer/handleImport.go +++ b/internal/importer/handleImport.go @@ -112,8 +112,8 @@ func HandleImportFlag(flag string) error { } for _, tag := range job.Tags { - if _, err := r.AddTagOrCreate(id, tag.Type, tag.Name); err != nil { - log.Error("Error while adding or creating tag") + if err := r.ImportTag(id, tag.Type, tag.Name, tag.Scope); err != nil { + log.Error("Error while adding or creating tag on import") return err } } diff --git a/internal/repository/tags.go b/internal/repository/tags.go index 8dace03..2b7e5d5 100644 --- a/internal/repository/tags.go +++ b/internal/repository/tags.go @@ -5,6 +5,8 @@ package repository import ( + "context" + "fmt" "strings" "github.com/ClusterCockpit/cc-backend/pkg/archive" @@ -14,7 +16,14 @@ import ( ) // Add the tag with id `tagId` to the job with the database id `jobId`. -func (r *JobRepository) AddTag(job int64, tag int64) ([]*schema.Tag, error) { +func (r *JobRepository) AddTag(ctx context.Context, job int64, tag int64) ([]*schema.Tag, error) { + + j, err := r.FindById(ctx, job) + if err != nil { + log.Warn("Error while finding job by id") + return nil, err + } + q := sq.Insert("jobtag").Columns("job_id", "tag_id").Values(job, tag) if _, err := q.RunWith(r.stmtCache).Exec(); err != nil { @@ -23,49 +32,56 @@ func (r *JobRepository) AddTag(job int64, tag int64) ([]*schema.Tag, error) { return nil, err } - j, err := r.FindByIdDirect(job) - if err != nil { - log.Warn("Error while finding job by id") - return nil, err - } - - tags, err := r.GetTags(&job) + tags, err := r.GetTags(ctx, &job) if err != nil { log.Warn("Error while getting tags for job") return nil, err } - return tags, archive.UpdateTags(j, tags) + archiveTags, err := r.getArchiveTags(&job) + if err != nil { + log.Warn("Error while getting tags for job") + return nil, err + } + + return tags, archive.UpdateTags(j, archiveTags) } // Removes a tag from a job -func (r *JobRepository) RemoveTag(job, tag int64) ([]*schema.Tag, error) { +func (r *JobRepository) RemoveTag(ctx context.Context, job, tag int64) ([]*schema.Tag, error) { + + j, err := r.FindById(ctx, job) + if err != nil { + log.Warn("Error while finding job by id") + return nil, err + } + q := sq.Delete("jobtag").Where("jobtag.job_id = ?", job).Where("jobtag.tag_id = ?", tag) if _, err := q.RunWith(r.stmtCache).Exec(); err != nil { s, _, _ := q.ToSql() - log.Errorf("Error adding tag with %s: %v", s, err) + log.Errorf("Error removing tag with %s: %v", s, err) return nil, err } - j, err := r.FindByIdDirect(job) - if err != nil { - log.Warn("Error while finding job by id") - return nil, err - } - - tags, err := r.GetTags(&job) + tags, err := r.GetTags(ctx, &job) if err != nil { log.Warn("Error while getting tags for job") return nil, err } - return tags, archive.UpdateTags(j, tags) + archiveTags, err := r.getArchiveTags(&job) + if err != nil { + log.Warn("Error while getting tags for job") + return nil, err + } + + return tags, archive.UpdateTags(j, archiveTags) } // CreateTag creates a new tag with the specified type and name and returns its database id. -func (r *JobRepository) CreateTag(tagType string, tagName string) (tagId int64, err error) { - q := sq.Insert("tag").Columns("tag_type", "tag_name").Values(tagType, tagName) +func (r *JobRepository) CreateTag(tagType string, tagName string, tagScope string) (tagId int64, err error) { + q := sq.Insert("tag").Columns("tag_type", "tag_name", "tag_scope").Values(tagType, tagName, tagScope) res, err := q.RunWith(r.stmtCache).Exec() if err != nil { @@ -77,9 +93,10 @@ func (r *JobRepository) CreateTag(tagType string, tagName string) (tagId int64, return res.LastInsertId() } -func (r *JobRepository) CountTags(user *schema.User) (tags []schema.Tag, counts map[string]int, err error) { +func (r *JobRepository) CountTags(ctx context.Context) (tags []schema.Tag, counts map[string]int, err error) { + // Fetch all Tags in DB for Display in Frontend Tag-View tags = make([]schema.Tag, 0, 100) - xrows, err := r.DB.Queryx("SELECT id, tag_type, tag_name FROM tag") + xrows, err := r.DB.Queryx("SELECT id, tag_type, tag_name, tag_scope FROM tag") if err != nil { return nil, nil, err } @@ -89,14 +106,36 @@ func (r *JobRepository) CountTags(user *schema.User) (tags []schema.Tag, counts if err = xrows.StructScan(&t); err != nil { return nil, nil, err } - tags = append(tags, t) + + // Handle Scope Filtering: Tag Scope is Global, Private (== Username) or User is auth'd to view Admin Tags + readable, err := r.checkScopeAuth(ctx, "read", t.Scope) + if err != nil { + return nil, nil, err + } + if readable { + tags = append(tags, t) + } } + user := GetUserFromContext(ctx) + + // Query and Count Jobs with attached Tags q := sq.Select("t.tag_name, count(jt.tag_id)"). From("tag t"). LeftJoin("jobtag jt ON t.id = jt.tag_id"). GroupBy("t.tag_name") + // Handle Scope Filtering + scopeList := "\"global\"" + if user != nil { + scopeList += ",\"" + user.Username + "\"" + } + if user.HasAnyRole([]schema.Role{schema.RoleAdmin, schema.RoleSupport}) { + scopeList += ",\"admin\"" + } + q = q.Where("t.tag_scope IN (" + scopeList + ")") + + // Handle Job Ownership if user != nil && user.HasAnyRole([]schema.Role{schema.RoleAdmin, schema.RoleSupport}) { // ADMIN || SUPPORT: Count all jobs log.Debug("CountTags: User Admin or Support -> Count all Jobs for Tags") // Unchanged: Needs to be own case still, due to UserRole/NoRole compatibility handling in else case @@ -123,21 +162,30 @@ func (r *JobRepository) CountTags(user *schema.User) (tags []schema.Tag, counts } err = rows.Err() - return + return tags, counts, err } // AddTagOrCreate adds the tag with the specified type and name to the job with the database id `jobId`. // If such a tag does not yet exist, it is created. -func (r *JobRepository) AddTagOrCreate(jobId int64, tagType string, tagName string) (tagId int64, err error) { - tagId, exists := r.TagId(tagType, tagName) +func (r *JobRepository) AddTagOrCreate(ctx context.Context, jobId int64, tagType string, tagName string, tagScope string) (tagId int64, err error) { + + writable, err := r.checkScopeAuth(ctx, "write", tagScope) + if err != nil { + return 0, err + } + if !writable { + return 0, fmt.Errorf("cannot write tag scope with current authorization") + } + + tagId, exists := r.TagId(tagType, tagName, tagScope) if !exists { - tagId, err = r.CreateTag(tagType, tagName) + tagId, err = r.CreateTag(tagType, tagName, tagScope) if err != nil { return 0, err } } - if _, err := r.AddTag(jobId, tagId); err != nil { + if _, err := r.AddTag(ctx, jobId, tagId); err != nil { return 0, err } @@ -145,19 +193,19 @@ func (r *JobRepository) AddTagOrCreate(jobId int64, tagType string, tagName stri } // TagId returns the database id of the tag with the specified type and name. -func (r *JobRepository) TagId(tagType string, tagName string) (tagId int64, exists bool) { +func (r *JobRepository) TagId(tagType string, tagName string, tagScope string) (tagId int64, exists bool) { exists = true if err := sq.Select("id").From("tag"). - Where("tag.tag_type = ?", tagType).Where("tag.tag_name = ?", tagName). + Where("tag.tag_type = ?", tagType).Where("tag.tag_name = ?", tagName).Where("tag.tag_scope = ?", tagScope). RunWith(r.stmtCache).QueryRow().Scan(&tagId); err != nil { exists = false } return } -// GetTags returns a list of all tags if job is nil or of the tags that the job with that database ID has. -func (r *JobRepository) GetTags(job *int64) ([]*schema.Tag, error) { - q := sq.Select("id", "tag_type", "tag_name").From("tag") +// GetTags returns a list of all scoped tags if job is nil or of the tags that the job with that database ID has. +func (r *JobRepository) GetTags(ctx context.Context, job *int64) ([]*schema.Tag, error) { + q := sq.Select("id", "tag_type", "tag_name", "tag_scope").From("tag") if job != nil { q = q.Join("jobtag ON jobtag.tag_id = tag.id").Where("jobtag.job_id = ?", *job) } @@ -172,7 +220,41 @@ func (r *JobRepository) GetTags(job *int64) ([]*schema.Tag, error) { tags := make([]*schema.Tag, 0) for rows.Next() { tag := &schema.Tag{} - if err := rows.Scan(&tag.ID, &tag.Type, &tag.Name); err != nil { + if err := rows.Scan(&tag.ID, &tag.Type, &tag.Name, &tag.Scope); err != nil { + log.Warn("Error while scanning rows") + return nil, err + } + // Handle Scope Filtering: Tag Scope is Global, Private (== Username) or User is auth'd to view Admin Tags + readable, err := r.checkScopeAuth(ctx, "read", tag.Scope) + if err != nil { + return nil, err + } + if readable { + tags = append(tags, tag) + } + } + + return tags, nil +} + +// GetArchiveTags returns a list of all tags *regardless of scope* if job is nil or of the tags that the job with that database ID has. +func (r *JobRepository) getArchiveTags(job *int64) ([]*schema.Tag, error) { + q := sq.Select("id", "tag_type", "tag_name", "tag_scope").From("tag") + if job != nil { + q = q.Join("jobtag ON jobtag.tag_id = tag.id").Where("jobtag.job_id = ?", *job) + } + + rows, err := q.RunWith(r.stmtCache).Query() + if err != nil { + s, _, _ := q.ToSql() + log.Errorf("Error get tags with %s: %v", s, err) + return nil, err + } + + tags := make([]*schema.Tag, 0) + for rows.Next() { + tag := &schema.Tag{} + if err := rows.Scan(&tag.ID, &tag.Type, &tag.Name, &tag.Scope); err != nil { log.Warn("Error while scanning rows") return nil, err } @@ -181,3 +263,60 @@ func (r *JobRepository) GetTags(job *int64) ([]*schema.Tag, error) { return tags, nil } + +func (r *JobRepository) ImportTag(jobId int64, tagType string, tagName string, tagScope string) (err error) { + // Import has no scope ctx, only import from metafile to DB (No recursive archive update required), only returns err + + tagId, exists := r.TagId(tagType, tagName, tagScope) + if !exists { + tagId, err = r.CreateTag(tagType, tagName, tagScope) + if err != nil { + return err + } + } + + q := sq.Insert("jobtag").Columns("job_id", "tag_id").Values(jobId, tagId) + + if _, err := q.RunWith(r.stmtCache).Exec(); err != nil { + s, _, _ := q.ToSql() + log.Errorf("Error adding tag on import with %s: %v", s, err) + return err + } + + return nil +} + +func (r *JobRepository) checkScopeAuth(ctx context.Context, operation string, scope string) (pass bool, err error) { + user := GetUserFromContext(ctx) + if user != nil { + switch { + case operation == "write" && scope == "admin": + if user.HasRole(schema.RoleAdmin) || (len(user.Roles) == 1 && user.HasRole(schema.RoleApi)) { + return true, nil + } + return false, nil + case operation == "write" && scope == "global": + if user.HasRole(schema.RoleAdmin) || (len(user.Roles) == 1 && user.HasRole(schema.RoleApi)) { + return true, nil + } + return false, nil + case operation == "write" && scope == user.Username: + return true, nil + case operation == "read" && scope == "admin": + return user.HasAnyRole([]schema.Role{schema.RoleAdmin, schema.RoleSupport}), nil + case operation == "read" && scope == "global": + return true, nil + case operation == "read" && scope == user.Username: + return true, nil + default: + if operation == "read" || operation == "write" { + // No acceptable scope: deny tag + return false, nil + } else { + return false, fmt.Errorf("error while checking tag operation auth: unknown operation (%s)", operation) + } + } + } else { + return false, fmt.Errorf("error while checking tag operation auth: no user in context") + } +} diff --git a/internal/routerConfig/routes.go b/internal/routerConfig/routes.go index 1dd6dee..cde1562 100644 --- a/internal/routerConfig/routes.go +++ b/internal/routerConfig/routes.go @@ -124,9 +124,8 @@ func setupAnalysisRoute(i InfoType, r *http.Request) InfoType { func setupTaglistRoute(i InfoType, r *http.Request) InfoType { jobRepo := repository.GetJobRepository() - user := repository.GetUserFromContext(r.Context()) - tags, counts, err := jobRepo.CountTags(user) + tags, counts, err := jobRepo.CountTags(r.Context()) tagMap := make(map[string][]map[string]interface{}) if err != nil { log.Warnf("GetTags failed: %s", err.Error()) @@ -138,6 +137,7 @@ func setupTaglistRoute(i InfoType, r *http.Request) InfoType { tagItem := map[string]interface{}{ "id": tag.ID, "name": tag.Name, + "scope": tag.Scope, "count": counts[tag.Name], } tagMap[tag.Type] = append(tagMap[tag.Type], tagItem) diff --git a/pkg/archive/archive.go b/pkg/archive/archive.go index 56c5d47..52a760f 100644 --- a/pkg/archive/archive.go +++ b/pkg/archive/archive.go @@ -171,8 +171,9 @@ func UpdateTags(job *schema.Job, tags []*schema.Tag) error { jobMeta.Tags = make([]*schema.Tag, 0) for _, tag := range tags { jobMeta.Tags = append(jobMeta.Tags, &schema.Tag{ - Name: tag.Name, - Type: tag.Type, + Name: tag.Name, + Type: tag.Type, + Scope: tag.Scope, }) } diff --git a/pkg/schema/job.go b/pkg/schema/job.go index 83064c7..638533f 100644 --- a/pkg/schema/job.go +++ b/pkg/schema/job.go @@ -117,9 +117,10 @@ type JobStatistics struct { // Tag model // @Description Defines a tag using name and type. type Tag struct { - Type string `json:"type" db:"tag_type" example:"Debug"` - Name string `json:"name" db:"tag_name" example:"Testjob"` - ID int64 `json:"id" db:"id"` + Type string `json:"type" db:"tag_type" example:"Debug"` + Name string `json:"name" db:"tag_name" example:"Testjob"` + Scope string `json:"scope" db:"tag_scope" example:"global"` + ID int64 `json:"id" db:"id"` } // Resource model diff --git a/web/frontend/src/Job.root.svelte b/web/frontend/src/Job.root.svelte index f991e4f..02ca22a 100644 --- a/web/frontend/src/Job.root.svelte +++ b/web/frontend/src/Job.root.svelte @@ -2,9 +2,9 @@ @component Main single job display component; displays plots for every metric as well as various information Properties: + - `dbid Number`: The jobs DB ID - `username String`: Empty string if auth. is disabled, otherwise the username as string - `authlevel Number`: The current users authentication level - - `clusters [String]`: List of cluster names - `roles [Number]`: Enum containing available roles --> @@ -45,6 +45,7 @@ import MetricSelection from "./generic/select/MetricSelection.svelte"; export let dbid; + export let username; export let authlevel; export let roles; @@ -58,8 +59,7 @@ selectedScopes = []; let plots = {}, - jobTags, - statsTable + jobTags let missingMetrics = [], missingHosts = [], @@ -322,7 +322,7 @@ {#if $initq.data} - + {/if} @@ -418,7 +418,6 @@ {#if $jobMetrics?.data?.jobMetrics} {#key $jobMetrics.data.jobMetrics} diff --git a/web/frontend/src/Node.root.svelte b/web/frontend/src/Node.root.svelte index ad6983b..2d58540 100644 --- a/web/frontend/src/Node.root.svelte +++ b/web/frontend/src/Node.root.svelte @@ -27,7 +27,7 @@ import { init, checkMetricDisabled, - } from "./utils.js"; + } from "./generic/utils.js"; import PlotTable from "./generic/PlotTable.svelte"; import MetricPlot from "./generic/plots/MetricPlot.svelte"; import TimeSelection from "./generic/select/TimeSelection.svelte"; diff --git a/web/frontend/src/generic/JobList.svelte b/web/frontend/src/generic/JobList.svelte index e3e3f40..1cbd4c6 100644 --- a/web/frontend/src/generic/JobList.svelte +++ b/web/frontend/src/generic/JobList.svelte @@ -81,6 +81,7 @@ id type name + scope } userData { name diff --git a/web/frontend/src/generic/helper/Tag.svelte b/web/frontend/src/generic/helper/Tag.svelte index 55dabcb..66b4312 100644 --- a/web/frontend/src/generic/helper/Tag.svelte +++ b/web/frontend/src/generic/helper/Tag.svelte @@ -37,7 +37,13 @@ {#if tag} - {tag.type}: {tag.name} + {#if tag?.scope === "global"} + {tag.type}: {tag.name} + {:else if tag.scope === "admin"} + {tag.type}: {tag.name} + {:else} + {tag.type}: {tag.name} + {/if} {:else} Loading... {/if} diff --git a/web/frontend/src/generic/utils.js b/web/frontend/src/generic/utils.js index bb63a4f..84c9ce0 100644 --- a/web/frontend/src/generic/utils.js +++ b/web/frontend/src/generic/utils.js @@ -77,7 +77,7 @@ export function init(extraInitQuery = "") { footprint } } - tags { id, name, type } + tags { id, name, type, scope } globalMetrics { name scope diff --git a/web/frontend/src/job.entrypoint.js b/web/frontend/src/job.entrypoint.js index 4641e92..f810955 100644 --- a/web/frontend/src/job.entrypoint.js +++ b/web/frontend/src/job.entrypoint.js @@ -5,6 +5,7 @@ new Job({ target: document.getElementById('svelte-app'), props: { dbid: jobInfos.id, + username: username, authlevel: authlevel, roles: roles }, diff --git a/web/frontend/src/job/TagManagement.svelte b/web/frontend/src/job/TagManagement.svelte index 34bbcea..408cc70 100644 --- a/web/frontend/src/job/TagManagement.svelte +++ b/web/frontend/src/job/TagManagement.svelte @@ -4,6 +4,9 @@ Properties: - `job Object`: The job object - `jobTags [Number]`: The array of currently designated tags + - `username String`: Empty string if auth. is disabled, otherwise the username as string + - `authlevel Number`: The current users authentication level + - `roles [Number]`: Enum containing available roles --> diff --git a/web/templates/monitoring/taglist.tmpl b/web/templates/monitoring/taglist.tmpl index 6e487dd..ea29cd7 100644 --- a/web/templates/monitoring/taglist.tmpl +++ b/web/templates/monitoring/taglist.tmpl @@ -7,8 +7,16 @@ {{ $tagType }} {{ range $tagList }} - - {{ .name }} {{ .count }} + {{if eq .scope "global"}} + + {{ .name }} {{ .count }} + {{else if eq .scope "admin"}} + + {{ .name }} {{ .count }} + {{else}} + + {{ .name }} {{ .count }} + {{end}} {{end}} {{end}} From ff3502c87a1af7641a103e06e60278a93c0b86fc Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Fri, 2 Aug 2024 16:11:47 +0200 Subject: [PATCH 02/25] fix: fix tag filter results - displayed multiple identical entries before - job count was incorrect before --- internal/repository/jobQuery.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/repository/jobQuery.go b/internal/repository/jobQuery.go index 7b575ef..b0a846c 100644 --- a/internal/repository/jobQuery.go +++ b/internal/repository/jobQuery.go @@ -89,7 +89,7 @@ func (r *JobRepository) CountJobs( ctx context.Context, filters []*model.JobFilter, ) (int, error) { - query, qerr := SecurityCheck(ctx, sq.Select("count(*)").From("job")) + query, qerr := SecurityCheck(ctx, sq.Select("count(DISTINCT job.id)").From("job")) if qerr != nil { return 0, qerr } @@ -136,7 +136,7 @@ func SecurityCheck(ctx context.Context, query sq.SelectBuilder) (sq.SelectBuilde // Build a sq.SelectBuilder out of a schema.JobFilter. func BuildWhereClause(filter *model.JobFilter, query sq.SelectBuilder) sq.SelectBuilder { if filter.Tags != nil { - query = query.Join("jobtag ON jobtag.job_id = job.id").Where(sq.Eq{"jobtag.tag_id": filter.Tags}) + query = query.Join("jobtag ON jobtag.job_id = job.id").Where(sq.Eq{"jobtag.tag_id": filter.Tags}).Distinct() } if filter.JobID != nil { query = buildStringCondition("job.job_id", filter.JobID, query) From e02575aad7d88ca5396e28df2c0276fb03201e68 Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Fri, 2 Aug 2024 16:42:55 +0200 Subject: [PATCH 03/25] adds comments --- internal/repository/jobQuery.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/internal/repository/jobQuery.go b/internal/repository/jobQuery.go index b0a846c..843d797 100644 --- a/internal/repository/jobQuery.go +++ b/internal/repository/jobQuery.go @@ -89,6 +89,7 @@ func (r *JobRepository) CountJobs( ctx context.Context, filters []*model.JobFilter, ) (int, error) { + // DISTICT count for tags filters, does not affect other queries query, qerr := SecurityCheck(ctx, sq.Select("count(DISTINCT job.id)").From("job")) if qerr != nil { return 0, qerr @@ -136,6 +137,7 @@ func SecurityCheck(ctx context.Context, query sq.SelectBuilder) (sq.SelectBuilde // Build a sq.SelectBuilder out of a schema.JobFilter. func BuildWhereClause(filter *model.JobFilter, query sq.SelectBuilder) sq.SelectBuilder { if filter.Tags != nil { + // This is an OR-Logic query: Returns all distinct jobs with at least one of the requested tags; TODO: AND-Logic query? query = query.Join("jobtag ON jobtag.job_id = job.id").Where(sq.Eq{"jobtag.tag_id": filter.Tags}).Distinct() } if filter.JobID != nil { From 2551921ed63280cfdbf5ec9fe96cdc76f10e7667 Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Fri, 2 Aug 2024 18:14:24 +0200 Subject: [PATCH 04/25] fix: wrong display of tag after filter select - exitent pills were non-updated on change of key --- web/frontend/src/generic/Filters.svelte | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/web/frontend/src/generic/Filters.svelte b/web/frontend/src/generic/Filters.svelte index a1839c7..d01c16b 100644 --- a/web/frontend/src/generic/Filters.svelte +++ b/web/frontend/src/generic/Filters.svelte @@ -322,7 +322,9 @@ {#if filters.tags.length != 0} (isTagsOpen = true)}> {#each filters.tags as tagId} - + {#key tagId} + + {/key} {/each} {/if} From e6ebec8c1e9a1c56a54bfb093a50354402d2a434 Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Mon, 5 Aug 2024 10:19:00 +0200 Subject: [PATCH 05/25] fix TestGetTags test, was missing scope and ctx --- internal/repository/job_test.go | 16 +++++++++++++++- internal/repository/testdata/job.db | Bin 114688 -> 118784 bytes 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/internal/repository/job_test.go b/internal/repository/job_test.go index 2589fb9..f7b3783 100644 --- a/internal/repository/job_test.go +++ b/internal/repository/job_test.go @@ -5,9 +5,11 @@ package repository import ( + "context" "fmt" "testing" + "github.com/ClusterCockpit/cc-backend/pkg/schema" _ "github.com/mattn/go-sqlite3" ) @@ -45,7 +47,19 @@ func TestFindById(t *testing.T) { func TestGetTags(t *testing.T) { r := setup(t) - tags, counts, err := r.CountTags(nil) + const contextUserKey ContextKey = "user" + contextUserValue := &schema.User{ + Username: "testuser", + Projects: make([]string, 0), + Roles: []string{"user"}, + AuthType: 0, + AuthSource: 2, + } + + ctx := context.WithValue(getContext(t), contextUserKey, contextUserValue) + + // Test Tag has Scope "global" + tags, counts, err := r.CountTags(ctx) if err != nil { t.Fatal(err) } diff --git a/internal/repository/testdata/job.db b/internal/repository/testdata/job.db index c345327e67e5c6dff1500c214831d7da28019c13..23eba6fb5066702a5bf6c48539e75e69da8cfd53 100644 GIT binary patch delta 466 zcmZo@U~gE!K0#VgjDdkc7K)jGwAMr&V^%Q+y{roxQx@5?Tw>tAv{_K#1i!QnBO8OL zyrgkyNoGz`VqQvlW=cs$dQN^)V$SBb{&EhRWhOk~pSVGA(*ni`(o6vi?2OtB{8RbW z7`6E>asTGN#I_0ybtg7dZJ%bqxQmg6x0fXfXm%LO_Elz#kJ*^`{%`NMWh`K10Xjo{ z;zH%^YW9rx3Y2)c<}vZ#VgNsusGewEh z(=Wu;-8D!D$OVeWmsA#{DsfIXG-nj&fr#WK=E6iYAtJ@e`5;9ht`Q*$e*Qol>f-~_ zpP5&jT2vBWQmn)|xt~$Nmop&9)7LR5Qo-9bQUhd}CQy~Lzh7`jkR#Aq1*N3a_|m-0 z!qQYFg-}1wz))8Wgd23gOt9seT$+|LllyrswhQlJe9O!O3?L9-1Y)kuf)n2JPYhrM E01o4Tu>b%7 delta 281 zcmZozz~0cnK0#Vgl!1Xk28g+Um=TDTC+ZlpiZbYxb#6>qWY2Pwf&b=aL4ou9BC3q6 z45ISJr6rj;Nr`zW<(VlZ8Jpkw%Q-M|Zf2bDg@4lm#tG8QP7Iuk+6??t`PG;L`2I6$ z^IhWp&3%b&6`StFhN|t;3>bGYP7YxD#KPOloUvVUFXIX(uI@xZc5z{0#_5`q&Dm7B zx{V;*Y2FYnGlV-e7tHMnfpYml-0iE(7$37SF)(cJw`D9~ocy2f&-UN;jGP5h+}t;r z_;2zz@LTdd=WF9L;l0M&$ScYt!VU7xI&OaNP<=**%)FG;isHf?OPR_2ycXMq_b|R? F1_0e7O=SQ8 From 9b5c6e3164186e9d48b9c0bb062a4ea9abd71a4d Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Mon, 5 Aug 2024 10:37:42 +0200 Subject: [PATCH 06/25] fix StartJobTest, add tag_scope to migration --- internal/api/api_test.go | 4 ++-- .../repository/migrations/sqlite3/08_add-footprint.up.sql | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/api/api_test.go b/internal/api/api_test.go index 80a7e64..423bf5c 100644 --- a/internal/api/api_test.go +++ b/internal/api/api_test.go @@ -212,7 +212,7 @@ func TestRestApi(t *testing.T) { "exclusive": 1, "monitoringStatus": 1, "smt": 1, - "tags": [{ "type": "testTagType", "name": "testTagName" }], + "tags": [{ "type": "testTagType", "name": "testTagName", "scope": "testuser" }], "resources": [ { "hostname": "host123", @@ -280,7 +280,7 @@ func TestRestApi(t *testing.T) { t.Fatalf("unexpected job properties: %#v", job) } - if len(job.Tags) != 1 || job.Tags[0].Type != "testTagType" || job.Tags[0].Name != "testTagName" { + if len(job.Tags) != 1 || job.Tags[0].Type != "testTagType" || job.Tags[0].Name != "testTagName" || job.Tags[0].Scope != "testuser" { t.Fatalf("unexpected tags: %#v", job.Tags) } diff --git a/internal/repository/migrations/sqlite3/08_add-footprint.up.sql b/internal/repository/migrations/sqlite3/08_add-footprint.up.sql index 643b87e..cf9c2b8 100644 --- a/internal/repository/migrations/sqlite3/08_add-footprint.up.sql +++ b/internal/repository/migrations/sqlite3/08_add-footprint.up.sql @@ -1,6 +1,6 @@ ALTER TABLE job ADD COLUMN energy REAL NOT NULL DEFAULT 0.0; - ALTER TABLE job ADD COLUMN footprint TEXT DEFAULT NULL; +ALTER TABLE tag ADD COLUMN tag_scope TEXT NOT NULL DEFAULT 'global'; UPDATE job SET footprint = '{"flops_any_avg": 0.0}'; UPDATE job SET footprint = json_replace(footprint, '$.flops_any_avg', job.flops_any_avg); UPDATE job SET footprint = json_insert(footprint, '$.mem_bw_avg', job.mem_bw_avg); From 398e3c1b91f2e96ac4a22cbaa9f306973449f6e1 Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Wed, 4 Sep 2024 10:23:23 +0200 Subject: [PATCH 07/25] feat: split concurrent jobs list to own scrollable component --- web/frontend/src/Job.root.svelte | 40 +--------- .../src/generic/helper/ConcurrentJobs.svelte | 79 +++++++++++++++++++ 2 files changed, 83 insertions(+), 36 deletions(-) create mode 100644 web/frontend/src/generic/helper/ConcurrentJobs.svelte diff --git a/web/frontend/src/Job.root.svelte b/web/frontend/src/Job.root.svelte index 213898e..ab0bfee 100644 --- a/web/frontend/src/Job.root.svelte +++ b/web/frontend/src/Job.root.svelte @@ -38,6 +38,7 @@ import TagManagement from "./job/TagManagement.svelte"; import StatsTable from "./job/StatsTable.svelte"; import JobFootprint from "./generic/helper/JobFootprint.svelte"; + import ConcurrentJobs from "./generic/helper/ConcurrentJobs.svelte"; import PlotTable from "./generic/PlotTable.svelte"; import Polar from "./generic/plots/Polar.svelte"; import Roofline from "./generic/plots/Roofline.svelte"; @@ -250,42 +251,9 @@ {/if} {#if $initq?.data && $jobMetrics?.data?.jobMetrics} {#if $initq.data.job.concurrentJobs != null && $initq.data.job.concurrentJobs.items.length != 0} - {#if authlevel > roles.manager} - -
- Concurrent Jobs -
-
    -
  • - See All -
    • - {#each $initq.data.job.concurrentJobs.items as pjob, index} -
  • - {pjob.jobId} -
    • - {/each} -
- - {:else} - -
- {$initq.data.job.concurrentJobs.items.length} Concurrent Jobs -
-

- Number of shared jobs on the same node with overlapping runtimes. -

- - {/if} + + roles.manager)}/> + {/if} + + + + + {#if displayTitle} + + + {cJobs.items.length} Concurrent Jobs + + + + {/if} + + {#if showLinks} + + {:else} + {#if displayTitle} +

+ Jobs running on the same node with overlapping runtimes using shared resources. +

+ {:else} +

+ {cJobs.items.length} + Jobs running on the same node with overlapping runtimes using shared resources. +

+ {/if} + {/if} + + + + From df484dc816e66bae078f74f457c8c7cb5992b8a2 Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Thu, 5 Sep 2024 16:44:03 +0200 Subject: [PATCH 08/25] rework job view header, change footprint to summary component --- web/frontend/src/Job.root.svelte | 264 ++++++++------ .../src/generic/helper/ConcurrentJobs.svelte | 94 +++-- .../src/generic/joblist/JobInfo.svelte | 2 +- web/frontend/src/generic/plots/Polar.svelte | 91 +++-- .../src/generic/plots/Roofline.svelte | 8 +- web/frontend/src/job/JobSummary.svelte | 340 ++++++++++++++++++ web/frontend/src/job/StatsTable.svelte | 4 +- 7 files changed, 626 insertions(+), 177 deletions(-) create mode 100644 web/frontend/src/job/JobSummary.svelte diff --git a/web/frontend/src/Job.root.svelte b/web/frontend/src/Job.root.svelte index ab0bfee..8ea259a 100644 --- a/web/frontend/src/Job.root.svelte +++ b/web/frontend/src/Job.root.svelte @@ -37,10 +37,9 @@ import Metric from "./job/Metric.svelte"; import TagManagement from "./job/TagManagement.svelte"; import StatsTable from "./job/StatsTable.svelte"; - import JobFootprint from "./generic/helper/JobFootprint.svelte"; + import JobSummary from "./job/JobSummary.svelte"; import ConcurrentJobs from "./generic/helper/ConcurrentJobs.svelte"; import PlotTable from "./generic/PlotTable.svelte"; - import Polar from "./generic/plots/Polar.svelte"; import Roofline from "./generic/plots/Roofline.svelte"; import JobInfo from "./generic/joblist/JobInfo.svelte"; import MetricSelection from "./generic/select/MetricSelection.svelte"; @@ -232,62 +231,96 @@ })); - - + + + {#if $initq.error} {$initq.error.message} {:else if $initq.data} - + + + + + + + + {#if $initq.data.job.concurrentJobs != null && $initq.data.job.concurrentJobs.items.length != 0} + + + {$initq.data.job.concurrentJobs.items.length} Concurrent Jobs + + + roles.manager)}/> + + + {/if} + {#if $initq.data?.job?.metaData?.message} + + +

This note was added by administrators:

+
+

{@html $initq.data.job.metaData.message}

+ + + {/if} + + {:else} {/if} - {#if $initq.data && showFootprint} - - - - {/if} - {#if $initq?.data && $jobMetrics?.data?.jobMetrics} - {#if $initq.data.job.concurrentJobs != null && $initq.data.job.concurrentJobs.items.length != 0} - - roles.manager)}/> - - {/if} - - - - - c.name == $initq.data.job.cluster) - .subClusters.find((sc) => sc.name == $initq.data.job.subCluster)} - data={transformDataForRoofline( - $jobMetrics.data.jobMetrics.find( - (m) => m.name == "flops_any" && m.scope == "node", - )?.metric, - $jobMetrics.data.jobMetrics.find( - (m) => m.name == "mem_bw" && m.scope == "node", - )?.metric, - )} - /> + + + {#if showFootprint} + + {#if $initq.error} + {$initq.error.message} + {:else if $initq?.data && $jobMetrics?.data} + + {:else} + + {/if} {:else} - - - + {/if} + + + + + + + {#if $initq.error || $jobMetrics.error} + +

Initq Error: {$initq.error?.message}

+

jobMetrics Error: {$jobMetrics.error?.message}

+ + {:else if $initq?.data && $jobMetrics?.data} + + c.name == $initq.data.job.cluster) + .subClusters.find((sc) => sc.name == $initq.data.job.subCluster)} + data={transformDataForRoofline( + $jobMetrics.data.jobMetrics.find( + (m) => m.name == "flops_any" && m.scope == "node", + )?.metric, + $jobMetrics.data.jobMetrics.find( + (m) => m.name == "mem_bw" && m.scope == "node", + )?.metric, + )} + /> + + {:else} + + {/if} + - + +
+ + {#if $initq.data} @@ -344,76 +377,81 @@ {/if} - + +
+ + {#if $initq.data} - - {#if somethingMissing} - -
- - - Missing Metrics/Resources - - - {#if missingMetrics.length > 0} -

- No data at all is available for the metrics: {missingMetrics.join( - ", ", - )} -

- {/if} - {#if missingHosts.length > 0} -

Some metrics are missing for the following hosts:

-
    - {#each missingHosts as missing} -
  • - {missing.hostname}: {missing.metrics.join(", ")} -
    • - {/each} -
- {/if} - - + + + {#if somethingMissing} + +
+ + + Missing Metrics/Resources + + + {#if missingMetrics.length > 0} +

+ No data at all is available for the metrics: {missingMetrics.join( + ", ", + )} +

+ {/if} + {#if missingHosts.length > 0} +

Some metrics are missing for the following hosts:

+
    + {#each missingHosts as missing} +
  • + {missing.hostname}: {missing.metrics.join(", ")} +
    • + {/each} +
+ {/if} + + +
+ + {/if} + + {#if $jobMetrics?.data?.jobMetrics} + {#key $jobMetrics.data.jobMetrics} + + {/key} + {/if} + + +
+ {#if $initq.data.job.metaData?.jobScript} + 
{$initq.data.job.metaData?.jobScript}
+ {:else} + No job script available + {/if}
- {/if} - - {#if $jobMetrics?.data?.jobMetrics} - {#key $jobMetrics.data.jobMetrics} - - {/key} - {/if} - - -
- {#if $initq.data.job.metaData?.jobScript} - 
{$initq.data.job.metaData?.jobScript}
- {:else} - No job script available - {/if} -
- - -
- {#if $initq.data.job.metaData?.slurmInfo} - 
{$initq.data.job.metaData?.slurmInfo}
- {:else} - No additional slurm information available - {/if} -
- - + +
+ {#if $initq.data.job.metaData?.slurmInfo} + 
{$initq.data.job.metaData?.slurmInfo}
+ {:else} + No additional slurm information available + {/if} +
+ + + {/if} diff --git a/web/frontend/src/generic/helper/ConcurrentJobs.svelte b/web/frontend/src/generic/helper/ConcurrentJobs.svelte index 79e1886..c0de0b6 100644 --- a/web/frontend/src/generic/helper/ConcurrentJobs.svelte +++ b/web/frontend/src/generic/helper/ConcurrentJobs.svelte @@ -13,62 +13,86 @@ import { Card, CardHeader, - CardTitle, CardBody, Icon } from "@sveltestrap/sveltestrap"; export let cJobs; export let showLinks = false; - export let displayTitle = true; + export let renderCard = false; export let width = "auto"; - export let height = "310px"; + export let height = "400px"; - - {#if displayTitle} - - +{#if renderCard} + + {cJobs.items.length} Concurrent Jobs - - {/if} - - {#if showLinks} -
    -
  • - See All -
    • - {#each cJobs.items as cJob} + + {#if showLinks} + - {:else} - {#if displayTitle} -

    - Jobs running on the same node with overlapping runtimes using shared resources. -

    + {#each cJobs.items as cJob} +
  • + {cJob.jobId} +
    • + {/each} +
{:else} -

- {cJobs.items.length} - Jobs running on the same node with overlapping runtimes using shared resources. -

+
    + {#each cJobs.items as cJob} +
  • + {cJob.jobId} +
    • + {/each} +
{/if} - {/if} - - + + +{:else} +

+ Jobs running on the same node with overlapping runtimes using shared resources. +

+
+ {#if showLinks} + + {:else} +
    + {#each cJobs.items as cJob} +
  • + {cJob.jobId} +
    • + {/each} +
+ {/if} +{/if} diff --git a/web/frontend/src/job/StatsTable.svelte b/web/frontend/src/job/StatsTable.svelte index 88777f6..606d05b 100644 --- a/web/frontend/src/job/StatsTable.svelte +++ b/web/frontend/src/job/StatsTable.svelte @@ -84,7 +84,7 @@ } - +
@@ -146,8 +146,6 @@
-
- Date: Fri, 6 Sep 2024 12:00:33 +0200 Subject: [PATCH 09/25] Improve grid scaling --- web/frontend/src/Job.root.svelte | 50 +++++++++---------- .../src/generic/helper/ConcurrentJobs.svelte | 20 ++++---- .../src/generic/plots/Roofline.svelte | 2 - 3 files changed, 34 insertions(+), 38 deletions(-) diff --git a/web/frontend/src/Job.root.svelte b/web/frontend/src/Job.root.svelte index 8ea259a..2afa5a8 100644 --- a/web/frontend/src/Job.root.svelte +++ b/web/frontend/src/Job.root.svelte @@ -59,7 +59,8 @@ let plots = {}, jobTags, - statsTable + statsTable, + roofWidth let missingMetrics = [], missingHosts = [], @@ -231,9 +232,9 @@ })); - + - + {#if $initq.error} {$initq.error.message} {:else if $initq.data} @@ -272,7 +273,7 @@ {#if showFootprint} - + {#if $initq.error} {$initq.error.message} {:else if $initq?.data && $jobMetrics?.data} @@ -281,15 +282,10 @@ {/if} - {:else} - {/if} - - - - - + + {#if $initq.error || $jobMetrics.error}

Initq Error: {$initq.error?.message}

@@ -297,20 +293,24 @@ {:else if $initq?.data && $jobMetrics?.data} - c.name == $initq.data.job.cluster) - .subClusters.find((sc) => sc.name == $initq.data.job.subCluster)} - data={transformDataForRoofline( - $jobMetrics.data.jobMetrics.find( - (m) => m.name == "flops_any" && m.scope == "node", - )?.metric, - $jobMetrics.data.jobMetrics.find( - (m) => m.name == "mem_bw" && m.scope == "node", - )?.metric, - )} - /> +
+ c.name == $initq.data.job.cluster) + .subClusters.find((sc) => sc.name == $initq.data.job.subCluster)} + data={transformDataForRoofline( + $jobMetrics.data.jobMetrics.find( + (m) => m.name == "flops_any" && m.scope == "node", + )?.metric, + $jobMetrics.data.jobMetrics.find( + (m) => m.name == "mem_bw" && m.scope == "node", + )?.metric, + )} + /> +
{:else} diff --git a/web/frontend/src/generic/helper/ConcurrentJobs.svelte b/web/frontend/src/generic/helper/ConcurrentJobs.svelte index c0de0b6..85bac83 100644 --- a/web/frontend/src/generic/helper/ConcurrentJobs.svelte +++ b/web/frontend/src/generic/helper/ConcurrentJobs.svelte @@ -4,7 +4,7 @@ Properties: - `cJobs JobLinkResultList`: List of concurrent Jobs - `showLinks Bool?`: Show list as clickable links [Default: false] - - `displayTitle Bool?`: If to display cardHeader with title [Default: true] + - `renderCard Bool?`: If to render component as content only or with card wrapping [Default: true] - `width String?`: Width of the card [Default: 'auto'] - `height String?`: Height of the card [Default: '310px'] --> @@ -64,17 +64,15 @@ {:else}

- Jobs running on the same node with overlapping runtimes using shared resources. + {cJobs.items.length} Jobs running on the same node with overlapping runtimes using shared resources. + ( See All )

{#if showLinks}
@@ -246,7 +246,7 @@ {fpd.message}{fpd.message} {/if} {/each} diff --git a/web/frontend/src/job/JobSummary.svelte b/web/frontend/src/job/JobSummary.svelte index 48f6e99..3da86a5 100644 --- a/web/frontend/src/job/JobSummary.svelte +++ b/web/frontend/src/job/JobSummary.svelte @@ -265,7 +265,7 @@ {fpd.message}{fpd.message} @@ -303,7 +303,7 @@ {fpd.message}{fpd.message} {/if} {/each} diff --git a/web/frontend/src/job/StatsTable.svelte b/web/frontend/src/job/StatsTable.svelte index 606d05b..a71f60f 100644 --- a/web/frontend/src/job/StatsTable.svelte +++ b/web/frontend/src/job/StatsTable.svelte @@ -89,7 +89,7 @@ {#each selectedMetrics as metric} diff --git a/web/frontend/src/job/TagManagement.svelte b/web/frontend/src/job/TagManagement.svelte index 408cc70..2147a8d 100644 --- a/web/frontend/src/job/TagManagement.svelte +++ b/web/frontend/src/job/TagManagement.svelte @@ -7,21 +7,28 @@ - `username String`: Empty string if auth. is disabled, otherwise the username as string - `authlevel Number`: The current users authentication level - `roles [Number]`: Enum containing available roles + - `renderModal Bool?`: If component is rendered as bootstrap modal button [Default: true] --> - (isOpen = !isOpen)}> - - Manage Tags - {#if pendingChange !== false} - - {:else} - - {/if} - - +{#if renderModal} + (isOpen = !isOpen)}> + + Manage Tags + {#if pendingChange !== false} + + {:else} + + {/if} + + + + + + Search using "type: name". If no tag matches your search, a + button for creating a new one will appear. + +
+
    + {#each allTagsFiltered as tag} + + + + + {#if pendingChange === tag.id} + + {:else if job.tags.find((t) => t.id == tag.id)} + + {:else} + + {/if} + + + {:else} + + No tags matching + + {/each} +
+
+ {#if newTagType && newTagName && isNewTag(newTagType, newTagName)} +
+ + {#if roles && authlevel >= roles.admin} + + {/if} +
+ {:else if allTagsFiltered.length == 0} + Search Term is not a valid Tag (type: name) + {/if} + + + + + + + +{:else} + + + + + + + Search using "type: name". If no tag matches your search, a + button for creating a new one will appear. + + -
- - - Search using "type: name". If no tag matches your search, a - button for creating a new one will appear. - - -
    - {#each allTagsFiltered as tag} - - + {#if usedTagsFiltered.length > 0} + + {#each usedTagsFiltered as utag} + + - {#if pendingChange === tag.id} + {#if pendingChange === utag.id} + + {:else} + + {/if} + + + {/each} + + {:else if filterTerm !== ""} + + + No used tags matching + + + {/if} + + {#if unusedTagsFiltered.length > 0} + + {#each unusedTagsFiltered as uutag} + + + + + {#if pendingChange === uutag.id} - {:else if job.tags.find((t) => t.id == tag.id)} - {:else} {/if} - {:else} - - No tags matching - {/each} -
-
- {#if newTagType && newTagName && isNewTag(newTagType, newTagName)} -
+ + {:else if filterTerm !== ""} + + + No unused tags matching + + + {/if} + + {#if newTagType && newTagName && isNewTag(newTagType, newTagName)} + + - {#if roles && authlevel >= roles.admin} - - - {/if} -
- {:else if allTagsFiltered.length == 0} - Search Term is not a valid Tag (type: name) - {/if} - - - - - - - + + + {/if} + + {:else if allTagsFiltered.length == 0} + Search Term is not a valid Tag (type: name) + {/if} +{/if} From ccbf3867e14b2b198177bad17b2d0b2a154dbcfd Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Mon, 16 Sep 2024 13:54:40 +0200 Subject: [PATCH 20/25] change global tag color from gray to magenta --- web/frontend/src/generic/helper/Tag.svelte | 4 ++-- web/templates/monitoring/taglist.tmpl | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/web/frontend/src/generic/helper/Tag.svelte b/web/frontend/src/generic/helper/Tag.svelte index 66b4312..2be9ee6 100644 --- a/web/frontend/src/generic/helper/Tag.svelte +++ b/web/frontend/src/generic/helper/Tag.svelte @@ -38,9 +38,9 @@ {#if tag} {#if tag?.scope === "global"} - {tag.type}: {tag.name} + {tag.type}: {tag.name} {:else if tag.scope === "admin"} - {tag.type}: {tag.name} + {tag.type}: {tag.name} {:else} {tag.type}: {tag.name} {/if} diff --git a/web/templates/monitoring/taglist.tmpl b/web/templates/monitoring/taglist.tmpl index ea29cd7..7831a64 100644 --- a/web/templates/monitoring/taglist.tmpl +++ b/web/templates/monitoring/taglist.tmpl @@ -8,10 +8,10 @@ {{ range $tagList }} {{if eq .scope "global"}} - + {{ .name }} {{ .count }} {{else if eq .scope "admin"}} - + {{ .name }} {{ .count }} {{else}} From 2736b5d1ef4287c10bcbc51d732555fc815bd348 Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Mon, 16 Sep 2024 15:00:42 +0200 Subject: [PATCH 21/25] change background color for tag listitems --- web/frontend/src/job/TagManagement.svelte | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/web/frontend/src/job/TagManagement.svelte b/web/frontend/src/job/TagManagement.svelte index f358c71..b6d290a 100644 --- a/web/frontend/src/job/TagManagement.svelte +++ b/web/frontend/src/job/TagManagement.svelte @@ -207,7 +207,7 @@ {#if usedTagsFiltered.length > 0} {#each usedTagsFiltered as utag} - + @@ -243,7 +243,7 @@ {#if unusedTagsFiltered.length > 0} {#each unusedTagsFiltered as uutag} - + @@ -343,7 +343,7 @@ {#if usedTagsFiltered.length > 0} {#each usedTagsFiltered as utag} - + @@ -379,7 +379,7 @@ {#if unusedTagsFiltered.length > 0} {#each unusedTagsFiltered as uutag} - + From e29be2f140dca8b0281c1d24efa95c848da44c2d Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Mon, 16 Sep 2024 15:03:38 +0200 Subject: [PATCH 22/25] fix missing scope field request for jobview --- web/frontend/src/Job.root.svelte | 2 +- web/frontend/src/generic/helper/Tag.svelte | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/web/frontend/src/Job.root.svelte b/web/frontend/src/Job.root.svelte index ca11692..517671d 100644 --- a/web/frontend/src/Job.root.svelte +++ b/web/frontend/src/Job.root.svelte @@ -75,7 +75,7 @@ duration, numNodes, numHWThreads, numAcc, SMT, exclusive, partition, subCluster, arrayJobId, monitoringStatus, state, walltime, - tags { id, type, name }, + tags { id, type, scope, name }, resources { hostname, hwthreads, accelerators }, metaData, userData { name, email }, diff --git a/web/frontend/src/generic/helper/Tag.svelte b/web/frontend/src/generic/helper/Tag.svelte index 2be9ee6..b3e2d6b 100644 --- a/web/frontend/src/generic/helper/Tag.svelte +++ b/web/frontend/src/generic/helper/Tag.svelte @@ -39,7 +39,7 @@ {#if tag} {#if tag?.scope === "global"} {tag.type}: {tag.name} - {:else if tag.scope === "admin"} + {:else if tag?.scope === "admin"} {tag.type}: {tag.name} {:else} {tag.type}: {tag.name} From d7a8bbf40b5126cdca4568ac84fc7fba8605eb86 Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Wed, 18 Sep 2024 17:23:29 +0200 Subject: [PATCH 23/25] Rework tag and tag edit placement, add other feedback - admin message shown primarily if exists - comment demo summary tab --- web/frontend/src/Job.root.svelte | 59 ++-- web/frontend/src/generic/helper/Tag.svelte | 22 +- .../helper}/TagManagement.svelte | 17 +- .../src/generic/joblist/JobInfo.svelte | 37 ++- web/frontend/src/job/JobSummary.svelte | 281 +++++++++--------- 5 files changed, 217 insertions(+), 199 deletions(-) rename web/frontend/src/{job => generic/helper}/TagManagement.svelte (97%) diff --git a/web/frontend/src/Job.root.svelte b/web/frontend/src/Job.root.svelte index 8e508fa..cddace1 100644 --- a/web/frontend/src/Job.root.svelte +++ b/web/frontend/src/Job.root.svelte @@ -25,7 +25,6 @@ CardHeader, CardTitle, Button, - Icon, } from "@sveltestrap/sveltestrap"; import { getContext } from "svelte"; import { @@ -35,7 +34,6 @@ transformDataForRoofline, } from "./generic/utils.js"; import Metric from "./job/Metric.svelte"; - import TagManagement from "./job/TagManagement.svelte"; import StatsTable from "./job/StatsTable.svelte"; import JobSummary from "./job/JobSummary.svelte"; import ConcurrentJobs from "./generic/helper/ConcurrentJobs.svelte"; @@ -54,12 +52,10 @@ const ccconfig = getContext("cc-config") let isMetricsSelectionOpen = false, - showFootprint = !!ccconfig[`job_view_showFootprint`], selectedMetrics = [], selectedScopes = []; let plots = {}, - jobTags, roofWidth let missingMetrics = [], @@ -240,14 +236,22 @@ {:else if $initq.data} - + {#if $initq.data?.job?.metaData?.message} + + + +
Job {$initq.data?.job?.jobId} ({$initq.data?.job?.cluster})
+ The following note was added by administrators: + + + {@html $initq.data.job.metaData.message} + + + + {/if} + - - - - - - + {#if $initq.data.job.concurrentJobs != null && $initq.data.job.concurrentJobs.items.length != 0} @@ -260,15 +264,6 @@ {/if} - {#if $initq.data?.job?.metaData?.message} - - -

This note was added by administrators:

-
-

{@html $initq.data.job.metaData.message}

- - - {/if} {:else} @@ -276,21 +271,19 @@ {/if} - - {#if showFootprint} - - {#if $initq.error} - {$initq.error.message} - {:else if $initq?.data && $jobMetrics?.data} - - {:else} - - {/if} - - {/if} + + + {#if $initq.error} + {$initq.error.message} + {:else if $initq?.data && $jobMetrics?.data} + + {:else} + + {/if} + - + {#if $initq.error || $jobMetrics.error}

Initq Error: {$initq.error?.message}

diff --git a/web/frontend/src/generic/helper/Tag.svelte b/web/frontend/src/generic/helper/Tag.svelte index b3e2d6b..7efaf63 100644 --- a/web/frontend/src/generic/helper/Tag.svelte +++ b/web/frontend/src/generic/helper/Tag.svelte @@ -23,12 +23,22 @@ if ($initialized && tag == null) tag = allTags.find(tag => tag.id == id) } + + function getScopeColor(scope) { + switch (scope) { + case "admin": + return "#19e5e6"; + case "global": + return "#c85fc8"; + default: + return "#ffc107"; + } + }