ClusterCockpit/cc-backend
1
0
Fork 0
mirror of https://github.com/ClusterCockpit/cc-backend synced 2024-12-25 04:49:05 +01:00
Code Issues Packages Releases Activity

Merge branch '275_tag_scope_jobview_rework' into dev

Browse Source
This commit is contained in:
Christoph Kluge 2024-09-24 17:25:20 +02:00
commit 827f6daabc
31 changed files with 1643 additions and 577 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
api/schema.graphqls
View File

@ -113,6 +113,7 @@ type Tag {
id: ID!
type: String!
name: String!
scope: String!
}
type Resource {
@ -235,7 +236,7 @@ type Query {
}
type Mutation {
createTag(type: String!, name: String!): Tag!
createTag(type: String!, name: String!, scope: String!): Tag!
deleteTag(id: ID!): ID!
addTagsToJob(job: ID!, tagIds: [ID!]!): [Tag!]!
removeTagsFromJob(job: ID!, tagIds: [ID!]!): [Tag!]!

4
internal/api/api_test.go
View File

@ -215,7 +215,7 @@ func TestRestApi(t *testing.T) {
"exclusive": 1,
"monitoringStatus": 1,
"smt": 1,
"tags": [{ "type": "testTagType", "name": "testTagName" }],
"tags": [{ "type": "testTagType", "name": "testTagName", "scope": "testuser" }],
"resources": [
{
"hostname": "host123",
@ -283,7 +283,7 @@ func TestRestApi(t *testing.T) {
t.Fatalf("unexpected job properties: %#v", job)
}
if len(job.Tags) != 1 || job.Tags[0].Type != "testTagType" || job.Tags[0].Name != "testTagName" {
if len(job.Tags) != 1 || job.Tags[0].Type != "testTagType" || job.Tags[0].Name != "testTagName" || job.Tags[0].Scope != "testuser" {
t.Fatalf("unexpected tags: %#v", job.Tags)
}

25
internal/api/rest.go
View File

@ -177,8 +177,9 @@ type ErrorResponse struct {
// ApiTag model
type ApiTag struct {
// Tag Type
Type string `json:"type" example:"Debug"`
Name string `json:"name" example:"Testjob"` // Tag Name
Type string `json:"type" example:"Debug"`
Name string `json:"name" example:"Testjob"` // Tag Name
Scope string `json:"scope" example:"global"` // Tag Scope for Frontend Display
}
// ApiMeta model
@ -420,7 +421,7 @@ func (api *RestApi) getJobs(rw http.ResponseWriter, r *http.Request) {
StartTime: job.StartTime.Unix(),
}
res.Tags, err = api.JobRepository.GetTags(&job.ID)
res.Tags, err = api.JobRepository.GetTags(r.Context(), &job.ID)
if err != nil {
handleError(err, http.StatusInternalServerError, rw)
return
@ -493,7 +494,7 @@ func (api *RestApi) getCompleteJobById(rw http.ResponseWriter, r *http.Request)
return
}
job.Tags, err = api.JobRepository.GetTags(&job.ID)
job.Tags, err = api.JobRepository.GetTags(r.Context(), &job.ID)
if err != nil {
handleError(err, http.StatusInternalServerError, rw)
return
@ -579,7 +580,7 @@ func (api *RestApi) getJobById(rw http.ResponseWriter, r *http.Request) {
return
}
job.Tags, err = api.JobRepository.GetTags(&job.ID)
job.Tags, err = api.JobRepository.GetTags(r.Context(), &job.ID)
if err != nil {
handleError(err, http.StatusInternalServerError, rw)
return
@ -687,6 +688,7 @@ func (api *RestApi) editMeta(rw http.ResponseWriter, r *http.Request) {
// @summary Adds one or more tags to a job
// @tags Job add and modify
// @description Adds tag(s) to a job specified by DB ID. Name and Type of Tag(s) can be chosen freely.
// @description Tag Scope for frontend visibility will default to "global" if none entered, other options: "admin" or specific username.
// @description If tagged job is already finished: Tag will be written directly to respective archive files.
// @accept json
// @produce json
@ -712,7 +714,7 @@ func (api *RestApi) tagJob(rw http.ResponseWriter, r *http.Request) {
return
}
job.Tags, err = api.JobRepository.GetTags(&job.ID)
job.Tags, err = api.JobRepository.GetTags(r.Context(), &job.ID)
if err != nil {
http.Error(rw, err.Error(), http.StatusInternalServerError)
return
@ -725,16 +727,17 @@ func (api *RestApi) tagJob(rw http.ResponseWriter, r *http.Request) {
}
for _, tag := range req {
tagId, err := api.JobRepository.AddTagOrCreate(job.ID, tag.Type, tag.Name)
tagId, err := api.JobRepository.AddTagOrCreate(r.Context(), job.ID, tag.Type, tag.Name, tag.Scope)
if err != nil {
http.Error(rw, err.Error(), http.StatusInternalServerError)
return
}
job.Tags = append(job.Tags, &schema.Tag{
ID: tagId,
Type: tag.Type,
Name: tag.Name,
ID: tagId,
Type: tag.Type,
Name: tag.Name,
Scope: tag.Scope,
})
}
@ -802,7 +805,7 @@ func (api *RestApi) startJob(rw http.ResponseWriter, r *http.Request) {
unlockOnce.Do(api.RepositoryMutex.Unlock)
for _, tag := range req.Tags {
if _, err := api.JobRepository.AddTagOrCreate(id, tag.Type, tag.Name); err != nil {
if _, err := api.JobRepository.AddTagOrCreate(r.Context(), id, tag.Type, tag.Name, tag.Scope); err != nil {
http.Error(rw, err.Error(), http.StatusInternalServerError)
handleError(fmt.Errorf("adding tag to new job %d failed: %w", id, err), http.StatusInternalServerError, rw)
return

93
internal/graph/generated/generated.go
View File

@ -229,7 +229,7 @@ type ComplexityRoot struct {
Mutation struct {
AddTagsToJob func(childComplexity int, job string, tagIds []string) int
CreateTag func(childComplexity int, typeArg string, name string) int
CreateTag func(childComplexity int, typeArg string, name string, scope string) int
DeleteTag func(childComplexity int, id string) int
RemoveTagsFromJob func(childComplexity int, job string, tagIds []string) int
UpdateConfiguration func(childComplexity int, name string, value string) int
@ -303,9 +303,10 @@ type ComplexityRoot struct {
}
Tag struct {
ID func(childComplexity int) int
Name func(childComplexity int) int
Type func(childComplexity int) int
ID func(childComplexity int) int
Name func(childComplexity int) int
Scope func(childComplexity int) int
Type func(childComplexity int) int
}
TimeRangeOutput struct {
@ -355,7 +356,7 @@ type MetricValueResolver interface {
Name(ctx context.Context, obj *schema.MetricValue) (*string, error)
}
type MutationResolver interface {
CreateTag(ctx context.Context, typeArg string, name string) (*schema.Tag, error)
CreateTag(ctx context.Context, typeArg string, name string, scope string) (*schema.Tag, error)
DeleteTag(ctx context.Context, id string) (string, error)
AddTagsToJob(ctx context.Context, job string, tagIds []string) ([]*schema.Tag, error)
RemoveTagsFromJob(ctx context.Context, job string, tagIds []string) ([]*schema.Tag, error)
@ -1183,7 +1184,7 @@ func (e *executableSchema) Complexity(typeName, field string, childComplexity in
return 0, false
}
return e.complexity.Mutation.CreateTag(childComplexity, args["type"].(string), args["name"].(string)), true
return e.complexity.Mutation.CreateTag(childComplexity, args["type"].(string), args["name"].(string), args["scope"].(string)), true
case "Mutation.deleteTag":
if e.complexity.Mutation.DeleteTag == nil {
@ -1602,6 +1603,13 @@ func (e *executableSchema) Complexity(typeName, field string, childComplexity in
return e.complexity.Tag.Name(childComplexity), true
case "Tag.scope":
if e.complexity.Tag.Scope == nil {
break
}
return e.complexity.Tag.Scope(childComplexity), true
case "Tag.type":
if e.complexity.Tag.Type == nil {
break
@ -1949,6 +1957,7 @@ type Tag {
id: ID!
type: String!
name: String!
scope: String!
}
type Resource {
@ -2071,7 +2080,7 @@ type Query {
}
type Mutation {
createTag(type: String!, name: String!): Tag!
createTag(type: String!, name: String!, scope: String!): Tag!
deleteTag(id: ID!): ID!
addTagsToJob(job: ID!, tagIds: [ID!]!): [Tag!]!
removeTagsFromJob(job: ID!, tagIds: [ID!]!): [Tag!]!
@ -2244,6 +2253,15 @@ func (ec *executionContext) field_Mutation_createTag_args(ctx context.Context, r
}
}
args["name"] = arg1
var arg2 string
if tmp, ok := rawArgs["scope"]; ok {
ctx := graphql.WithPathContext(ctx, graphql.NewPathWithField("scope"))
arg2, err = ec.unmarshalNString2string(ctx, tmp)
if err != nil {
return nil, err
}
}
args["scope"] = arg2
return args, nil
}
@ -4622,6 +4640,8 @@ func (ec *executionContext) fieldContext_Job_tags(_ context.Context, field graph
return ec.fieldContext_Tag_type(ctx, field)
case "name":
return ec.fieldContext_Tag_name(ctx, field)
case "scope":
return ec.fieldContext_Tag_scope(ctx, field)
}
return nil, fmt.Errorf("no field named %q was found under type Tag", field.Name)
},
@ -7680,7 +7700,7 @@ func (ec *executionContext) _Mutation_createTag(ctx context.Context, field graph
}()
resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (interface{}, error) {
ctx = rctx // use context from middleware stack in children
return ec.resolvers.Mutation().CreateTag(rctx, fc.Args["type"].(string), fc.Args["name"].(string))
return ec.resolvers.Mutation().CreateTag(rctx, fc.Args["type"].(string), fc.Args["name"].(string), fc.Args["scope"].(string))
})
if err != nil {
ec.Error(ctx, err)
@ -7711,6 +7731,8 @@ func (ec *executionContext) fieldContext_Mutation_createTag(ctx context.Context,
return ec.fieldContext_Tag_type(ctx, field)
case "name":
return ec.fieldContext_Tag_name(ctx, field)
case "scope":
return ec.fieldContext_Tag_scope(ctx, field)
}
return nil, fmt.Errorf("no field named %q was found under type Tag", field.Name)
},
@ -7829,6 +7851,8 @@ func (ec *executionContext) fieldContext_Mutation_addTagsToJob(ctx context.Conte
return ec.fieldContext_Tag_type(ctx, field)
case "name":
return ec.fieldContext_Tag_name(ctx, field)
case "scope":
return ec.fieldContext_Tag_scope(ctx, field)
}
return nil, fmt.Errorf("no field named %q was found under type Tag", field.Name)
},
@ -7892,6 +7916,8 @@ func (ec *executionContext) fieldContext_Mutation_removeTagsFromJob(ctx context.
return ec.fieldContext_Tag_type(ctx, field)
case "name":
return ec.fieldContext_Tag_name(ctx, field)
case "scope":
return ec.fieldContext_Tag_scope(ctx, field)
}
return nil, fmt.Errorf("no field named %q was found under type Tag", field.Name)
},
@ -8199,6 +8225,8 @@ func (ec *executionContext) fieldContext_Query_tags(_ context.Context, field gra
return ec.fieldContext_Tag_type(ctx, field)
case "name":
return ec.fieldContext_Tag_name(ctx, field)
case "scope":
return ec.fieldContext_Tag_scope(ctx, field)
}
return nil, fmt.Errorf("no field named %q was found under type Tag", field.Name)
},
@ -10547,6 +10575,50 @@ func (ec *executionContext) fieldContext_Tag_name(_ context.Context, field graph
return fc, nil
}
func (ec *executionContext) _Tag_scope(ctx context.Context, field graphql.CollectedField, obj *schema.Tag) (ret graphql.Marshaler) {
fc, err := ec.fieldContext_Tag_scope(ctx, field)
if err != nil {
return graphql.Null
}
ctx = graphql.WithFieldContext(ctx, fc)
defer func() {
if r := recover(); r != nil {
ec.Error(ctx, ec.Recover(ctx, r))
ret = graphql.Null
}
}()
resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (interface{}, error) {
ctx = rctx // use context from middleware stack in children
return obj.Scope, nil
})
if err != nil {
ec.Error(ctx, err)
return graphql.Null
}
if resTmp == nil {
if !graphql.HasFieldError(ctx, fc) {
ec.Errorf(ctx, "must not be null")
}
return graphql.Null
}
res := resTmp.(string)
fc.Result = res
return ec.marshalNString2string(ctx, field.Selections, res)
}
func (ec *executionContext) fieldContext_Tag_scope(_ context.Context, field graphql.CollectedField) (fc *graphql.FieldContext, err error) {
fc = &graphql.FieldContext{
Object: "Tag",
Field: field,
IsMethod: false,
IsResolver: false,
Child: func(ctx context.Context, field graphql.CollectedField) (*graphql.FieldContext, error) {
return nil, errors.New("field of type String does not have child fields")
},
}
return fc, nil
}
func (ec *executionContext) _TimeRangeOutput_from(ctx context.Context, field graphql.CollectedField, obj *model.TimeRangeOutput) (ret graphql.Marshaler) {
fc, err := ec.fieldContext_TimeRangeOutput_from(ctx, field)
if err != nil {
@ -15666,6 +15738,11 @@ func (ec *executionContext) _Tag(ctx context.Context, sel ast.SelectionSet, obj
if out.Values[i] == graphql.Null {
out.Invalids++
}
case "scope":
out.Values[i] = ec._Tag_scope(ctx, field, obj)
if out.Values[i] == graphql.Null {
out.Invalids++
}
default:
panic("unknown field " + strconv.Quote(field.Name))
}

16
internal/graph/schema.resolvers.go
View File

@ -29,7 +29,7 @@ func (r *clusterResolver) Partitions(ctx context.Context, obj *schema.Cluster) (
// Tags is the resolver for the tags field.
func (r *jobResolver) Tags(ctx context.Context, obj *schema.Job) ([]*schema.Tag, error) {
return r.Repo.GetTags(&obj.ID)
return r.Repo.GetTags(ctx, &obj.ID)
}
// ConcurrentJobs is the resolver for the concurrentJobs field.
@ -86,14 +86,14 @@ func (r *metricValueResolver) Name(ctx context.Context, obj *schema.MetricValue)
}
// CreateTag is the resolver for the createTag field.
func (r *mutationResolver) CreateTag(ctx context.Context, typeArg string, name string) (*schema.Tag, error) {
id, err := r.Repo.CreateTag(typeArg, name)
func (r *mutationResolver) CreateTag(ctx context.Context, typeArg string, name string, scope string) (*schema.Tag, error) {
id, err := r.Repo.CreateTag(typeArg, name, scope)
if err != nil {
log.Warn("Error while creating tag")
return nil, err
}
return &schema.Tag{ID: id, Type: typeArg, Name: name}, nil
return &schema.Tag{ID: id, Type: typeArg, Name: name, Scope: scope}, nil
}
// DeleteTag is the resolver for the deleteTag field.
@ -103,6 +103,7 @@ func (r *mutationResolver) DeleteTag(ctx context.Context, id string) (string, er
// AddTagsToJob is the resolver for the addTagsToJob field.
func (r *mutationResolver) AddTagsToJob(ctx context.Context, job string, tagIds []string) ([]*schema.Tag, error) {
// Selectable Tags Pre-Filtered by Scope in Frontend: No backend check required
jid, err := strconv.ParseInt(job, 10, 64)
if err != nil {
log.Warn("Error while adding tag to job")
@ -117,7 +118,7 @@ func (r *mutationResolver) AddTagsToJob(ctx context.Context, job string, tagIds
return nil, err
}
if tags, err = r.Repo.AddTag(jid, tid); err != nil {
if tags, err = r.Repo.AddTag(ctx, jid, tid); err != nil {
log.Warn("Error while adding tag")
return nil, err
}
@ -128,6 +129,7 @@ func (r *mutationResolver) AddTagsToJob(ctx context.Context, job string, tagIds
// RemoveTagsFromJob is the resolver for the removeTagsFromJob field.
func (r *mutationResolver) RemoveTagsFromJob(ctx context.Context, job string, tagIds []string) ([]*schema.Tag, error) {
// Removable Tags Pre-Filtered by Scope in Frontend: No backend check required
jid, err := strconv.ParseInt(job, 10, 64)
if err != nil {
log.Warn("Error while parsing job id")
@ -142,7 +144,7 @@ func (r *mutationResolver) RemoveTagsFromJob(ctx context.Context, job string, ta
return nil, err
}
if tags, err = r.Repo.RemoveTag(jid, tid); err != nil {
if tags, err = r.Repo.RemoveTag(ctx, jid, tid); err != nil {
log.Warn("Error while removing tag")
return nil, err
}
@ -168,7 +170,7 @@ func (r *queryResolver) Clusters(ctx context.Context) ([]*schema.Cluster, error)
// Tags is the resolver for the tags field.
func (r *queryResolver) Tags(ctx context.Context) ([]*schema.Tag, error) {
return r.Repo.GetTags(nil)
return r.Repo.GetTags(ctx, nil)
}
// GlobalMetrics is the resolver for the globalMetrics field.

4
internal/importer/handleImport.go
View File

@ -120,8 +120,8 @@ func HandleImportFlag(flag string) error {
}
for _, tag := range job.Tags {
if _, err := r.AddTagOrCreate(id, tag.Type, tag.Name); err != nil {
log.Error("Error while adding or creating tag")
if err := r.ImportTag(id, tag.Type, tag.Name, tag.Scope); err != nil {
log.Error("Error while adding or creating tag on import")
return err
}
}

6
internal/repository/jobQuery.go
View File

@ -89,7 +89,8 @@ func (r *JobRepository) CountJobs(
ctx context.Context,
filters []*model.JobFilter,
) (int, error) {
query, qerr := SecurityCheck(ctx, sq.Select("count(*)").From("job"))
// DISTICT count for tags filters, does not affect other queries
query, qerr := SecurityCheck(ctx, sq.Select("count(DISTINCT job.id)").From("job"))
if qerr != nil {
return 0, qerr
}
@ -136,7 +137,8 @@ func SecurityCheck(ctx context.Context, query sq.SelectBuilder) (sq.SelectBuilde
// Build a sq.SelectBuilder out of a schema.JobFilter.
func BuildWhereClause(filter *model.JobFilter, query sq.SelectBuilder) sq.SelectBuilder {
if filter.Tags != nil {
query = query.Join("jobtag ON jobtag.job_id = job.id").Where(sq.Eq{"jobtag.tag_id": filter.Tags})
// This is an OR-Logic query: Returns all distinct jobs with at least one of the requested tags; TODO: AND-Logic query?
query = query.Join("jobtag ON jobtag.job_id = job.id").Where(sq.Eq{"jobtag.tag_id": filter.Tags}).Distinct()
}
if filter.JobID != nil {
query = buildStringCondition("job.job_id", filter.JobID, query)

16
internal/repository/job_test.go
View File

@ -5,9 +5,11 @@
package repository
import (
"context"
"fmt"
"testing"
"github.com/ClusterCockpit/cc-backend/pkg/schema"
_ "github.com/mattn/go-sqlite3"
)
@ -45,7 +47,19 @@ func TestFindById(t *testing.T) {
func TestGetTags(t *testing.T) {
r := setup(t)
tags, counts, err := r.CountTags(nil)
const contextUserKey ContextKey = "user"
contextUserValue := &schema.User{
Username: "testuser",
Projects: make([]string, 0),
Roles: []string{"user"},
AuthType: 0,
AuthSource: 2,
}
ctx := context.WithValue(getContext(t), contextUserKey, contextUserValue)
// Test Tag has Scope "global"
tags, counts, err := r.CountTags(ctx)
if err != nil {
t.Fatal(err)
}

3
internal/repository/migrations/sqlite3/08_add-footprint.up.sql
View File

@ -11,8 +11,9 @@ ALTER TABLE job ADD COLUMN energy REAL NOT NULL DEFAULT 0.0;
ALTER TABLE job ADD COLUMN energy_footprint TEXT DEFAULT NULL;
ALTER TABLE job ADD COLUMN footprint TEXT DEFAULT NULL;
UPDATE job SET footprint = '{"flops_any_avg": 0.0}';
ALTER TABLE tag ADD COLUMN tag_scope TEXT NOT NULL DEFAULT 'global';
UPDATE job SET footprint = '{"flops_any_avg": 0.0}';
UPDATE job SET footprint = json_replace(footprint, '$.flops_any_avg', job.flops_any_avg);
UPDATE job SET footprint = json_insert(footprint, '$.mem_bw_avg', job.mem_bw_avg);
UPDATE job SET footprint = json_insert(footprint, '$.mem_used_max', job.mem_used_max);

230
internal/repository/tags.go
View File

@ -5,6 +5,8 @@
package repository
import (
"context"
"fmt"
"strings"
"github.com/ClusterCockpit/cc-backend/pkg/archive"
@ -14,7 +16,14 @@ import (
)
// Add the tag with id `tagId` to the job with the database id `jobId`.
func (r *JobRepository) AddTag(job int64, tag int64) ([]*schema.Tag, error) {
func (r *JobRepository) AddTag(ctx context.Context, job int64, tag int64) ([]*schema.Tag, error) {
j, err := r.FindById(ctx, job)
if err != nil {
log.Warn("Error while finding job by id")
return nil, err
}
q := sq.Insert("jobtag").Columns("job_id", "tag_id").Values(job, tag)
if _, err := q.RunWith(r.stmtCache).Exec(); err != nil {
@ -23,49 +32,62 @@ func (r *JobRepository) AddTag(job int64, tag int64) ([]*schema.Tag, error) {
return nil, err
}
j, err := r.FindByIdDirect(job)
if err != nil {
log.Warn("Error while finding job by id")
return nil, err
}
tags, err := r.GetTags(&job)
tags, err := r.GetTags(ctx, &job)
if err != nil {
log.Warn("Error while getting tags for job")
return nil, err
}
return tags, archive.UpdateTags(j, tags)
archiveTags, err := r.getArchiveTags(&job)
if err != nil {
log.Warn("Error while getting tags for job")
return nil, err
}
return tags, archive.UpdateTags(j, archiveTags)
}
// Removes a tag from a job
func (r *JobRepository) RemoveTag(job, tag int64) ([]*schema.Tag, error) {
func (r *JobRepository) RemoveTag(ctx context.Context, job, tag int64) ([]*schema.Tag, error) {
j, err := r.FindById(ctx, job)
if err != nil {
log.Warn("Error while finding job by id")
return nil, err
}
q := sq.Delete("jobtag").Where("jobtag.job_id = ?", job).Where("jobtag.tag_id = ?", tag)
if _, err := q.RunWith(r.stmtCache).Exec(); err != nil {
s, _, _ := q.ToSql()
log.Errorf("Error adding tag with %s: %v", s, err)
log.Errorf("Error removing tag with %s: %v", s, err)
return nil, err
}
j, err := r.FindByIdDirect(job)
if err != nil {
log.Warn("Error while finding job by id")
return nil, err
}
tags, err := r.GetTags(&job)
tags, err := r.GetTags(ctx, &job)
if err != nil {
log.Warn("Error while getting tags for job")
return nil, err
}
return tags, archive.UpdateTags(j, tags)
archiveTags, err := r.getArchiveTags(&job)
if err != nil {
log.Warn("Error while getting tags for job")
return nil, err
}
return tags, archive.UpdateTags(j, archiveTags)
}
// CreateTag creates a new tag with the specified type and name and returns its database id.
func (r *JobRepository) CreateTag(tagType string, tagName string) (tagId int64, err error) {
q := sq.Insert("tag").Columns("tag_type", "tag_name").Values(tagType, tagName)
func (r *JobRepository) CreateTag(tagType string, tagName string, tagScope string) (tagId int64, err error) {
// Default to "Global" scope if none defined
if tagScope == "" {
tagScope = "global"
}
q := sq.Insert("tag").Columns("tag_type", "tag_name", "tag_scope").Values(tagType, tagName, tagScope)
res, err := q.RunWith(r.stmtCache).Exec()
if err != nil {
@ -77,9 +99,10 @@ func (r *JobRepository) CreateTag(tagType string, tagName string) (tagId int64,
return res.LastInsertId()
}
func (r *JobRepository) CountTags(user *schema.User) (tags []schema.Tag, counts map[string]int, err error) {
func (r *JobRepository) CountTags(ctx context.Context) (tags []schema.Tag, counts map[string]int, err error) {
// Fetch all Tags in DB for Display in Frontend Tag-View
tags = make([]schema.Tag, 0, 100)
xrows, err := r.DB.Queryx("SELECT id, tag_type, tag_name FROM tag")
xrows, err := r.DB.Queryx("SELECT id, tag_type, tag_name, tag_scope FROM tag")
if err != nil {
return nil, nil, err
}
@ -89,16 +112,38 @@ func (r *JobRepository) CountTags(user *schema.User) (tags []schema.Tag, counts
if err = xrows.StructScan(&t); err != nil {
return nil, nil, err
}
tags = append(tags, t)
// Handle Scope Filtering: Tag Scope is Global, Private (== Username) or User is auth'd to view Admin Tags
readable, err := r.checkScopeAuth(ctx, "read", t.Scope)
if err != nil {
return nil, nil, err
}
if readable {
tags = append(tags, t)
}
}
q := sq.Select("t.tag_name, count(jt.tag_id)").
user := GetUserFromContext(ctx)
// Query and Count Jobs with attached Tags
q := sq.Select("t.tag_name, t.id, count(jt.tag_id)").
From("tag t").
LeftJoin("jobtag jt ON t.id = jt.tag_id").
GroupBy("t.tag_name")
// Handle Scope Filtering
scopeList := "\"global\""
if user != nil {
scopeList += ",\"" + user.Username + "\""
}
if user.HasAnyRole([]schema.Role{schema.RoleAdmin, schema.RoleSupport}) {
scopeList += ",\"admin\""
}
q = q.Where("t.tag_scope IN (" + scopeList + ")")
// Handle Job Ownership
if user != nil && user.HasAnyRole([]schema.Role{schema.RoleAdmin, schema.RoleSupport}) { // ADMIN || SUPPORT: Count all jobs
log.Debug("CountTags: User Admin or Support -> Count all Jobs for Tags")
// log.Debug("CountTags: User Admin or Support -> Count all Jobs for Tags")
// Unchanged: Needs to be own case still, due to UserRole/NoRole compatibility handling in else case
} else if user != nil && user.HasRole(schema.RoleManager) { // MANAGER: Count own jobs plus project's jobs
// Build ("project1", "project2", ...) list of variable length directly in SQL string
@ -115,29 +160,45 @@ func (r *JobRepository) CountTags(user *schema.User) (tags []schema.Tag, counts
counts = make(map[string]int)
for rows.Next() {
var tagName string
var tagId int
var count int
if err = rows.Scan(&tagName, &count); err != nil {
if err = rows.Scan(&tagName, &tagId, &count); err != nil {
return nil, nil, err
}
counts[tagName] = count
// Use tagId as second Map-Key component to differentiate tags with identical names
counts[fmt.Sprint(tagName, tagId)] = count
}
err = rows.Err()
return
return tags, counts, err
}
// AddTagOrCreate adds the tag with the specified type and name to the job with the database id `jobId`.
// If such a tag does not yet exist, it is created.
func (r *JobRepository) AddTagOrCreate(jobId int64, tagType string, tagName string) (tagId int64, err error) {
tagId, exists := r.TagId(tagType, tagName)
func (r *JobRepository) AddTagOrCreate(ctx context.Context, jobId int64, tagType string, tagName string, tagScope string) (tagId int64, err error) {
// Default to "Global" scope if none defined
if tagScope == "" {
tagScope = "global"
}
writable, err := r.checkScopeAuth(ctx, "write", tagScope)
if err != nil {
return 0, err
}
if !writable {
return 0, fmt.Errorf("cannot write tag scope with current authorization")
}
tagId, exists := r.TagId(tagType, tagName, tagScope)
if !exists {
tagId, err = r.CreateTag(tagType, tagName)
tagId, err = r.CreateTag(tagType, tagName, tagScope)
if err != nil {
return 0, err
}
}
if _, err := r.AddTag(jobId, tagId); err != nil {
if _, err := r.AddTag(ctx, jobId, tagId); err != nil {
return 0, err
}
@ -145,19 +206,19 @@ func (r *JobRepository) AddTagOrCreate(jobId int64, tagType string, tagName stri
}
// TagId returns the database id of the tag with the specified type and name.
func (r *JobRepository) TagId(tagType string, tagName string) (tagId int64, exists bool) {
func (r *JobRepository) TagId(tagType string, tagName string, tagScope string) (tagId int64, exists bool) {
exists = true
if err := sq.Select("id").From("tag").
Where("tag.tag_type = ?", tagType).Where("tag.tag_name = ?", tagName).
Where("tag.tag_type = ?", tagType).Where("tag.tag_name = ?", tagName).Where("tag.tag_scope = ?", tagScope).
RunWith(r.stmtCache).QueryRow().Scan(&tagId); err != nil {
exists = false
}
return
}
// GetTags returns a list of all tags if job is nil or of the tags that the job with that database ID has.
func (r *JobRepository) GetTags(job *int64) ([]*schema.Tag, error) {
q := sq.Select("id", "tag_type", "tag_name").From("tag")
// GetTags returns a list of all scoped tags if job is nil or of the tags that the job with that database ID has.
func (r *JobRepository) GetTags(ctx context.Context, job *int64) ([]*schema.Tag, error) {
q := sq.Select("id", "tag_type", "tag_name", "tag_scope").From("tag")
if job != nil {
q = q.Join("jobtag ON jobtag.tag_id = tag.id").Where("jobtag.job_id = ?", *job)
}
@ -172,7 +233,41 @@ func (r *JobRepository) GetTags(job *int64) ([]*schema.Tag, error) {
tags := make([]*schema.Tag, 0)
for rows.Next() {
tag := &schema.Tag{}
if err := rows.Scan(&tag.ID, &tag.Type, &tag.Name); err != nil {
if err := rows.Scan(&tag.ID, &tag.Type, &tag.Name, &tag.Scope); err != nil {
log.Warn("Error while scanning rows")
return nil, err
}
// Handle Scope Filtering: Tag Scope is Global, Private (== Username) or User is auth'd to view Admin Tags
readable, err := r.checkScopeAuth(ctx, "read", tag.Scope)
if err != nil {
return nil, err
}
if readable {
tags = append(tags, tag)
}
}
return tags, nil
}
// GetArchiveTags returns a list of all tags *regardless of scope* for archiving if job is nil or of the tags that the job with that database ID has.
func (r *JobRepository) getArchiveTags(job *int64) ([]*schema.Tag, error) {
q := sq.Select("id", "tag_type", "tag_name", "tag_scope").From("tag")
if job != nil {
q = q.Join("jobtag ON jobtag.tag_id = tag.id").Where("jobtag.job_id = ?", *job)
}
rows, err := q.RunWith(r.stmtCache).Query()
if err != nil {
s, _, _ := q.ToSql()
log.Errorf("Error get tags with %s: %v", s, err)
return nil, err
}
tags := make([]*schema.Tag, 0)
for rows.Next() {
tag := &schema.Tag{}
if err := rows.Scan(&tag.ID, &tag.Type, &tag.Name, &tag.Scope); err != nil {
log.Warn("Error while scanning rows")
return nil, err
}
@ -181,3 +276,60 @@ func (r *JobRepository) GetTags(job *int64) ([]*schema.Tag, error) {
return tags, nil
}
func (r *JobRepository) ImportTag(jobId int64, tagType string, tagName string, tagScope string) (err error) {
// Import has no scope ctx, only import from metafile to DB (No recursive archive update required), only returns err
tagId, exists := r.TagId(tagType, tagName, tagScope)
if !exists {
tagId, err = r.CreateTag(tagType, tagName, tagScope)
if err != nil {
return err
}
}
q := sq.Insert("jobtag").Columns("job_id", "tag_id").Values(jobId, tagId)
if _, err := q.RunWith(r.stmtCache).Exec(); err != nil {
s, _, _ := q.ToSql()
log.Errorf("Error adding tag on import with %s: %v", s, err)
return err
}
return nil
}
func (r *JobRepository) checkScopeAuth(ctx context.Context, operation string, scope string) (pass bool, err error) {
user := GetUserFromContext(ctx)
if user != nil {
switch {
case operation == "write" && scope == "admin":
if user.HasRole(schema.RoleAdmin) || (len(user.Roles) == 1 && user.HasRole(schema.RoleApi)) {
return true, nil
}
return false, nil
case operation == "write" && scope == "global":
if user.HasRole(schema.RoleAdmin) || (len(user.Roles) == 1 && user.HasRole(schema.RoleApi)) {
return true, nil
}
return false, nil
case operation == "write" && scope == user.Username:
return true, nil
case operation == "read" && scope == "admin":
return user.HasAnyRole([]schema.Role{schema.RoleAdmin, schema.RoleSupport}), nil
case operation == "read" && scope == "global":
return true, nil
case operation == "read" && scope == user.Username:
return true, nil
default:
if operation == "read" || operation == "write" {
// No acceptable scope: deny tag
return false, nil
} else {
return false, fmt.Errorf("error while checking tag operation auth: unknown operation (%s)", operation)
}
}
} else {
return false, fmt.Errorf("error while checking tag operation auth: no user in context")
}
}

BIN
internal/repository/testdata/job.db vendored
View File

Binary file not shown.

7
internal/routerConfig/routes.go
View File

@ -124,9 +124,8 @@ func setupAnalysisRoute(i InfoType, r *http.Request) InfoType {
func setupTaglistRoute(i InfoType, r *http.Request) InfoType {
jobRepo := repository.GetJobRepository()
user := repository.GetUserFromContext(r.Context())
tags, counts, err := jobRepo.CountTags(user)
tags, counts, err := jobRepo.CountTags(r.Context())
tagMap := make(map[string][]map[string]interface{})
if err != nil {
log.Warnf("GetTags failed: %s", err.Error())
@ -134,11 +133,13 @@ func setupTaglistRoute(i InfoType, r *http.Request) InfoType {
return i
}
// Uses tag.ID as second Map-Key component to differentiate tags with identical names
for _, tag := range tags {
tagItem := map[string]interface{}{
"id": tag.ID,
"name": tag.Name,
"count": counts[tag.Name],
"scope": tag.Scope,
"count": counts[fmt.Sprint(tag.Name, tag.ID)],
}
tagMap[tag.Type] = append(tagMap[tag.Type], tagItem)
}

5
pkg/archive/archive.go
View File

@ -171,8 +171,9 @@ func UpdateTags(job *schema.Job, tags []*schema.Tag) error {
jobMeta.Tags = make([]*schema.Tag, 0)
for _, tag := range tags {
jobMeta.Tags = append(jobMeta.Tags, &schema.Tag{
Name: tag.Name,
Type: tag.Type,
Name: tag.Name,
Type: tag.Type,
Scope: tag.Scope,
})
}

7
pkg/schema/job.go
View File

@ -117,9 +117,10 @@ type JobStatistics struct {
// Tag model
// @Description Defines a tag using name and type.
type Tag struct {
Type string `json:"type" db:"tag_type" example:"Debug"`
Name string `json:"name" db:"tag_name" example:"Testjob"`
ID int64 `json:"id" db:"id"`
Type string `json:"type" db:"tag_type" example:"Debug"`
Name string `json:"name" db:"tag_name" example:"Testjob"`
Scope string `json:"scope" db:"tag_scope" example:"global"`
ID int64 `json:"id" db:"id"`
}
// Resource model

410
web/frontend/src/Job.root.svelte
View File

@ -2,9 +2,9 @@
@component Main single job display component; displays plots for every metric as well as various information
Properties:
- `dbid Number`: The jobs DB ID
- `username String`: Empty string if auth. is disabled, otherwise the username as string
- `authlevel Number`: The current users authentication level
- `clusters [String]`: List of cluster names
- `roles [Number]`: Enum containing available roles
-->
@ -25,7 +25,6 @@
CardHeader,
CardTitle,
Button,
Icon,
} from "@sveltestrap/sveltestrap";
import { getContext } from "svelte";
import {
@ -35,16 +34,16 @@
transformDataForRoofline,
} from "./generic/utils.js";
import Metric from "./job/Metric.svelte";
import TagManagement from "./job/TagManagement.svelte";
import StatsTable from "./job/StatsTable.svelte";
import JobFootprint from "./generic/helper/JobFootprint.svelte";
import JobSummary from "./job/JobSummary.svelte";
import ConcurrentJobs from "./generic/helper/ConcurrentJobs.svelte";
import PlotTable from "./generic/PlotTable.svelte";
import Polar from "./generic/plots/Polar.svelte";
import Roofline from "./generic/plots/Roofline.svelte";
import JobInfo from "./generic/joblist/JobInfo.svelte";
import MetricSelection from "./generic/select/MetricSelection.svelte";
export let dbid;
export let username;
export let authlevel;
export let roles;
@ -53,13 +52,11 @@
const ccconfig = getContext("cc-config")
let isMetricsSelectionOpen = false,
showFootprint = !!ccconfig[`job_view_showFootprint`],
selectedMetrics = [],
selectedScopes = [];
let plots = {},
jobTags,
statsTable
roofWidth
let missingMetrics = [],
missingHosts = [],
@ -75,7 +72,7 @@
duration, numNodes, numHWThreads, numAcc,
SMT, exclusive, partition, subCluster, arrayJobId,
monitoringStatus, state, walltime,
tags { id, type, name },
tags { id, type, scope, name },
resources { hostname, hwthreads, accelerators },
metaData,
userData { name, email },
@ -231,221 +228,224 @@
}));
</script>
<Row>
<Col>
<Row class="mb-3">
<!-- Column 1: Job Info, Job Tags, Concurrent Jobs, Admin Message if found-->
<Col xs={12} md={6} xl={3} class="mb-3 mb-xxl-0">
{#if $initq.error}
<Card body color="danger">{$initq.error.message}</Card>
{:else if $initq.data}
<JobInfo job={$initq.data.job} {jobTags} />
<Card class="overflow-auto" style="height: 400px;">
<TabContent> <!-- on:tab={(e) => (status = e.detail)} -->
{#if $initq.data?.job?.metaData?.message}
<TabPane tabId="admin-msg" tab="Admin Note" active>
<CardBody>
<Card body class="mb-2" color="warning">
<h5>Job {$initq.data?.job?.jobId} ({$initq.data?.job?.cluster})</h5>
The following note was added by administrators:
</Card>
<Card body>
{@html $initq.data.job.metaData.message}
</Card>
</CardBody>
</TabPane>
{/if}
<TabPane tabId="meta-info" tab="Job Info" active={$initq.data?.job?.metaData?.message?false:true}>
<CardBody class="pb-2">
<JobInfo job={$initq.data.job} {username} {authlevel} {roles} showTags={false} showTagedit/>
</CardBody>
</TabPane>
{#if $initq.data.job.concurrentJobs != null && $initq.data.job.concurrentJobs.items.length != 0}
<TabPane tabId="shared-jobs">
<span slot="tab">
{$initq.data.job.concurrentJobs.items.length} Concurrent Jobs
</span>
<CardBody>
<ConcurrentJobs cJobs={$initq.data.job.concurrentJobs} showLinks={(authlevel > roles.manager)}/>
</CardBody>
</TabPane>
{/if}
</TabContent>
</Card>
{:else}
<Spinner secondary />
{/if}
</Col>
{#if $initq.data && showFootprint}
<Col>
<JobFootprint
job={$initq.data.job}
/>
</Col>
{/if}
{#if $initq?.data && $jobMetrics?.data?.jobMetrics}
{#if $initq.data.job.concurrentJobs != null && $initq.data.job.concurrentJobs.items.length != 0}
{#if authlevel > roles.manager}
<Col>
<h5>
Concurrent Jobs <Icon
name="info-circle"
style="cursor:help;"
title="Shared jobs running on the same node with overlapping runtimes"
/>
</h5>
<ul>
<li>
<a
href="/monitoring/jobs/?{$initq.data.job.concurrentJobs
.listQuery}"
target="_blank">See All</a
>
</li>
{#each $initq.data.job.concurrentJobs.items as pjob, index}
<li>
<a href="/monitoring/job/{pjob.id}" target="_blank"
>{pjob.jobId}</a
>
</li>
{/each}
</ul>
</Col>
{:else}
<Col>
<h5>
{$initq.data.job.concurrentJobs.items.length} Concurrent Jobs
</h5>
<p>
Number of shared jobs on the same node with overlapping runtimes.
</p>
</Col>
{/if}
{/if}
<Col>
<Polar
metrics={ccconfig[
`job_view_polarPlotMetrics:${$initq.data.job.cluster}`
] || ccconfig[`job_view_polarPlotMetrics`]}
cluster={$initq.data.job.cluster}
subCluster={$initq.data.job.subCluster}
jobMetrics={$jobMetrics.data.jobMetrics}
/>
</Col>
<Col>
<Roofline
renderTime={true}
subCluster={$initq.data.clusters
.find((c) => c.name == $initq.data.job.cluster)
.subClusters.find((sc) => sc.name == $initq.data.job.subCluster)}
data={transformDataForRoofline(
$jobMetrics.data?.jobMetrics?.find(
(m) => m.name == "flops_any" && m.scope == "node",
)?.metric,
$jobMetrics.data?.jobMetrics?.find(
(m) => m.name == "mem_bw" && m.scope == "node",
)?.metric,
)}
/>
</Col>
{:else}
<Col />
<!-- Column 2: Job Footprint, Polar Representation, Heuristic Summary -->
<Col xs={12} md={6} xl={4} xxl={3} class="mb-3 mb-xxl-0">
{#if $initq.error}
<Card body color="danger">{$initq.error.message}</Card>
{:else if $initq?.data && $jobMetrics?.data}
<JobSummary job={$initq.data.job} jobMetrics={$jobMetrics.data.jobMetrics}/>
{:else}
<Spinner secondary />
<Col />
{/if}
</Row>
<Row class="mb-3">
<Col xs="auto">
{#if $initq.data}
<TagManagement job={$initq.data.job} bind:jobTags />
{/if}
</Col>
<Col xs="auto">
{#if $initq.data}
<Button outline on:click={() => (isMetricsSelectionOpen = true)}>
<Icon name="graph-up" /> Metrics
</Button>
{/if}
</Col>
</Row>
<Row>
<Col>
{#if $jobMetrics.error}
{#if $initq.data.job.monitoringStatus == 0 || $initq.data.job.monitoringStatus == 2}
<Card body color="warning">Not monitored or archiving failed</Card>
<br />
{/if}
<Card body color="danger">{$jobMetrics.error.message}</Card>
{:else if $jobMetrics.fetching}
<Spinner secondary />
{:else if $initq?.data && $jobMetrics?.data?.jobMetrics}
<PlotTable
let:item
let:width
renderFor="job"
items={orderAndMap(
groupByScope($jobMetrics.data.jobMetrics),
selectedMetrics,
)}
itemsPerRow={ccconfig.plot_view_plotsPerRow}
>
{#if item.data}
<Metric
bind:this={plots[item.metric]}
on:load-all={loadAllScopes}
job={$initq.data.job}
metricName={item.metric}
metricUnit={$initq.data.globalMetrics.find((gm) => gm.name == item.metric)?.unit}
nativeScope={$initq.data.globalMetrics.find((gm) => gm.name == item.metric)?.scope}
rawData={item.data.map((x) => x.metric)}
scopes={item.data.map((x) => x.scope)}
{width}
isShared={$initq.data.job.exclusive != 1}
<!-- Column 3: Job Roofline; If footprint Enabled: full width, else half width -->
<Col xs={12} md={12} xl={5} xxl={6}>
{#if $initq.error || $jobMetrics.error}
<Card body color="danger">
<p>Initq Error: {$initq.error?.message}</p>
<p>jobMetrics Error: {$jobMetrics.error?.message}</p>
</Card>
{:else if $initq?.data && $jobMetrics?.data}
<Card style="height: 400px;">
<div bind:clientWidth={roofWidth}>
<Roofline
allowSizeChange={true}
width={roofWidth}
renderTime={true}
subCluster={$initq.data.clusters
.find((c) => c.name == $initq.data.job.cluster)
.subClusters.find((sc) => sc.name == $initq.data.job.subCluster)}
data={transformDataForRoofline(
$jobMetrics.data?.jobMetrics?.find(
(m) => m.name == "flops_any" && m.scope == "node",
)?.metric,
$jobMetrics.data?.jobMetrics?.find(
(m) => m.name == "mem_bw" && m.scope == "node",
)?.metric,
)}
/>
{:else}
<Card body color="warning"
>No dataset returned for <code>{item.metric}</code></Card
>
{/if}
</PlotTable>
</div>
</Card>
{:else}
<Spinner secondary />
{/if}
</Col>
</Row>
<Row class="mt-2">
<Col>
{#if $initq.data}
<TabContent>
{#if somethingMissing}
<TabPane tabId="resources" tab="Resources" active={somethingMissing}>
<div style="margin: 10px;">
<Card color="warning">
<CardHeader>
<CardTitle>Missing Metrics/Resources</CardTitle>
</CardHeader>
<CardBody>
{#if missingMetrics.length > 0}
<p>
No data at all is available for the metrics: {missingMetrics.join(
", ",
)}
</p>
{/if}
{#if missingHosts.length > 0}
<p>Some metrics are missing for the following hosts:</p>
<ul>
{#each missingHosts as missing}
<li>
{missing.hostname}: {missing.metrics.join(", ")}
</li>
{/each}
</ul>
{/if}
</CardBody>
</Card>
</div>
</TabPane>
{/if}
<TabPane
tabId="stats"
tab="Statistics Table"
active={!somethingMissing}
>
{#if $jobMetrics?.data?.jobMetrics}
{#key $jobMetrics.data.jobMetrics}
<StatsTable
bind:this={statsTable}
job={$initq.data.job}
jobMetrics={$jobMetrics.data.jobMetrics}
/>
{/key}
<Card class="mb-3">
<CardBody>
<Row class="mb-2">
{#if $initq.data}
<Col xs="auto">
<Button outline on:click={() => (isMetricsSelectionOpen = true)} color="primary">
Select Metrics
</Button>
</Col>
{/if}
</Row>
<hr/>
<Row>
<Col>
{#if $jobMetrics.error}
{#if $initq.data.job.monitoringStatus == 0 || $initq.data.job.monitoringStatus == 2}
<Card body color="warning">Not monitored or archiving failed</Card>
<br />
{/if}
</TabPane>
<TabPane tabId="job-script" tab="Job Script">
<div class="pre-wrapper">
{#if $initq.data.job.metaData?.jobScript}
<pre><code>{$initq.data.job.metaData?.jobScript}</code></pre>
{:else}
<Card body color="warning">No job script available</Card>
{/if}
</div>
</TabPane>
<TabPane tabId="slurm-info" tab="Slurm Info">
<div class="pre-wrapper">
{#if $initq.data.job.metaData?.slurmInfo}
<pre><code>{$initq.data.job.metaData?.slurmInfo}</code></pre>
<Card body color="danger">{$jobMetrics.error.message}</Card>
{:else if $jobMetrics.fetching}
<Spinner secondary />
{:else if $initq?.data && $jobMetrics?.data?.jobMetrics}
<PlotTable
let:item
let:width
renderFor="job"
items={orderAndMap(
groupByScope($jobMetrics.data.jobMetrics),
selectedMetrics,
)}
itemsPerRow={ccconfig.plot_view_plotsPerRow}
>
{#if item.data}
<Metric
bind:this={plots[item.metric]}
on:load-all={loadAllScopes}
job={$initq.data.job}
metricName={item.metric}
metricUnit={$initq.data.globalMetrics.find((gm) => gm.name == item.metric)?.unit}
nativeScope={$initq.data.globalMetrics.find((gm) => gm.name == item.metric)?.scope}
rawData={item.data.map((x) => x.metric)}
scopes={item.data.map((x) => x.scope)}
{width}
isShared={$initq.data.job.exclusive != 1}
/>
{:else}
<Card body color="warning"
>No additional slurm information available</Card
>No dataset returned for <code>{item.metric}</code></Card
>
{/if}
</div>
</TabPane>
</TabContent>
</PlotTable>
{/if}
</Col>
</Row>
</CardBody>
</Card>
<Row class="mb-3">
<Col>
{#if $initq.data}
<Card>
<TabContent>
{#if somethingMissing}
<TabPane tabId="resources" tab="Resources" active={somethingMissing}>
<div style="margin: 10px;">
<Card color="warning">
<CardHeader>
<CardTitle>Missing Metrics/Resources</CardTitle>
</CardHeader>
<CardBody>
{#if missingMetrics.length > 0}
<p>
No data at all is available for the metrics: {missingMetrics.join(
", ",
)}
</p>
{/if}
{#if missingHosts.length > 0}
<p>Some metrics are missing for the following hosts:</p>
<ul>
{#each missingHosts as missing}
<li>
{missing.hostname}: {missing.metrics.join(", ")}
</li>
{/each}
</ul>
{/if}
</CardBody>
</Card>
</div>
</TabPane>
{/if}
<TabPane
tabId="stats"
tab="Statistics Table"
class="overflow-x-auto"
active={!somethingMissing}
>
{#if $jobMetrics?.data?.jobMetrics}
{#key $jobMetrics.data.jobMetrics}
<StatsTable
job={$initq.data.job}
jobMetrics={$jobMetrics.data.jobMetrics}
/>
{/key}
{/if}
</TabPane>
<TabPane tabId="job-script" tab="Job Script">
<div class="pre-wrapper">
{#if $initq.data.job.metaData?.jobScript}
<pre><code>{$initq.data.job.metaData?.jobScript}</code></pre>
{:else}
<Card body color="warning">No job script available</Card>
{/if}
</div>
</TabPane>
<TabPane tabId="slurm-info" tab="Slurm Info">
<div class="pre-wrapper">
{#if $initq.data.job.metaData?.slurmInfo}
<pre><code>{$initq.data.job.metaData?.slurmInfo}</code></pre>
{:else}
<Card body color="warning"
>No additional slurm information available</Card
>
{/if}
</div>
</TabPane>
</TabContent>
</Card>
{/if}
</Col>
</Row>

4
web/frontend/src/generic/Filters.svelte
View File

@ -322,7 +322,9 @@
{#if filters.tags.length != 0}
<Info icon="tags" on:click={() => (isTagsOpen = true)}>
{#each filters.tags as tagId}
<Tag id={tagId} clickable={false} />
{#key tagId}
<Tag id={tagId} clickable={false} />
{/key}
{/each}
</Info>
{/if}

1
web/frontend/src/generic/JobList.svelte
View File

@ -81,6 +81,7 @@
id
type
name
scope
}
userData {
name

101
web/frontend/src/generic/helper/ConcurrentJobs.svelte Normal file
View File

@ -0,0 +1,101 @@
<!--
@component Concurrent Jobs Component; Lists all concurrent jobs in one scrollable card.
Properties:
- `cJobs JobLinkResultList`: List of concurrent Jobs
- `showLinks Bool?`: Show list as clickable links [Default: false]
- `renderCard Bool?`: If to render component as content only or with card wrapping [Default: true]
- `width String?`: Width of the card [Default: 'auto']
- `height String?`: Height of the card [Default: '310px']
-->
<script>
import {
Card,
CardHeader,
CardBody,
Icon
} from "@sveltestrap/sveltestrap";
export let cJobs;
export let showLinks = false;
export let renderCard = false;
export let width = "auto";
export let height = "400px";
</script>
{#if renderCard}
<Card class="overflow-auto" style="width: {width}; height: {height}">
<CardHeader class="mb-0 d-flex justify-content-center">
{cJobs.items.length} Concurrent Jobs
<Icon
style="cursor:help; margin-left:0.5rem;"
name="info-circle"
title="Jobs running on the same node with overlapping runtimes using shared resources"
/>
</CardHeader>
<CardBody>
{#if showLinks}
<ul>
<li>
<a
href="/monitoring/jobs/?{cJobs.listQuery}"
target="_blank">See All</a
>
</li>
{#each cJobs.items as cJob}
<li>
<a href="/monitoring/job/{cJob.id}" target="_blank"
>{cJob.jobId}</a
>
</li>
{/each}
</ul>
{:else}
<ul>
{#each cJobs.items as cJob}
<li>
{cJob.jobId}
</li>
{/each}
</ul>
{/if}
</CardBody>
</Card>
{:else}
<p>
{cJobs.items.length} Jobs running on the same node with overlapping runtimes using shared resources.
( <a
href="/monitoring/jobs/?{cJobs.listQuery}"
target="_blank">See All</a
> )
</p>
<hr/>
{#if showLinks}
<ul>
{#each cJobs.items as cJob}
<li>
<a href="/monitoring/job/{cJob.id}" target="_blank"
>{cJob.jobId}</a
>
</li>
{/each}
</ul>
{:else}
<ul>
{#each cJobs.items as cJob}
<li>
{cJob.jobId}
</li>
{/each}
</ul>
{/if}
{/if}
<style>
ul {
columns: 3;
-webkit-columns: 3;
-moz-columns: 3;
}
</style>

4
web/frontend/src/generic/helper/JobFootprint.svelte
View File

@ -208,7 +208,7 @@
<Tooltip
target={`footprint-${job.jobId}-${index}`}
placement="right"
offset={[0, 20]}>{fpd.message}</Tooltip
>{fpd.message}</Tooltip
>
</div>
<Row cols={12} class="{(footprintData.length == (index + 1)) ? 'mb-0' : 'mb-2'}">
@ -246,7 +246,7 @@
<Tooltip
target={`footprint-${job.jobId}-${index}`}
placement="right"
offset={[0, 20]}>{fpd.message}</Tooltip
>{fpd.message}</Tooltip
>
{/if}
{/each}

16
web/frontend/src/generic/helper/Tag.svelte
View File

@ -23,12 +23,22 @@
if ($initialized && tag == null)
tag = allTags.find(tag => tag.id == id)
}
function getScopeColor(scope) {
switch (scope) {
case "admin":
return "#19e5e6";
case "global":
return "#c85fc8";
default:
return "#ffc107";
}
}
</script>
<style>
a {
margin-left: 0.5rem;
line-height: 2;
margin-right: 0.5rem;
}
span {
font-size: 0.9rem;
@ -37,7 +47,7 @@
<a target={clickable ? "_blank" : null} href={clickable ? `/monitoring/jobs/?tag=${id}` : null}>
{#if tag}
<span class="badge bg-warning text-dark">{tag.type}: {tag.name}</span>
<span style="background-color:{getScopeColor(tag?.scope)};" class="my-1 badge text-dark">{tag.type}: {tag.name}</span>
{:else}
Loading...
{/if}

510
web/frontend/src/generic/helper/TagManagement.svelte Normal file
View File

@ -0,0 +1,510 @@
<!--
@component Job Info Subcomponent; allows management of job tags by deletion or new entries
Properties:
- `job Object`: The job object
- `jobTags [Number]`: The array of currently designated tags
- `username String`: Empty string if auth. is disabled, otherwise the username as string
- `authlevel Number`: The current users authentication level
- `roles [Number]`: Enum containing available roles
- `renderModal Bool?`: If component is rendered as bootstrap modal button [Default: true]
-->
<script>
import { getContext } from "svelte";
import { gql, getContextClient, mutationStore } from "@urql/svelte";
import {
Row,
Col,
Icon,
Button,
ListGroup,
ListGroupItem,
Input,
InputGroup,
InputGroupText,
Spinner,
Modal,
ModalBody,
ModalHeader,
ModalFooter,
Alert,
Tooltip,
} from "@sveltestrap/sveltestrap";
import { fuzzySearchTags } from "../utils.js";
import Tag from "./Tag.svelte";
export let job;
export let jobTags = job.tags;
export let username;
export let authlevel;
export let roles;
export let renderModal = true;
let allTags = getContext("tags"),
initialized = getContext("initialized");
let newTagType = "",
newTagName = "",
newTagScope = username;
let filterTerm = "";
let pendingChange = false;
let isOpen = false;
const isAdmin = (roles && authlevel >= roles.admin);
const client = getContextClient();
const createTagMutation = ({ type, name, scope }) => {
return mutationStore({
client: client,
query: gql`
mutation ($type: String!, $name: String!, $scope: String!) {
createTag(type: $type, name: $name, scope: $scope) {
id
type
name
scope
}
}
`,
variables: { type, name, scope },
});
};
const addTagsToJobMutation = ({ job, tagIds }) => {
return mutationStore({
client: client,
query: gql`
mutation ($job: ID!, $tagIds: [ID!]!) {
addTagsToJob(job: $job, tagIds: $tagIds) {
id
type
name
scope
}
}
`,
variables: { job, tagIds },
});
};
const removeTagsFromJobMutation = ({ job, tagIds }) => {
return mutationStore({
client: client,
query: gql`
mutation ($job: ID!, $tagIds: [ID!]!) {
removeTagsFromJob(job: $job, tagIds: $tagIds) {
id
type
name
scope
}
}
`,
variables: { job, tagIds },
});
};
$: allTagsFiltered = ($initialized, fuzzySearchTags(filterTerm, allTags));
$: usedTagsFiltered = matchJobTags(jobTags, allTagsFiltered, 'used', isAdmin);
$: unusedTagsFiltered = matchJobTags(jobTags, allTagsFiltered, 'unused', isAdmin);
$: {
newTagType = "";
newTagName = "";
let parts = filterTerm.split(":").map((s) => s.trim());
if (parts.length == 2 && parts.every((s) => s.length > 0)) {
newTagType = parts[0];
newTagName = parts[1];
}
}
function matchJobTags(tags, availableTags, type, isAdmin) {
const jobTagIds = tags.map((t) => t.id)
if (type == 'used') {
return availableTags.filter((at) => jobTagIds.includes(at.id))
} else if (type == 'unused' && isAdmin) {
return availableTags.filter((at) => !jobTagIds.includes(at.id))
} else if (type == 'unused' && !isAdmin) { // Normal Users should not see unused global tags here
return availableTags.filter((at) => !jobTagIds.includes(at.id) && at.scope !== "global")
}
return []
}
function isNewTag(type, name) {
for (let tag of allTagsFiltered)
if (tag.type == type && tag.name == name) return false;
return true;
}
function createTag(type, name, scope) {
pendingChange = true;
createTagMutation({ type: type, name: name, scope: scope }).subscribe((res) => {
if (res.fetching === false && !res.error) {
pendingChange = false;
allTags = [...allTags, res.data.createTag];
newTagType = "";
newTagName = "";
addTagToJob(res.data.createTag);
} else if (res.fetching === false && res.error) {
throw res.error;
}
});
}
function addTagToJob(tag) {
pendingChange = tag.id;
addTagsToJobMutation({ job: job.id, tagIds: [tag.id] }).subscribe((res) => {
if (res.fetching === false && !res.error) {
jobTags = job.tags = res.data.addTagsToJob;
pendingChange = false;
} else if (res.fetching === false && res.error) {
throw res.error;
}
});
}
function removeTagFromJob(tag) {
pendingChange = tag.id;
removeTagsFromJobMutation({ job: job.id, tagIds: [tag.id] }).subscribe(
(res) => {
if (res.fetching === false && !res.error) {
jobTags = job.tags = res.data.removeTagsFromJob;
pendingChange = false;
} else if (res.fetching === false && res.error) {
throw res.error;
}
},
);
}
</script>
{#if renderModal}
<Modal {isOpen} toggle={() => (isOpen = !isOpen)}>
<ModalHeader>
Manage Tags <Icon name="tags"/>
</ModalHeader>
<ModalBody>
<InputGroup class="mb-3">
<Input
type="text"
placeholder="Search Tags"
bind:value={filterTerm}
/>
<InputGroupText id={`tag-management-info-modal`} style="cursor:help; font-size:larger;align-content:center;">
<Icon name=info-circle/>
</InputGroupText>
<Tooltip
target={`tag-management-info-modal`}
placement="right">
Search using "type: name". If no tag matches your search, a
button for creating a new one will appear.
</Tooltip>
</InputGroup>
<div class="mb-3 scroll-group">
{#if usedTagsFiltered.length > 0}
<ListGroup class="mb-3">
{#each usedTagsFiltered as utag}
<ListGroupItem color="light">
<Tag tag={utag} />
<span style="float: right;">
{#if pendingChange === utag.id}
<Spinner size="sm" secondary />
{:else}
{#if utag.scope === 'global' || utag.scope === 'admin'}
{#if isAdmin}
<Button
size="sm"
color="danger"
on:click={() => removeTagFromJob(utag)}
>
<Icon name="x" />
</Button>
{:else}
<Button
size="sm"
color="dark"
outline
disabled
>
Global Tag
</Button>
{/if}
{:else}
<Button
size="sm"
color="danger"
on:click={() => removeTagFromJob(utag)}
>
<Icon name="x" />
</Button>
{/if}
{/if}
</span>
</ListGroupItem>
{/each}
</ListGroup>
{:else if filterTerm !== ""}
<ListGroup class="mb-3">
<ListGroupItem disabled>
<i>No attached tags matching.</i>
</ListGroupItem>
</ListGroup>
{:else}
<ListGroup class="mb-3">
<ListGroupItem disabled>
<i>Job has no attached tags.</i>
</ListGroupItem>
</ListGroup>
{/if}
{#if unusedTagsFiltered.length > 0}
<ListGroup class="">
{#each unusedTagsFiltered as uutag}
<ListGroupItem color="secondary">
<Tag tag={uutag} />
<span style="float: right;">
{#if pendingChange === uutag.id}
<Spinner size="sm" secondary />
{:else}
{#if uutag.scope === 'global' || uutag.scope === 'admin'}
{#if isAdmin}
<Button
size="sm"
color="success"
on:click={() => addTagToJob(uutag)}
>
<Icon name="plus" />
</Button>
{/if}
{:else}
<Button
size="sm"
color="success"
on:click={() => addTagToJob(uutag)}
>
<Icon name="plus" />
</Button>
{/if}
{/if}
</span>
</ListGroupItem>
{/each}
</ListGroup>
{:else if filterTerm !== ""}
<ListGroup class="">
<ListGroupItem disabled>
<i>No unused tags matching.</i>
</ListGroupItem>
</ListGroup>
{:else}
<ListGroup class="">
<ListGroupItem disabled>
<i>No unused tags available.</i>
</ListGroupItem>
</ListGroup>
{/if}
</div>
{#if newTagType && newTagName && isNewTag(newTagType, newTagName)}
<Row>
<Col xs={isAdmin ? 7 : 12} md={12} lg={isAdmin ? 7 : 12} xl={12} xxl={isAdmin ? 7 : 12} class="mb-2">
<Button
outline
style="width:100%;"
color="success"
on:click={(e) => (
e.preventDefault(), createTag(newTagType, newTagName, newTagScope)
)}
>
Add new tag:
<Tag tag={{ type: newTagType, name: newTagName, scope: newTagScope }} clickable={false}/>
</Button>
</Col>
{#if isAdmin}
<Col xs={5} md={12} lg={5} xl={12} xxl={5} class="mb-2" style="align-content:center;">
<Input type="select" bind:value={newTagScope}>
<option value={username}>Scope: Private</option>
<option value={"global"}>Scope: Global</option>
<option value={"admin"}>Scope: Admin</option>
</Input>
</Col>
{/if}
</Row>
{:else if filterTerm !== "" && allTagsFiltered.length == 0}
<Alert color="info">
Search Term is not a valid Tag (<code>type: name</code>)
</Alert>
{:else if filterTerm == "" && unusedTagsFiltered.length == 0}
<Alert color="info">
Type "<code>type: name</code>" into the search field to create a new tag.
</Alert>
{/if}
</ModalBody>
<ModalFooter>
<Button color="primary" on:click={() => (isOpen = false)}>Close</Button>
</ModalFooter>
</Modal>
<Button outline on:click={() => (isOpen = true)} size="sm" color="primary">
Manage {jobTags?.length ? jobTags.length : ''} Tags
</Button>
{:else}
<InputGroup class="mb-3">
<Input
type="text"
placeholder="Search Tags"
bind:value={filterTerm}
/>
<InputGroupText id={`tag-management-info`} style="cursor:help; font-size:larger;align-content:center;">
<Icon name=info-circle/>
</InputGroupText>
<Tooltip
target={`tag-management-info`}
placement="right">
Search using "type: name". If no tag matches your search, a
button for creating a new one will appear.
</Tooltip>
</InputGroup>
{#if usedTagsFiltered.length > 0}
<ListGroup class="mb-3">
{#each usedTagsFiltered as utag}
<ListGroupItem color="light">
<Tag tag={utag} />
<span style="float: right;">
{#if pendingChange === utag.id}
<Spinner size="sm" secondary />
{:else}
{#if utag.scope === 'global' || utag.scope === 'admin'}
{#if isAdmin}
<Button
size="sm"
color="danger"
on:click={() => removeTagFromJob(utag)}
>
<Icon name="x" />
</Button>
{/if}
{:else}
<Button
size="sm"
color="danger"
on:click={() => removeTagFromJob(utag)}
>
<Icon name="x" />
</Button>
{/if}
{/if}
</span>
</ListGroupItem>
{/each}
</ListGroup>
{:else if filterTerm !== ""}
<ListGroup class="mb-3">
<ListGroupItem disabled>
<i>No attached tags matching.</i>
</ListGroupItem>
</ListGroup>
{:else}
<ListGroup class="mb-3">
<ListGroupItem disabled>
<i>Job has no attached tags.</i>
</ListGroupItem>
</ListGroup>
{/if}
{#if unusedTagsFiltered.length > 0}
<ListGroup class="mb-3">
{#each unusedTagsFiltered as uutag}
<ListGroupItem color="secondary">
<Tag tag={uutag} />
<span style="float: right;">
{#if pendingChange === uutag.id}
<Spinner size="sm" secondary />
{:else}
{#if uutag.scope === 'global' || uutag.scope === 'admin'}
{#if isAdmin}
<Button
size="sm"
color="success"
on:click={() => addTagToJob(uutag)}
>
<Icon name="plus" />
</Button>
{/if}
{:else}
<Button
size="sm"
color="success"
on:click={() => addTagToJob(uutag)}
>
<Icon name="plus" />
</Button>
{/if}
{/if}
</span>
</ListGroupItem>
{/each}
</ListGroup>
{:else if filterTerm !== ""}
<ListGroup class="mb-3">
<ListGroupItem disabled>
<i>No unused tags matching.</i>
</ListGroupItem>
</ListGroup>
{:else}
<ListGroup class="mb-3">
<ListGroupItem disabled>
<i>No unused tags available.</i>
</ListGroupItem>
</ListGroup>
{/if}
{#if newTagType && newTagName && isNewTag(newTagType, newTagName)}
<Row>
<Col xs={isAdmin ? 7 : 12} md={12} lg={isAdmin ? 7 : 12} xl={12} xxl={isAdmin ? 7 : 12} class="mb-2">
<Button
outline
style="width:100%;"
color="success"
on:click={(e) => (
e.preventDefault(), createTag(newTagType, newTagName, newTagScope)
)}
>
Add new tag:
<Tag tag={{ type: newTagType, name: newTagName, scope: newTagScope }} clickable={false}/>
</Button>
</Col>
{#if isAdmin}
<Col xs={5} md={12} lg={5} xl={12} xxl={5} class="mb-2" style="align-content:center;">
<Input type="select" bind:value={newTagScope}>
<option value={username}>Scope: Private</option>
<option value={"global"}>Scope: Global</option>
<option value={"admin"}>Scope: Admin</option>
</Input>
</Col>
{/if}
</Row>
{:else if filterTerm !== "" && allTagsFiltered.length == 0}
<Alert color="info">
Search Term is not a valid Tag (<code>type: name</code>)
</Alert>
{:else if filterTerm == "" && unusedTagsFiltered.length == 0}
<Alert color="info">
Type "<code>type: name</code>" into the search field to create a new tag.
</Alert>
{/if}
{/if}
<style>
.scroll-group {
max-height: 500px;
overflow-y: auto;
}
</style>

37
web/frontend/src/generic/joblist/JobInfo.svelte
View File

@ -10,9 +10,14 @@
import { Badge, Icon } from "@sveltestrap/sveltestrap";
import { scrambleNames, scramble } from "../utils.js";
import Tag from "../helper/Tag.svelte";
import TagManagement from "../helper/TagManagement.svelte";
export let job;
export let jobTags = job.tags;
export let showTagedit = false;
export let username = null;
export let authlevel= null;
export let roles = null;
function formatDuration(duration) {
const hours = Math.floor(duration / 3600);
@ -36,7 +41,7 @@
</script>
<div>
<p>
<p class="mb-2">
<span class="fw-bold"
><a href="/monitoring/job/{job.id}" target="_blank">{job.jobId}</a>
({job.cluster})</span
@ -63,7 +68,7 @@
{/if}
</p>
<p>
<p class="mb-2">
<Icon name="person-fill" />
<a class="fst-italic" href="/monitoring/user/{job.user}" target="_blank">
{scrambleNames ? scramble(job.user) : job.user}
@ -84,7 +89,7 @@
{/if}
</p>
<p>
<p class="mb-2">
{#if job.numNodes == 1}
{job.resources[0].hostname}
{:else}
@ -104,7 +109,7 @@
{job.subCluster}
</p>
<p>
<p class="mb-2">
Start: <span class="fw-bold"
>{new Date(job.startTime).toLocaleString()}</span
>
@ -117,11 +122,25 @@
{/if}
</p>
<p>
{#each jobTags as tag}
<Tag {tag} />
{/each}
</p>
{#if showTagedit}
<hr class="mt-0 mb-2"/>
<p class="mb-1">
<TagManagement bind:jobTags {job} {username} {authlevel} {roles} renderModal/> :
{#if jobTags?.length > 0}
{#each jobTags as tag}
<Tag {tag}/>
{/each}
{:else}
<span style="font-size: 0.9rem; background-color: lightgray;" class="my-1 badge text-dark">No Tags</span>
{/if}
</p>
{:else}
<p class="mb-1">
{#each jobTags as tag}
<Tag {tag} />
{/each}
</p>
{/if}
</div>
<style>

91
web/frontend/src/generic/plots/Polar.svelte
View File

@ -2,10 +2,11 @@
@component Polar Plot based on chartJS Radar
Properties:
- `metrics [String]`: Metric names to display as polar plot
- `cluster GraphQL.Cluster`: Cluster Object of the parent job
- `subCluster GraphQL.SubCluster`: SubCluster Object of the parent job
- `jobMetrics [GraphQL.JobMetricWithName]`: Metric data
- `footprintData [Object]?`: job.footprint content, evaluated in regards to peak config in jobSummary.svelte [Default: null]
- `metrics [String]?`: Metric names to display as polar plot [Default: null]
- `cluster GraphQL.Cluster?`: Cluster Object of the parent job [Default: null]
- `subCluster GraphQL.SubCluster?`: SubCluster Object of the parent job [Default: null]
- `jobMetrics [GraphQL.JobMetricWithName]?`: Metric data [Default: null]
- `height Number?`: Plot height [Default: 365]
-->
@ -33,24 +34,52 @@
LineElement
);
export let metrics
export let cluster
export let subCluster
export let jobMetrics
export let height = 365
export let footprintData = null;
export let metrics = null;
export let cluster = null;
export let subCluster = null;
export let jobMetrics = null;
export let height = 350;
const getMetricConfig = getContext("getMetricConfig")
const labels = metrics.filter(name => {
if (!jobMetrics.find(m => m.name == name && m.scope == "node")) {
console.warn(`PolarPlot: No metric data for '${name}'`)
return false
function getLabels() {
if (footprintData) {
return footprintData.filter(fpd => {
if (!jobMetrics.find(m => m.name == fpd.name && m.scope == "node" || fpd.impact == 4)) {
console.warn(`PolarPlot: No metric data (or config) for '${fpd.name}'`)
return false
}
return true
})
.map(filtered => filtered.name)
.sort(function (a, b) {
return ((a > b) ? 1 : ((b > a) ? -1 : 0));
});
} else {
return metrics.filter(name => {
if (!jobMetrics.find(m => m.name == name && m.scope == "node")) {
console.warn(`PolarPlot: No metric data for '${name}'`)
return false
}
return true
})
.sort(function (a, b) {
return ((a > b) ? 1 : ((b > a) ? -1 : 0));
});
}
return true
}
const labels = getLabels();
const getMetricConfig = getContext("getMetricConfig");
const getValuesForStatGeneric = (getStat) => labels.map(name => {
const peak = getMetricConfig(cluster, subCluster, name).peak
const metric = jobMetrics.find(m => m.name == name && m.scope == "node")
const value = getStat(metric.metric) / peak
return value <= 1. ? value : 1.
})
const getValuesForStat = (getStat) => labels.map(name => {
const peak = getMetricConfig(cluster, subCluster, name).peak
const getValuesForStatFootprint = (getStat) => labels.map(name => {
const peak = footprintData.find(fpd => fpd.name === name).peak
const metric = jobMetrics.find(m => m.name == name && m.scope == "node")
const value = getStat(metric.metric) / peak
return value <= 1. ? value : 1.
@ -70,12 +99,32 @@
return avg / metric.series.length
}
function loadDataGeneric(type) {
if (type === 'avg') {
return getValuesForStatGeneric(getAvg)
} else if (type === 'max') {
return getValuesForStatGeneric(getMax)
}
console.log('Unknown Type For Polar Data')
return []
}
function loadDataForFootprint(type) {
if (type === 'avg') {
return getValuesForStatFootprint(getAvg)
} else if (type === 'max') {
return getValuesForStatFootprint(getMax)
}
console.log('Unknown Type For Polar Data')
return []
}
const data = {
labels: labels,
datasets: [
{
label: 'Max',
data: getValuesForStat(getMax),
data: footprintData ? loadDataForFootprint('max') : loadDataGeneric('max'), //
fill: 1,
backgroundColor: 'rgba(0, 102, 255, 0.25)',
borderColor: 'rgb(0, 102, 255)',
@ -86,7 +135,7 @@
},
{
label: 'Avg',
data: getValuesForStat(getAvg),
data: footprintData ? loadDataForFootprint('avg') : loadDataGeneric('avg'), // getValuesForStat(getAvg)
fill: true,
backgroundColor: 'rgba(255, 153, 0, 0.25)',
borderColor: 'rgb(255, 153, 0)',
@ -100,7 +149,7 @@
// No custom defined options but keep for clarity
const options = {
maintainAspectRatio: false,
maintainAspectRatio: true,
animation: false,
scales: { // fix scale
r: {

10
web/frontend/src/generic/plots/Roofline.svelte
View File

@ -7,7 +7,7 @@
- `allowSizeChange Bool?`: If dimensions of rendered plot can change [Default: false]
- `subCluster GraphQL.SubCluster?`: SubCluster Object; contains required topology information [Default: null]
- `width Number?`: Plot width (reactively adaptive) [Default: 600]
- `height Number?`: Plot height (reactively adaptive) [Default: 350]
- `height Number?`: Plot height (reactively adaptive) [Default: 380]
Data Format:
- `data = [null, [], []]`
@ -33,7 +33,7 @@
export let allowSizeChange = false;
export let subCluster = null;
export let width = 600;
export let height = 350;
export let height = 380;
let plotWrapper = null;
let uplot = null;
@ -41,8 +41,6 @@
const lineWidth = clusterCockpitConfig.plot_general_lineWidth;
// Helpers
function getGradientR(x) {
if (x < 0.5) return 0;
@ -317,7 +315,7 @@
// The Color Scale For Time Information
const posX = u.valToPos(0.1, "x", true)
const posXLimit = u.valToPos(100, "x", true)
const posY = u.valToPos(15000.0, "y", true)
const posY = u.valToPos(14000.0, "y", true)
u.ctx.fillStyle = 'black'
u.ctx.fillText('Start', posX, posY)
const start = posX + 10
@ -364,7 +362,7 @@
</script>
{#if data != null}
<div bind:this={plotWrapper} />
<div bind:this={plotWrapper} class="p-2"/>
{:else}
<Card class="mx-4" body color="warning">Cannot render roofline: No data!</Card
>

2
web/frontend/src/generic/utils.js
View File

@ -77,7 +77,7 @@ export function init(extraInitQuery = "") {
footprint
}
}
tags { id, name, type }
tags { id, name, type, scope }
globalMetrics {
name
scope

1
web/frontend/src/job.entrypoint.js
View File

@ -5,6 +5,7 @@ new Job({
target: document.getElementById('svelte-app'),
props: {
dbid: jobInfos.id,
username: username,
authlevel: authlevel,
roles: roles
},

347
web/frontend/src/job/JobSummary.svelte Normal file
View File

@ -0,0 +1,347 @@
<!--
@component Job Summary component; Displays job.footprint data as bars in relation to thresholds, as polar plot, and summariziong comment
Properties:
- `job Object`: The GQL job object
- `displayTitle Bool?`: If to display cardHeader with title [Default: true]
- `width String?`: Width of the card [Default: 'auto']
- `height String?`: Height of the card [Default: '310px']
-->
<script context="module">
function findJobThresholds(job, metricConfig) {
if (!job || !metricConfig) {
console.warn("Argument missing for findJobThresholds!");
return null;
}
// metricConfig is on subCluster-Level
const defaultThresholds = {
peak: metricConfig.peak,
normal: metricConfig.normal,
caution: metricConfig.caution,
alert: metricConfig.alert
};
// Job_Exclusivity does not matter, only aggregation
if (metricConfig.aggregation === "avg") {
return defaultThresholds;
} else if (metricConfig.aggregation === "sum") {
const topol = getContext("getHardwareTopology")(job.cluster, job.subCluster)
const jobFraction = job.numHWThreads / topol.node.length;
return {
peak: round(defaultThresholds.peak * jobFraction, 0),
normal: round(defaultThresholds.normal * jobFraction, 0),
caution: round(defaultThresholds.caution * jobFraction, 0),
alert: round(defaultThresholds.alert * jobFraction, 0),
};
} else {
console.warn(
"Missing or unkown aggregation mode (sum/avg) for metric:",
metricConfig,
);
return defaultThresholds;
}
}
</script>
<script>
import { getContext } from "svelte";
import {
Card,
CardBody,
Progress,
Icon,
Tooltip,
Row,
Col,
TabContent,
TabPane
} from "@sveltestrap/sveltestrap";
import Polar from "../generic/plots/Polar.svelte";
import { round } from "mathjs";
export let job;
export let jobMetrics;
export let width = "auto";
export let height = "400px";
const ccconfig = getContext("cc-config")
const showFootprint = !!ccconfig[`job_view_showFootprint`];
const footprintData = job?.footprint?.map((jf) => {
const fmc = getContext("getMetricConfig")(job.cluster, job.subCluster, jf.name);
if (fmc) {
// Unit
const unit = (fmc?.unit?.prefix ? fmc.unit.prefix : "") + (fmc?.unit?.base ? fmc.unit.base : "")
// Threshold / -Differences
const fmt = findJobThresholds(job, fmc);
if (jf.name === "flops_any") fmt.peak = round(fmt.peak * 0.85, 0);
// Define basic data -> Value: Use as Provided
const fmBase = {
name: jf.name,
stat: jf.stat,
value: jf.value,
unit: unit,
peak: fmt.peak,
dir: fmc.lowerIsBetter
};
if (evalFootprint(jf.value, fmt, fmc.lowerIsBetter, "alert")) {
return {
...fmBase,
color: "danger",
message: `Metric average way ${fmc.lowerIsBetter ? "above" : "below"} expected normal thresholds.`,
impact: 3
};
} else if (evalFootprint(jf.value, fmt, fmc.lowerIsBetter, "caution")) {
return {
...fmBase,
color: "warning",
message: `Metric average ${fmc.lowerIsBetter ? "above" : "below"} expected normal thresholds.`,
impact: 2,
};
} else if (evalFootprint(jf.value, fmt, fmc.lowerIsBetter, "normal")) {
return {
...fmBase,
color: "success",
message: "Metric average within expected thresholds.",
impact: 1,
};
} else if (evalFootprint(jf.value, fmt, fmc.lowerIsBetter, "peak")) {
return {
...fmBase,
color: "info",
message:
"Metric average above expected normal thresholds: Check for artifacts recommended.",
impact: 0,
};
} else {
return {
...fmBase,
color: "secondary",
message:
"Metric average above expected peak threshold: Check for artifacts!",
impact: -1,
};
}
} else { // No matching metric config: display as single value
return {
name: jf.name,
stat: jf.stat,
value: jf.value,
message:
`No config for metric ${jf.name} found.`,
impact: 4,
};
}
}).sort(function (a, b) { // Sort by impact value primarily, within impact sort name alphabetically
return a.impact - b.impact || ((a.name > b.name) ? 1 : ((b.name > a.name) ? -1 : 0));
});;
function evalFootprint(mean, thresholds, lowerIsBetter, level) {
// Handle Metrics in which less value is better
switch (level) {
case "peak":
if (lowerIsBetter)
return false; // metric over peak -> return false to trigger impact -1
else return mean <= thresholds.peak && mean > thresholds.normal;
case "alert":
if (lowerIsBetter)
return mean <= thresholds.peak && mean >= thresholds.alert;
else return mean <= thresholds.alert && mean >= 0;
case "caution":
if (lowerIsBetter)
return mean < thresholds.alert && mean >= thresholds.caution;
else return mean <= thresholds.caution && mean > thresholds.alert;
case "normal":
if (lowerIsBetter)
return mean < thresholds.caution && mean >= 0;
else return mean <= thresholds.normal && mean > thresholds.caution;
default:
return false;
}
}
/*
function writeSummary(fpd) {
// Hardcoded! Needs to be retrieved from globalMetrics
const performanceMetrics = ['flops_any', 'mem_bw'];
const utilizationMetrics = ['cpu_load', 'acc_utilization'];
const energyMetrics = ['cpu_power'];
let performanceScore = 0;
let utilizationScore = 0;
let energyScore = 0;
let performanceMetricsCounted = 0;
let utilizationMetricsCounted = 0;
let energyMetricsCounted = 0;
fpd.forEach(metric => {
console.log('Metric, Impact', metric.name, metric.impact)
if (performanceMetrics.includes(metric.name)) {
performanceScore += metric.impact
performanceMetricsCounted += 1
} else if (utilizationMetrics.includes(metric.name)) {
utilizationScore += metric.impact
utilizationMetricsCounted += 1
} else if (energyMetrics.includes(metric.name)) {
energyScore += metric.impact
energyMetricsCounted += 1
}
});
performanceScore = (performanceMetricsCounted == 0) ? performanceScore : (performanceScore / performanceMetricsCounted);
utilizationScore = (utilizationMetricsCounted == 0) ? utilizationScore : (utilizationScore / utilizationMetricsCounted);
energyScore = (energyMetricsCounted == 0) ? energyScore : (energyScore / energyMetricsCounted);
let res = [];
console.log('Perf', performanceScore, performanceMetricsCounted)
console.log('Util', utilizationScore, utilizationMetricsCounted)
console.log('Energy', energyScore, energyMetricsCounted)
if (performanceScore == 1) {
res.push('<b>Performance:</b> Your job performs well.')
} else if (performanceScore != 0) {
res.push('<b>Performance:</b> Your job performs suboptimal.')
}
if (utilizationScore == 1) {
res.push('<b>Utilization:</b> Your job utilizes resources well.')
} else if (utilizationScore != 0) {
res.push('<b>Utilization:</b> Your job utilizes resources suboptimal.')
}
if (energyScore == 1) {
res.push('<b>Energy:</b> Your job has good energy values.')
} else if (energyScore != 0) {
res.push('<b>Energy:</b> Your job consumes more energy than necessary.')
}
return res;
};
$: summaryMessages = writeSummary(footprintData)
*/
</script>
<Card class="overflow-auto" style="width: {width}; height: {height}">
<TabContent> <!-- on:tab={(e) => (status = e.detail)} -->
{#if showFootprint}
<TabPane tabId="foot" tab="Footprint" active>
<CardBody>
{#each footprintData as fpd, index}
{#if fpd.impact !== 4}
<div class="mb-1 d-flex justify-content-between">
<div>&nbsp;<b>{fpd.name} ({fpd.stat})</b></div>
<div
class="cursor-help d-inline-flex"
id={`footprint-${job.jobId}-${index}`}
>
<div class="mx-1">
{#if fpd.impact === 3 || fpd.impact === -1}
<Icon name="exclamation-triangle-fill" class="text-danger" />
{:else if fpd.impact === 2}
<Icon name="exclamation-triangle" class="text-warning" />
{/if}
{#if fpd.impact === 3}
<Icon name="emoji-frown" class="text-danger" />
{:else if fpd.impact === 2}
<Icon name="emoji-neutral" class="text-warning" />
{:else if fpd.impact === 1}
<Icon name="emoji-smile" class="text-success" />
{:else if fpd.impact === 0}
<Icon name="emoji-laughing" class="text-info" />
{:else if fpd.impact === -1}
<Icon name="emoji-dizzy" class="text-danger" />
{/if}
</div>
<div>
{fpd.value} / {fpd.peak}
{fpd.unit} &nbsp;
</div>
</div>
<Tooltip
target={`footprint-${job.jobId}-${index}`}
placement="right"
>{fpd.message}</Tooltip
>
</div>
<Row cols={12} class="{(footprintData.length == (index + 1)) ? 'mb-0' : 'mb-2'}">
{#if fpd.dir}
<Col xs="1">
<Icon name="caret-left-fill" />
</Col>
{/if}
<Col xs="11" class="align-content-center">
<Progress value={fpd.value} max={fpd.peak} color={fpd.color} />
</Col>
{#if !fpd.dir}
<Col xs="1">
<Icon name="caret-right-fill" />
</Col>
{/if}
</Row>
{:else}
<div class="mb-1 d-flex justify-content-between">
<div>
&nbsp;<b>{fpd.name} ({fpd.stat})</b>
</div>
<div
class="cursor-help d-inline-flex"
id={`footprint-${job.jobId}-${index}`}
>
<div class="mx-1">
<Icon name="info-circle"/>
</div>
<div>
{fpd.value}&nbsp;
</div>
</div>
</div>
<Tooltip
target={`footprint-${job.jobId}-${index}`}
placement="right"
>{fpd.message}</Tooltip
>
{/if}
{/each}
</CardBody>
</TabPane>
{/if}
<TabPane tabId="polar" tab="Polar" active={!showFootprint}>
<CardBody>
<Polar
{footprintData}
{jobMetrics}
/>
</CardBody>
</TabPane>
<!--
<TabPane tabId="summary" tab="Summary">
<CardBody>
<p>Based on footprint data, this job performs as follows:</p>
<hr/>
<ul>
{#each summaryMessages as sm}
<li>
{@html sm}
</li>
{/each}
</ul>
</CardBody>
</TabPane>
-->
</TabContent>
</Card>
<style>
.cursor-help {
cursor: help;
}
</style>

16
web/frontend/src/job/StatsTable.svelte
View File

@ -11,6 +11,7 @@
import {
Button,
Table,
Input,
InputGroup,
InputGroupText,
Icon,
@ -84,29 +85,32 @@
}
</script>
<Table>
<Table class="mb-0">
<thead>
<!-- Header Row 1: Selectors -->
<tr>
<th>
<Button outline on:click={() => (isMetricSelectionOpen = true)}>
Metrics
<Button outline on:click={() => (isMetricSelectionOpen = true)} class="w-100 px-2" color="primary">
Select Metrics
</Button>
</th>
{#each selectedMetrics as metric}
<!-- To Match Row-2 Header Field Count-->
<th colspan={selectedScopes[metric] == "node" ? 3 : 4}>
<InputGroup>
<InputGroupText>
{metric}
</InputGroupText>
<select class="form-select" bind:value={selectedScopes[metric]}>
<Input type="select" bind:value={selectedScopes[metric]}>
{#each scopesForMetric(metric, jobMetrics) as scope}
<option value={scope}>{scope}</option>
{/each}
</select>
</Input>
</InputGroup>
</th>
{/each}
</tr>
<!-- Header Row 2: Fields -->
<tr>
<th>Node</th>
{#each selectedMetrics as metric}
@ -146,8 +150,6 @@
</tbody>
</Table>
<br />
<MetricSelection
cluster={job.cluster}
configName="job_view_nodestats_selectedMetrics"

238
web/frontend/src/job/TagManagement.svelte
View File

@ -1,238 +0,0 @@
<!--
@component Job View Subcomponent; allows management of job tags by deletion or new entries
Properties:
- `job Object`: The job object
- `jobTags [Number]`: The array of currently designated tags
-->
<script>
import { getContext } from "svelte";
import { gql, getContextClient, mutationStore } from "@urql/svelte";
import {
Icon,
Button,
ListGroupItem,
Spinner,
Modal,
Input,
ModalBody,
ModalHeader,
ModalFooter,
Alert,
} from "@sveltestrap/sveltestrap";
import { fuzzySearchTags } from "../generic/utils.js";
import Tag from "../generic/helper/Tag.svelte";
export let job;
export let jobTags = job.tags;
let allTags = getContext("tags"),
initialized = getContext("initialized");
let newTagType = "",
newTagName = "";
let filterTerm = "";
let pendingChange = false;
let isOpen = false;
const client = getContextClient();
const createTagMutation = ({ type, name }) => {
return mutationStore({
client: client,
query: gql`
mutation ($type: String!, $name: String!) {
createTag(type: $type, name: $name) {
id
type
name
}
}
`,
variables: { type, name },
});
};
const addTagsToJobMutation = ({ job, tagIds }) => {
return mutationStore({
client: client,
query: gql`
mutation ($job: ID!, $tagIds: [ID!]!) {
addTagsToJob(job: $job, tagIds: $tagIds) {
id
type
name
}
}
`,
variables: { job, tagIds },
});
};
const removeTagsFromJobMutation = ({ job, tagIds }) => {
return mutationStore({
client: client,
query: gql`
mutation ($job: ID!, $tagIds: [ID!]!) {
removeTagsFromJob(job: $job, tagIds: $tagIds) {
id
type
name
}
}
`,
variables: { job, tagIds },
});
};
let allTagsFiltered; // $initialized is in there because when it becomes true, allTags is initailzed.
$: allTagsFiltered = ($initialized, fuzzySearchTags(filterTerm, allTags));
$: {
newTagType = "";
newTagName = "";
let parts = filterTerm.split(":").map((s) => s.trim());
if (parts.length == 2 && parts.every((s) => s.length > 0)) {
newTagType = parts[0];
newTagName = parts[1];
}
}
function isNewTag(type, name) {
for (let tag of allTagsFiltered)
if (tag.type == type && tag.name == name) return false;
return true;
}
function createTag(type, name) {
pendingChange = true;
createTagMutation({ type: type, name: name }).subscribe((res) => {
if (res.fetching === false && !res.error) {
pendingChange = false;
allTags = [...allTags, res.data.createTag];
newTagType = "";
newTagName = "";
addTagToJob(res.data.createTag);
} else if (res.fetching === false && res.error) {
throw res.error;
}
});
}
function addTagToJob(tag) {
pendingChange = tag.id;
addTagsToJobMutation({ job: job.id, tagIds: [tag.id] }).subscribe((res) => {
if (res.fetching === false && !res.error) {
jobTags = job.tags = res.data.addTagsToJob;
pendingChange = false;
} else if (res.fetching === false && res.error) {
throw res.error;
}
});
}
function removeTagFromJob(tag) {
pendingChange = tag.id;
removeTagsFromJobMutation({ job: job.id, tagIds: [tag.id] }).subscribe(
(res) => {
if (res.fetching === false && !res.error) {
jobTags = job.tags = res.data.removeTagsFromJob;
pendingChange = false;
} else if (res.fetching === false && res.error) {
throw res.error;
}
},
);
}
</script>
<Modal {isOpen} toggle={() => (isOpen = !isOpen)}>
<ModalHeader>
Manage Tags
{#if pendingChange !== false}
<Spinner size="sm" secondary />
{:else}
<Icon name="tags" />
{/if}
</ModalHeader>
<ModalBody>
<Input
style="width: 100%;"
type="text"
placeholder="Search Tags"
bind:value={filterTerm}
/>
<br />
<Alert color="info">
Search using "<code>type: name</code>". If no tag matches your search, a
button for creating a new one will appear.
</Alert>
<ul class="list-group">
{#each allTagsFiltered as tag}
<ListGroupItem>
<Tag {tag} />
<span style="float: right;">
{#if pendingChange === tag.id}
<Spinner size="sm" secondary />
{:else if job.tags.find((t) => t.id == tag.id)}
<Button
size="sm"
outline
color="danger"
on:click={() => removeTagFromJob(tag)}
>
<Icon name="x" />
</Button>
{:else}
<Button
size="sm"
outline
color="success"
on:click={() => addTagToJob(tag)}
>
<Icon name="plus" />
</Button>
{/if}
</span>
</ListGroupItem>
{:else}
<ListGroupItem disabled>
<i>No tags matching</i>
</ListGroupItem>
{/each}
</ul>
<br />
{#if newTagType && newTagName && isNewTag(newTagType, newTagName)}
<Button
outline
color="success"
on:click={(e) => (
e.preventDefault(), createTag(newTagType, newTagName)
)}
>
Create & Add Tag:
<Tag tag={{ type: newTagType, name: newTagName }} clickable={false} />
</Button>
{:else if allTagsFiltered.length == 0}
<Alert>Search Term is not a valid Tag (<code>type: name</code>)</Alert>
{/if}
</ModalBody>
<ModalFooter>
<Button color="primary" on:click={() => (isOpen = false)}>Close</Button>
</ModalFooter>
</Modal>
<Button outline on:click={() => (isOpen = true)}>
Manage Tags <Icon name="tags" />
</Button>
<style>
ul.list-group {
max-height: 450px;
margin-bottom: 10px;
overflow: scroll;
}
</style>

1
web/templates/monitoring/job.tmpl
View File

@ -11,6 +11,7 @@
id: "{{ .Infos.id }}",
};
const clusterCockpitConfig = {{ .Config }};
const username = {{ .User.Username }};
const authlevel = {{ .User.GetAuthLevel }};
const roles = {{ .Roles }};
</script>

12
web/templates/monitoring/taglist.tmpl
View File

@ -7,8 +7,16 @@
{{ $tagType }}
</div>
{{ range $tagList }}
<a class="btn btn-lg btn-warning" href="/monitoring/jobs/?tag={{ .id }}" role="button">
{{ .name }} <span class="badge bg-light text-dark">{{ .count }}</span> </a>
{{if eq .scope "global"}}
<a style="background-color:#c85fc8;" class="btn btn-lg" href="/monitoring/jobs/?tag={{ .id }}" role="button">
{{ .name }} <span class="badge bg-light text-dark">{{ .count }}</span> </a>
{{else if eq .scope "admin"}}
<a style="background-color:#19e5e6;" class="btn btn-lg" href="/monitoring/jobs/?tag={{ .id }}" role="button">
{{ .name }} <span class="badge bg-light text-dark">{{ .count }}</span> </a>
{{else}}
<a class="btn btn-lg btn-warning" href="/monitoring/jobs/?tag={{ .id }}" role="button">
{{ .name }} <span class="badge bg-light text-dark">{{ .count }}</span> </a>
{{end}}
{{end}}
{{end}}
</div>