From 9535c11dc4b9b861ca83725a6d9e0831e4dfa372 Mon Sep 17 00:00:00 2001 From: Lou Knauer Date: Tue, 8 Mar 2022 11:53:24 +0100 Subject: [PATCH] Make metaData a map[string]string; Resolve explicitly --- api/rest.go | 6 ------ api_test.go | 12 ++++++++++-- frontend | 2 +- gqlgen.yml | 2 ++ graph/generated/generated.go | 18 ++++++++++++++---- graph/schema.resolvers.go | 4 ++++ init-db.go | 5 +++++ repository/import.go | 4 ++++ repository/job.go | 32 +++++++++++++++++++++++++++++-- schema/job.go | 37 ++++++++++++++++++------------------ 10 files changed, 89 insertions(+), 33 deletions(-) diff --git a/api/rest.go b/api/rest.go index 96aeb1b..9cf441a 100644 --- a/api/rest.go +++ b/api/rest.go @@ -285,12 +285,6 @@ func (api *RestApi) startJob(rw http.ResponseWriter, r *http.Request) { return } - req.RawResources, err = json.Marshal(req.Resources) - if err != nil { - handleError(fmt.Errorf("basically impossible: %w", err), http.StatusBadRequest, rw) - return - } - id, err := api.JobRepository.Start(&req) if err != nil { handleError(fmt.Errorf("insert into database failed: %w", err), http.StatusInternalServerError, rw) diff --git a/api_test.go b/api_test.go index 179add6..7fd3fa3 100644 --- a/api_test.go +++ b/api_test.go @@ -177,7 +177,7 @@ func TestRestApi(t *testing.T) { "hwthreads": [0, 1, 2, 3, 4, 5, 6, 7] } ], - "metaData": null, + "metaData": { "jobScript": "blablabla..." }, "startTime": 123456789 }` @@ -260,7 +260,6 @@ func TestRestApi(t *testing.T) { } if job.State != schema.JobStateCompleted { - print("STATE:" + job.State) t.Fatal("expected job to be completed") } @@ -268,6 +267,15 @@ func TestRestApi(t *testing.T) { t.Fatalf("unexpected job properties: %#v", job) } + job.MetaData, err = restapi.JobRepository.FetchMetadata(job) + if err != nil { + t.Fatal(err) + } + + if !reflect.DeepEqual(job.MetaData, map[string]string{"jobScript": "blablabla..."}) { + t.Fatalf("unexpected job.metaData: %#v", job.MetaData) + } + stoppedJob = job }); !ok { return diff --git a/frontend b/frontend index b7e422e..fdf89ac 160000 --- a/frontend +++ b/frontend @@ -1 +1 @@ -Subproject commit b7e422e49ece2d9eeef7354ee595c54c474a4697 +Subproject commit fdf89ac36ff7eb12fabd483ad6e50d55cc118b8b diff --git a/gqlgen.yml b/gqlgen.yml index 59381ee..576f2bb 100644 --- a/gqlgen.yml +++ b/gqlgen.yml @@ -59,6 +59,8 @@ models: fields: tags: resolver: true + metaData: + resolver: true NullableFloat: { model: "github.com/ClusterCockpit/cc-backend/schema.Float" } MetricScope: { model: "github.com/ClusterCockpit/cc-backend/schema.MetricScope" } JobStatistics: { model: "github.com/ClusterCockpit/cc-backend/schema.JobStatistics" } diff --git a/graph/generated/generated.go b/graph/generated/generated.go index c83c99a..204e623 100644 --- a/graph/generated/generated.go +++ b/graph/generated/generated.go @@ -234,6 +234,7 @@ type ComplexityRoot struct { } type JobResolver interface { + MetaData(ctx context.Context, obj *schema.Job) (interface{}, error) Tags(ctx context.Context, obj *schema.Job) ([]*schema.Tag, error) } type MutationResolver interface { @@ -3043,14 +3044,14 @@ func (ec *executionContext) _Job_metaData(ctx context.Context, field graphql.Col Object: "Job", Field: field, Args: nil, - IsMethod: false, - IsResolver: false, + IsMethod: true, + IsResolver: true, } ctx = graphql.WithFieldContext(ctx, fc) resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (interface{}, error) { ctx = rctx // use context from middleware stack in children - return obj.MetaData, nil + return ec.resolvers.Job().MetaData(rctx, obj) }) if err != nil { ec.Error(ctx, err) @@ -7750,7 +7751,16 @@ func (ec *executionContext) _Job(ctx context.Context, sel ast.SelectionSet, obj atomic.AddUint32(&invalids, 1) } case "metaData": - out.Values[i] = ec._Job_metaData(ctx, field, obj) + field := field + out.Concurrently(i, func() (res graphql.Marshaler) { + defer func() { + if r := recover(); r != nil { + ec.Error(ctx, ec.Recover(ctx, r)) + } + }() + res = ec._Job_metaData(ctx, field, obj) + return res + }) case "tags": field := field out.Concurrently(i, func() (res graphql.Marshaler) { diff --git a/graph/schema.resolvers.go b/graph/schema.resolvers.go index 73fd5ef..3fa95b0 100644 --- a/graph/schema.resolvers.go +++ b/graph/schema.resolvers.go @@ -18,6 +18,10 @@ import ( "github.com/ClusterCockpit/cc-backend/schema" ) +func (r *jobResolver) MetaData(ctx context.Context, obj *schema.Job) (interface{}, error) { + return r.Repo.FetchMetadata(obj) +} + func (r *jobResolver) Tags(ctx context.Context, obj *schema.Job) ([]*schema.Tag, error) { return r.Repo.GetTags(&obj.ID) } diff --git a/init-db.go b/init-db.go index 4e124e7..b9fd9f7 100644 --- a/init-db.go +++ b/init-db.go @@ -219,6 +219,11 @@ func loadJob(tx *sqlx.Tx, stmt *sqlx.NamedStmt, tags map[string]int64, path stri return err } + job.RawMetaData, err = json.Marshal(job.MetaData) + if err != nil { + return err + } + if err := repository.SanityChecks(&job.BaseJob); err != nil { return err } diff --git a/repository/import.go b/repository/import.go index 5689278..ff996a4 100644 --- a/repository/import.go +++ b/repository/import.go @@ -93,6 +93,10 @@ func (r *JobRepository) ImportJob(jobMeta *schema.JobMeta, jobData *schema.JobDa if err != nil { return err } + job.RawMetaData, err = json.Marshal(job.MetaData) + if err != nil { + return err + } if err := SanityChecks(&job.BaseJob); err != nil { return err diff --git a/repository/job.go b/repository/job.go index 9145396..7f3f9ee 100644 --- a/repository/job.go +++ b/repository/job.go @@ -5,6 +5,7 @@ import ( "database/sql" "encoding/json" "errors" + "fmt" "strconv" "time" @@ -29,7 +30,7 @@ func (r *JobRepository) Init() error { var jobColumns []string = []string{ "job.id", "job.job_id", "job.user", "job.project", "job.cluster", "job.start_time", "job.partition", "job.array_job_id", "job.num_nodes", "job.num_hwthreads", "job.num_acc", "job.exclusive", "job.monitoring_status", "job.smt", "job.job_state", - "job.duration", "job.resources", "job.meta_data", + "job.duration", "job.resources", // "job.meta_data", } func scanJob(row interface{ Scan(...interface{}) error }) (*schema.Job, error) { @@ -37,7 +38,7 @@ func scanJob(row interface{ Scan(...interface{}) error }) (*schema.Job, error) { if err := row.Scan( &job.ID, &job.JobID, &job.User, &job.Project, &job.Cluster, &job.StartTimeUnix, &job.Partition, &job.ArrayJobId, &job.NumNodes, &job.NumHWThreads, &job.NumAcc, &job.Exclusive, &job.MonitoringStatus, &job.SMT, &job.State, - &job.Duration, &job.RawResources, &job.MetaData); err != nil { + &job.Duration, &job.RawResources /*&job.MetaData*/); err != nil { return nil, err } @@ -54,6 +55,23 @@ func scanJob(row interface{ Scan(...interface{}) error }) (*schema.Job, error) { return job, nil } +func (r *JobRepository) FetchMetadata(job *schema.Job) (map[string]string, error) { + if err := sq.Select("job.meta_data").From("job").Where("job.id = ?", job.ID). + RunWith(r.stmtCache).QueryRow().Scan(&job.RawMetaData); err != nil { + return nil, err + } + + if len(job.RawMetaData) == 0 { + return nil, nil + } + + if err := json.Unmarshal(job.RawMetaData, &job.MetaData); err != nil { + return nil, err + } + + return job.MetaData, nil +} + // Find executes a SQL query to find a specific batch job. // The job is queried using the batch job id, the cluster name, // and the start time of the job in UNIX epoch time seconds. @@ -91,6 +109,16 @@ func (r *JobRepository) FindById( // Start inserts a new job in the table, returning the unique job ID. // Statistics are not transfered! func (r *JobRepository) Start(job *schema.JobMeta) (id int64, err error) { + job.RawResources, err = json.Marshal(job.Resources) + if err != nil { + return -1, fmt.Errorf("encoding resources field failed: %w", err) + } + + job.RawMetaData, err = json.Marshal(job.MetaData) + if err != nil { + return -1, fmt.Errorf("encoding metaData field failed: %w", err) + } + res, err := r.DB.NamedExec(`INSERT INTO job ( job_id, user, project, cluster, `+"`partition`"+`, array_job_id, num_nodes, num_hwthreads, num_acc, exclusive, monitoring_status, smt, job_state, start_time, duration, resources, meta_data diff --git a/schema/job.go b/schema/job.go index 9bb4c99..e6e9b25 100644 --- a/schema/job.go +++ b/schema/job.go @@ -10,24 +10,25 @@ import ( // Common subset of Job and JobMeta. Use one of those, not // this type directly. type BaseJob struct { - JobID int64 `json:"jobId" db:"job_id"` - User string `json:"user" db:"user"` - Project string `json:"project" db:"project"` - Cluster string `json:"cluster" db:"cluster"` - Partition string `json:"partition" db:"partition"` - ArrayJobId int32 `json:"arrayJobId" db:"array_job_id"` - NumNodes int32 `json:"numNodes" db:"num_nodes"` - NumHWThreads int32 `json:"numHwthreads" db:"num_hwthreads"` - NumAcc int32 `json:"numAcc" db:"num_acc"` - Exclusive int32 `json:"exclusive" db:"exclusive"` - MonitoringStatus int32 `json:"monitoringStatus" db:"monitoring_status"` - SMT int32 `json:"smt" db:"smt"` - State JobState `json:"jobState" db:"job_state"` - Duration int32 `json:"duration" db:"duration"` - Tags []*Tag `json:"tags"` - RawResources []byte `json:"-" db:"resources"` - Resources []*Resource `json:"resources"` - MetaData interface{} `json:"metaData" db:"meta_data"` + JobID int64 `json:"jobId" db:"job_id"` + User string `json:"user" db:"user"` + Project string `json:"project" db:"project"` + Cluster string `json:"cluster" db:"cluster"` + Partition string `json:"partition" db:"partition"` + ArrayJobId int32 `json:"arrayJobId" db:"array_job_id"` + NumNodes int32 `json:"numNodes" db:"num_nodes"` + NumHWThreads int32 `json:"numHwthreads" db:"num_hwthreads"` + NumAcc int32 `json:"numAcc" db:"num_acc"` + Exclusive int32 `json:"exclusive" db:"exclusive"` + MonitoringStatus int32 `json:"monitoringStatus" db:"monitoring_status"` + SMT int32 `json:"smt" db:"smt"` + State JobState `json:"jobState" db:"job_state"` + Duration int32 `json:"duration" db:"duration"` + Tags []*Tag `json:"tags"` + RawResources []byte `json:"-" db:"resources"` + Resources []*Resource `json:"resources"` + RawMetaData []byte `json:"-" db:"meta_data"` + MetaData map[string]string `json:"metaData"` } // This type is used as the GraphQL interface and using sqlx as a table row.