Merge branch 'master' into 231-sql-statement-syntax

api/swagger.json

@@ -327,6 +327,76 @@
                 }
             }
         },
+        "/jobs/edit_meta/{id}": {
+            "post": {
+                "security": [
+                    {
+                        "ApiKeyAuth": []
+                    }
+                ],
+                "description": "Edit key value pairs in job metadata json\nIf a key already exists its content will be overwritten",
+                "consumes": [
+                    "application/json"
+                ],
+                "produces": [
+                    "application/json"
+                ],
+                "tags": [
+                    "Job add and modify"
+                ],
+                "summary": "Edit meta-data json",
+                "parameters": [
+                    {
+                        "type": "integer",
+                        "description": "Job Database ID",
+                        "name": "id",
+                        "in": "path",
+                        "required": true
+                    },
+                    {
+                        "description": "Kay value pair to add",
+                        "name": "request",
+                        "in": "body",
+                        "required": true,
+                        "schema": {
+                            "$ref": "#/definitions/api.EditMetaRequest"
+                        }
+                    }
+                ],
+                "responses": {
+                    "200": {
+                        "description": "Updated job resource",
+                        "schema": {
+                            "$ref": "#/definitions/schema.Job"
+                        }
+                    },
+                    "400": {
+                        "description": "Bad Request",
+                        "schema": {
+                            "$ref": "#/definitions/api.ErrorResponse"
+                        }
+                    },
+                    "401": {
+                        "description": "Unauthorized",
+                        "schema": {
+                            "$ref": "#/definitions/api.ErrorResponse"
+                        }
+                    },
+                    "404": {
+                        "description": "Job does not exist",
+                        "schema": {
+                            "$ref": "#/definitions/api.ErrorResponse"
+                        }
+                    },
+                    "500": {
+                        "description": "Internal Server Error",
+                        "schema": {
+                            "$ref": "#/definitions/api.ErrorResponse"
+                        }
+                    }
+                }
+            }
+        },
         "/jobs/start_job/": {
             "post": {
                 "security": [
@@ -1114,6 +1184,19 @@
                 }
             }
         },
+        "api.EditMetaRequest": {
+            "type": "object",
+            "properties": {
+                "key": {
+                    "type": "string",
+                    "example": "jobScript"
+                },
+                "value": {
+                    "type": "string",
+                    "example": "bash script"
+                }
+            }
+        },
         "api.ErrorResponse": {
             "type": "object",
             "properties": {
@@ -1252,6 +1335,10 @@
                     "minimum": 0,
                     "example": 1
                 },
+                "flopsAnyAvg": {
+                    "description": "FlopsAnyAvg as Float64",
+                    "type": "number"
+                },
                 "id": {
                     "description": "The unique identifier of a job in the database",
                     "type": "integer"
@@ -1278,6 +1365,18 @@
                     ],
                     "example": "completed"
                 },
+                "loadAvg": {
+                    "description": "LoadAvg as Float64",
+                    "type": "number"
+                },
+                "memBwAvg": {
+                    "description": "MemBwAvg as Float64",
+                    "type": "number"
+                },
+                "memUsedMax": {
+                    "description": "MemUsedMax as Float64",
+                    "type": "number"
+                },
                 "metaData": {
                     "description": "Additional information about the job",
                     "type": "object",

api/swagger.yaml

@@ -50,6 +50,15 @@ definitions:
       msg:
         type: string
     type: object
+  api.EditMetaRequest:
+    properties:
+      key:
+        example: jobScript
+        type: string
+      value:
+        example: bash script
+        type: string
+    type: object
   api.ErrorResponse:
     properties:
       error:
@@ -150,6 +159,9 @@ definitions:
         maximum: 2
         minimum: 0
         type: integer
+      flopsAnyAvg:
+        description: FlopsAnyAvg as Float64
+        type: number
       id:
         description: The unique identifier of a job in the database
         type: integer
@@ -169,6 +181,15 @@ definitions:
         - timeout
         - out_of_memory
         example: completed
+      loadAvg:
+        description: LoadAvg as Float64
+        type: number
+      memBwAvg:
+        description: MemBwAvg as Float64
+        type: number
+      memUsedMax:
+        description: MemUsedMax as Float64
+        type: number
       metaData:
         additionalProperties:
           type: string
@@ -810,6 +831,53 @@ paths:
       summary: Remove a job from the sql database
       tags:
       - Job remove
+  /jobs/edit_meta/{id}:
+    post:
+      consumes:
+      - application/json
+      description: |-
+        Edit key value pairs in job metadata json
+        If a key already exists its content will be overwritten
+      parameters:
+      - description: Job Database ID
+        in: path
+        name: id
+        required: true
+        type: integer
+      - description: Kay value pair to add
+        in: body
+        name: request
+        required: true
+        schema:
+          $ref: '#/definitions/api.EditMetaRequest'
+      produces:
+      - application/json
+      responses:
+        "200":
+          description: Updated job resource
+          schema:
+            $ref: '#/definitions/schema.Job'
+        "400":
+          description: Bad Request
+          schema:
+            $ref: '#/definitions/api.ErrorResponse'
+        "401":
+          description: Unauthorized
+          schema:
+            $ref: '#/definitions/api.ErrorResponse'
+        "404":
+          description: Job does not exist
+          schema:
+            $ref: '#/definitions/api.ErrorResponse'
+        "500":
+          description: Internal Server Error
+          schema:
+            $ref: '#/definitions/api.ErrorResponse'
+      security:
+      - ApiKeyAuth: []
+      summary: Edit meta-data json
+      tags:
+      - Job add and modify
   /jobs/start_job/:
     post:
       consumes:

internal/api/docs.go

@@ -1,5 +1,4 @@
-// Code generated by swaggo/swag. DO NOT EDIT.
+// Package api Code generated by swaggo/swag. DO NOT EDIT
-
 package api
 
 import "github.com/swaggo/swag"
@@ -334,6 +333,76 @@ const docTemplate = `{
                 }
             }
         },
+        "/jobs/edit_meta/{id}": {
+            "post": {
+                "security": [
+                    {
+                        "ApiKeyAuth": []
+                    }
+                ],
+                "description": "Edit key value pairs in job metadata json\nIf a key already exists its content will be overwritten",
+                "consumes": [
+                    "application/json"
+                ],
+                "produces": [
+                    "application/json"
+                ],
+                "tags": [
+                    "Job add and modify"
+                ],
+                "summary": "Edit meta-data json",
+                "parameters": [
+                    {
+                        "type": "integer",
+                        "description": "Job Database ID",
+                        "name": "id",
+                        "in": "path",
+                        "required": true
+                    },
+                    {
+                        "description": "Kay value pair to add",
+                        "name": "request",
+                        "in": "body",
+                        "required": true,
+                        "schema": {
+                            "$ref": "#/definitions/api.EditMetaRequest"
+                        }
+                    }
+                ],
+                "responses": {
+                    "200": {
+                        "description": "Updated job resource",
+                        "schema": {
+                            "$ref": "#/definitions/schema.Job"
+                        }
+                    },
+                    "400": {
+                        "description": "Bad Request",
+                        "schema": {
+                            "$ref": "#/definitions/api.ErrorResponse"
+                        }
+                    },
+                    "401": {
+                        "description": "Unauthorized",
+                        "schema": {
+                            "$ref": "#/definitions/api.ErrorResponse"
+                        }
+                    },
+                    "404": {
+                        "description": "Job does not exist",
+                        "schema": {
+                            "$ref": "#/definitions/api.ErrorResponse"
+                        }
+                    },
+                    "500": {
+                        "description": "Internal Server Error",
+                        "schema": {
+                            "$ref": "#/definitions/api.ErrorResponse"
+                        }
+                    }
+                }
+            }
+        },
         "/jobs/start_job/": {
             "post": {
                 "security": [
@@ -1121,6 +1190,19 @@ const docTemplate = `{
                 }
             }
         },
+        "api.EditMetaRequest": {
+            "type": "object",
+            "properties": {
+                "key": {
+                    "type": "string",
+                    "example": "jobScript"
+                },
+                "value": {
+                    "type": "string",
+                    "example": "bash script"
+                }
+            }
+        },
         "api.ErrorResponse": {
             "type": "object",
             "properties": {
@@ -1259,6 +1341,10 @@ const docTemplate = `{
                     "minimum": 0,
                     "example": 1
                 },
+                "flopsAnyAvg": {
+                    "description": "FlopsAnyAvg as Float64",
+                    "type": "number"
+                },
                 "id": {
                     "description": "The unique identifier of a job in the database",
                     "type": "integer"
@@ -1285,6 +1371,18 @@ const docTemplate = `{
                     ],
                     "example": "completed"
                 },
+                "loadAvg": {
+                    "description": "LoadAvg as Float64",
+                    "type": "number"
+                },
+                "memBwAvg": {
+                    "description": "MemBwAvg as Float64",
+                    "type": "number"
+                },
+                "memUsedMax": {
+                    "description": "MemUsedMax as Float64",
+                    "type": "number"
+                },
                 "metaData": {
                     "description": "Additional information about the job",
                     "type": "object",

internal/api/rest.go

@@ -71,6 +71,7 @@ func (api *RestApi) MountRoutes(r *mux.Router) {
 	r.HandleFunc("/jobs/", api.getJobs).Methods(http.MethodGet)
 	r.HandleFunc("/jobs/{id}", api.getJobById).Methods(http.MethodPost)
 	r.HandleFunc("/jobs/tag_job/{id}", api.tagJob).Methods(http.MethodPost, http.MethodPatch)
+	r.HandleFunc("/jobs/edit_meta/{id}", api.editMeta).Methods(http.MethodPost, http.MethodPatch)
 	r.HandleFunc("/jobs/metrics/{id}", api.getJobMetrics).Methods(http.MethodGet)
 	r.HandleFunc("/jobs/delete_job/", api.deleteJobByRequest).Methods(http.MethodDelete)
 	r.HandleFunc("/jobs/delete_job/{id}", api.deleteJobById).Methods(http.MethodDelete)
@@ -146,6 +147,12 @@ type ApiTag struct {
 	Name string `json:"name" example:"Testjob"` // Tag Name
 }
 
+// ApiMeta model
+type EditMetaRequest struct {
+	Key   string `json:"key" example:"jobScript"`
+	Value string `json:"value" example:"bash script"`
+}
+
 type TagJobApiRequest []*ApiTag
 
 type GetJobApiRequest []string
@@ -243,7 +250,6 @@ func securedCheck(r *http.Request) error {
 // @security    ApiKeyAuth
 // @router      /jobs/ [get]
 func (api *RestApi) getJobs(rw http.ResponseWriter, r *http.Request) {
-
 	if user := repository.GetUserFromContext(r.Context()); user != nil &&
 		!user.HasRole(schema.RoleApi) {
 
@@ -464,6 +470,57 @@ func (api *RestApi) getJobById(rw http.ResponseWriter, r *http.Request) {
 	}
 }
 
+// editMeta godoc
+// @summary    Edit meta-data json
+// @tags Job add and modify
+// @description Edit key value pairs in job metadata json
+// @description If a key already exists its content will be overwritten
+// @accept      json
+// @produce     json
+// @param       id      path     int                  true "Job Database ID"
+// @param       request body     api.EditMetaRequest  true "Kay value pair to add"
+// @success     200     {object} schema.Job                "Updated job resource"
+// @failure     400     {object} api.ErrorResponse         "Bad Request"
+// @failure     401     {object} api.ErrorResponse         "Unauthorized"
+// @failure     404     {object} api.ErrorResponse         "Job does not exist"
+// @failure     500     {object} api.ErrorResponse         "Internal Server Error"
+// @security    ApiKeyAuth
+// @router      /jobs/edit_meta/{id} [post]
+func (api *RestApi) editMeta(rw http.ResponseWriter, r *http.Request) {
+	if user := repository.GetUserFromContext(r.Context()); user != nil &&
+		!user.HasRole(schema.RoleApi) {
+		handleError(fmt.Errorf("missing role: %v", schema.GetRoleString(schema.RoleApi)), http.StatusForbidden, rw)
+		return
+	}
+
+	iid, err := strconv.ParseInt(mux.Vars(r)["id"], 10, 64)
+	if err != nil {
+		http.Error(rw, err.Error(), http.StatusBadRequest)
+		return
+	}
+
+	job, err := api.JobRepository.FindById(iid)
+	if err != nil {
+		http.Error(rw, err.Error(), http.StatusNotFound)
+		return
+	}
+
+	var req EditMetaRequest
+	if err := decode(r.Body, &req); err != nil {
+		http.Error(rw, err.Error(), http.StatusBadRequest)
+		return
+	}
+
+	if err := api.JobRepository.UpdateMetadata(job, req.Key, req.Value); err != nil {
+		http.Error(rw, err.Error(), http.StatusInternalServerError)
+		return
+	}
+
+	rw.Header().Add("Content-Type", "application/json")
+	rw.WriteHeader(http.StatusOK)
+	json.NewEncoder(rw).Encode(job)
+}
+
 // tagJob godoc
 // @summary     Adds one or more tags to a job
 // @tags Job add and modify
@@ -873,7 +930,6 @@ func (api *RestApi) deleteJobBefore(rw http.ResponseWriter, r *http.Request) {
 }
 
 func (api *RestApi) checkAndHandleStopJob(rw http.ResponseWriter, job *schema.Job, req StopJobApiRequest) {
-
 	// Sanity checks
 	if job == nil || job.StartTime.Unix() >= req.StopTime || job.State != schema.JobStateRunning {
 		handleError(errors.New("stopTime must be larger than startTime and only running jobs can be stopped"), http.StatusBadRequest, rw)
@@ -1015,7 +1071,8 @@ func (api *RestApi) createUser(rw http.ResponseWriter, r *http.Request) {
 		Password: password,
 		Email:    email,
 		Projects: []string{project},
-		Roles:    []string{role}}); err != nil {
+		Roles:    []string{role},
+	}); err != nil {
 		http.Error(rw, err.Error(), http.StatusUnprocessableEntity)
 		return
 	}

internal/repository/job.go

@@ -16,6 +16,7 @@ import (
 
 	"github.com/ClusterCockpit/cc-backend/internal/graph/model"
 	"github.com/ClusterCockpit/cc-backend/internal/metricdata"
+	"github.com/ClusterCockpit/cc-backend/pkg/archive"
 	"github.com/ClusterCockpit/cc-backend/pkg/log"
 	"github.com/ClusterCockpit/cc-backend/pkg/lrucache"
 	"github.com/ClusterCockpit/cc-backend/pkg/schema"
@@ -212,7 +213,7 @@ func (r *JobRepository) UpdateMetadata(job *schema.Job, key, val string) (err er
 	}
 
 	r.cache.Put(cachekey, job.MetaData, len(job.RawMetaData), 24*time.Hour)
-	return nil
+	return archive.UpdateMetadata(job, job.MetaData)
 }
 
 // Find executes a SQL query to find a specific batch job.

pkg/archive/archive.go

@@ -52,9 +52,11 @@ type JobContainer struct {
 	Data *schema.JobData
 }
 
-var cache *lrucache.Cache = lrucache.New(128 * 1024 * 1024)
+var (
-var ar ArchiveBackend
+	cache      *lrucache.Cache = lrucache.New(128 * 1024 * 1024)
-var useArchive bool
+	ar         ArchiveBackend
+	useArchive bool
+)
 
 func Init(rawConfig json.RawMessage, disableArchive bool) error {
 	useArchive = !disableArchive
@@ -95,8 +97,8 @@ func GetHandle() ArchiveBackend {
 func LoadAveragesFromArchive(
 	job *schema.Job,
 	metrics []string,
-	data [][]schema.Float) error {
+	data [][]schema.Float,
-
+) error {
 	metaFile, err := ar.LoadJobMeta(job)
 	if err != nil {
 		log.Warn("Error while loading job metadata from archiveBackend")
@@ -115,7 +117,6 @@ func LoadAveragesFromArchive(
 }
 
 func GetStatistics(job *schema.Job) (map[string]schema.JobStatistics, error) {
-
 	metaFile, err := ar.LoadJobMeta(job)
 	if err != nil {
 		log.Warn("Error while loading job metadata from archiveBackend")
@@ -125,10 +126,29 @@ func GetStatistics(job *schema.Job) (map[string]schema.JobStatistics, error) {
 	return metaFile.Statistics, nil
 }
 
+// If the job is archived, find its `meta.json` file and override the Metadata
+// in that JSON file. If the job is not archived, nothing is done.
+func UpdateMetadata(job *schema.Job, metadata map[string]string) error {
+	if job.State == schema.JobStateRunning || !useArchive {
+		return nil
+	}
+
+	jobMeta, err := ar.LoadJobMeta(job)
+	if err != nil {
+		log.Warn("Error while loading job metadata from archiveBackend")
+		return err
+	}
+
+	for k, v := range metadata {
+		jobMeta.MetaData[k] = v
+	}
+
+	return ar.StoreJobMeta(jobMeta)
+}
+
 // If the job is archived, find its `meta.json` file and override the tags list
 // in that JSON file. If the job is not archived, nothing is done.
 func UpdateTags(job *schema.Job, tags []*schema.Tag) error {
-
 	if job.State == schema.JobStateRunning || !useArchive {
 		return nil
 	}