Merge branch 'master' of github.com:ClusterCockpit/cc-backend

Makefile

@@ -1,5 +1,6 @@
 TARGET = ./cc-backend
 VAR = ./var
+DB = ./var/job.db
 FRONTEND = ./web/frontend
 VERSION = 0.1
 GIT_HASH := $(shell git rev-parse --short HEAD || echo 'development')
@@ -27,10 +28,9 @@ SVELTE_SRC = $(wildcard $(FRONTEND)/src/*.svelte)         \
 
 .NOTPARALLEL:
 
-$(TARGET): $(VAR) $(SVELTE_TARGETS)
+$(TARGET): $(VAR) $(DB) $(SVELTE_TARGETS)
 	$(info ===>  BUILD cc-backend)
 	@go build -ldflags=${LD_FLAGS} ./cmd/cc-backend
-	./cc-backend --migrate-db
 
 clean:
 	$(info ===>  CLEAN)
@@ -47,10 +47,13 @@ tags:
 	$(info ===>  TAGS)
 	@ctags -R
 
-$(SVELTE_TARGETS): $(SVELTE_SRC)
-	$(info ===>  BUILD frontend)
-	cd web/frontend && yarn build
-
 $(VAR):
 	@mkdir $(VAR)
 	cd web/frontend && yarn install
+
+$(DB):
+	./cc-backend --migrate-db
+
+$(SVELTE_TARGETS): $(SVELTE_SRC)
+	$(info ===>  BUILD frontend)
+	cd web/frontend && yarn build

README.md

@@ -136,7 +136,9 @@ The swagger doc files can be found in `./api/`.
 You can generate the configuration of swagger-ui by running `go run github.com/swaggo/swag/cmd/swag init -d ./internal/api,./pkg/schema  -g rest.go -o ./api `.
 You need to move the generated `./api/doc.go` to `./internal/api/doc.go`.
 If you start cc-backend with flag `--dev` the Swagger UI is available at http://localhost:8080/swagger/ .
-You have to enter a JWT key for a user with role API. This user must not be logged in the same browser (have a running session), otherwise Swagger requests will not work.
+You have to enter a JWT key for a user with role API.
+
+**NOTICE** The user owning the JWT token must not be logged in the same browser (have a running session), otherwise Swagger requests will not work. It is recommended to create a separate user that has just the API role.
 
 ## Project Structure
 

api/swagger.json

@@ -80,12 +80,9 @@
                 ],
                 "responses": {
                     "200": {
-                        "description": "Array of matching jobs",
+                        "description": "Job array and page info",
                         "schema": {
-                            "type": "array",
+                            "$ref": "#/definitions/api.GetJobsApiResponse"
-                            "items": {
-                                "$ref": "#/definitions/schema.Job"
-                            }
                         }
                     },
                     "400": {
@@ -100,6 +97,12 @@
                             "$ref": "#/definitions/api.ErrorResponse"
                         }
                     },
+                    "403": {
+                        "description": "Forbidden",
+                        "schema": {
+                            "$ref": "#/definitions/api.ErrorResponse"
+                        }
+                    },
                     "500": {
                         "description": "Internal Server Error",
                         "schema": {
@@ -681,6 +684,26 @@
                 }
             }
         },
+        "api.GetJobsApiResponse": {
+            "type": "object",
+            "properties": {
+                "items": {
+                    "description": "Number of jobs returned",
+                    "type": "integer"
+                },
+                "jobs": {
+                    "description": "Array of jobs",
+                    "type": "array",
+                    "items": {
+                        "$ref": "#/definitions/schema.JobMeta"
+                    }
+                },
+                "page": {
+                    "description": "Page id returned",
+                    "type": "integer"
+                }
+            }
+        },
         "api.StartJobApiResponse": {
             "type": "object",
             "properties": {
@@ -709,13 +732,6 @@
                 },
                 "jobState": {
                     "description": "Final job state",
-                    "enum": [
-                        "completed",
-                        "failed",
-                        "cancelled",
-                        "stopped",
-                        "timeout"
-                    ],
                     "allOf": [
                         {
                             "$ref": "#/definitions/schema.JobState"
@@ -773,14 +789,6 @@
                 },
                 "jobState": {
                     "description": "Final state of job",
-                    "enum": [
-                        "completed",
-                        "failed",
-                        "cancelled",
-                        "stopped",
-                        "timeout",
-                        "out_of_memory"
-                    ],
                     "allOf": [
                         {
                             "$ref": "#/definitions/schema.JobState"
@@ -909,14 +917,6 @@
                 },
                 "jobState": {
                     "description": "Final state of job",
-                    "enum": [
-                        "completed",
-                        "failed",
-                        "cancelled",
-                        "stopped",
-                        "timeout",
-                        "out_of_memory"
-                    ],
                     "allOf": [
                         {
                             "$ref": "#/definitions/schema.JobState"

api/swagger.yaml

@@ -42,6 +42,20 @@ definitions:
         description: Statustext of Errorcode
         type: string
     type: object
+  api.GetJobsApiResponse:
+    properties:
+      items:
+        description: Number of jobs returned
+        type: integer
+      jobs:
+        description: Array of jobs
+        items:
+          $ref: '#/definitions/schema.JobMeta'
+        type: array
+      page:
+        description: Page id returned
+        type: integer
+    type: object
   api.StartJobApiResponse:
     properties:
       id:
@@ -62,12 +76,6 @@ definitions:
         allOf:
         - $ref: '#/definitions/schema.JobState'
         description: Final job state
-        enum:
-        - completed
-        - failed
-        - cancelled
-        - stopped
-        - timeout
         example: completed
       startTime:
         description: Start Time of job as epoch
@@ -116,13 +124,6 @@ definitions:
         allOf:
         - $ref: '#/definitions/schema.JobState'
         description: Final state of job
-        enum:
-        - completed
-        - failed
-        - cancelled
-        - stopped
-        - timeout
-        - out_of_memory
         example: completed
       metaData:
         additionalProperties:
@@ -225,13 +226,6 @@ definitions:
         allOf:
         - $ref: '#/definitions/schema.JobState'
         description: Final state of job
-        enum:
-        - completed
-        - failed
-        - cancelled
-        - stopped
-        - timeout
-        - out_of_memory
         example: completed
       metaData:
         additionalProperties:
@@ -438,11 +432,9 @@ paths:
       - application/json
       responses:
         "200":
-          description: Array of matching jobs
+          description: Job array and page info
           schema:
-            items:
+            $ref: '#/definitions/api.GetJobsApiResponse'
-              $ref: '#/definitions/schema.Job'
-            type: array
         "400":
           description: Bad Request
           schema:
@@ -451,6 +443,10 @@ paths:
           description: Unauthorized
           schema:
             $ref: '#/definitions/api.ErrorResponse'
+        "403":
+          description: Forbidden
+          schema:
+            $ref: '#/definitions/api.ErrorResponse'
         "500":
           description: Internal Server Error
           schema:

cmd/cc-backend/main.go

@@ -76,7 +76,7 @@ func main() {
 	flag.StringVar(&flagDelUser, "del-user", "", "Remove user by `username`")
 	flag.StringVar(&flagGenJWT, "jwt", "", "Generate and print a JWT for the user specified by its `username`")
 	flag.StringVar(&flagImportJob, "import-job", "", "Import a job. Argument format: `<path-to-meta.json>:<path-to-data.json>,...`")
-	flag.StringVar(&flagLogLevel, "loglevel", "debug", "Sets the logging level: `[debug (default),info,warn,err,fatal,crit]`")
+	flag.StringVar(&flagLogLevel, "loglevel", "warn", "Sets the logging level: `[debug,info,warn (default),err,fatal,crit]`")
 	flag.Parse()
 
 	if flagVersion {
@@ -375,9 +375,9 @@ func main() {
 			MinVersion:               tls.VersionTLS12,
 			PreferServerCipherSuites: true,
 		})
-		log.Printf("HTTPS server listening at %s...", config.Keys.Addr)
+		fmt.Printf("HTTPS server listening at %s...", config.Keys.Addr)
 	} else {
-		log.Printf("HTTP server listening at %s...", config.Keys.Addr)
+		fmt.Printf("HTTP server listening at %s...", config.Keys.Addr)
 	}
 
 	// Because this program will want to bind to a privileged port (like 80), the listener must

internal/api/docs.go

@@ -86,12 +86,9 @@ const docTemplate = `{
                 ],
                 "responses": {
                     "200": {
-                        "description": "Array of matching jobs",
+                        "description": "Job array and page info",
                         "schema": {
-                            "type": "array",
+                            "$ref": "#/definitions/api.GetJobsApiResponse"
-                            "items": {
-                                "$ref": "#/definitions/schema.Job"
-                            }
                         }
                     },
                     "400": {
@@ -106,6 +103,12 @@ const docTemplate = `{
                             "$ref": "#/definitions/api.ErrorResponse"
                         }
                     },
+                    "403": {
+                        "description": "Forbidden",
+                        "schema": {
+                            "$ref": "#/definitions/api.ErrorResponse"
+                        }
+                    },
                     "500": {
                         "description": "Internal Server Error",
                         "schema": {
@@ -687,6 +690,26 @@ const docTemplate = `{
                 }
             }
         },
+        "api.GetJobsApiResponse": {
+            "type": "object",
+            "properties": {
+                "items": {
+                    "description": "Number of jobs returned",
+                    "type": "integer"
+                },
+                "jobs": {
+                    "description": "Array of jobs",
+                    "type": "array",
+                    "items": {
+                        "$ref": "#/definitions/schema.JobMeta"
+                    }
+                },
+                "page": {
+                    "description": "Page id returned",
+                    "type": "integer"
+                }
+            }
+        },
         "api.StartJobApiResponse": {
             "type": "object",
             "properties": {
@@ -715,13 +738,6 @@ const docTemplate = `{
                 },
                 "jobState": {
                     "description": "Final job state",
-                    "enum": [
-                        "completed",
-                        "failed",
-                        "cancelled",
-                        "stopped",
-                        "timeout"
-                    ],
                     "allOf": [
                         {
                             "$ref": "#/definitions/schema.JobState"
@@ -779,14 +795,6 @@ const docTemplate = `{
                 },
                 "jobState": {
                     "description": "Final state of job",
-                    "enum": [
-                        "completed",
-                        "failed",
-                        "cancelled",
-                        "stopped",
-                        "timeout",
-                        "out_of_memory"
-                    ],
                     "allOf": [
                         {
                             "$ref": "#/definitions/schema.JobState"
@@ -915,14 +923,6 @@ const docTemplate = `{
                 },
                 "jobState": {
                     "description": "Final state of job",
-                    "enum": [
-                        "completed",
-                        "failed",
-                        "cancelled",
-                        "stopped",
-                        "timeout",
-                        "out_of_memory"
-                    ],
                     "allOf": [
                         {
                             "$ref": "#/definitions/schema.JobState"

internal/api/rest.go

@@ -104,10 +104,10 @@ type DeleteJobApiResponse struct {
 type StopJobApiRequest struct {
 	// Stop Time of job as epoch
 	StopTime  int64           `json:"stopTime" validate:"required" example:"1649763839"`
-	State     schema.JobState `json:"jobState" validate:"required" example:"completed" enums:"completed,failed,cancelled,stopped,timeout"` // Final job state
+	State     schema.JobState `json:"jobState" validate:"required" example:"completed"` // Final job state
-	JobId     *int64          `json:"jobId" example:"123000"`                                                                              // Cluster Job ID of job
+	JobId     *int64          `json:"jobId" example:"123000"`                           // Cluster Job ID of job
-	Cluster   *string         `json:"cluster" example:"fritz"`                                                                             // Cluster of job
+	Cluster   *string         `json:"cluster" example:"fritz"`                          // Cluster of job
-	StartTime *int64          `json:"startTime" example:"1649723812"`                                                                      // Start Time of job as epoch
+	StartTime *int64          `json:"startTime" example:"1649723812"`                   // Start Time of job as epoch
 }
 
 // DeleteJobApiRequest model
@@ -171,6 +171,7 @@ func decode(r io.Reader, val interface{}) error {
 // @success     200            {object} api.GetJobsApiResponse  "Job array and page info"
 // @failure     400            {object} api.ErrorResponse       "Bad Request"
 // @failure     401   		   {object} api.ErrorResponse       "Unauthorized"
+// @failure     403            {object} api.ErrorResponse       "Forbidden"
 // @failure     500            {object} api.ErrorResponse       "Internal Server Error"
 // @security    ApiKeyAuth
 // @router      /jobs/ [get]
@@ -738,29 +739,6 @@ func (api *RestApi) checkAndHandleStopJob(rw http.ResponseWriter, job *schema.Jo
 	api.JobRepository.TriggerArchiving(job)
 }
 
-// func (api *RestApi) importJob(rw http.ResponseWriter, r *http.Request) {
-// 	if user := auth.GetUser(r.Context()); user != nil && !user.HasRole(auth.RoleApi) {
-// 		handleError(fmt.Errorf("missing role: %v", auth.RoleApi), http.StatusForbidden, rw)
-// 		return
-// 	}
-
-// 	var body struct {
-// 		Meta *schema.JobMeta `json:"meta"`
-// 		Data *schema.JobData `json:"data"`
-// 	}
-// 	if err := decode(r.Body, &body); err != nil {
-// 		handleError(fmt.Errorf("import failed: %s", err.Error()), http.StatusBadRequest, rw)
-// 		return
-// 	}
-
-// 	if err := api.JobRepository.ImportJob(body.Meta, body.Data); err != nil {
-// 		handleError(fmt.Errorf("import failed: %s", err.Error()), http.StatusUnprocessableEntity, rw)
-// 		return
-// 	}
-
-// 	rw.Write([]byte(`{ "status": "OK" }`))
-// }
-
 func (api *RestApi) getJobMetrics(rw http.ResponseWriter, r *http.Request) {
 	id := mux.Vars(r)["id"]
 	metrics := r.URL.Query()["metric"]

internal/repository/migration.go

@@ -102,7 +102,11 @@ func MigrateDB(backend string, db string) {
 	}
 
 	if err := m.Up(); err != nil {
-		log.Fatal(err)
+		if err == migrate.ErrNoChange {
+			log.Info("DB already up to date!")
+		} else {
+			log.Fatal(err)
+		}
 	}
 
 	m.Close()

pkg/schema/job.go

@@ -17,26 +17,26 @@ import (
 type BaseJob struct {
 	// The unique identifier of a job
 	JobID            int64             `json:"jobId" db:"job_id" example:"123000"`
-	User             string            `json:"user" db:"user" example:"abcd100h"`                                                                            // The unique identifier of a user
+	User             string            `json:"user" db:"user" example:"abcd100h"`                                           // The unique identifier of a user
-	Project          string            `json:"project" db:"project" example:"abcd200"`                                                                       // The unique identifier of a project
+	Project          string            `json:"project" db:"project" example:"abcd200"`                                      // The unique identifier of a project
-	Cluster          string            `json:"cluster" db:"cluster" example:"fritz"`                                                                         // The unique identifier of a cluster
+	Cluster          string            `json:"cluster" db:"cluster" example:"fritz"`                                        // The unique identifier of a cluster
-	SubCluster       string            `json:"subCluster" db:"subcluster" example:"main"`                                                                    // The unique identifier of a sub cluster
+	SubCluster       string            `json:"subCluster" db:"subcluster" example:"main"`                                   // The unique identifier of a sub cluster
-	Partition        string            `json:"partition" db:"partition" example:"main"`                                                                      // The Slurm partition to which the job was submitted
+	Partition        string            `json:"partition" db:"partition" example:"main"`                                     // The Slurm partition to which the job was submitted
-	ArrayJobId       int64             `json:"arrayJobId" db:"array_job_id" example:"123000"`                                                                // The unique identifier of an array job
+	ArrayJobId       int64             `json:"arrayJobId" db:"array_job_id" example:"123000"`                               // The unique identifier of an array job
-	NumNodes         int32             `json:"numNodes" db:"num_nodes" example:"2" minimum:"1"`                                                              // Number of nodes used (Min > 0)
+	NumNodes         int32             `json:"numNodes" db:"num_nodes" example:"2" minimum:"1"`                             // Number of nodes used (Min > 0)
-	NumHWThreads     int32             `json:"numHwthreads" db:"num_hwthreads" example:"20" minimum:"1"`                                                     // Number of HWThreads used (Min > 0)
+	NumHWThreads     int32             `json:"numHwthreads" db:"num_hwthreads" example:"20" minimum:"1"`                    // Number of HWThreads used (Min > 0)
-	NumAcc           int32             `json:"numAcc" db:"num_acc" example:"2" minimum:"1"`                                                                  // Number of accelerators used (Min > 0)
+	NumAcc           int32             `json:"numAcc" db:"num_acc" example:"2" minimum:"1"`                                 // Number of accelerators used (Min > 0)
-	Exclusive        int32             `json:"exclusive" db:"exclusive" example:"1" minimum:"0" maximum:"2"`                                                 // Specifies how nodes are shared: 0 - Shared among multiple jobs of multiple users, 1 - Job exclusive (Default), 2 - Shared among multiple jobs of same user
+	Exclusive        int32             `json:"exclusive" db:"exclusive" example:"1" minimum:"0" maximum:"2"`                // Specifies how nodes are shared: 0 - Shared among multiple jobs of multiple users, 1 - Job exclusive (Default), 2 - Shared among multiple jobs of same user
-	MonitoringStatus int32             `json:"monitoringStatus" db:"monitoring_status" example:"1" minimum:"0" maximum:"3"`                                  // State of monitoring system during job run: 0 - Disabled, 1 - Running or Archiving (Default), 2 - Archiving Failed, 3 - Archiving Successfull
+	MonitoringStatus int32             `json:"monitoringStatus" db:"monitoring_status" example:"1" minimum:"0" maximum:"3"` // State of monitoring system during job run: 0 - Disabled, 1 - Running or Archiving (Default), 2 - Archiving Failed, 3 - Archiving Successfull
-	SMT              int32             `json:"smt" db:"smt" example:"4"`                                                                                     // SMT threads used by job
+	SMT              int32             `json:"smt" db:"smt" example:"4"`                                                    // SMT threads used by job
-	State            JobState          `json:"jobState" db:"job_state" example:"completed" enums:"completed,failed,cancelled,stopped,timeout,out_of_memory"` // Final state of job
+	State            JobState          `json:"jobState" db:"job_state" example:"completed"`                                 // Final state of job
-	Duration         int32             `json:"duration" db:"duration" example:"43200" minimum:"1"`                                                           // Duration of job in seconds (Min > 0)
+	Duration         int32             `json:"duration" db:"duration" example:"43200" minimum:"1"`                          // Duration of job in seconds (Min > 0)
-	Walltime         int64             `json:"walltime" db:"walltime" example:"86400" minimum:"1"`                                                           // Requested walltime of job in seconds (Min > 0)
+	Walltime         int64             `json:"walltime" db:"walltime" example:"86400" minimum:"1"`                          // Requested walltime of job in seconds (Min > 0)
-	Tags             []*Tag            `json:"tags"`                                                                                                         // List of tags
+	Tags             []*Tag            `json:"tags"`                                                                        // List of tags
-	RawResources     []byte            `json:"-" db:"resources"`                                                                                             // Resources used by job [As Bytes]
+	RawResources     []byte            `json:"-" db:"resources"`                                                            // Resources used by job [As Bytes]
-	Resources        []*Resource       `json:"resources"`                                                                                                    // Resources used by job
+	Resources        []*Resource       `json:"resources"`                                                                   // Resources used by job
-	RawMetaData      []byte            `json:"-" db:"meta_data"`                                                                                             // Additional information about the job [As Bytes]
+	RawMetaData      []byte            `json:"-" db:"meta_data"`                                                            // Additional information about the job [As Bytes]
-	MetaData         map[string]string `json:"metaData"`                                                                                                     // Additional information about the job
+	MetaData         map[string]string `json:"metaData"`                                                                    // Additional information about the job
 }
 
 // Non-Swaggered Comment: Job

startDemo.sh

@@ -4,21 +4,16 @@ if [ -d './var' ]; then
     echo 'Directory ./var already exists! Skipping initialization.'
     ./cc-backend --server --dev
 else
-    mkdir ./var
+    make
-    cd ./var
 
+    cd var
     wget https://hpc-mover.rrze.uni-erlangen.de/HPC-Data/0x7b58aefb/eig7ahyo6fo2bais0ephuf2aitohv1ai/job-archive-dev.tar.xz
     tar xJf job-archive-dev.tar.xz
     rm ./job-archive-dev.tar.xz
+    cd ../
 
-    cd ../web/frontend
-    yarn install
-    yarn build
-
-    cd ../..
     cp ./configs/env-template.txt .env
     cp ./docs/config.json config.json
-    go build ./cmd/cc-backend
 
     ./cc-backend --migrate-db
     ./cc-backend --server --dev --init-db --add-user demo:admin:AdminDev