Merge branch 'master' into 40_45_82_update_roles

Makefile

@@ -1,5 +1,6 @@
 TARGET = ./cc-backend
 VAR = ./var
+DB = ./var/job.db
 FRONTEND = ./web/frontend
 VERSION = 0.1
 GIT_HASH := $(shell git rev-parse --short HEAD || echo 'development')
@@ -27,10 +28,9 @@ SVELTE_SRC = $(wildcard $(FRONTEND)/src/*.svelte)         \
 
 .NOTPARALLEL:
 
-$(TARGET): $(VAR) $(SVELTE_TARGETS)
+$(TARGET): $(VAR) $(DB) $(SVELTE_TARGETS)
 	$(info ===>  BUILD cc-backend)
 	@go build -ldflags=${LD_FLAGS} ./cmd/cc-backend
-	./cc-backend --migrate-db
 
 clean:
 	$(info ===>  CLEAN)
@@ -43,10 +43,13 @@ test:
 	@go vet ./...
 	@go test ./...
 
-$(SVELTE_TARGETS): $(SVELTE_SRC)
-	$(info ===>  BUILD frontend)
-	cd web/frontend && yarn build
-
 $(VAR):
 	@mkdir $(VAR)
 	cd web/frontend && yarn install
+
+$(DB):
+	./cc-backend --migrate-db
+
+$(SVELTE_TARGETS): $(SVELTE_SRC)
+	$(info ===>  BUILD frontend)
+	cd web/frontend && yarn build

README.md

@@ -136,7 +136,9 @@ The swagger doc files can be found in `./api/`.
 You can generate the configuration of swagger-ui by running `go run github.com/swaggo/swag/cmd/swag init -d ./internal/api,./pkg/schema  -g rest.go -o ./api `.
 You need to move the generated `./api/doc.go` to `./internal/api/doc.go`.
 If you start cc-backend with flag `--dev` the Swagger UI is available at http://localhost:8080/swagger/ .
-You have to enter a JWT key for a user with role API. This user must not be logged in the same browser (have a running session), otherwise Swagger requests will not work.
+You have to enter a JWT key for a user with role API.
+
+**NOTICE** The user owning the JWT token must not be logged in the same browser (have a running session), otherwise Swagger requests will not work. It is recommended to create a separate user that has just the API role.
 
 ## Project Structure
 

api/swagger.json

@@ -80,12 +80,9 @@
                 ],
                 "responses": {
                     "200": {
-                        "description": "Array of matching jobs",
+                        "description": "Job array and page info",
                         "schema": {
-                            "type": "array",
+                            "$ref": "#/definitions/api.GetJobsApiResponse"
-                            "items": {
-                                "$ref": "#/definitions/schema.Job"
-                            }
                         }
                     },
                     "400": {
@@ -681,6 +678,26 @@
                 }
             }
         },
+        "api.GetJobsApiResponse": {
+            "type": "object",
+            "properties": {
+                "items": {
+                    "description": "Number of jobs returned",
+                    "type": "integer"
+                },
+                "jobs": {
+                    "description": "Array of jobs",
+                    "type": "array",
+                    "items": {
+                        "$ref": "#/definitions/schema.JobMeta"
+                    }
+                },
+                "page": {
+                    "description": "Page id returned",
+                    "type": "integer"
+                }
+            }
+        },
         "api.StartJobApiResponse": {
             "type": "object",
             "properties": {

api/swagger.yaml

@@ -42,6 +42,20 @@ definitions:
         description: Statustext of Errorcode
         type: string
     type: object
+  api.GetJobsApiResponse:
+    properties:
+      items:
+        description: Number of jobs returned
+        type: integer
+      jobs:
+        description: Array of jobs
+        items:
+          $ref: '#/definitions/schema.JobMeta'
+        type: array
+      page:
+        description: Page id returned
+        type: integer
+    type: object
   api.StartJobApiResponse:
     properties:
       id:
@@ -438,11 +452,9 @@ paths:
       - application/json
       responses:
         "200":
-          description: Array of matching jobs
+          description: Job array and page info
           schema:
-            items:
+            $ref: '#/definitions/api.GetJobsApiResponse'
-              $ref: '#/definitions/schema.Job'
-            type: array
         "400":
           description: Bad Request
           schema:

cmd/cc-backend/main.go

@@ -76,7 +76,7 @@ func main() {
 	flag.StringVar(&flagDelUser, "del-user", "", "Remove user by `username`")
 	flag.StringVar(&flagGenJWT, "jwt", "", "Generate and print a JWT for the user specified by its `username`")
 	flag.StringVar(&flagImportJob, "import-job", "", "Import a job. Argument format: `<path-to-meta.json>:<path-to-data.json>,...`")
-	flag.StringVar(&flagLogLevel, "loglevel", "debug", "Sets the logging level: `[debug (default),info,warn,err,fatal,crit]`")
+	flag.StringVar(&flagLogLevel, "loglevel", "warn", "Sets the logging level: `[debug,info,warn (default),err,fatal,crit]`")
 	flag.Parse()
 
 	if flagVersion {
@@ -375,9 +375,9 @@ func main() {
 			MinVersion:               tls.VersionTLS12,
 			PreferServerCipherSuites: true,
 		})
-		log.Printf("HTTPS server listening at %s...", config.Keys.Addr)
+		fmt.Printf("HTTPS server listening at %s...", config.Keys.Addr)
 	} else {
-		log.Printf("HTTP server listening at %s...", config.Keys.Addr)
+		fmt.Printf("HTTP server listening at %s...", config.Keys.Addr)
 	}
 
 	// Because this program will want to bind to a privileged port (like 80), the listener must

configs/README.md

@@ -1,6 +1,6 @@
 ## Intro
 
-cc-backend requires a configuration file speciyfing the cluster systems to be used. Still many  default
+cc-backend requires a configuration file specifying the cluster systems to be used. Still many  default
 options documented below are used. cc-backend tries to load a config.json from the working directory per default.
 To overwrite the default specify a json config file location using the command line option `--config <filepath>`.
 All security relevant configuration. e.g., keys and passwords, are set using environment variables.
@@ -44,9 +44,10 @@ It is supported to specify these by means of an `.env` file located in the proje
                 "startTime": { "from": "2022-01-01T00:00:00Z", "to": null }
             }
    ```
-* `ui-defaults`: Type object. Default configuration for ui views. If overwriten, all options  must be provided! Most options can be overwritten by the user via the web interface.
+* `ui-defaults`: Type object. Default configuration for ui views. If overwritten, all options  must be provided! Most options can be overwritten by the user via the web interface.
    - `analysis_view_histogramMetrics`: Type string array. Metrics to show as job count histograms in analysis view. Default `["flops_any", "mem_bw", "mem_used"]`.
-   - `analysis_view_scatterPlotMetrics`: Type array of string array. Initial scatter plto configuration in analysis view. Default `[["flops_any", "mem_bw"], ["flops_any", "cpu_load"], ["cpu_load", "mem_bw"]]`.
+   - `analysis_view_scatterPlotMetrics`: Type array of string array. Initial
+   scatter plot configuration in analysis view. Default `[["flops_any", "mem_bw"], ["flops_any", "cpu_load"], ["cpu_load", "mem_bw"]]`.
    - `job_view_nodestats_selectedMetrics`: Type string array. Initial metrics shown in node statistics table of single job view. Default `["flops_any", "mem_bw", "mem_used"]`.
    - `job_view_polarPlotMetrics`: Type string array. Metrics shown in polar plot of single job view. Default `["flops_any", "mem_bw", "mem_used", "net_bw", "file_bw"]`.
    - `job_view_selectedMetrics`: Type string array.  Default `["flops_any", "mem_bw", "mem_used"]`.

internal/api/docs.go

@@ -86,12 +86,9 @@ const docTemplate = `{
                 ],
                 "responses": {
                     "200": {
-                        "description": "Array of matching jobs",
+                        "description": "Job array and page info",
                         "schema": {
-                            "type": "array",
+                            "$ref": "#/definitions/api.GetJobsApiResponse"
-                            "items": {
-                                "$ref": "#/definitions/schema.Job"
-                            }
                         }
                     },
                     "400": {
@@ -687,6 +684,26 @@ const docTemplate = `{
                 }
             }
         },
+        "api.GetJobsApiResponse": {
+            "type": "object",
+            "properties": {
+                "items": {
+                    "description": "Number of jobs returned",
+                    "type": "integer"
+                },
+                "jobs": {
+                    "description": "Array of jobs",
+                    "type": "array",
+                    "items": {
+                        "$ref": "#/definitions/schema.JobMeta"
+                    }
+                },
+                "page": {
+                    "description": "Page id returned",
+                    "type": "integer"
+                }
+            }
+        },
         "api.StartJobApiResponse": {
             "type": "object",
             "properties": {

internal/api/rest.go

@@ -118,6 +118,13 @@ type DeleteJobApiRequest struct {
 	StartTime *int64  `json:"startTime" example:"1649723812"`             // Start Time of job as epoch
 }
 
+// GetJobsApiResponse model
+type GetJobsApiResponse struct {
+	Jobs  []*schema.JobMeta `json:"jobs"`  // Array of jobs
+	Items int               `json:"items"` // Number of jobs returned
+	Page  int               `json:"page"`  // Page id returned
+}
+
 // ErrorResponse model
 type ErrorResponse struct {
 	// Statustext of Errorcode
@@ -162,9 +169,10 @@ func decode(r io.Reader, val interface{}) error {
 // @param       items-per-page query    int               false "Items per page (Default: 25)"
 // @param       page           query    int               false "Page Number (Default: 1)"
 // @param       with-metadata  query    bool              false "Include metadata (e.g. jobScript) in response"
-// @success     200            {array}  schema.Job              "Array of matching jobs"
+// @success     200            {object} api.GetJobsApiResponse  "Job array and page info"
 // @failure     400            {object} api.ErrorResponse       "Bad Request"
 // @failure     401   		   {object} api.ErrorResponse       "Unauthorized"
+// @failure     403            {object} api.ErrorResponse       "Forbidden"
 // @failure     500            {object} api.ErrorResponse       "Internal Server Error"
 // @security    ApiKeyAuth
 // @router      /jobs/ [get]
@@ -185,7 +193,8 @@ func (api *RestApi) getJobs(rw http.ResponseWriter, r *http.Request) {
 			for _, s := range vals {
 				state := schema.JobState(s)
 				if !state.Valid() {
-					http.Error(rw, "invalid query parameter value: state", http.StatusBadRequest)
+					handleError(fmt.Errorf("invalid query parameter value: state"),
+						http.StatusBadRequest, rw)
 					return
 				}
 				filter.State = append(filter.State, state)
@@ -195,17 +204,18 @@ func (api *RestApi) getJobs(rw http.ResponseWriter, r *http.Request) {
 		case "start-time":
 			st := strings.Split(vals[0], "-")
 			if len(st) != 2 {
-				http.Error(rw, "invalid query parameter value: startTime", http.StatusBadRequest)
+				handleError(fmt.Errorf("invalid query parameter value: startTime"),
+					http.StatusBadRequest, rw)
 				return
 			}
 			from, err := strconv.ParseInt(st[0], 10, 64)
 			if err != nil {
-				http.Error(rw, err.Error(), http.StatusBadRequest)
+				handleError(err, http.StatusBadRequest, rw)
 				return
 			}
 			to, err := strconv.ParseInt(st[1], 10, 64)
 			if err != nil {
-				http.Error(rw, err.Error(), http.StatusBadRequest)
+				handleError(err, http.StatusBadRequest, rw)
 				return
 			}
 			ufrom, uto := time.Unix(from, 0), time.Unix(to, 0)
@@ -213,28 +223,29 @@ func (api *RestApi) getJobs(rw http.ResponseWriter, r *http.Request) {
 		case "page":
 			x, err := strconv.Atoi(vals[0])
 			if err != nil {
-				http.Error(rw, err.Error(), http.StatusBadRequest)
+				handleError(err, http.StatusBadRequest, rw)
 				return
 			}
 			page.Page = x
 		case "items-per-page":
 			x, err := strconv.Atoi(vals[0])
 			if err != nil {
-				http.Error(rw, err.Error(), http.StatusBadRequest)
+				handleError(err, http.StatusBadRequest, rw)
 				return
 			}
 			page.ItemsPerPage = x
 		case "with-metadata":
 			withMetadata = true
 		default:
-			http.Error(rw, "invalid query parameter: "+key, http.StatusBadRequest)
+			handleError(fmt.Errorf("invalid query parameter: %s", key),
+				http.StatusBadRequest, rw)
 			return
 		}
 	}
 
 	jobs, err := api.JobRepository.QueryJobs(r.Context(), []*model.JobFilter{filter}, page, order)
 	if err != nil {
-		http.Error(rw, err.Error(), http.StatusInternalServerError)
+		handleError(err, http.StatusInternalServerError, rw)
 		return
 	}
 
@@ -242,7 +253,7 @@ func (api *RestApi) getJobs(rw http.ResponseWriter, r *http.Request) {
 	for _, job := range jobs {
 		if withMetadata {
 			if _, err := api.JobRepository.FetchMetadata(job); err != nil {
-				http.Error(rw, err.Error(), http.StatusInternalServerError)
+				handleError(err, http.StatusInternalServerError, rw)
 				return
 			}
 		}
@@ -255,7 +266,7 @@ func (api *RestApi) getJobs(rw http.ResponseWriter, r *http.Request) {
 
 		res.Tags, err = api.JobRepository.GetTags(&job.ID)
 		if err != nil {
-			http.Error(rw, err.Error(), http.StatusInternalServerError)
+			handleError(err, http.StatusInternalServerError, rw)
 			return
 		}
 
@@ -263,7 +274,7 @@ func (api *RestApi) getJobs(rw http.ResponseWriter, r *http.Request) {
 			res.Statistics, err = archive.GetStatistics(job)
 			if err != nil {
 				if err != nil {
-					http.Error(rw, err.Error(), http.StatusInternalServerError)
+					handleError(err, http.StatusInternalServerError, rw)
 					return
 				}
 			}
@@ -273,12 +284,18 @@ func (api *RestApi) getJobs(rw http.ResponseWriter, r *http.Request) {
 	}
 
 	log.Debugf("/api/jobs: %d jobs returned", len(results))
+	rw.Header().Add("Content-Type", "application/json")
 	bw := bufio.NewWriter(rw)
 	defer bw.Flush()
-	if err := json.NewEncoder(bw).Encode(map[string]interface{}{
+
-		"jobs": results,
+	payload := GetJobsApiResponse{
-	}); err != nil {
+		Jobs:  results,
-		http.Error(rw, err.Error(), http.StatusInternalServerError)
+		Items: page.ItemsPerPage,
+		Page:  page.Page,
+	}
+
+	if err := json.NewEncoder(bw).Encode(payload); err != nil {
+		handleError(err, http.StatusInternalServerError, rw)
 		return
 	}
 }
@@ -723,29 +740,6 @@ func (api *RestApi) checkAndHandleStopJob(rw http.ResponseWriter, job *schema.Jo
 	api.JobRepository.TriggerArchiving(job)
 }
 
-// func (api *RestApi) importJob(rw http.ResponseWriter, r *http.Request) {
-// 	if user := auth.GetUser(r.Context()); user != nil && !user.HasRole(auth.RoleApi) {
-// 		handleError(fmt.Errorf("missing role: %v", auth.RoleApi), http.StatusForbidden, rw)
-// 		return
-// 	}
-
-// 	var body struct {
-// 		Meta *schema.JobMeta `json:"meta"`
-// 		Data *schema.JobData `json:"data"`
-// 	}
-// 	if err := decode(r.Body, &body); err != nil {
-// 		handleError(fmt.Errorf("import failed: %s", err.Error()), http.StatusBadRequest, rw)
-// 		return
-// 	}
-
-// 	if err := api.JobRepository.ImportJob(body.Meta, body.Data); err != nil {
-// 		handleError(fmt.Errorf("import failed: %s", err.Error()), http.StatusUnprocessableEntity, rw)
-// 		return
-// 	}
-
-// 	rw.Write([]byte(`{ "status": "OK" }`))
-// }
-
 func (api *RestApi) getJobMetrics(rw http.ResponseWriter, r *http.Request) {
 	id := mux.Vars(r)["id"]
 	metrics := r.URL.Query()["metric"]
@@ -794,7 +788,8 @@ func (api *RestApi) getJWT(rw http.ResponseWriter, r *http.Request) {
 	me := auth.GetUser(r.Context())
 	if !me.HasRole(auth.RoleAdmin) {
 		if username != me.Username {
-			http.Error(rw, "Only admins are allowed to sign JWTs not for themselves", http.StatusForbidden)
+			http.Error(rw, "Only admins are allowed to sign JWTs not for themselves",
+				http.StatusForbidden)
 			return
 		}
 	}

internal/repository/migration.go

@@ -102,7 +102,11 @@ func MigrateDB(backend string, db string) {
 	}
 
 	if err := m.Up(); err != nil {
-		log.Fatal(err)
+		if err == migrate.ErrNoChange {
+			log.Info("DB already up to date!")
+		} else {
+			log.Fatal(err)
+		}
 	}
 
 	m.Close()

startDemo.sh

@@ -4,21 +4,16 @@ if [ -d './var' ]; then
     echo 'Directory ./var already exists! Skipping initialization.'
     ./cc-backend --server --dev
 else
-    mkdir ./var
+    make
-    cd ./var
 
+    cd var
     wget https://hpc-mover.rrze.uni-erlangen.de/HPC-Data/0x7b58aefb/eig7ahyo6fo2bais0ephuf2aitohv1ai/job-archive-dev.tar.xz
     tar xJf job-archive-dev.tar.xz
     rm ./job-archive-dev.tar.xz
+    cd ../
 
-    cd ../web/frontend
-    yarn install
-    yarn build
-
-    cd ../..
     cp ./configs/env-template.txt .env
     cp ./docs/config.json config.json
-    go build ./cmd/cc-backend
 
     ./cc-backend --migrate-db
     ./cc-backend --server --dev --init-db --add-user demo:admin:AdminDev

test/integration_test.go

@@ -416,7 +416,7 @@ func TestRestApi(t *testing.T) {
 	}
 
 	const stopJobBody string = `{
-"jobId":     123,
+        "jobId":     123,
 		"startTime": 123456789,
 		"cluster":   "testcluster",