From 1e3e2ab222f87b227528524dd7eb7a95a8f74c6e Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Sat, 25 Feb 2023 07:59:37 +0100 Subject: [PATCH 01/10] Change default loglevel warn --- cmd/cc-backend/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/cc-backend/main.go b/cmd/cc-backend/main.go index ab684f5..6af39b2 100644 --- a/cmd/cc-backend/main.go +++ b/cmd/cc-backend/main.go @@ -76,7 +76,7 @@ func main() { flag.StringVar(&flagDelUser, "del-user", "", "Remove user by `username`") flag.StringVar(&flagGenJWT, "jwt", "", "Generate and print a JWT for the user specified by its `username`") flag.StringVar(&flagImportJob, "import-job", "", "Import a job. Argument format: `:,...`") - flag.StringVar(&flagLogLevel, "loglevel", "debug", "Sets the logging level: `[debug (default),info,warn,err,fatal,crit]`") + flag.StringVar(&flagLogLevel, "loglevel", "warn", "Sets the logging level: `[debug,info,warn (default),err,fatal,crit]`") flag.Parse() if flagVersion { From 252ec3909afb83534df03ae474fc92cbdd9e6c2c Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Sat, 25 Feb 2023 08:17:58 +0100 Subject: [PATCH 02/10] Add correct dependency on job.db in Makefile --- Makefile | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index 40af0e1..ff32c6c 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,6 @@ TARGET = ./cc-backend VAR = ./var +DB = ./var/job.db FRONTEND = ./web/frontend VERSION = 0.1 GIT_HASH := $(shell git rev-parse --short HEAD || echo 'development') @@ -27,10 +28,9 @@ SVELTE_SRC = $(wildcard $(FRONTEND)/src/*.svelte) \ .NOTPARALLEL: -$(TARGET): $(VAR) $(SVELTE_TARGETS) +$(TARGET): $(VAR) $(DB) $(SVELTE_TARGETS) $(info ===> BUILD cc-backend) @go build -ldflags=${LD_FLAGS} ./cmd/cc-backend - ./cc-backend --migrate-db clean: $(info ===> CLEAN) @@ -43,10 +43,13 @@ test: @go vet ./... @go test ./... -$(SVELTE_TARGETS): $(SVELTE_SRC) - $(info ===> BUILD frontend) - cd web/frontend && yarn build - $(VAR): @mkdir $(VAR) cd web/frontend && yarn install + +$(DB): + ./cc-backend --migrate-db + +$(SVELTE_TARGETS): $(SVELTE_SRC) + $(info ===> BUILD frontend) + cd web/frontend && yarn build From 89b1d8f7dafb684feffa58f003ab6f653b501198 Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Sat, 25 Feb 2023 08:26:56 +0100 Subject: [PATCH 03/10] Treat error no change as non fatal --- internal/repository/migration.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/internal/repository/migration.go b/internal/repository/migration.go index 258593d..934a002 100644 --- a/internal/repository/migration.go +++ b/internal/repository/migration.go @@ -102,7 +102,11 @@ func MigrateDB(backend string, db string) { } if err := m.Up(); err != nil { - log.Fatal(err) + if err == migrate.ErrNoChange { + log.Info("DB already up to date!") + } else { + log.Fatal(err) + } } m.Close() From 8139673a7fdac056a0b39595f668136bbf87eec3 Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Sat, 25 Feb 2023 08:28:43 +0100 Subject: [PATCH 04/10] Always print Listen to URL --- cmd/cc-backend/main.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/cc-backend/main.go b/cmd/cc-backend/main.go index 6af39b2..1a8aa4c 100644 --- a/cmd/cc-backend/main.go +++ b/cmd/cc-backend/main.go @@ -375,9 +375,9 @@ func main() { MinVersion: tls.VersionTLS12, PreferServerCipherSuites: true, }) - log.Printf("HTTPS server listening at %s...", config.Keys.Addr) + fmt.Printf("HTTPS server listening at %s...", config.Keys.Addr) } else { - log.Printf("HTTP server listening at %s...", config.Keys.Addr) + fmt.Printf("HTTP server listening at %s...", config.Keys.Addr) } // Because this program will want to bind to a privileged port (like 80), the listener must From 2de4d03c8d1eaeea8140f2ff1c1c5eca942067dd Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Sat, 25 Feb 2023 08:29:08 +0100 Subject: [PATCH 05/10] Make startDemo.sh use Makefile --- startDemo.sh | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/startDemo.sh b/startDemo.sh index d210772..8667df0 100755 --- a/startDemo.sh +++ b/startDemo.sh @@ -4,21 +4,16 @@ if [ -d './var' ]; then echo 'Directory ./var already exists! Skipping initialization.' ./cc-backend --server --dev else - mkdir ./var - cd ./var + make + cd var wget https://hpc-mover.rrze.uni-erlangen.de/HPC-Data/0x7b58aefb/eig7ahyo6fo2bais0ephuf2aitohv1ai/job-archive-dev.tar.xz tar xJf job-archive-dev.tar.xz rm ./job-archive-dev.tar.xz + cd ../ - cd ../web/frontend - yarn install - yarn build - - cd ../.. cp ./configs/env-template.txt .env cp ./docs/config.json config.json - go build ./cmd/cc-backend ./cc-backend --migrate-db ./cc-backend --server --dev --init-db --add-user demo:admin:AdminDev From e13f184414475052de5160d5ad9e04f3cd185709 Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Mon, 27 Feb 2023 07:49:37 +0100 Subject: [PATCH 06/10] Regenerate Swagger docs Relates to #94 and #95 --- api/swagger.json | 27 ++++++++++++++++++++++----- api/swagger.yaml | 20 ++++++++++++++++---- internal/api/docs.go | 27 ++++++++++++++++++++++----- internal/api/rest.go | 23 ----------------------- 4 files changed, 60 insertions(+), 37 deletions(-) diff --git a/api/swagger.json b/api/swagger.json index 6436484..8de2d8a 100644 --- a/api/swagger.json +++ b/api/swagger.json @@ -80,12 +80,9 @@ ], "responses": { "200": { - "description": "Array of matching jobs", + "description": "Job array and page info", "schema": { - "type": "array", - "items": { - "$ref": "#/definitions/schema.Job" - } + "$ref": "#/definitions/api.GetJobsApiResponse" } }, "400": { @@ -681,6 +678,26 @@ } } }, + "api.GetJobsApiResponse": { + "type": "object", + "properties": { + "items": { + "description": "Number of jobs returned", + "type": "integer" + }, + "jobs": { + "description": "Array of jobs", + "type": "array", + "items": { + "$ref": "#/definitions/schema.JobMeta" + } + }, + "page": { + "description": "Page id returned", + "type": "integer" + } + } + }, "api.StartJobApiResponse": { "type": "object", "properties": { diff --git a/api/swagger.yaml b/api/swagger.yaml index ca04575..65806ed 100644 --- a/api/swagger.yaml +++ b/api/swagger.yaml @@ -42,6 +42,20 @@ definitions: description: Statustext of Errorcode type: string type: object + api.GetJobsApiResponse: + properties: + items: + description: Number of jobs returned + type: integer + jobs: + description: Array of jobs + items: + $ref: '#/definitions/schema.JobMeta' + type: array + page: + description: Page id returned + type: integer + type: object api.StartJobApiResponse: properties: id: @@ -438,11 +452,9 @@ paths: - application/json responses: "200": - description: Array of matching jobs + description: Job array and page info schema: - items: - $ref: '#/definitions/schema.Job' - type: array + $ref: '#/definitions/api.GetJobsApiResponse' "400": description: Bad Request schema: diff --git a/internal/api/docs.go b/internal/api/docs.go index 4ac70a6..62d05e2 100644 --- a/internal/api/docs.go +++ b/internal/api/docs.go @@ -86,12 +86,9 @@ const docTemplate = `{ ], "responses": { "200": { - "description": "Array of matching jobs", + "description": "Job array and page info", "schema": { - "type": "array", - "items": { - "$ref": "#/definitions/schema.Job" - } + "$ref": "#/definitions/api.GetJobsApiResponse" } }, "400": { @@ -687,6 +684,26 @@ const docTemplate = `{ } } }, + "api.GetJobsApiResponse": { + "type": "object", + "properties": { + "items": { + "description": "Number of jobs returned", + "type": "integer" + }, + "jobs": { + "description": "Array of jobs", + "type": "array", + "items": { + "$ref": "#/definitions/schema.JobMeta" + } + }, + "page": { + "description": "Page id returned", + "type": "integer" + } + } + }, "api.StartJobApiResponse": { "type": "object", "properties": { diff --git a/internal/api/rest.go b/internal/api/rest.go index 6572147..95b87b2 100644 --- a/internal/api/rest.go +++ b/internal/api/rest.go @@ -738,29 +738,6 @@ func (api *RestApi) checkAndHandleStopJob(rw http.ResponseWriter, job *schema.Jo api.JobRepository.TriggerArchiving(job) } -// func (api *RestApi) importJob(rw http.ResponseWriter, r *http.Request) { -// if user := auth.GetUser(r.Context()); user != nil && !user.HasRole(auth.RoleApi) { -// handleError(fmt.Errorf("missing role: %v", auth.RoleApi), http.StatusForbidden, rw) -// return -// } - -// var body struct { -// Meta *schema.JobMeta `json:"meta"` -// Data *schema.JobData `json:"data"` -// } -// if err := decode(r.Body, &body); err != nil { -// handleError(fmt.Errorf("import failed: %s", err.Error()), http.StatusBadRequest, rw) -// return -// } - -// if err := api.JobRepository.ImportJob(body.Meta, body.Data); err != nil { -// handleError(fmt.Errorf("import failed: %s", err.Error()), http.StatusUnprocessableEntity, rw) -// return -// } - -// rw.Write([]byte(`{ "status": "OK" }`)) -// } - func (api *RestApi) getJobMetrics(rw http.ResponseWriter, r *http.Request) { id := mux.Vars(r)["id"] metrics := r.URL.Query()["metric"] From 2472698367688b3bb85872aa9dfc38c6a6926517 Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Mon, 27 Feb 2023 09:12:04 +0100 Subject: [PATCH 07/10] Update README.md --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 6d3536b..ae8d74a 100644 --- a/README.md +++ b/README.md @@ -136,7 +136,8 @@ The swagger doc files can be found in `./api/`. You can generate the configuration of swagger-ui by running `go run github.com/swaggo/swag/cmd/swag init -d ./internal/api,./pkg/schema -g rest.go -o ./api `. You need to move the generated `./api/doc.go` to `./internal/api/doc.go`. If you start cc-backend with flag `--dev` the Swagger UI is available at http://localhost:8080/swagger/ . -You have to enter a JWT key for a user with role API. This user must not be logged in the same browser (have a running session), otherwise Swagger requests will not work. +You have to enter a JWT key for a user with role API. +**NOTICE** The user owning the JWT token must not be logged in the same browser (have a running session), otherwise Swagger requests will not work. It is recommended to create a separate user that has just the API role. ## Project Structure From 5af1fa19b9ee0276aab0d4b679a93446f154f049 Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Mon, 27 Feb 2023 09:12:26 +0100 Subject: [PATCH 08/10] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index ae8d74a..9823694 100644 --- a/README.md +++ b/README.md @@ -137,6 +137,7 @@ You can generate the configuration of swagger-ui by running `go run github.com/s You need to move the generated `./api/doc.go` to `./internal/api/doc.go`. If you start cc-backend with flag `--dev` the Swagger UI is available at http://localhost:8080/swagger/ . You have to enter a JWT key for a user with role API. + **NOTICE** The user owning the JWT token must not be logged in the same browser (have a running session), otherwise Swagger requests will not work. It is recommended to create a separate user that has just the API role. ## Project Structure From 1fc090965d70ca3e78fabbd557f13f4157499256 Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Mon, 27 Feb 2023 09:16:18 +0100 Subject: [PATCH 09/10] Add missing error code --- internal/api/rest.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/api/rest.go b/internal/api/rest.go index 95b87b2..a262fca 100644 --- a/internal/api/rest.go +++ b/internal/api/rest.go @@ -171,6 +171,7 @@ func decode(r io.Reader, val interface{}) error { // @success 200 {object} api.GetJobsApiResponse "Job array and page info" // @failure 400 {object} api.ErrorResponse "Bad Request" // @failure 401 {object} api.ErrorResponse "Unauthorized" +// @failure 403 {object} api.ErrorResponse "Forbidden" // @failure 500 {object} api.ErrorResponse "Internal Server Error" // @security ApiKeyAuth // @router /jobs/ [get] From 9d28cb83f08ac2aaf425edc951e3caa97e7f09c9 Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Wed, 1 Mar 2023 10:49:08 +0100 Subject: [PATCH 10/10] Remove swagger enum comment and rebuild Swagger docs Relates to #95 --- api/swagger.json | 29 ++++++----------------------- api/swagger.yaml | 24 ++++-------------------- internal/api/docs.go | 29 ++++++----------------------- internal/api/rest.go | 8 ++++---- pkg/schema/job.go | 40 ++++++++++++++++++++-------------------- 5 files changed, 40 insertions(+), 90 deletions(-) diff --git a/api/swagger.json b/api/swagger.json index 8de2d8a..5c32a2d 100644 --- a/api/swagger.json +++ b/api/swagger.json @@ -97,6 +97,12 @@ "$ref": "#/definitions/api.ErrorResponse" } }, + "403": { + "description": "Forbidden", + "schema": { + "$ref": "#/definitions/api.ErrorResponse" + } + }, "500": { "description": "Internal Server Error", "schema": { @@ -726,13 +732,6 @@ }, "jobState": { "description": "Final job state", - "enum": [ - "completed", - "failed", - "cancelled", - "stopped", - "timeout" - ], "allOf": [ { "$ref": "#/definitions/schema.JobState" @@ -790,14 +789,6 @@ }, "jobState": { "description": "Final state of job", - "enum": [ - "completed", - "failed", - "cancelled", - "stopped", - "timeout", - "out_of_memory" - ], "allOf": [ { "$ref": "#/definitions/schema.JobState" @@ -926,14 +917,6 @@ }, "jobState": { "description": "Final state of job", - "enum": [ - "completed", - "failed", - "cancelled", - "stopped", - "timeout", - "out_of_memory" - ], "allOf": [ { "$ref": "#/definitions/schema.JobState" diff --git a/api/swagger.yaml b/api/swagger.yaml index 65806ed..7d008b8 100644 --- a/api/swagger.yaml +++ b/api/swagger.yaml @@ -76,12 +76,6 @@ definitions: allOf: - $ref: '#/definitions/schema.JobState' description: Final job state - enum: - - completed - - failed - - cancelled - - stopped - - timeout example: completed startTime: description: Start Time of job as epoch @@ -130,13 +124,6 @@ definitions: allOf: - $ref: '#/definitions/schema.JobState' description: Final state of job - enum: - - completed - - failed - - cancelled - - stopped - - timeout - - out_of_memory example: completed metaData: additionalProperties: @@ -239,13 +226,6 @@ definitions: allOf: - $ref: '#/definitions/schema.JobState' description: Final state of job - enum: - - completed - - failed - - cancelled - - stopped - - timeout - - out_of_memory example: completed metaData: additionalProperties: @@ -463,6 +443,10 @@ paths: description: Unauthorized schema: $ref: '#/definitions/api.ErrorResponse' + "403": + description: Forbidden + schema: + $ref: '#/definitions/api.ErrorResponse' "500": description: Internal Server Error schema: diff --git a/internal/api/docs.go b/internal/api/docs.go index 62d05e2..af6745f 100644 --- a/internal/api/docs.go +++ b/internal/api/docs.go @@ -103,6 +103,12 @@ const docTemplate = `{ "$ref": "#/definitions/api.ErrorResponse" } }, + "403": { + "description": "Forbidden", + "schema": { + "$ref": "#/definitions/api.ErrorResponse" + } + }, "500": { "description": "Internal Server Error", "schema": { @@ -732,13 +738,6 @@ const docTemplate = `{ }, "jobState": { "description": "Final job state", - "enum": [ - "completed", - "failed", - "cancelled", - "stopped", - "timeout" - ], "allOf": [ { "$ref": "#/definitions/schema.JobState" @@ -796,14 +795,6 @@ const docTemplate = `{ }, "jobState": { "description": "Final state of job", - "enum": [ - "completed", - "failed", - "cancelled", - "stopped", - "timeout", - "out_of_memory" - ], "allOf": [ { "$ref": "#/definitions/schema.JobState" @@ -932,14 +923,6 @@ const docTemplate = `{ }, "jobState": { "description": "Final state of job", - "enum": [ - "completed", - "failed", - "cancelled", - "stopped", - "timeout", - "out_of_memory" - ], "allOf": [ { "$ref": "#/definitions/schema.JobState" diff --git a/internal/api/rest.go b/internal/api/rest.go index a262fca..dc2f3d8 100644 --- a/internal/api/rest.go +++ b/internal/api/rest.go @@ -104,10 +104,10 @@ type DeleteJobApiResponse struct { type StopJobApiRequest struct { // Stop Time of job as epoch StopTime int64 `json:"stopTime" validate:"required" example:"1649763839"` - State schema.JobState `json:"jobState" validate:"required" example:"completed" enums:"completed,failed,cancelled,stopped,timeout"` // Final job state - JobId *int64 `json:"jobId" example:"123000"` // Cluster Job ID of job - Cluster *string `json:"cluster" example:"fritz"` // Cluster of job - StartTime *int64 `json:"startTime" example:"1649723812"` // Start Time of job as epoch + State schema.JobState `json:"jobState" validate:"required" example:"completed"` // Final job state + JobId *int64 `json:"jobId" example:"123000"` // Cluster Job ID of job + Cluster *string `json:"cluster" example:"fritz"` // Cluster of job + StartTime *int64 `json:"startTime" example:"1649723812"` // Start Time of job as epoch } // DeleteJobApiRequest model diff --git a/pkg/schema/job.go b/pkg/schema/job.go index 9b99d99..208af11 100644 --- a/pkg/schema/job.go +++ b/pkg/schema/job.go @@ -17,26 +17,26 @@ import ( type BaseJob struct { // The unique identifier of a job JobID int64 `json:"jobId" db:"job_id" example:"123000"` - User string `json:"user" db:"user" example:"abcd100h"` // The unique identifier of a user - Project string `json:"project" db:"project" example:"abcd200"` // The unique identifier of a project - Cluster string `json:"cluster" db:"cluster" example:"fritz"` // The unique identifier of a cluster - SubCluster string `json:"subCluster" db:"subcluster" example:"main"` // The unique identifier of a sub cluster - Partition string `json:"partition" db:"partition" example:"main"` // The Slurm partition to which the job was submitted - ArrayJobId int64 `json:"arrayJobId" db:"array_job_id" example:"123000"` // The unique identifier of an array job - NumNodes int32 `json:"numNodes" db:"num_nodes" example:"2" minimum:"1"` // Number of nodes used (Min > 0) - NumHWThreads int32 `json:"numHwthreads" db:"num_hwthreads" example:"20" minimum:"1"` // Number of HWThreads used (Min > 0) - NumAcc int32 `json:"numAcc" db:"num_acc" example:"2" minimum:"1"` // Number of accelerators used (Min > 0) - Exclusive int32 `json:"exclusive" db:"exclusive" example:"1" minimum:"0" maximum:"2"` // Specifies how nodes are shared: 0 - Shared among multiple jobs of multiple users, 1 - Job exclusive (Default), 2 - Shared among multiple jobs of same user - MonitoringStatus int32 `json:"monitoringStatus" db:"monitoring_status" example:"1" minimum:"0" maximum:"3"` // State of monitoring system during job run: 0 - Disabled, 1 - Running or Archiving (Default), 2 - Archiving Failed, 3 - Archiving Successfull - SMT int32 `json:"smt" db:"smt" example:"4"` // SMT threads used by job - State JobState `json:"jobState" db:"job_state" example:"completed" enums:"completed,failed,cancelled,stopped,timeout,out_of_memory"` // Final state of job - Duration int32 `json:"duration" db:"duration" example:"43200" minimum:"1"` // Duration of job in seconds (Min > 0) - Walltime int64 `json:"walltime" db:"walltime" example:"86400" minimum:"1"` // Requested walltime of job in seconds (Min > 0) - Tags []*Tag `json:"tags"` // List of tags - RawResources []byte `json:"-" db:"resources"` // Resources used by job [As Bytes] - Resources []*Resource `json:"resources"` // Resources used by job - RawMetaData []byte `json:"-" db:"meta_data"` // Additional information about the job [As Bytes] - MetaData map[string]string `json:"metaData"` // Additional information about the job + User string `json:"user" db:"user" example:"abcd100h"` // The unique identifier of a user + Project string `json:"project" db:"project" example:"abcd200"` // The unique identifier of a project + Cluster string `json:"cluster" db:"cluster" example:"fritz"` // The unique identifier of a cluster + SubCluster string `json:"subCluster" db:"subcluster" example:"main"` // The unique identifier of a sub cluster + Partition string `json:"partition" db:"partition" example:"main"` // The Slurm partition to which the job was submitted + ArrayJobId int64 `json:"arrayJobId" db:"array_job_id" example:"123000"` // The unique identifier of an array job + NumNodes int32 `json:"numNodes" db:"num_nodes" example:"2" minimum:"1"` // Number of nodes used (Min > 0) + NumHWThreads int32 `json:"numHwthreads" db:"num_hwthreads" example:"20" minimum:"1"` // Number of HWThreads used (Min > 0) + NumAcc int32 `json:"numAcc" db:"num_acc" example:"2" minimum:"1"` // Number of accelerators used (Min > 0) + Exclusive int32 `json:"exclusive" db:"exclusive" example:"1" minimum:"0" maximum:"2"` // Specifies how nodes are shared: 0 - Shared among multiple jobs of multiple users, 1 - Job exclusive (Default), 2 - Shared among multiple jobs of same user + MonitoringStatus int32 `json:"monitoringStatus" db:"monitoring_status" example:"1" minimum:"0" maximum:"3"` // State of monitoring system during job run: 0 - Disabled, 1 - Running or Archiving (Default), 2 - Archiving Failed, 3 - Archiving Successfull + SMT int32 `json:"smt" db:"smt" example:"4"` // SMT threads used by job + State JobState `json:"jobState" db:"job_state" example:"completed"` // Final state of job + Duration int32 `json:"duration" db:"duration" example:"43200" minimum:"1"` // Duration of job in seconds (Min > 0) + Walltime int64 `json:"walltime" db:"walltime" example:"86400" minimum:"1"` // Requested walltime of job in seconds (Min > 0) + Tags []*Tag `json:"tags"` // List of tags + RawResources []byte `json:"-" db:"resources"` // Resources used by job [As Bytes] + Resources []*Resource `json:"resources"` // Resources used by job + RawMetaData []byte `json:"-" db:"meta_data"` // Additional information about the job [As Bytes] + MetaData map[string]string `json:"metaData"` // Additional information about the job } // Non-Swaggered Comment: Job