diff --git a/Makefile b/Makefile index d576045..e44f692 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,6 @@ TARGET = ./cc-backend VAR = ./var +DB = ./var/job.db FRONTEND = ./web/frontend VERSION = 0.1 GIT_HASH := $(shell git rev-parse --short HEAD || echo 'development') @@ -27,10 +28,9 @@ SVELTE_SRC = $(wildcard $(FRONTEND)/src/*.svelte) \ .NOTPARALLEL: -$(TARGET): $(VAR) $(SVELTE_TARGETS) +$(TARGET): $(VAR) $(DB) $(SVELTE_TARGETS) $(info ===> BUILD cc-backend) @go build -ldflags=${LD_FLAGS} ./cmd/cc-backend - ./cc-backend --migrate-db clean: $(info ===> CLEAN) @@ -47,10 +47,13 @@ tags: $(info ===> TAGS) @ctags -R -$(SVELTE_TARGETS): $(SVELTE_SRC) - $(info ===> BUILD frontend) - cd web/frontend && yarn build - $(VAR): @mkdir $(VAR) cd web/frontend && yarn install + +$(DB): + ./cc-backend --migrate-db + +$(SVELTE_TARGETS): $(SVELTE_SRC) + $(info ===> BUILD frontend) + cd web/frontend && yarn build diff --git a/README.md b/README.md index 6d3536b..9823694 100644 --- a/README.md +++ b/README.md @@ -136,7 +136,9 @@ The swagger doc files can be found in `./api/`. You can generate the configuration of swagger-ui by running `go run github.com/swaggo/swag/cmd/swag init -d ./internal/api,./pkg/schema -g rest.go -o ./api `. You need to move the generated `./api/doc.go` to `./internal/api/doc.go`. If you start cc-backend with flag `--dev` the Swagger UI is available at http://localhost:8080/swagger/ . -You have to enter a JWT key for a user with role API. This user must not be logged in the same browser (have a running session), otherwise Swagger requests will not work. +You have to enter a JWT key for a user with role API. + +**NOTICE** The user owning the JWT token must not be logged in the same browser (have a running session), otherwise Swagger requests will not work. It is recommended to create a separate user that has just the API role. ## Project Structure diff --git a/api/swagger.json b/api/swagger.json index 6436484..5c32a2d 100644 --- a/api/swagger.json +++ b/api/swagger.json @@ -80,12 +80,9 @@ ], "responses": { "200": { - "description": "Array of matching jobs", + "description": "Job array and page info", "schema": { - "type": "array", - "items": { - "$ref": "#/definitions/schema.Job" - } + "$ref": "#/definitions/api.GetJobsApiResponse" } }, "400": { @@ -100,6 +97,12 @@ "$ref": "#/definitions/api.ErrorResponse" } }, + "403": { + "description": "Forbidden", + "schema": { + "$ref": "#/definitions/api.ErrorResponse" + } + }, "500": { "description": "Internal Server Error", "schema": { @@ -681,6 +684,26 @@ } } }, + "api.GetJobsApiResponse": { + "type": "object", + "properties": { + "items": { + "description": "Number of jobs returned", + "type": "integer" + }, + "jobs": { + "description": "Array of jobs", + "type": "array", + "items": { + "$ref": "#/definitions/schema.JobMeta" + } + }, + "page": { + "description": "Page id returned", + "type": "integer" + } + } + }, "api.StartJobApiResponse": { "type": "object", "properties": { @@ -709,13 +732,6 @@ }, "jobState": { "description": "Final job state", - "enum": [ - "completed", - "failed", - "cancelled", - "stopped", - "timeout" - ], "allOf": [ { "$ref": "#/definitions/schema.JobState" @@ -773,14 +789,6 @@ }, "jobState": { "description": "Final state of job", - "enum": [ - "completed", - "failed", - "cancelled", - "stopped", - "timeout", - "out_of_memory" - ], "allOf": [ { "$ref": "#/definitions/schema.JobState" @@ -909,14 +917,6 @@ }, "jobState": { "description": "Final state of job", - "enum": [ - "completed", - "failed", - "cancelled", - "stopped", - "timeout", - "out_of_memory" - ], "allOf": [ { "$ref": "#/definitions/schema.JobState" diff --git a/api/swagger.yaml b/api/swagger.yaml index ca04575..7d008b8 100644 --- a/api/swagger.yaml +++ b/api/swagger.yaml @@ -42,6 +42,20 @@ definitions: description: Statustext of Errorcode type: string type: object + api.GetJobsApiResponse: + properties: + items: + description: Number of jobs returned + type: integer + jobs: + description: Array of jobs + items: + $ref: '#/definitions/schema.JobMeta' + type: array + page: + description: Page id returned + type: integer + type: object api.StartJobApiResponse: properties: id: @@ -62,12 +76,6 @@ definitions: allOf: - $ref: '#/definitions/schema.JobState' description: Final job state - enum: - - completed - - failed - - cancelled - - stopped - - timeout example: completed startTime: description: Start Time of job as epoch @@ -116,13 +124,6 @@ definitions: allOf: - $ref: '#/definitions/schema.JobState' description: Final state of job - enum: - - completed - - failed - - cancelled - - stopped - - timeout - - out_of_memory example: completed metaData: additionalProperties: @@ -225,13 +226,6 @@ definitions: allOf: - $ref: '#/definitions/schema.JobState' description: Final state of job - enum: - - completed - - failed - - cancelled - - stopped - - timeout - - out_of_memory example: completed metaData: additionalProperties: @@ -438,11 +432,9 @@ paths: - application/json responses: "200": - description: Array of matching jobs + description: Job array and page info schema: - items: - $ref: '#/definitions/schema.Job' - type: array + $ref: '#/definitions/api.GetJobsApiResponse' "400": description: Bad Request schema: @@ -451,6 +443,10 @@ paths: description: Unauthorized schema: $ref: '#/definitions/api.ErrorResponse' + "403": + description: Forbidden + schema: + $ref: '#/definitions/api.ErrorResponse' "500": description: Internal Server Error schema: diff --git a/cmd/cc-backend/main.go b/cmd/cc-backend/main.go index ab684f5..1a8aa4c 100644 --- a/cmd/cc-backend/main.go +++ b/cmd/cc-backend/main.go @@ -76,7 +76,7 @@ func main() { flag.StringVar(&flagDelUser, "del-user", "", "Remove user by `username`") flag.StringVar(&flagGenJWT, "jwt", "", "Generate and print a JWT for the user specified by its `username`") flag.StringVar(&flagImportJob, "import-job", "", "Import a job. Argument format: `:,...`") - flag.StringVar(&flagLogLevel, "loglevel", "debug", "Sets the logging level: `[debug (default),info,warn,err,fatal,crit]`") + flag.StringVar(&flagLogLevel, "loglevel", "warn", "Sets the logging level: `[debug,info,warn (default),err,fatal,crit]`") flag.Parse() if flagVersion { @@ -375,9 +375,9 @@ func main() { MinVersion: tls.VersionTLS12, PreferServerCipherSuites: true, }) - log.Printf("HTTPS server listening at %s...", config.Keys.Addr) + fmt.Printf("HTTPS server listening at %s...", config.Keys.Addr) } else { - log.Printf("HTTP server listening at %s...", config.Keys.Addr) + fmt.Printf("HTTP server listening at %s...", config.Keys.Addr) } // Because this program will want to bind to a privileged port (like 80), the listener must diff --git a/internal/api/docs.go b/internal/api/docs.go index 4ac70a6..af6745f 100644 --- a/internal/api/docs.go +++ b/internal/api/docs.go @@ -86,12 +86,9 @@ const docTemplate = `{ ], "responses": { "200": { - "description": "Array of matching jobs", + "description": "Job array and page info", "schema": { - "type": "array", - "items": { - "$ref": "#/definitions/schema.Job" - } + "$ref": "#/definitions/api.GetJobsApiResponse" } }, "400": { @@ -106,6 +103,12 @@ const docTemplate = `{ "$ref": "#/definitions/api.ErrorResponse" } }, + "403": { + "description": "Forbidden", + "schema": { + "$ref": "#/definitions/api.ErrorResponse" + } + }, "500": { "description": "Internal Server Error", "schema": { @@ -687,6 +690,26 @@ const docTemplate = `{ } } }, + "api.GetJobsApiResponse": { + "type": "object", + "properties": { + "items": { + "description": "Number of jobs returned", + "type": "integer" + }, + "jobs": { + "description": "Array of jobs", + "type": "array", + "items": { + "$ref": "#/definitions/schema.JobMeta" + } + }, + "page": { + "description": "Page id returned", + "type": "integer" + } + } + }, "api.StartJobApiResponse": { "type": "object", "properties": { @@ -715,13 +738,6 @@ const docTemplate = `{ }, "jobState": { "description": "Final job state", - "enum": [ - "completed", - "failed", - "cancelled", - "stopped", - "timeout" - ], "allOf": [ { "$ref": "#/definitions/schema.JobState" @@ -779,14 +795,6 @@ const docTemplate = `{ }, "jobState": { "description": "Final state of job", - "enum": [ - "completed", - "failed", - "cancelled", - "stopped", - "timeout", - "out_of_memory" - ], "allOf": [ { "$ref": "#/definitions/schema.JobState" @@ -915,14 +923,6 @@ const docTemplate = `{ }, "jobState": { "description": "Final state of job", - "enum": [ - "completed", - "failed", - "cancelled", - "stopped", - "timeout", - "out_of_memory" - ], "allOf": [ { "$ref": "#/definitions/schema.JobState" diff --git a/internal/api/rest.go b/internal/api/rest.go index 6572147..dc2f3d8 100644 --- a/internal/api/rest.go +++ b/internal/api/rest.go @@ -104,10 +104,10 @@ type DeleteJobApiResponse struct { type StopJobApiRequest struct { // Stop Time of job as epoch StopTime int64 `json:"stopTime" validate:"required" example:"1649763839"` - State schema.JobState `json:"jobState" validate:"required" example:"completed" enums:"completed,failed,cancelled,stopped,timeout"` // Final job state - JobId *int64 `json:"jobId" example:"123000"` // Cluster Job ID of job - Cluster *string `json:"cluster" example:"fritz"` // Cluster of job - StartTime *int64 `json:"startTime" example:"1649723812"` // Start Time of job as epoch + State schema.JobState `json:"jobState" validate:"required" example:"completed"` // Final job state + JobId *int64 `json:"jobId" example:"123000"` // Cluster Job ID of job + Cluster *string `json:"cluster" example:"fritz"` // Cluster of job + StartTime *int64 `json:"startTime" example:"1649723812"` // Start Time of job as epoch } // DeleteJobApiRequest model @@ -171,6 +171,7 @@ func decode(r io.Reader, val interface{}) error { // @success 200 {object} api.GetJobsApiResponse "Job array and page info" // @failure 400 {object} api.ErrorResponse "Bad Request" // @failure 401 {object} api.ErrorResponse "Unauthorized" +// @failure 403 {object} api.ErrorResponse "Forbidden" // @failure 500 {object} api.ErrorResponse "Internal Server Error" // @security ApiKeyAuth // @router /jobs/ [get] @@ -738,29 +739,6 @@ func (api *RestApi) checkAndHandleStopJob(rw http.ResponseWriter, job *schema.Jo api.JobRepository.TriggerArchiving(job) } -// func (api *RestApi) importJob(rw http.ResponseWriter, r *http.Request) { -// if user := auth.GetUser(r.Context()); user != nil && !user.HasRole(auth.RoleApi) { -// handleError(fmt.Errorf("missing role: %v", auth.RoleApi), http.StatusForbidden, rw) -// return -// } - -// var body struct { -// Meta *schema.JobMeta `json:"meta"` -// Data *schema.JobData `json:"data"` -// } -// if err := decode(r.Body, &body); err != nil { -// handleError(fmt.Errorf("import failed: %s", err.Error()), http.StatusBadRequest, rw) -// return -// } - -// if err := api.JobRepository.ImportJob(body.Meta, body.Data); err != nil { -// handleError(fmt.Errorf("import failed: %s", err.Error()), http.StatusUnprocessableEntity, rw) -// return -// } - -// rw.Write([]byte(`{ "status": "OK" }`)) -// } - func (api *RestApi) getJobMetrics(rw http.ResponseWriter, r *http.Request) { id := mux.Vars(r)["id"] metrics := r.URL.Query()["metric"] diff --git a/internal/repository/migration.go b/internal/repository/migration.go index 258593d..934a002 100644 --- a/internal/repository/migration.go +++ b/internal/repository/migration.go @@ -102,7 +102,11 @@ func MigrateDB(backend string, db string) { } if err := m.Up(); err != nil { - log.Fatal(err) + if err == migrate.ErrNoChange { + log.Info("DB already up to date!") + } else { + log.Fatal(err) + } } m.Close() diff --git a/pkg/schema/job.go b/pkg/schema/job.go index 9b99d99..208af11 100644 --- a/pkg/schema/job.go +++ b/pkg/schema/job.go @@ -17,26 +17,26 @@ import ( type BaseJob struct { // The unique identifier of a job JobID int64 `json:"jobId" db:"job_id" example:"123000"` - User string `json:"user" db:"user" example:"abcd100h"` // The unique identifier of a user - Project string `json:"project" db:"project" example:"abcd200"` // The unique identifier of a project - Cluster string `json:"cluster" db:"cluster" example:"fritz"` // The unique identifier of a cluster - SubCluster string `json:"subCluster" db:"subcluster" example:"main"` // The unique identifier of a sub cluster - Partition string `json:"partition" db:"partition" example:"main"` // The Slurm partition to which the job was submitted - ArrayJobId int64 `json:"arrayJobId" db:"array_job_id" example:"123000"` // The unique identifier of an array job - NumNodes int32 `json:"numNodes" db:"num_nodes" example:"2" minimum:"1"` // Number of nodes used (Min > 0) - NumHWThreads int32 `json:"numHwthreads" db:"num_hwthreads" example:"20" minimum:"1"` // Number of HWThreads used (Min > 0) - NumAcc int32 `json:"numAcc" db:"num_acc" example:"2" minimum:"1"` // Number of accelerators used (Min > 0) - Exclusive int32 `json:"exclusive" db:"exclusive" example:"1" minimum:"0" maximum:"2"` // Specifies how nodes are shared: 0 - Shared among multiple jobs of multiple users, 1 - Job exclusive (Default), 2 - Shared among multiple jobs of same user - MonitoringStatus int32 `json:"monitoringStatus" db:"monitoring_status" example:"1" minimum:"0" maximum:"3"` // State of monitoring system during job run: 0 - Disabled, 1 - Running or Archiving (Default), 2 - Archiving Failed, 3 - Archiving Successfull - SMT int32 `json:"smt" db:"smt" example:"4"` // SMT threads used by job - State JobState `json:"jobState" db:"job_state" example:"completed" enums:"completed,failed,cancelled,stopped,timeout,out_of_memory"` // Final state of job - Duration int32 `json:"duration" db:"duration" example:"43200" minimum:"1"` // Duration of job in seconds (Min > 0) - Walltime int64 `json:"walltime" db:"walltime" example:"86400" minimum:"1"` // Requested walltime of job in seconds (Min > 0) - Tags []*Tag `json:"tags"` // List of tags - RawResources []byte `json:"-" db:"resources"` // Resources used by job [As Bytes] - Resources []*Resource `json:"resources"` // Resources used by job - RawMetaData []byte `json:"-" db:"meta_data"` // Additional information about the job [As Bytes] - MetaData map[string]string `json:"metaData"` // Additional information about the job + User string `json:"user" db:"user" example:"abcd100h"` // The unique identifier of a user + Project string `json:"project" db:"project" example:"abcd200"` // The unique identifier of a project + Cluster string `json:"cluster" db:"cluster" example:"fritz"` // The unique identifier of a cluster + SubCluster string `json:"subCluster" db:"subcluster" example:"main"` // The unique identifier of a sub cluster + Partition string `json:"partition" db:"partition" example:"main"` // The Slurm partition to which the job was submitted + ArrayJobId int64 `json:"arrayJobId" db:"array_job_id" example:"123000"` // The unique identifier of an array job + NumNodes int32 `json:"numNodes" db:"num_nodes" example:"2" minimum:"1"` // Number of nodes used (Min > 0) + NumHWThreads int32 `json:"numHwthreads" db:"num_hwthreads" example:"20" minimum:"1"` // Number of HWThreads used (Min > 0) + NumAcc int32 `json:"numAcc" db:"num_acc" example:"2" minimum:"1"` // Number of accelerators used (Min > 0) + Exclusive int32 `json:"exclusive" db:"exclusive" example:"1" minimum:"0" maximum:"2"` // Specifies how nodes are shared: 0 - Shared among multiple jobs of multiple users, 1 - Job exclusive (Default), 2 - Shared among multiple jobs of same user + MonitoringStatus int32 `json:"monitoringStatus" db:"monitoring_status" example:"1" minimum:"0" maximum:"3"` // State of monitoring system during job run: 0 - Disabled, 1 - Running or Archiving (Default), 2 - Archiving Failed, 3 - Archiving Successfull + SMT int32 `json:"smt" db:"smt" example:"4"` // SMT threads used by job + State JobState `json:"jobState" db:"job_state" example:"completed"` // Final state of job + Duration int32 `json:"duration" db:"duration" example:"43200" minimum:"1"` // Duration of job in seconds (Min > 0) + Walltime int64 `json:"walltime" db:"walltime" example:"86400" minimum:"1"` // Requested walltime of job in seconds (Min > 0) + Tags []*Tag `json:"tags"` // List of tags + RawResources []byte `json:"-" db:"resources"` // Resources used by job [As Bytes] + Resources []*Resource `json:"resources"` // Resources used by job + RawMetaData []byte `json:"-" db:"meta_data"` // Additional information about the job [As Bytes] + MetaData map[string]string `json:"metaData"` // Additional information about the job } // Non-Swaggered Comment: Job diff --git a/startDemo.sh b/startDemo.sh index d210772..8667df0 100755 --- a/startDemo.sh +++ b/startDemo.sh @@ -4,21 +4,16 @@ if [ -d './var' ]; then echo 'Directory ./var already exists! Skipping initialization.' ./cc-backend --server --dev else - mkdir ./var - cd ./var + make + cd var wget https://hpc-mover.rrze.uni-erlangen.de/HPC-Data/0x7b58aefb/eig7ahyo6fo2bais0ephuf2aitohv1ai/job-archive-dev.tar.xz tar xJf job-archive-dev.tar.xz rm ./job-archive-dev.tar.xz + cd ../ - cd ../web/frontend - yarn install - yarn build - - cd ../.. cp ./configs/env-template.txt .env cp ./docs/config.json config.json - go build ./cmd/cc-backend ./cc-backend --migrate-db ./cc-backend --server --dev --init-db --add-user demo:admin:AdminDev