mirror of
https://github.com/ClusterCockpit/cc-backend
synced 2024-12-25 04:49:05 +01:00
Merge branch 'master' of github.com:ClusterCockpit/cc-backend
This commit is contained in:
commit
8a1288db1a
15
Makefile
15
Makefile
@ -1,5 +1,6 @@
|
||||
TARGET = ./cc-backend
|
||||
VAR = ./var
|
||||
DB = ./var/job.db
|
||||
FRONTEND = ./web/frontend
|
||||
VERSION = 0.1
|
||||
GIT_HASH := $(shell git rev-parse --short HEAD || echo 'development')
|
||||
@ -27,10 +28,9 @@ SVELTE_SRC = $(wildcard $(FRONTEND)/src/*.svelte) \
|
||||
|
||||
.NOTPARALLEL:
|
||||
|
||||
$(TARGET): $(VAR) $(SVELTE_TARGETS)
|
||||
$(TARGET): $(VAR) $(DB) $(SVELTE_TARGETS)
|
||||
$(info ===> BUILD cc-backend)
|
||||
@go build -ldflags=${LD_FLAGS} ./cmd/cc-backend
|
||||
./cc-backend --migrate-db
|
||||
|
||||
clean:
|
||||
$(info ===> CLEAN)
|
||||
@ -47,10 +47,13 @@ tags:
|
||||
$(info ===> TAGS)
|
||||
@ctags -R
|
||||
|
||||
$(SVELTE_TARGETS): $(SVELTE_SRC)
|
||||
$(info ===> BUILD frontend)
|
||||
cd web/frontend && yarn build
|
||||
|
||||
$(VAR):
|
||||
@mkdir $(VAR)
|
||||
cd web/frontend && yarn install
|
||||
|
||||
$(DB):
|
||||
./cc-backend --migrate-db
|
||||
|
||||
$(SVELTE_TARGETS): $(SVELTE_SRC)
|
||||
$(info ===> BUILD frontend)
|
||||
cd web/frontend && yarn build
|
||||
|
||||
@ -136,7 +136,9 @@ The swagger doc files can be found in `./api/`.
|
||||
You can generate the configuration of swagger-ui by running `go run github.com/swaggo/swag/cmd/swag init -d ./internal/api,./pkg/schema -g rest.go -o ./api `.
|
||||
You need to move the generated `./api/doc.go` to `./internal/api/doc.go`.
|
||||
If you start cc-backend with flag `--dev` the Swagger UI is available at http://localhost:8080/swagger/ .
|
||||
You have to enter a JWT key for a user with role API. This user must not be logged in the same browser (have a running session), otherwise Swagger requests will not work.
|
||||
You have to enter a JWT key for a user with role API.
|
||||
|
||||
**NOTICE** The user owning the JWT token must not be logged in the same browser (have a running session), otherwise Swagger requests will not work. It is recommended to create a separate user that has just the API role.
|
||||
|
||||
## Project Structure
|
||||
|
||||
|
||||
@ -80,12 +80,9 @@
|
||||
],
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "Array of matching jobs",
|
||||
"description": "Job array and page info",
|
||||
"schema": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/definitions/schema.Job"
|
||||
}
|
||||
"$ref": "#/definitions/api.GetJobsApiResponse"
|
||||
}
|
||||
},
|
||||
"400": {
|
||||
@ -100,6 +97,12 @@
|
||||
"$ref": "#/definitions/api.ErrorResponse"
|
||||
}
|
||||
},
|
||||
"403": {
|
||||
"description": "Forbidden",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/api.ErrorResponse"
|
||||
}
|
||||
},
|
||||
"500": {
|
||||
"description": "Internal Server Error",
|
||||
"schema": {
|
||||
@ -681,6 +684,26 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"api.GetJobsApiResponse": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"items": {
|
||||
"description": "Number of jobs returned",
|
||||
"type": "integer"
|
||||
},
|
||||
"jobs": {
|
||||
"description": "Array of jobs",
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/definitions/schema.JobMeta"
|
||||
}
|
||||
},
|
||||
"page": {
|
||||
"description": "Page id returned",
|
||||
"type": "integer"
|
||||
}
|
||||
}
|
||||
},
|
||||
"api.StartJobApiResponse": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
@ -709,13 +732,6 @@
|
||||
},
|
||||
"jobState": {
|
||||
"description": "Final job state",
|
||||
"enum": [
|
||||
"completed",
|
||||
"failed",
|
||||
"cancelled",
|
||||
"stopped",
|
||||
"timeout"
|
||||
],
|
||||
"allOf": [
|
||||
{
|
||||
"$ref": "#/definitions/schema.JobState"
|
||||
@ -773,14 +789,6 @@
|
||||
},
|
||||
"jobState": {
|
||||
"description": "Final state of job",
|
||||
"enum": [
|
||||
"completed",
|
||||
"failed",
|
||||
"cancelled",
|
||||
"stopped",
|
||||
"timeout",
|
||||
"out_of_memory"
|
||||
],
|
||||
"allOf": [
|
||||
{
|
||||
"$ref": "#/definitions/schema.JobState"
|
||||
@ -909,14 +917,6 @@
|
||||
},
|
||||
"jobState": {
|
||||
"description": "Final state of job",
|
||||
"enum": [
|
||||
"completed",
|
||||
"failed",
|
||||
"cancelled",
|
||||
"stopped",
|
||||
"timeout",
|
||||
"out_of_memory"
|
||||
],
|
||||
"allOf": [
|
||||
{
|
||||
"$ref": "#/definitions/schema.JobState"
|
||||
|
||||
@ -42,6 +42,20 @@ definitions:
|
||||
description: Statustext of Errorcode
|
||||
type: string
|
||||
type: object
|
||||
api.GetJobsApiResponse:
|
||||
properties:
|
||||
items:
|
||||
description: Number of jobs returned
|
||||
type: integer
|
||||
jobs:
|
||||
description: Array of jobs
|
||||
items:
|
||||
$ref: '#/definitions/schema.JobMeta'
|
||||
type: array
|
||||
page:
|
||||
description: Page id returned
|
||||
type: integer
|
||||
type: object
|
||||
api.StartJobApiResponse:
|
||||
properties:
|
||||
id:
|
||||
@ -62,12 +76,6 @@ definitions:
|
||||
allOf:
|
||||
- $ref: '#/definitions/schema.JobState'
|
||||
description: Final job state
|
||||
enum:
|
||||
- completed
|
||||
- failed
|
||||
- cancelled
|
||||
- stopped
|
||||
- timeout
|
||||
example: completed
|
||||
startTime:
|
||||
description: Start Time of job as epoch
|
||||
@ -116,13 +124,6 @@ definitions:
|
||||
allOf:
|
||||
- $ref: '#/definitions/schema.JobState'
|
||||
description: Final state of job
|
||||
enum:
|
||||
- completed
|
||||
- failed
|
||||
- cancelled
|
||||
- stopped
|
||||
- timeout
|
||||
- out_of_memory
|
||||
example: completed
|
||||
metaData:
|
||||
additionalProperties:
|
||||
@ -225,13 +226,6 @@ definitions:
|
||||
allOf:
|
||||
- $ref: '#/definitions/schema.JobState'
|
||||
description: Final state of job
|
||||
enum:
|
||||
- completed
|
||||
- failed
|
||||
- cancelled
|
||||
- stopped
|
||||
- timeout
|
||||
- out_of_memory
|
||||
example: completed
|
||||
metaData:
|
||||
additionalProperties:
|
||||
@ -438,11 +432,9 @@ paths:
|
||||
- application/json
|
||||
responses:
|
||||
"200":
|
||||
description: Array of matching jobs
|
||||
description: Job array and page info
|
||||
schema:
|
||||
items:
|
||||
$ref: '#/definitions/schema.Job'
|
||||
type: array
|
||||
$ref: '#/definitions/api.GetJobsApiResponse'
|
||||
"400":
|
||||
description: Bad Request
|
||||
schema:
|
||||
@ -451,6 +443,10 @@ paths:
|
||||
description: Unauthorized
|
||||
schema:
|
||||
$ref: '#/definitions/api.ErrorResponse'
|
||||
"403":
|
||||
description: Forbidden
|
||||
schema:
|
||||
$ref: '#/definitions/api.ErrorResponse'
|
||||
"500":
|
||||
description: Internal Server Error
|
||||
schema:
|
||||
|
||||
@ -76,7 +76,7 @@ func main() {
|
||||
flag.StringVar(&flagDelUser, "del-user", "", "Remove user by `username`")
|
||||
flag.StringVar(&flagGenJWT, "jwt", "", "Generate and print a JWT for the user specified by its `username`")
|
||||
flag.StringVar(&flagImportJob, "import-job", "", "Import a job. Argument format: `<path-to-meta.json>:<path-to-data.json>,...`")
|
||||
flag.StringVar(&flagLogLevel, "loglevel", "debug", "Sets the logging level: `[debug (default),info,warn,err,fatal,crit]`")
|
||||
flag.StringVar(&flagLogLevel, "loglevel", "warn", "Sets the logging level: `[debug,info,warn (default),err,fatal,crit]`")
|
||||
flag.Parse()
|
||||
|
||||
if flagVersion {
|
||||
@ -375,9 +375,9 @@ func main() {
|
||||
MinVersion: tls.VersionTLS12,
|
||||
PreferServerCipherSuites: true,
|
||||
})
|
||||
log.Printf("HTTPS server listening at %s...", config.Keys.Addr)
|
||||
fmt.Printf("HTTPS server listening at %s...", config.Keys.Addr)
|
||||
} else {
|
||||
log.Printf("HTTP server listening at %s...", config.Keys.Addr)
|
||||
fmt.Printf("HTTP server listening at %s...", config.Keys.Addr)
|
||||
}
|
||||
|
||||
// Because this program will want to bind to a privileged port (like 80), the listener must
|
||||
|
||||
@ -86,12 +86,9 @@ const docTemplate = `{
|
||||
],
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "Array of matching jobs",
|
||||
"description": "Job array and page info",
|
||||
"schema": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/definitions/schema.Job"
|
||||
}
|
||||
"$ref": "#/definitions/api.GetJobsApiResponse"
|
||||
}
|
||||
},
|
||||
"400": {
|
||||
@ -106,6 +103,12 @@ const docTemplate = `{
|
||||
"$ref": "#/definitions/api.ErrorResponse"
|
||||
}
|
||||
},
|
||||
"403": {
|
||||
"description": "Forbidden",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/api.ErrorResponse"
|
||||
}
|
||||
},
|
||||
"500": {
|
||||
"description": "Internal Server Error",
|
||||
"schema": {
|
||||
@ -687,6 +690,26 @@ const docTemplate = `{
|
||||
}
|
||||
}
|
||||
},
|
||||
"api.GetJobsApiResponse": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"items": {
|
||||
"description": "Number of jobs returned",
|
||||
"type": "integer"
|
||||
},
|
||||
"jobs": {
|
||||
"description": "Array of jobs",
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/definitions/schema.JobMeta"
|
||||
}
|
||||
},
|
||||
"page": {
|
||||
"description": "Page id returned",
|
||||
"type": "integer"
|
||||
}
|
||||
}
|
||||
},
|
||||
"api.StartJobApiResponse": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
@ -715,13 +738,6 @@ const docTemplate = `{
|
||||
},
|
||||
"jobState": {
|
||||
"description": "Final job state",
|
||||
"enum": [
|
||||
"completed",
|
||||
"failed",
|
||||
"cancelled",
|
||||
"stopped",
|
||||
"timeout"
|
||||
],
|
||||
"allOf": [
|
||||
{
|
||||
"$ref": "#/definitions/schema.JobState"
|
||||
@ -779,14 +795,6 @@ const docTemplate = `{
|
||||
},
|
||||
"jobState": {
|
||||
"description": "Final state of job",
|
||||
"enum": [
|
||||
"completed",
|
||||
"failed",
|
||||
"cancelled",
|
||||
"stopped",
|
||||
"timeout",
|
||||
"out_of_memory"
|
||||
],
|
||||
"allOf": [
|
||||
{
|
||||
"$ref": "#/definitions/schema.JobState"
|
||||
@ -915,14 +923,6 @@ const docTemplate = `{
|
||||
},
|
||||
"jobState": {
|
||||
"description": "Final state of job",
|
||||
"enum": [
|
||||
"completed",
|
||||
"failed",
|
||||
"cancelled",
|
||||
"stopped",
|
||||
"timeout",
|
||||
"out_of_memory"
|
||||
],
|
||||
"allOf": [
|
||||
{
|
||||
"$ref": "#/definitions/schema.JobState"
|
||||
|
||||
@ -104,10 +104,10 @@ type DeleteJobApiResponse struct {
|
||||
type StopJobApiRequest struct {
|
||||
// Stop Time of job as epoch
|
||||
StopTime int64 `json:"stopTime" validate:"required" example:"1649763839"`
|
||||
State schema.JobState `json:"jobState" validate:"required" example:"completed" enums:"completed,failed,cancelled,stopped,timeout"` // Final job state
|
||||
JobId *int64 `json:"jobId" example:"123000"` // Cluster Job ID of job
|
||||
Cluster *string `json:"cluster" example:"fritz"` // Cluster of job
|
||||
StartTime *int64 `json:"startTime" example:"1649723812"` // Start Time of job as epoch
|
||||
State schema.JobState `json:"jobState" validate:"required" example:"completed"` // Final job state
|
||||
JobId *int64 `json:"jobId" example:"123000"` // Cluster Job ID of job
|
||||
Cluster *string `json:"cluster" example:"fritz"` // Cluster of job
|
||||
StartTime *int64 `json:"startTime" example:"1649723812"` // Start Time of job as epoch
|
||||
}
|
||||
|
||||
// DeleteJobApiRequest model
|
||||
@ -171,6 +171,7 @@ func decode(r io.Reader, val interface{}) error {
|
||||
// @success 200 {object} api.GetJobsApiResponse "Job array and page info"
|
||||
// @failure 400 {object} api.ErrorResponse "Bad Request"
|
||||
// @failure 401 {object} api.ErrorResponse "Unauthorized"
|
||||
// @failure 403 {object} api.ErrorResponse "Forbidden"
|
||||
// @failure 500 {object} api.ErrorResponse "Internal Server Error"
|
||||
// @security ApiKeyAuth
|
||||
// @router /jobs/ [get]
|
||||
@ -738,29 +739,6 @@ func (api *RestApi) checkAndHandleStopJob(rw http.ResponseWriter, job *schema.Jo
|
||||
api.JobRepository.TriggerArchiving(job)
|
||||
}
|
||||
|
||||
// func (api *RestApi) importJob(rw http.ResponseWriter, r *http.Request) {
|
||||
// if user := auth.GetUser(r.Context()); user != nil && !user.HasRole(auth.RoleApi) {
|
||||
// handleError(fmt.Errorf("missing role: %v", auth.RoleApi), http.StatusForbidden, rw)
|
||||
// return
|
||||
// }
|
||||
|
||||
// var body struct {
|
||||
// Meta *schema.JobMeta `json:"meta"`
|
||||
// Data *schema.JobData `json:"data"`
|
||||
// }
|
||||
// if err := decode(r.Body, &body); err != nil {
|
||||
// handleError(fmt.Errorf("import failed: %s", err.Error()), http.StatusBadRequest, rw)
|
||||
// return
|
||||
// }
|
||||
|
||||
// if err := api.JobRepository.ImportJob(body.Meta, body.Data); err != nil {
|
||||
// handleError(fmt.Errorf("import failed: %s", err.Error()), http.StatusUnprocessableEntity, rw)
|
||||
// return
|
||||
// }
|
||||
|
||||
// rw.Write([]byte(`{ "status": "OK" }`))
|
||||
// }
|
||||
|
||||
func (api *RestApi) getJobMetrics(rw http.ResponseWriter, r *http.Request) {
|
||||
id := mux.Vars(r)["id"]
|
||||
metrics := r.URL.Query()["metric"]
|
||||
|
||||
@ -102,7 +102,11 @@ func MigrateDB(backend string, db string) {
|
||||
}
|
||||
|
||||
if err := m.Up(); err != nil {
|
||||
log.Fatal(err)
|
||||
if err == migrate.ErrNoChange {
|
||||
log.Info("DB already up to date!")
|
||||
} else {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
m.Close()
|
||||
|
||||
@ -17,26 +17,26 @@ import (
|
||||
type BaseJob struct {
|
||||
// The unique identifier of a job
|
||||
JobID int64 `json:"jobId" db:"job_id" example:"123000"`
|
||||
User string `json:"user" db:"user" example:"abcd100h"` // The unique identifier of a user
|
||||
Project string `json:"project" db:"project" example:"abcd200"` // The unique identifier of a project
|
||||
Cluster string `json:"cluster" db:"cluster" example:"fritz"` // The unique identifier of a cluster
|
||||
SubCluster string `json:"subCluster" db:"subcluster" example:"main"` // The unique identifier of a sub cluster
|
||||
Partition string `json:"partition" db:"partition" example:"main"` // The Slurm partition to which the job was submitted
|
||||
ArrayJobId int64 `json:"arrayJobId" db:"array_job_id" example:"123000"` // The unique identifier of an array job
|
||||
NumNodes int32 `json:"numNodes" db:"num_nodes" example:"2" minimum:"1"` // Number of nodes used (Min > 0)
|
||||
NumHWThreads int32 `json:"numHwthreads" db:"num_hwthreads" example:"20" minimum:"1"` // Number of HWThreads used (Min > 0)
|
||||
NumAcc int32 `json:"numAcc" db:"num_acc" example:"2" minimum:"1"` // Number of accelerators used (Min > 0)
|
||||
Exclusive int32 `json:"exclusive" db:"exclusive" example:"1" minimum:"0" maximum:"2"` // Specifies how nodes are shared: 0 - Shared among multiple jobs of multiple users, 1 - Job exclusive (Default), 2 - Shared among multiple jobs of same user
|
||||
MonitoringStatus int32 `json:"monitoringStatus" db:"monitoring_status" example:"1" minimum:"0" maximum:"3"` // State of monitoring system during job run: 0 - Disabled, 1 - Running or Archiving (Default), 2 - Archiving Failed, 3 - Archiving Successfull
|
||||
SMT int32 `json:"smt" db:"smt" example:"4"` // SMT threads used by job
|
||||
State JobState `json:"jobState" db:"job_state" example:"completed" enums:"completed,failed,cancelled,stopped,timeout,out_of_memory"` // Final state of job
|
||||
Duration int32 `json:"duration" db:"duration" example:"43200" minimum:"1"` // Duration of job in seconds (Min > 0)
|
||||
Walltime int64 `json:"walltime" db:"walltime" example:"86400" minimum:"1"` // Requested walltime of job in seconds (Min > 0)
|
||||
Tags []*Tag `json:"tags"` // List of tags
|
||||
RawResources []byte `json:"-" db:"resources"` // Resources used by job [As Bytes]
|
||||
Resources []*Resource `json:"resources"` // Resources used by job
|
||||
RawMetaData []byte `json:"-" db:"meta_data"` // Additional information about the job [As Bytes]
|
||||
MetaData map[string]string `json:"metaData"` // Additional information about the job
|
||||
User string `json:"user" db:"user" example:"abcd100h"` // The unique identifier of a user
|
||||
Project string `json:"project" db:"project" example:"abcd200"` // The unique identifier of a project
|
||||
Cluster string `json:"cluster" db:"cluster" example:"fritz"` // The unique identifier of a cluster
|
||||
SubCluster string `json:"subCluster" db:"subcluster" example:"main"` // The unique identifier of a sub cluster
|
||||
Partition string `json:"partition" db:"partition" example:"main"` // The Slurm partition to which the job was submitted
|
||||
ArrayJobId int64 `json:"arrayJobId" db:"array_job_id" example:"123000"` // The unique identifier of an array job
|
||||
NumNodes int32 `json:"numNodes" db:"num_nodes" example:"2" minimum:"1"` // Number of nodes used (Min > 0)
|
||||
NumHWThreads int32 `json:"numHwthreads" db:"num_hwthreads" example:"20" minimum:"1"` // Number of HWThreads used (Min > 0)
|
||||
NumAcc int32 `json:"numAcc" db:"num_acc" example:"2" minimum:"1"` // Number of accelerators used (Min > 0)
|
||||
Exclusive int32 `json:"exclusive" db:"exclusive" example:"1" minimum:"0" maximum:"2"` // Specifies how nodes are shared: 0 - Shared among multiple jobs of multiple users, 1 - Job exclusive (Default), 2 - Shared among multiple jobs of same user
|
||||
MonitoringStatus int32 `json:"monitoringStatus" db:"monitoring_status" example:"1" minimum:"0" maximum:"3"` // State of monitoring system during job run: 0 - Disabled, 1 - Running or Archiving (Default), 2 - Archiving Failed, 3 - Archiving Successfull
|
||||
SMT int32 `json:"smt" db:"smt" example:"4"` // SMT threads used by job
|
||||
State JobState `json:"jobState" db:"job_state" example:"completed"` // Final state of job
|
||||
Duration int32 `json:"duration" db:"duration" example:"43200" minimum:"1"` // Duration of job in seconds (Min > 0)
|
||||
Walltime int64 `json:"walltime" db:"walltime" example:"86400" minimum:"1"` // Requested walltime of job in seconds (Min > 0)
|
||||
Tags []*Tag `json:"tags"` // List of tags
|
||||
RawResources []byte `json:"-" db:"resources"` // Resources used by job [As Bytes]
|
||||
Resources []*Resource `json:"resources"` // Resources used by job
|
||||
RawMetaData []byte `json:"-" db:"meta_data"` // Additional information about the job [As Bytes]
|
||||
MetaData map[string]string `json:"metaData"` // Additional information about the job
|
||||
}
|
||||
|
||||
// Non-Swaggered Comment: Job
|
||||
|
||||
11
startDemo.sh
11
startDemo.sh
@ -4,21 +4,16 @@ if [ -d './var' ]; then
|
||||
echo 'Directory ./var already exists! Skipping initialization.'
|
||||
./cc-backend --server --dev
|
||||
else
|
||||
mkdir ./var
|
||||
cd ./var
|
||||
make
|
||||
|
||||
cd var
|
||||
wget https://hpc-mover.rrze.uni-erlangen.de/HPC-Data/0x7b58aefb/eig7ahyo6fo2bais0ephuf2aitohv1ai/job-archive-dev.tar.xz
|
||||
tar xJf job-archive-dev.tar.xz
|
||||
rm ./job-archive-dev.tar.xz
|
||||
cd ../
|
||||
|
||||
cd ../web/frontend
|
||||
yarn install
|
||||
yarn build
|
||||
|
||||
cd ../..
|
||||
cp ./configs/env-template.txt .env
|
||||
cp ./docs/config.json config.json
|
||||
go build ./cmd/cc-backend
|
||||
|
||||
./cc-backend --migrate-db
|
||||
./cc-backend --server --dev --init-db --add-user demo:admin:AdminDev
|
||||
|
||||
Loading…
Reference in New Issue
Block a user