ClusterCockpit/cc-backend
1
0
Fork 0
mirror of https://github.com/ClusterCockpit/cc-backend synced 2024-09-20 03:17:26 +02:00
Code Issues Packages Releases Activity

Merge branch 'master' of github.com:ClusterCockpit/cc-backend

Browse Source
This commit is contained in:
Jan Eitzinger 2023-03-02 14:13:34 +01:00
commit 8a1288db1a
10 changed files with 124 additions and 146 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
Makefile
View File

@ -1,5 +1,6 @@
TARGET = ./cc-backend
VAR = ./var
DB = ./var/job.db
FRONTEND = ./web/frontend
VERSION = 0.1
GIT_HASH := $(shell git rev-parse --short HEAD || echo 'development')
@ -27,10 +28,9 @@ SVELTE_SRC = $(wildcard $(FRONTEND)/src/*.svelte) \
.NOTPARALLEL:
$(TARGET): $(VAR) $(SVELTE_TARGETS)
$(TARGET): $(VAR) $(DB) $(SVELTE_TARGETS)
$(info ===> BUILD cc-backend)
@go build -ldflags=${LD_FLAGS} ./cmd/cc-backend
./cc-backend --migrate-db
clean:
$(info ===> CLEAN)
@ -47,10 +47,13 @@ tags:
$(info ===> TAGS)
@ctags -R
$(SVELTE_TARGETS): $(SVELTE_SRC)
$(info ===> BUILD frontend)
cd web/frontend && yarn build
$(VAR):
@mkdir $(VAR)
cd web/frontend && yarn install
$(DB):
./cc-backend --migrate-db
$(SVELTE_TARGETS): $(SVELTE_SRC)
$(info ===> BUILD frontend)
cd web/frontend && yarn build

4
README.md
View File

@ -136,7 +136,9 @@ The swagger doc files can be found in `./api/`.
You can generate the configuration of swagger-ui by running `go run github.com/swaggo/swag/cmd/swag init -d ./internal/api,./pkg/schema -g rest.go -o ./api `.
You need to move the generated `./api/doc.go` to `./internal/api/doc.go`.
If you start cc-backend with flag `--dev` the Swagger UI is available at http://localhost:8080/swagger/ .
You have to enter a JWT key for a user with role API. This user must not be logged in the same browser (have a running session), otherwise Swagger requests will not work.
You have to enter a JWT key for a user with role API.
**NOTICE** The user owning the JWT token must not be logged in the same browser (have a running session), otherwise Swagger requests will not work. It is recommended to create a separate user that has just the API role.
## Project Structure

56
api/swagger.json
View File

@ -80,12 +80,9 @@
],
"responses": {
"200": {
"description": "Array of matching jobs",
"description": "Job array and page info",
"schema": {
"type": "array",
"items": {
"$ref": "#/definitions/schema.Job"
}
"$ref": "#/definitions/api.GetJobsApiResponse"
}
},
"400": {
@ -100,6 +97,12 @@
"$ref": "#/definitions/api.ErrorResponse"
}
},
"403": {
"description": "Forbidden",
"schema": {
"$ref": "#/definitions/api.ErrorResponse"
}
},
"500": {
"description": "Internal Server Error",
"schema": {
@ -681,6 +684,26 @@
}
}
},
"api.GetJobsApiResponse": {
"type": "object",
"properties": {
"items": {
"description": "Number of jobs returned",
"type": "integer"
},
"jobs": {
"description": "Array of jobs",
"type": "array",
"items": {
"$ref": "#/definitions/schema.JobMeta"
}
},
"page": {
"description": "Page id returned",
"type": "integer"
}
}
},
"api.StartJobApiResponse": {
"type": "object",
"properties": {
@ -709,13 +732,6 @@
},
"jobState": {
"description": "Final job state",
"enum": [
"completed",
"failed",
"cancelled",
"stopped",
"timeout"
],
"allOf": [
{
"$ref": "#/definitions/schema.JobState"
@ -773,14 +789,6 @@
},
"jobState": {
"description": "Final state of job",
"enum": [
"completed",
"failed",
"cancelled",
"stopped",
"timeout",
"out_of_memory"
],
"allOf": [
{
"$ref": "#/definitions/schema.JobState"
@ -909,14 +917,6 @@
},
"jobState": {
"description": "Final state of job",
"enum": [
"completed",
"failed",
"cancelled",
"stopped",
"timeout",
"out_of_memory"
],
"allOf": [
{
"$ref": "#/definitions/schema.JobState"

44
api/swagger.yaml
View File

@ -42,6 +42,20 @@ definitions:
description: Statustext of Errorcode
type: string
type: object
api.GetJobsApiResponse:
properties:
items:
description: Number of jobs returned
type: integer
jobs:
description: Array of jobs
items:
$ref: '#/definitions/schema.JobMeta'
type: array
page:
description: Page id returned
type: integer
type: object
api.StartJobApiResponse:
properties:
id:
@ -62,12 +76,6 @@ definitions:
allOf:
- $ref: '#/definitions/schema.JobState'
description: Final job state
enum:
- completed
- failed
- cancelled
- stopped
- timeout
example: completed
startTime:
description: Start Time of job as epoch
@ -116,13 +124,6 @@ definitions:
allOf:
- $ref: '#/definitions/schema.JobState'
description: Final state of job
enum:
- completed
- failed
- cancelled
- stopped
- timeout
- out_of_memory
example: completed
metaData:
additionalProperties:
@ -225,13 +226,6 @@ definitions:
allOf:
- $ref: '#/definitions/schema.JobState'
description: Final state of job
enum:
- completed
- failed
- cancelled
- stopped
- timeout
- out_of_memory
example: completed
metaData:
additionalProperties:
@ -438,11 +432,9 @@ paths:
- application/json
responses:
"200":
description: Array of matching jobs
description: Job array and page info
schema:
items:
$ref: '#/definitions/schema.Job'
type: array
$ref: '#/definitions/api.GetJobsApiResponse'
"400":
description: Bad Request
schema:
@ -451,6 +443,10 @@ paths:
description: Unauthorized
schema:
$ref: '#/definitions/api.ErrorResponse'
"403":
description: Forbidden
schema:
$ref: '#/definitions/api.ErrorResponse'
"500":
description: Internal Server Error
schema:

6
cmd/cc-backend/main.go
View File

@ -76,7 +76,7 @@ func main() {
flag.StringVar(&flagDelUser, "del-user", "", "Remove user by `username`")
flag.StringVar(&flagGenJWT, "jwt", "", "Generate and print a JWT for the user specified by its `username`")
flag.StringVar(&flagImportJob, "import-job", "", "Import a job. Argument format: `<path-to-meta.json>:<path-to-data.json>,...`")
flag.StringVar(&flagLogLevel, "loglevel", "debug", "Sets the logging level: `[debug (default),info,warn,err,fatal,crit]`")
flag.StringVar(&flagLogLevel, "loglevel", "warn", "Sets the logging level: `[debug,info,warn (default),err,fatal,crit]`")
flag.Parse()
if flagVersion {
@ -375,9 +375,9 @@ func main() {
MinVersion: tls.VersionTLS12,
PreferServerCipherSuites: true,
})
log.Printf("HTTPS server listening at %s...", config.Keys.Addr)
fmt.Printf("HTTPS server listening at %s...", config.Keys.Addr)
} else {
log.Printf("HTTP server listening at %s...", config.Keys.Addr)
fmt.Printf("HTTP server listening at %s...", config.Keys.Addr)
}
// Because this program will want to bind to a privileged port (like 80), the listener must

56
internal/api/docs.go
View File

@ -86,12 +86,9 @@ const docTemplate = `{
],
"responses": {
"200": {
"description": "Array of matching jobs",
"description": "Job array and page info",
"schema": {
"type": "array",
"items": {
"$ref": "#/definitions/schema.Job"
}
"$ref": "#/definitions/api.GetJobsApiResponse"
}
},
"400": {
@ -106,6 +103,12 @@ const docTemplate = `{
"$ref": "#/definitions/api.ErrorResponse"
}
},
"403": {
"description": "Forbidden",
"schema": {
"$ref": "#/definitions/api.ErrorResponse"
}
},
"500": {
"description": "Internal Server Error",
"schema": {
@ -687,6 +690,26 @@ const docTemplate = `{
}
}
},
"api.GetJobsApiResponse": {
"type": "object",
"properties": {
"items": {
"description": "Number of jobs returned",
"type": "integer"
},
"jobs": {
"description": "Array of jobs",
"type": "array",
"items": {
"$ref": "#/definitions/schema.JobMeta"
}
},
"page": {
"description": "Page id returned",
"type": "integer"
}
}
},
"api.StartJobApiResponse": {
"type": "object",
"properties": {
@ -715,13 +738,6 @@ const docTemplate = `{
},
"jobState": {
"description": "Final job state",
"enum": [
"completed",
"failed",
"cancelled",
"stopped",
"timeout"
],
"allOf": [
{
"$ref": "#/definitions/schema.JobState"
@ -779,14 +795,6 @@ const docTemplate = `{
},
"jobState": {
"description": "Final state of job",
"enum": [
"completed",
"failed",
"cancelled",
"stopped",
"timeout",
"out_of_memory"
],
"allOf": [
{
"$ref": "#/definitions/schema.JobState"
@ -915,14 +923,6 @@ const docTemplate = `{
},
"jobState": {
"description": "Final state of job",
"enum": [
"completed",
"failed",
"cancelled",
"stopped",
"timeout",
"out_of_memory"
],
"allOf": [
{
"$ref": "#/definitions/schema.JobState"

32
internal/api/rest.go
View File

@ -104,10 +104,10 @@ type DeleteJobApiResponse struct {
type StopJobApiRequest struct {
// Stop Time of job as epoch
StopTime int64 `json:"stopTime" validate:"required" example:"1649763839"`
State schema.JobState `json:"jobState" validate:"required" example:"completed" enums:"completed,failed,cancelled,stopped,timeout"` // Final job state
JobId *int64 `json:"jobId" example:"123000"` // Cluster Job ID of job
Cluster *string `json:"cluster" example:"fritz"` // Cluster of job
StartTime *int64 `json:"startTime" example:"1649723812"` // Start Time of job as epoch
State schema.JobState `json:"jobState" validate:"required" example:"completed"` // Final job state
JobId *int64 `json:"jobId" example:"123000"` // Cluster Job ID of job
Cluster *string `json:"cluster" example:"fritz"` // Cluster of job
StartTime *int64 `json:"startTime" example:"1649723812"` // Start Time of job as epoch
}
// DeleteJobApiRequest model
@ -171,6 +171,7 @@ func decode(r io.Reader, val interface{}) error {
// @success 200 {object} api.GetJobsApiResponse "Job array and page info"
// @failure 400 {object} api.ErrorResponse "Bad Request"
// @failure 401 {object} api.ErrorResponse "Unauthorized"
// @failure 403 {object} api.ErrorResponse "Forbidden"
// @failure 500 {object} api.ErrorResponse "Internal Server Error"
// @security ApiKeyAuth
// @router /jobs/ [get]
@ -738,29 +739,6 @@ func (api *RestApi) checkAndHandleStopJob(rw http.ResponseWriter, job *schema.Jo
api.JobRepository.TriggerArchiving(job)
}
// func (api *RestApi) importJob(rw http.ResponseWriter, r *http.Request) {
// if user := auth.GetUser(r.Context()); user != nil && !user.HasRole(auth.RoleApi) {
// handleError(fmt.Errorf("missing role: %v", auth.RoleApi), http.StatusForbidden, rw)
// return
// }
// var body struct {
// Meta *schema.JobMeta `json:"meta"`
// Data *schema.JobData `json:"data"`
// }
// if err := decode(r.Body, &body); err != nil {
// handleError(fmt.Errorf("import failed: %s", err.Error()), http.StatusBadRequest, rw)
// return
// }
// if err := api.JobRepository.ImportJob(body.Meta, body.Data); err != nil {
// handleError(fmt.Errorf("import failed: %s", err.Error()), http.StatusUnprocessableEntity, rw)
// return
// }
// rw.Write([]byte(`{ "status": "OK" }`))
// }
func (api *RestApi) getJobMetrics(rw http.ResponseWriter, r *http.Request) {
id := mux.Vars(r)["id"]
metrics := r.URL.Query()["metric"]

6
internal/repository/migration.go
View File

@ -102,7 +102,11 @@ func MigrateDB(backend string, db string) {
}
if err := m.Up(); err != nil {
log.Fatal(err)
if err == migrate.ErrNoChange {
log.Info("DB already up to date!")
} else {
log.Fatal(err)
}
}
m.Close()

40
pkg/schema/job.go
View File

@ -17,26 +17,26 @@ import (
type BaseJob struct {
// The unique identifier of a job
JobID int64 `json:"jobId" db:"job_id" example:"123000"`
User string `json:"user" db:"user" example:"abcd100h"` // The unique identifier of a user
Project string `json:"project" db:"project" example:"abcd200"` // The unique identifier of a project
Cluster string `json:"cluster" db:"cluster" example:"fritz"` // The unique identifier of a cluster
SubCluster string `json:"subCluster" db:"subcluster" example:"main"` // The unique identifier of a sub cluster
Partition string `json:"partition" db:"partition" example:"main"` // The Slurm partition to which the job was submitted
ArrayJobId int64 `json:"arrayJobId" db:"array_job_id" example:"123000"` // The unique identifier of an array job
NumNodes int32 `json:"numNodes" db:"num_nodes" example:"2" minimum:"1"` // Number of nodes used (Min > 0)
NumHWThreads int32 `json:"numHwthreads" db:"num_hwthreads" example:"20" minimum:"1"` // Number of HWThreads used (Min > 0)
NumAcc int32 `json:"numAcc" db:"num_acc" example:"2" minimum:"1"` // Number of accelerators used (Min > 0)
Exclusive int32 `json:"exclusive" db:"exclusive" example:"1" minimum:"0" maximum:"2"` // Specifies how nodes are shared: 0 - Shared among multiple jobs of multiple users, 1 - Job exclusive (Default), 2 - Shared among multiple jobs of same user
MonitoringStatus int32 `json:"monitoringStatus" db:"monitoring_status" example:"1" minimum:"0" maximum:"3"` // State of monitoring system during job run: 0 - Disabled, 1 - Running or Archiving (Default), 2 - Archiving Failed, 3 - Archiving Successfull
SMT int32 `json:"smt" db:"smt" example:"4"` // SMT threads used by job
State JobState `json:"jobState" db:"job_state" example:"completed" enums:"completed,failed,cancelled,stopped,timeout,out_of_memory"` // Final state of job
Duration int32 `json:"duration" db:"duration" example:"43200" minimum:"1"` // Duration of job in seconds (Min > 0)
Walltime int64 `json:"walltime" db:"walltime" example:"86400" minimum:"1"` // Requested walltime of job in seconds (Min > 0)
Tags []*Tag `json:"tags"` // List of tags
RawResources []byte `json:"-" db:"resources"` // Resources used by job [As Bytes]
Resources []*Resource `json:"resources"` // Resources used by job
RawMetaData []byte `json:"-" db:"meta_data"` // Additional information about the job [As Bytes]
MetaData map[string]string `json:"metaData"` // Additional information about the job
User string `json:"user" db:"user" example:"abcd100h"` // The unique identifier of a user
Project string `json:"project" db:"project" example:"abcd200"` // The unique identifier of a project
Cluster string `json:"cluster" db:"cluster" example:"fritz"` // The unique identifier of a cluster
SubCluster string `json:"subCluster" db:"subcluster" example:"main"` // The unique identifier of a sub cluster
Partition string `json:"partition" db:"partition" example:"main"` // The Slurm partition to which the job was submitted
ArrayJobId int64 `json:"arrayJobId" db:"array_job_id" example:"123000"` // The unique identifier of an array job
NumNodes int32 `json:"numNodes" db:"num_nodes" example:"2" minimum:"1"` // Number of nodes used (Min > 0)
NumHWThreads int32 `json:"numHwthreads" db:"num_hwthreads" example:"20" minimum:"1"` // Number of HWThreads used (Min > 0)
NumAcc int32 `json:"numAcc" db:"num_acc" example:"2" minimum:"1"` // Number of accelerators used (Min > 0)
Exclusive int32 `json:"exclusive" db:"exclusive" example:"1" minimum:"0" maximum:"2"` // Specifies how nodes are shared: 0 - Shared among multiple jobs of multiple users, 1 - Job exclusive (Default), 2 - Shared among multiple jobs of same user
MonitoringStatus int32 `json:"monitoringStatus" db:"monitoring_status" example:"1" minimum:"0" maximum:"3"` // State of monitoring system during job run: 0 - Disabled, 1 - Running or Archiving (Default), 2 - Archiving Failed, 3 - Archiving Successfull
SMT int32 `json:"smt" db:"smt" example:"4"` // SMT threads used by job
State JobState `json:"jobState" db:"job_state" example:"completed"` // Final state of job
Duration int32 `json:"duration" db:"duration" example:"43200" minimum:"1"` // Duration of job in seconds (Min > 0)
Walltime int64 `json:"walltime" db:"walltime" example:"86400" minimum:"1"` // Requested walltime of job in seconds (Min > 0)
Tags []*Tag `json:"tags"` // List of tags
RawResources []byte `json:"-" db:"resources"` // Resources used by job [As Bytes]
Resources []*Resource `json:"resources"` // Resources used by job
RawMetaData []byte `json:"-" db:"meta_data"` // Additional information about the job [As Bytes]
MetaData map[string]string `json:"metaData"` // Additional information about the job
}
// Non-Swaggered Comment: Job

11
startDemo.sh
View File

@ -4,21 +4,16 @@ if [ -d './var' ]; then
echo 'Directory ./var already exists! Skipping initialization.'
./cc-backend --server --dev
else
mkdir ./var
cd ./var
make
cd var
wget https://hpc-mover.rrze.uni-erlangen.de/HPC-Data/0x7b58aefb/eig7ahyo6fo2bais0ephuf2aitohv1ai/job-archive-dev.tar.xz
tar xJf job-archive-dev.tar.xz
rm ./job-archive-dev.tar.xz
cd ../
cd ../web/frontend
yarn install
yarn build
cd ../..
cp ./configs/env-template.txt .env
cp ./docs/config.json config.json
go build ./cmd/cc-backend
./cc-backend --migrate-db
./cc-backend --server --dev --init-db --add-user demo:admin:AdminDev