mirror of
https://github.com/ClusterCockpit/cc-backend
synced 2024-12-25 04:49:05 +01:00
Merge branch 'master' into 40_45_82_update_roles
This commit is contained in:
commit
df44bd9621
15
Makefile
15
Makefile
@ -1,5 +1,6 @@
|
||||
TARGET = ./cc-backend
|
||||
VAR = ./var
|
||||
DB = ./var/job.db
|
||||
FRONTEND = ./web/frontend
|
||||
VERSION = 0.1
|
||||
GIT_HASH := $(shell git rev-parse --short HEAD || echo 'development')
|
||||
@ -27,10 +28,9 @@ SVELTE_SRC = $(wildcard $(FRONTEND)/src/*.svelte) \
|
||||
|
||||
.NOTPARALLEL:
|
||||
|
||||
$(TARGET): $(VAR) $(SVELTE_TARGETS)
|
||||
$(TARGET): $(VAR) $(DB) $(SVELTE_TARGETS)
|
||||
$(info ===> BUILD cc-backend)
|
||||
@go build -ldflags=${LD_FLAGS} ./cmd/cc-backend
|
||||
./cc-backend --migrate-db
|
||||
|
||||
clean:
|
||||
$(info ===> CLEAN)
|
||||
@ -43,10 +43,13 @@ test:
|
||||
@go vet ./...
|
||||
@go test ./...
|
||||
|
||||
$(SVELTE_TARGETS): $(SVELTE_SRC)
|
||||
$(info ===> BUILD frontend)
|
||||
cd web/frontend && yarn build
|
||||
|
||||
$(VAR):
|
||||
@mkdir $(VAR)
|
||||
cd web/frontend && yarn install
|
||||
|
||||
$(DB):
|
||||
./cc-backend --migrate-db
|
||||
|
||||
$(SVELTE_TARGETS): $(SVELTE_SRC)
|
||||
$(info ===> BUILD frontend)
|
||||
cd web/frontend && yarn build
|
||||
|
@ -136,7 +136,9 @@ The swagger doc files can be found in `./api/`.
|
||||
You can generate the configuration of swagger-ui by running `go run github.com/swaggo/swag/cmd/swag init -d ./internal/api,./pkg/schema -g rest.go -o ./api `.
|
||||
You need to move the generated `./api/doc.go` to `./internal/api/doc.go`.
|
||||
If you start cc-backend with flag `--dev` the Swagger UI is available at http://localhost:8080/swagger/ .
|
||||
You have to enter a JWT key for a user with role API. This user must not be logged in the same browser (have a running session), otherwise Swagger requests will not work.
|
||||
You have to enter a JWT key for a user with role API.
|
||||
|
||||
**NOTICE** The user owning the JWT token must not be logged in the same browser (have a running session), otherwise Swagger requests will not work. It is recommended to create a separate user that has just the API role.
|
||||
|
||||
## Project Structure
|
||||
|
||||
|
@ -80,12 +80,9 @@
|
||||
],
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "Array of matching jobs",
|
||||
"description": "Job array and page info",
|
||||
"schema": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/definitions/schema.Job"
|
||||
}
|
||||
"$ref": "#/definitions/api.GetJobsApiResponse"
|
||||
}
|
||||
},
|
||||
"400": {
|
||||
@ -681,6 +678,26 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"api.GetJobsApiResponse": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"items": {
|
||||
"description": "Number of jobs returned",
|
||||
"type": "integer"
|
||||
},
|
||||
"jobs": {
|
||||
"description": "Array of jobs",
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/definitions/schema.JobMeta"
|
||||
}
|
||||
},
|
||||
"page": {
|
||||
"description": "Page id returned",
|
||||
"type": "integer"
|
||||
}
|
||||
}
|
||||
},
|
||||
"api.StartJobApiResponse": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
|
@ -42,6 +42,20 @@ definitions:
|
||||
description: Statustext of Errorcode
|
||||
type: string
|
||||
type: object
|
||||
api.GetJobsApiResponse:
|
||||
properties:
|
||||
items:
|
||||
description: Number of jobs returned
|
||||
type: integer
|
||||
jobs:
|
||||
description: Array of jobs
|
||||
items:
|
||||
$ref: '#/definitions/schema.JobMeta'
|
||||
type: array
|
||||
page:
|
||||
description: Page id returned
|
||||
type: integer
|
||||
type: object
|
||||
api.StartJobApiResponse:
|
||||
properties:
|
||||
id:
|
||||
@ -438,11 +452,9 @@ paths:
|
||||
- application/json
|
||||
responses:
|
||||
"200":
|
||||
description: Array of matching jobs
|
||||
description: Job array and page info
|
||||
schema:
|
||||
items:
|
||||
$ref: '#/definitions/schema.Job'
|
||||
type: array
|
||||
$ref: '#/definitions/api.GetJobsApiResponse'
|
||||
"400":
|
||||
description: Bad Request
|
||||
schema:
|
||||
|
@ -76,7 +76,7 @@ func main() {
|
||||
flag.StringVar(&flagDelUser, "del-user", "", "Remove user by `username`")
|
||||
flag.StringVar(&flagGenJWT, "jwt", "", "Generate and print a JWT for the user specified by its `username`")
|
||||
flag.StringVar(&flagImportJob, "import-job", "", "Import a job. Argument format: `<path-to-meta.json>:<path-to-data.json>,...`")
|
||||
flag.StringVar(&flagLogLevel, "loglevel", "debug", "Sets the logging level: `[debug (default),info,warn,err,fatal,crit]`")
|
||||
flag.StringVar(&flagLogLevel, "loglevel", "warn", "Sets the logging level: `[debug,info,warn (default),err,fatal,crit]`")
|
||||
flag.Parse()
|
||||
|
||||
if flagVersion {
|
||||
@ -375,9 +375,9 @@ func main() {
|
||||
MinVersion: tls.VersionTLS12,
|
||||
PreferServerCipherSuites: true,
|
||||
})
|
||||
log.Printf("HTTPS server listening at %s...", config.Keys.Addr)
|
||||
fmt.Printf("HTTPS server listening at %s...", config.Keys.Addr)
|
||||
} else {
|
||||
log.Printf("HTTP server listening at %s...", config.Keys.Addr)
|
||||
fmt.Printf("HTTP server listening at %s...", config.Keys.Addr)
|
||||
}
|
||||
|
||||
// Because this program will want to bind to a privileged port (like 80), the listener must
|
||||
|
@ -1,6 +1,6 @@
|
||||
## Intro
|
||||
|
||||
cc-backend requires a configuration file speciyfing the cluster systems to be used. Still many default
|
||||
cc-backend requires a configuration file specifying the cluster systems to be used. Still many default
|
||||
options documented below are used. cc-backend tries to load a config.json from the working directory per default.
|
||||
To overwrite the default specify a json config file location using the command line option `--config <filepath>`.
|
||||
All security relevant configuration. e.g., keys and passwords, are set using environment variables.
|
||||
@ -44,9 +44,10 @@ It is supported to specify these by means of an `.env` file located in the proje
|
||||
"startTime": { "from": "2022-01-01T00:00:00Z", "to": null }
|
||||
}
|
||||
```
|
||||
* `ui-defaults`: Type object. Default configuration for ui views. If overwriten, all options must be provided! Most options can be overwritten by the user via the web interface.
|
||||
* `ui-defaults`: Type object. Default configuration for ui views. If overwritten, all options must be provided! Most options can be overwritten by the user via the web interface.
|
||||
- `analysis_view_histogramMetrics`: Type string array. Metrics to show as job count histograms in analysis view. Default `["flops_any", "mem_bw", "mem_used"]`.
|
||||
- `analysis_view_scatterPlotMetrics`: Type array of string array. Initial scatter plto configuration in analysis view. Default `[["flops_any", "mem_bw"], ["flops_any", "cpu_load"], ["cpu_load", "mem_bw"]]`.
|
||||
- `analysis_view_scatterPlotMetrics`: Type array of string array. Initial
|
||||
scatter plot configuration in analysis view. Default `[["flops_any", "mem_bw"], ["flops_any", "cpu_load"], ["cpu_load", "mem_bw"]]`.
|
||||
- `job_view_nodestats_selectedMetrics`: Type string array. Initial metrics shown in node statistics table of single job view. Default `["flops_any", "mem_bw", "mem_used"]`.
|
||||
- `job_view_polarPlotMetrics`: Type string array. Metrics shown in polar plot of single job view. Default `["flops_any", "mem_bw", "mem_used", "net_bw", "file_bw"]`.
|
||||
- `job_view_selectedMetrics`: Type string array. Default `["flops_any", "mem_bw", "mem_used"]`.
|
||||
|
@ -86,12 +86,9 @@ const docTemplate = `{
|
||||
],
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "Array of matching jobs",
|
||||
"description": "Job array and page info",
|
||||
"schema": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/definitions/schema.Job"
|
||||
}
|
||||
"$ref": "#/definitions/api.GetJobsApiResponse"
|
||||
}
|
||||
},
|
||||
"400": {
|
||||
@ -687,6 +684,26 @@ const docTemplate = `{
|
||||
}
|
||||
}
|
||||
},
|
||||
"api.GetJobsApiResponse": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"items": {
|
||||
"description": "Number of jobs returned",
|
||||
"type": "integer"
|
||||
},
|
||||
"jobs": {
|
||||
"description": "Array of jobs",
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/definitions/schema.JobMeta"
|
||||
}
|
||||
},
|
||||
"page": {
|
||||
"description": "Page id returned",
|
||||
"type": "integer"
|
||||
}
|
||||
}
|
||||
},
|
||||
"api.StartJobApiResponse": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
|
@ -118,6 +118,13 @@ type DeleteJobApiRequest struct {
|
||||
StartTime *int64 `json:"startTime" example:"1649723812"` // Start Time of job as epoch
|
||||
}
|
||||
|
||||
// GetJobsApiResponse model
|
||||
type GetJobsApiResponse struct {
|
||||
Jobs []*schema.JobMeta `json:"jobs"` // Array of jobs
|
||||
Items int `json:"items"` // Number of jobs returned
|
||||
Page int `json:"page"` // Page id returned
|
||||
}
|
||||
|
||||
// ErrorResponse model
|
||||
type ErrorResponse struct {
|
||||
// Statustext of Errorcode
|
||||
@ -162,9 +169,10 @@ func decode(r io.Reader, val interface{}) error {
|
||||
// @param items-per-page query int false "Items per page (Default: 25)"
|
||||
// @param page query int false "Page Number (Default: 1)"
|
||||
// @param with-metadata query bool false "Include metadata (e.g. jobScript) in response"
|
||||
// @success 200 {array} schema.Job "Array of matching jobs"
|
||||
// @success 200 {object} api.GetJobsApiResponse "Job array and page info"
|
||||
// @failure 400 {object} api.ErrorResponse "Bad Request"
|
||||
// @failure 401 {object} api.ErrorResponse "Unauthorized"
|
||||
// @failure 403 {object} api.ErrorResponse "Forbidden"
|
||||
// @failure 500 {object} api.ErrorResponse "Internal Server Error"
|
||||
// @security ApiKeyAuth
|
||||
// @router /jobs/ [get]
|
||||
@ -185,7 +193,8 @@ func (api *RestApi) getJobs(rw http.ResponseWriter, r *http.Request) {
|
||||
for _, s := range vals {
|
||||
state := schema.JobState(s)
|
||||
if !state.Valid() {
|
||||
http.Error(rw, "invalid query parameter value: state", http.StatusBadRequest)
|
||||
handleError(fmt.Errorf("invalid query parameter value: state"),
|
||||
http.StatusBadRequest, rw)
|
||||
return
|
||||
}
|
||||
filter.State = append(filter.State, state)
|
||||
@ -195,17 +204,18 @@ func (api *RestApi) getJobs(rw http.ResponseWriter, r *http.Request) {
|
||||
case "start-time":
|
||||
st := strings.Split(vals[0], "-")
|
||||
if len(st) != 2 {
|
||||
http.Error(rw, "invalid query parameter value: startTime", http.StatusBadRequest)
|
||||
handleError(fmt.Errorf("invalid query parameter value: startTime"),
|
||||
http.StatusBadRequest, rw)
|
||||
return
|
||||
}
|
||||
from, err := strconv.ParseInt(st[0], 10, 64)
|
||||
if err != nil {
|
||||
http.Error(rw, err.Error(), http.StatusBadRequest)
|
||||
handleError(err, http.StatusBadRequest, rw)
|
||||
return
|
||||
}
|
||||
to, err := strconv.ParseInt(st[1], 10, 64)
|
||||
if err != nil {
|
||||
http.Error(rw, err.Error(), http.StatusBadRequest)
|
||||
handleError(err, http.StatusBadRequest, rw)
|
||||
return
|
||||
}
|
||||
ufrom, uto := time.Unix(from, 0), time.Unix(to, 0)
|
||||
@ -213,28 +223,29 @@ func (api *RestApi) getJobs(rw http.ResponseWriter, r *http.Request) {
|
||||
case "page":
|
||||
x, err := strconv.Atoi(vals[0])
|
||||
if err != nil {
|
||||
http.Error(rw, err.Error(), http.StatusBadRequest)
|
||||
handleError(err, http.StatusBadRequest, rw)
|
||||
return
|
||||
}
|
||||
page.Page = x
|
||||
case "items-per-page":
|
||||
x, err := strconv.Atoi(vals[0])
|
||||
if err != nil {
|
||||
http.Error(rw, err.Error(), http.StatusBadRequest)
|
||||
handleError(err, http.StatusBadRequest, rw)
|
||||
return
|
||||
}
|
||||
page.ItemsPerPage = x
|
||||
case "with-metadata":
|
||||
withMetadata = true
|
||||
default:
|
||||
http.Error(rw, "invalid query parameter: "+key, http.StatusBadRequest)
|
||||
handleError(fmt.Errorf("invalid query parameter: %s", key),
|
||||
http.StatusBadRequest, rw)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
jobs, err := api.JobRepository.QueryJobs(r.Context(), []*model.JobFilter{filter}, page, order)
|
||||
if err != nil {
|
||||
http.Error(rw, err.Error(), http.StatusInternalServerError)
|
||||
handleError(err, http.StatusInternalServerError, rw)
|
||||
return
|
||||
}
|
||||
|
||||
@ -242,7 +253,7 @@ func (api *RestApi) getJobs(rw http.ResponseWriter, r *http.Request) {
|
||||
for _, job := range jobs {
|
||||
if withMetadata {
|
||||
if _, err := api.JobRepository.FetchMetadata(job); err != nil {
|
||||
http.Error(rw, err.Error(), http.StatusInternalServerError)
|
||||
handleError(err, http.StatusInternalServerError, rw)
|
||||
return
|
||||
}
|
||||
}
|
||||
@ -255,7 +266,7 @@ func (api *RestApi) getJobs(rw http.ResponseWriter, r *http.Request) {
|
||||
|
||||
res.Tags, err = api.JobRepository.GetTags(&job.ID)
|
||||
if err != nil {
|
||||
http.Error(rw, err.Error(), http.StatusInternalServerError)
|
||||
handleError(err, http.StatusInternalServerError, rw)
|
||||
return
|
||||
}
|
||||
|
||||
@ -263,7 +274,7 @@ func (api *RestApi) getJobs(rw http.ResponseWriter, r *http.Request) {
|
||||
res.Statistics, err = archive.GetStatistics(job)
|
||||
if err != nil {
|
||||
if err != nil {
|
||||
http.Error(rw, err.Error(), http.StatusInternalServerError)
|
||||
handleError(err, http.StatusInternalServerError, rw)
|
||||
return
|
||||
}
|
||||
}
|
||||
@ -273,12 +284,18 @@ func (api *RestApi) getJobs(rw http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
|
||||
log.Debugf("/api/jobs: %d jobs returned", len(results))
|
||||
rw.Header().Add("Content-Type", "application/json")
|
||||
bw := bufio.NewWriter(rw)
|
||||
defer bw.Flush()
|
||||
if err := json.NewEncoder(bw).Encode(map[string]interface{}{
|
||||
"jobs": results,
|
||||
}); err != nil {
|
||||
http.Error(rw, err.Error(), http.StatusInternalServerError)
|
||||
|
||||
payload := GetJobsApiResponse{
|
||||
Jobs: results,
|
||||
Items: page.ItemsPerPage,
|
||||
Page: page.Page,
|
||||
}
|
||||
|
||||
if err := json.NewEncoder(bw).Encode(payload); err != nil {
|
||||
handleError(err, http.StatusInternalServerError, rw)
|
||||
return
|
||||
}
|
||||
}
|
||||
@ -723,29 +740,6 @@ func (api *RestApi) checkAndHandleStopJob(rw http.ResponseWriter, job *schema.Jo
|
||||
api.JobRepository.TriggerArchiving(job)
|
||||
}
|
||||
|
||||
// func (api *RestApi) importJob(rw http.ResponseWriter, r *http.Request) {
|
||||
// if user := auth.GetUser(r.Context()); user != nil && !user.HasRole(auth.RoleApi) {
|
||||
// handleError(fmt.Errorf("missing role: %v", auth.RoleApi), http.StatusForbidden, rw)
|
||||
// return
|
||||
// }
|
||||
|
||||
// var body struct {
|
||||
// Meta *schema.JobMeta `json:"meta"`
|
||||
// Data *schema.JobData `json:"data"`
|
||||
// }
|
||||
// if err := decode(r.Body, &body); err != nil {
|
||||
// handleError(fmt.Errorf("import failed: %s", err.Error()), http.StatusBadRequest, rw)
|
||||
// return
|
||||
// }
|
||||
|
||||
// if err := api.JobRepository.ImportJob(body.Meta, body.Data); err != nil {
|
||||
// handleError(fmt.Errorf("import failed: %s", err.Error()), http.StatusUnprocessableEntity, rw)
|
||||
// return
|
||||
// }
|
||||
|
||||
// rw.Write([]byte(`{ "status": "OK" }`))
|
||||
// }
|
||||
|
||||
func (api *RestApi) getJobMetrics(rw http.ResponseWriter, r *http.Request) {
|
||||
id := mux.Vars(r)["id"]
|
||||
metrics := r.URL.Query()["metric"]
|
||||
@ -794,7 +788,8 @@ func (api *RestApi) getJWT(rw http.ResponseWriter, r *http.Request) {
|
||||
me := auth.GetUser(r.Context())
|
||||
if !me.HasRole(auth.RoleAdmin) {
|
||||
if username != me.Username {
|
||||
http.Error(rw, "Only admins are allowed to sign JWTs not for themselves", http.StatusForbidden)
|
||||
http.Error(rw, "Only admins are allowed to sign JWTs not for themselves",
|
||||
http.StatusForbidden)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
@ -102,8 +102,12 @@ func MigrateDB(backend string, db string) {
|
||||
}
|
||||
|
||||
if err := m.Up(); err != nil {
|
||||
if err == migrate.ErrNoChange {
|
||||
log.Info("DB already up to date!")
|
||||
} else {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
m.Close()
|
||||
}
|
||||
|
11
startDemo.sh
11
startDemo.sh
@ -4,21 +4,16 @@ if [ -d './var' ]; then
|
||||
echo 'Directory ./var already exists! Skipping initialization.'
|
||||
./cc-backend --server --dev
|
||||
else
|
||||
mkdir ./var
|
||||
cd ./var
|
||||
make
|
||||
|
||||
cd var
|
||||
wget https://hpc-mover.rrze.uni-erlangen.de/HPC-Data/0x7b58aefb/eig7ahyo6fo2bais0ephuf2aitohv1ai/job-archive-dev.tar.xz
|
||||
tar xJf job-archive-dev.tar.xz
|
||||
rm ./job-archive-dev.tar.xz
|
||||
cd ../
|
||||
|
||||
cd ../web/frontend
|
||||
yarn install
|
||||
yarn build
|
||||
|
||||
cd ../..
|
||||
cp ./configs/env-template.txt .env
|
||||
cp ./docs/config.json config.json
|
||||
go build ./cmd/cc-backend
|
||||
|
||||
./cc-backend --migrate-db
|
||||
./cc-backend --server --dev --init-db --add-user demo:admin:AdminDev
|
||||
|
@ -416,7 +416,7 @@ func TestRestApi(t *testing.T) {
|
||||
}
|
||||
|
||||
const stopJobBody string = `{
|
||||
"jobId": 123,
|
||||
"jobId": 123,
|
||||
"startTime": 123456789,
|
||||
"cluster": "testcluster",
|
||||
|
||||
|
Loading…
Reference in New Issue
Block a user