ClusterCockpit/cc-backend
1
0
Fork 0
mirror of https://github.com/ClusterCockpit/cc-backend synced 2024-12-24 12:29:05 +01:00
Code Issues Packages Releases Activity

Merge branch 'master' into 40_45_82_update_roles

Browse Source
This commit is contained in:
Christoph Kluge 2023-02-28 11:18:46 +01:00
commit df44bd9621
11 changed files with 123 additions and 77 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
Makefile
View File

@ -1,5 +1,6 @@
TARGET = ./cc-backend
VAR = ./var
DB = ./var/job.db
FRONTEND = ./web/frontend
VERSION = 0.1
GIT_HASH := $(shell git rev-parse --short HEAD || echo 'development')
@ -27,10 +28,9 @@ SVELTE_SRC = $(wildcard $(FRONTEND)/src/*.svelte) \
.NOTPARALLEL:
$(TARGET): $(VAR) $(SVELTE_TARGETS)
$(TARGET): $(VAR) $(DB) $(SVELTE_TARGETS)
$(info ===> BUILD cc-backend)
@go build -ldflags=${LD_FLAGS} ./cmd/cc-backend
./cc-backend --migrate-db
clean:
$(info ===> CLEAN)
@ -43,10 +43,13 @@ test:
@go vet ./...
@go test ./...
$(SVELTE_TARGETS): $(SVELTE_SRC)
$(info ===> BUILD frontend)
cd web/frontend && yarn build
$(VAR):
@mkdir $(VAR)
cd web/frontend && yarn install
$(DB):
./cc-backend --migrate-db
$(SVELTE_TARGETS): $(SVELTE_SRC)
$(info ===> BUILD frontend)
cd web/frontend && yarn build

4
README.md
View File

@ -136,7 +136,9 @@ The swagger doc files can be found in `./api/`.
You can generate the configuration of swagger-ui by running `go run github.com/swaggo/swag/cmd/swag init -d ./internal/api,./pkg/schema -g rest.go -o ./api `.
You need to move the generated `./api/doc.go` to `./internal/api/doc.go`.
If you start cc-backend with flag `--dev` the Swagger UI is available at http://localhost:8080/swagger/ .
You have to enter a JWT key for a user with role API. This user must not be logged in the same browser (have a running session), otherwise Swagger requests will not work.
You have to enter a JWT key for a user with role API.
**NOTICE** The user owning the JWT token must not be logged in the same browser (have a running session), otherwise Swagger requests will not work. It is recommended to create a separate user that has just the API role.
## Project Structure

27
api/swagger.json
View File

@ -80,12 +80,9 @@
],
"responses": {
"200": {
"description": "Array of matching jobs",
"description": "Job array and page info",
"schema": {
"type": "array",
"items": {
"$ref": "#/definitions/schema.Job"
}
"$ref": "#/definitions/api.GetJobsApiResponse"
}
},
"400": {
@ -681,6 +678,26 @@
}
}
},
"api.GetJobsApiResponse": {
"type": "object",
"properties": {
"items": {
"description": "Number of jobs returned",
"type": "integer"
},
"jobs": {
"description": "Array of jobs",
"type": "array",
"items": {
"$ref": "#/definitions/schema.JobMeta"
}
},
"page": {
"description": "Page id returned",
"type": "integer"
}
}
},
"api.StartJobApiResponse": {
"type": "object",
"properties": {

20
api/swagger.yaml
View File

@ -42,6 +42,20 @@ definitions:
description: Statustext of Errorcode
type: string
type: object
api.GetJobsApiResponse:
properties:
items:
description: Number of jobs returned
type: integer
jobs:
description: Array of jobs
items:
$ref: '#/definitions/schema.JobMeta'
type: array
page:
description: Page id returned
type: integer
type: object
api.StartJobApiResponse:
properties:
id:
@ -438,11 +452,9 @@ paths:
- application/json
responses:
"200":
description: Array of matching jobs
description: Job array and page info
schema:
items:
$ref: '#/definitions/schema.Job'
type: array
$ref: '#/definitions/api.GetJobsApiResponse'
"400":
description: Bad Request
schema:

6
cmd/cc-backend/main.go
View File

@ -76,7 +76,7 @@ func main() {
flag.StringVar(&flagDelUser, "del-user", "", "Remove user by `username`")
flag.StringVar(&flagGenJWT, "jwt", "", "Generate and print a JWT for the user specified by its `username`")
flag.StringVar(&flagImportJob, "import-job", "", "Import a job. Argument format: `<path-to-meta.json>:<path-to-data.json>,...`")
flag.StringVar(&flagLogLevel, "loglevel", "debug", "Sets the logging level: `[debug (default),info,warn,err,fatal,crit]`")
flag.StringVar(&flagLogLevel, "loglevel", "warn", "Sets the logging level: `[debug,info,warn (default),err,fatal,crit]`")
flag.Parse()
if flagVersion {
@ -375,9 +375,9 @@ func main() {
MinVersion: tls.VersionTLS12,
PreferServerCipherSuites: true,
})
log.Printf("HTTPS server listening at %s...", config.Keys.Addr)
fmt.Printf("HTTPS server listening at %s...", config.Keys.Addr)
} else {
log.Printf("HTTP server listening at %s...", config.Keys.Addr)
fmt.Printf("HTTP server listening at %s...", config.Keys.Addr)
}
// Because this program will want to bind to a privileged port (like 80), the listener must

7
configs/README.md
View File

@ -1,6 +1,6 @@
## Intro
cc-backend requires a configuration file speciyfing the cluster systems to be used. Still many default
cc-backend requires a configuration file specifying the cluster systems to be used. Still many default
options documented below are used. cc-backend tries to load a config.json from the working directory per default.
To overwrite the default specify a json config file location using the command line option `--config <filepath>`.
All security relevant configuration. e.g., keys and passwords, are set using environment variables.
@ -44,9 +44,10 @@ It is supported to specify these by means of an `.env` file located in the proje
"startTime": { "from": "2022-01-01T00:00:00Z", "to": null }
}
```
* `ui-defaults`: Type object. Default configuration for ui views. If overwriten, all options must be provided! Most options can be overwritten by the user via the web interface.
* `ui-defaults`: Type object. Default configuration for ui views. If overwritten, all options must be provided! Most options can be overwritten by the user via the web interface.
- `analysis_view_histogramMetrics`: Type string array. Metrics to show as job count histograms in analysis view. Default `["flops_any", "mem_bw", "mem_used"]`.
- `analysis_view_scatterPlotMetrics`: Type array of string array. Initial scatter plto configuration in analysis view. Default `[["flops_any", "mem_bw"], ["flops_any", "cpu_load"], ["cpu_load", "mem_bw"]]`.
- `analysis_view_scatterPlotMetrics`: Type array of string array. Initial
scatter plot configuration in analysis view. Default `[["flops_any", "mem_bw"], ["flops_any", "cpu_load"], ["cpu_load", "mem_bw"]]`.
- `job_view_nodestats_selectedMetrics`: Type string array. Initial metrics shown in node statistics table of single job view. Default `["flops_any", "mem_bw", "mem_used"]`.
- `job_view_polarPlotMetrics`: Type string array. Metrics shown in polar plot of single job view. Default `["flops_any", "mem_bw", "mem_used", "net_bw", "file_bw"]`.
- `job_view_selectedMetrics`: Type string array. Default `["flops_any", "mem_bw", "mem_used"]`.

27
internal/api/docs.go
View File

@ -86,12 +86,9 @@ const docTemplate = `{
],
"responses": {
"200": {
"description": "Array of matching jobs",
"description": "Job array and page info",
"schema": {
"type": "array",
"items": {
"$ref": "#/definitions/schema.Job"
}
"$ref": "#/definitions/api.GetJobsApiResponse"
}
},
"400": {
@ -687,6 +684,26 @@ const docTemplate = `{
}
}
},
"api.GetJobsApiResponse": {
"type": "object",
"properties": {
"items": {
"description": "Number of jobs returned",
"type": "integer"
},
"jobs": {
"description": "Array of jobs",
"type": "array",
"items": {
"$ref": "#/definitions/schema.JobMeta"
}
},
"page": {
"description": "Page id returned",
"type": "integer"
}
}
},
"api.StartJobApiResponse": {
"type": "object",
"properties": {

75
internal/api/rest.go
View File

@ -118,6 +118,13 @@ type DeleteJobApiRequest struct {
StartTime *int64 `json:"startTime" example:"1649723812"` // Start Time of job as epoch
}
// GetJobsApiResponse model
type GetJobsApiResponse struct {
Jobs []*schema.JobMeta `json:"jobs"` // Array of jobs
Items int `json:"items"` // Number of jobs returned
Page int `json:"page"` // Page id returned
}
// ErrorResponse model
type ErrorResponse struct {
// Statustext of Errorcode
@ -162,9 +169,10 @@ func decode(r io.Reader, val interface{}) error {
// @param items-per-page query int false "Items per page (Default: 25)"
// @param page query int false "Page Number (Default: 1)"
// @param with-metadata query bool false "Include metadata (e.g. jobScript) in response"
// @success 200 {array} schema.Job "Array of matching jobs"
// @success 200 {object} api.GetJobsApiResponse "Job array and page info"
// @failure 400 {object} api.ErrorResponse "Bad Request"
// @failure 401 {object} api.ErrorResponse "Unauthorized"
// @failure 403 {object} api.ErrorResponse "Forbidden"
// @failure 500 {object} api.ErrorResponse "Internal Server Error"
// @security ApiKeyAuth
// @router /jobs/ [get]
@ -185,7 +193,8 @@ func (api *RestApi) getJobs(rw http.ResponseWriter, r *http.Request) {
for _, s := range vals {
state := schema.JobState(s)
if !state.Valid() {
http.Error(rw, "invalid query parameter value: state", http.StatusBadRequest)
handleError(fmt.Errorf("invalid query parameter value: state"),
http.StatusBadRequest, rw)
return
}
filter.State = append(filter.State, state)
@ -195,17 +204,18 @@ func (api *RestApi) getJobs(rw http.ResponseWriter, r *http.Request) {
case "start-time":
st := strings.Split(vals[0], "-")
if len(st) != 2 {
http.Error(rw, "invalid query parameter value: startTime", http.StatusBadRequest)
handleError(fmt.Errorf("invalid query parameter value: startTime"),
http.StatusBadRequest, rw)
return
}
from, err := strconv.ParseInt(st[0], 10, 64)
if err != nil {
http.Error(rw, err.Error(), http.StatusBadRequest)
handleError(err, http.StatusBadRequest, rw)
return
}
to, err := strconv.ParseInt(st[1], 10, 64)
if err != nil {
http.Error(rw, err.Error(), http.StatusBadRequest)
handleError(err, http.StatusBadRequest, rw)
return
}
ufrom, uto := time.Unix(from, 0), time.Unix(to, 0)
@ -213,28 +223,29 @@ func (api *RestApi) getJobs(rw http.ResponseWriter, r *http.Request) {
case "page":
x, err := strconv.Atoi(vals[0])
if err != nil {
http.Error(rw, err.Error(), http.StatusBadRequest)
handleError(err, http.StatusBadRequest, rw)
return
}
page.Page = x
case "items-per-page":
x, err := strconv.Atoi(vals[0])
if err != nil {
http.Error(rw, err.Error(), http.StatusBadRequest)
handleError(err, http.StatusBadRequest, rw)
return
}
page.ItemsPerPage = x
case "with-metadata":
withMetadata = true
default:
http.Error(rw, "invalid query parameter: "+key, http.StatusBadRequest)
handleError(fmt.Errorf("invalid query parameter: %s", key),
http.StatusBadRequest, rw)
return
}
}
jobs, err := api.JobRepository.QueryJobs(r.Context(), []*model.JobFilter{filter}, page, order)
if err != nil {
http.Error(rw, err.Error(), http.StatusInternalServerError)
handleError(err, http.StatusInternalServerError, rw)
return
}
@ -242,7 +253,7 @@ func (api *RestApi) getJobs(rw http.ResponseWriter, r *http.Request) {
for _, job := range jobs {
if withMetadata {
if _, err := api.JobRepository.FetchMetadata(job); err != nil {
http.Error(rw, err.Error(), http.StatusInternalServerError)
handleError(err, http.StatusInternalServerError, rw)
return
}
}
@ -255,7 +266,7 @@ func (api *RestApi) getJobs(rw http.ResponseWriter, r *http.Request) {
res.Tags, err = api.JobRepository.GetTags(&job.ID)
if err != nil {
http.Error(rw, err.Error(), http.StatusInternalServerError)
handleError(err, http.StatusInternalServerError, rw)
return
}
@ -263,7 +274,7 @@ func (api *RestApi) getJobs(rw http.ResponseWriter, r *http.Request) {
res.Statistics, err = archive.GetStatistics(job)
if err != nil {
if err != nil {
http.Error(rw, err.Error(), http.StatusInternalServerError)
handleError(err, http.StatusInternalServerError, rw)
return
}
}
@ -273,12 +284,18 @@ func (api *RestApi) getJobs(rw http.ResponseWriter, r *http.Request) {
}
log.Debugf("/api/jobs: %d jobs returned", len(results))
rw.Header().Add("Content-Type", "application/json")
bw := bufio.NewWriter(rw)
defer bw.Flush()
if err := json.NewEncoder(bw).Encode(map[string]interface{}{
"jobs": results,
}); err != nil {
http.Error(rw, err.Error(), http.StatusInternalServerError)
payload := GetJobsApiResponse{
Jobs: results,
Items: page.ItemsPerPage,
Page: page.Page,
}
if err := json.NewEncoder(bw).Encode(payload); err != nil {
handleError(err, http.StatusInternalServerError, rw)
return
}
}
@ -723,29 +740,6 @@ func (api *RestApi) checkAndHandleStopJob(rw http.ResponseWriter, job *schema.Jo
api.JobRepository.TriggerArchiving(job)
}
// func (api *RestApi) importJob(rw http.ResponseWriter, r *http.Request) {
// if user := auth.GetUser(r.Context()); user != nil && !user.HasRole(auth.RoleApi) {
// handleError(fmt.Errorf("missing role: %v", auth.RoleApi), http.StatusForbidden, rw)
// return
// }
// var body struct {
// Meta *schema.JobMeta `json:"meta"`
// Data *schema.JobData `json:"data"`
// }
// if err := decode(r.Body, &body); err != nil {
// handleError(fmt.Errorf("import failed: %s", err.Error()), http.StatusBadRequest, rw)
// return
// }
// if err := api.JobRepository.ImportJob(body.Meta, body.Data); err != nil {
// handleError(fmt.Errorf("import failed: %s", err.Error()), http.StatusUnprocessableEntity, rw)
// return
// }
// rw.Write([]byte(`{ "status": "OK" }`))
// }
func (api *RestApi) getJobMetrics(rw http.ResponseWriter, r *http.Request) {
id := mux.Vars(r)["id"]
metrics := r.URL.Query()["metric"]
@ -794,7 +788,8 @@ func (api *RestApi) getJWT(rw http.ResponseWriter, r *http.Request) {
me := auth.GetUser(r.Context())
if !me.HasRole(auth.RoleAdmin) {
if username != me.Username {
http.Error(rw, "Only admins are allowed to sign JWTs not for themselves", http.StatusForbidden)
http.Error(rw, "Only admins are allowed to sign JWTs not for themselves",
http.StatusForbidden)
return
}
}

6
internal/repository/migration.go
View File

@ -102,7 +102,11 @@ func MigrateDB(backend string, db string) {
}
if err := m.Up(); err != nil {
log.Fatal(err)
if err == migrate.ErrNoChange {
log.Info("DB already up to date!")
} else {
log.Fatal(err)
}
}
m.Close()

11
startDemo.sh
View File

@ -4,21 +4,16 @@ if [ -d './var' ]; then
echo 'Directory ./var already exists! Skipping initialization.'
./cc-backend --server --dev
else
mkdir ./var
cd ./var
make
cd var
wget https://hpc-mover.rrze.uni-erlangen.de/HPC-Data/0x7b58aefb/eig7ahyo6fo2bais0ephuf2aitohv1ai/job-archive-dev.tar.xz
tar xJf job-archive-dev.tar.xz
rm ./job-archive-dev.tar.xz
cd ../
cd ../web/frontend
yarn install
yarn build
cd ../..
cp ./configs/env-template.txt .env
cp ./docs/config.json config.json
go build ./cmd/cc-backend
./cc-backend --migrate-db
./cc-backend --server --dev --init-db --add-user demo:admin:AdminDev

2
test/integration_test.go
View File

@ -416,7 +416,7 @@ func TestRestApi(t *testing.T) {
}
const stopJobBody string = `{
"jobId": 123,
"jobId": 123,
"startTime": 123456789,
"cluster": "testcluster",