Merge branch 'dev' into add_editmeta_byrequest

This commit is contained in:
Christoph Kluge
2025-07-03 14:02:54 +02:00
242 changed files with 15413 additions and 15625 deletions

View File

@@ -1,5 +1,5 @@
// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package api_test
@@ -27,8 +27,8 @@ import (
"github.com/ClusterCockpit/cc-backend/internal/metricdata"
"github.com/ClusterCockpit/cc-backend/internal/repository"
"github.com/ClusterCockpit/cc-backend/pkg/archive"
"github.com/ClusterCockpit/cc-backend/pkg/log"
"github.com/ClusterCockpit/cc-backend/pkg/schema"
cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
"github.com/ClusterCockpit/cc-lib/schema"
"github.com/gorilla/mux"
_ "github.com/mattn/go-sqlite3"
@@ -116,14 +116,14 @@ func setup(t *testing.T) *api.RestApi {
]
}`
log.Init("info", true)
cclog.Init("info", true)
tmpdir := t.TempDir()
jobarchive := filepath.Join(tmpdir, "job-archive")
if err := os.Mkdir(jobarchive, 0777); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(jobarchive, "version.txt"), []byte(fmt.Sprintf("%d", 2)), 0666); err != nil {
if err := os.WriteFile(filepath.Join(jobarchive, "version.txt"), fmt.Appendf(nil, "%d", 2), 0666); err != nil {
t.Fatal(err)
}
@@ -204,11 +204,11 @@ func TestRestApi(t *testing.T) {
restapi.MountApiRoutes(r)
var TestJobId int64 = 123
var TestClusterName string = "testcluster"
TestClusterName := "testcluster"
var TestStartTime int64 = 123456789
const startJobBody string = `{
"jobId": 123,
"jobId": 123,
"user": "testuser",
"project": "testproj",
"cluster": "testcluster",
@@ -221,7 +221,6 @@ func TestRestApi(t *testing.T) {
"exclusive": 1,
"monitoringStatus": 1,
"smt": 1,
"tags": [{ "type": "testTagType", "name": "testTagName", "scope": "testuser" }],
"resources": [
{
"hostname": "host123",
@@ -252,16 +251,17 @@ func TestRestApi(t *testing.T) {
if response.StatusCode != http.StatusCreated {
t.Fatal(response.Status, recorder.Body.String())
}
resolver := graph.GetResolverInstance()
// resolver := graph.GetResolverInstance()
restapi.JobRepository.SyncJobs()
job, err := restapi.JobRepository.Find(&TestJobId, &TestClusterName, &TestStartTime)
if err != nil {
t.Fatal(err)
}
job.Tags, err = resolver.Job().Tags(ctx, job)
if err != nil {
t.Fatal(err)
}
// job.Tags, err = resolver.Job().Tags(ctx, job)
// if err != nil {
// t.Fatal(err)
// }
if job.JobID != 123 ||
job.User != "testuser" ||
@@ -278,13 +278,13 @@ func TestRestApi(t *testing.T) {
job.MonitoringStatus != 1 ||
job.SMT != 1 ||
!reflect.DeepEqual(job.Resources, []*schema.Resource{{Hostname: "host123", HWThreads: []int{0, 1, 2, 3, 4, 5, 6, 7}}}) ||
job.StartTime.Unix() != 123456789 {
job.StartTime != 123456789 {
t.Fatalf("unexpected job properties: %#v", job)
}
if len(job.Tags) != 1 || job.Tags[0].Type != "testTagType" || job.Tags[0].Name != "testTagName" || job.Tags[0].Scope != "testuser" {
t.Fatalf("unexpected tags: %#v", job.Tags)
}
// if len(job.Tags) != 1 || job.Tags[0].Type != "testTagType" || job.Tags[0].Name != "testTagName" || job.Tags[0].Scope != "testuser" {
// t.Fatalf("unexpected tags: %#v", job.Tags)
// }
}); !ok {
return
}
@@ -352,7 +352,7 @@ func TestRestApi(t *testing.T) {
t.Run("CheckDoubleStart", func(t *testing.T) {
// Starting a job with the same jobId and cluster should only be allowed if the startTime is far appart!
body := strings.Replace(startJobBody, `"startTime": 123456789`, `"startTime": 123456790`, -1)
body := strings.ReplaceAll(startJobBody, `"startTime": 123456789`, `"startTime": 123456790`)
req := httptest.NewRequest(http.MethodPost, "/jobs/start_job/", bytes.NewBuffer([]byte(body)))
recorder := httptest.NewRecorder()
@@ -402,6 +402,7 @@ func TestRestApi(t *testing.T) {
}
time.Sleep(1 * time.Second)
restapi.JobRepository.SyncJobs()
const stopJobBodyFailed string = `{
"jobId": 12345,

70
internal/api/cluster.go Normal file
View File

@@ -0,0 +1,70 @@
// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package api
import (
"bufio"
"encoding/json"
"fmt"
"net/http"
"github.com/ClusterCockpit/cc-backend/internal/repository"
"github.com/ClusterCockpit/cc-backend/pkg/archive"
"github.com/ClusterCockpit/cc-lib/schema"
)
// GetClustersApiResponse model
type GetClustersApiResponse struct {
Clusters []*schema.Cluster `json:"clusters"` // Array of clusters
}
// getClusters godoc
// @summary Lists all cluster configs
// @tags Cluster query
// @description Get a list of all cluster configs. Specific cluster can be requested using query parameter.
// @produce json
// @param cluster query string false "Job Cluster"
// @success 200 {object} api.GetClustersApiResponse "Array of clusters"
// @failure 400 {object} api.ErrorResponse "Bad Request"
// @failure 401 {object} api.ErrorResponse "Unauthorized"
// @failure 403 {object} api.ErrorResponse "Forbidden"
// @failure 500 {object} api.ErrorResponse "Internal Server Error"
// @security ApiKeyAuth
// @router /api/clusters/ [get]
func (api *RestApi) getClusters(rw http.ResponseWriter, r *http.Request) {
if user := repository.GetUserFromContext(r.Context()); user != nil &&
!user.HasRole(schema.RoleApi) {
handleError(fmt.Errorf("missing role: %v", schema.GetRoleString(schema.RoleApi)), http.StatusForbidden, rw)
return
}
rw.Header().Add("Content-Type", "application/json")
bw := bufio.NewWriter(rw)
defer bw.Flush()
var clusters []*schema.Cluster
if r.URL.Query().Has("cluster") {
name := r.URL.Query().Get("cluster")
cluster := archive.GetCluster(name)
if cluster == nil {
handleError(fmt.Errorf("unknown cluster: %s", name), http.StatusBadRequest, rw)
return
}
clusters = append(clusters, cluster)
} else {
clusters = archive.Clusters
}
payload := GetClustersApiResponse{
Clusters: clusters,
}
if err := json.NewEncoder(bw).Encode(payload); err != nil {
handleError(err, http.StatusInternalServerError, rw)
return
}
}

View File

@@ -208,7 +208,7 @@ const docTemplate = `{
"200": {
"description": "Success message",
"schema": {
"$ref": "#/definitions/api.DefaultJobApiResponse"
"$ref": "#/definitions/api.DefaultApiResponse"
}
},
"400": {
@@ -278,7 +278,7 @@ const docTemplate = `{
"200": {
"description": "Success message",
"schema": {
"$ref": "#/definitions/api.DefaultJobApiResponse"
"$ref": "#/definitions/api.DefaultApiResponse"
}
},
"400": {
@@ -348,7 +348,7 @@ const docTemplate = `{
"200": {
"description": "Success message",
"schema": {
"$ref": "#/definitions/api.DefaultJobApiResponse"
"$ref": "#/definitions/api.DefaultApiResponse"
}
},
"400": {
@@ -530,7 +530,7 @@ const docTemplate = `{
"ApiKeyAuth": []
}
],
"description": "Job specified in request body will be saved to database as \"running\" with new DB ID.\nJob specifications follow the 'JobMeta' scheme, API will fail to execute if requirements are not met.",
"description": "Job specified in request body will be saved to database as \"running\" with new DB ID.\nJob specifications follow the 'Job' scheme, API will fail to execute if requirements are not met.",
"consumes": [
"application/json"
],
@@ -548,7 +548,7 @@ const docTemplate = `{
"in": "body",
"required": true,
"schema": {
"$ref": "#/definitions/schema.JobMeta"
"$ref": "#/definitions/schema.Job"
}
}
],
@@ -556,7 +556,7 @@ const docTemplate = `{
"201": {
"description": "Job added successfully",
"schema": {
"$ref": "#/definitions/api.DefaultJobApiResponse"
"$ref": "#/definitions/api.DefaultApiResponse"
}
},
"400": {
@@ -599,7 +599,7 @@ const docTemplate = `{
"ApiKeyAuth": []
}
],
"description": "Job to stop is specified by request body. All fields are required in this case.\nReturns full job resource information according to 'JobMeta' scheme.",
"description": "Job to stop is specified by request body. All fields are required in this case.\nReturns full job resource information according to 'Job' scheme.",
"produces": [
"application/json"
],
@@ -622,7 +622,7 @@ const docTemplate = `{
"200": {
"description": "Success message",
"schema": {
"$ref": "#/definitions/schema.JobMeta"
"$ref": "#/definitions/schema.Job"
}
},
"400": {
@@ -744,7 +744,7 @@ const docTemplate = `{
"ApiKeyAuth": []
}
],
"description": "Job to get is specified by database ID\nReturns full job resource information according to 'JobMeta' scheme and all metrics according to 'JobData'.",
"description": "Job to get is specified by database ID\nReturns full job resource information according to 'Job' scheme and all metrics according to 'JobData'.",
"produces": [
"application/json"
],
@@ -818,7 +818,7 @@ const docTemplate = `{
"ApiKeyAuth": []
}
],
"description": "Job to get is specified by database ID\nReturns full job resource information according to 'JobMeta' scheme and all metrics according to 'JobData'.",
"description": "Job to get is specified by database ID\nReturns full job resource information according to 'Job' scheme and all metrics according to 'JobData'.",
"consumes": [
"application/json"
],
@@ -896,6 +896,66 @@ const docTemplate = `{
}
}
},
"/api/nodestats/": {
"post": {
"security": [
{
"ApiKeyAuth": []
}
],
"description": "Returns a JSON-encoded list of users.\nRequired query-parameter defines if all users or only users with additional special roles are returned.",
"produces": [
"application/json"
],
"tags": [
"Nodestates"
],
"summary": "Deliver updated Slurm node states",
"parameters": [
{
"description": "Request body containing nodes and their states",
"name": "request",
"in": "body",
"required": true,
"schema": {
"$ref": "#/definitions/api.UpdateNodeStatesRequest"
}
}
],
"responses": {
"200": {
"description": "Success message",
"schema": {
"$ref": "#/definitions/api.DefaultApiResponse"
}
},
"400": {
"description": "Bad Request",
"schema": {
"$ref": "#/definitions/api.ErrorResponse"
}
},
"401": {
"description": "Unauthorized",
"schema": {
"$ref": "#/definitions/api.ErrorResponse"
}
},
"403": {
"description": "Forbidden",
"schema": {
"$ref": "#/definitions/api.ErrorResponse"
}
},
"500": {
"description": "Internal Server Error",
"schema": {
"$ref": "#/definitions/api.ErrorResponse"
}
}
}
}
},
"/api/users/": {
"get": {
"security": [
@@ -1144,7 +1204,7 @@ const docTemplate = `{
}
}
},
"api.DefaultJobApiResponse": {
"api.DefaultApiResponse": {
"type": "object",
"properties": {
"msg": {
@@ -1238,7 +1298,7 @@ const docTemplate = `{
"description": "Array of jobs",
"type": "array",
"items": {
"$ref": "#/definitions/schema.JobMeta"
"$ref": "#/definitions/schema.Job"
}
},
"page": {
@@ -1292,6 +1352,20 @@ const docTemplate = `{
}
}
},
"api.Node": {
"type": "object",
"properties": {
"hostname": {
"type": "string"
},
"states": {
"type": "array",
"items": {
"type": "string"
}
}
}
},
"api.StopJobApiRequest": {
"type": "object",
"required": [
@@ -1325,6 +1399,21 @@ const docTemplate = `{
}
}
},
"api.UpdateNodeStatesRequest": {
"type": "object",
"properties": {
"cluster": {
"type": "string",
"example": "fritz"
},
"nodes": {
"type": "array",
"items": {
"$ref": "#/definitions/api.Node"
}
}
}
},
"schema.Accelerator": {
"type": "object",
"properties": {
@@ -1360,7 +1449,6 @@ const docTemplate = `{
}
},
"schema.Job": {
"description": "Information of a HPC job.",
"type": "object",
"properties": {
"arrayJobId": {
@@ -1458,6 +1546,12 @@ const docTemplate = `{
"type": "string",
"example": "abcd200"
},
"requestedMemory": {
"description": "in MB",
"type": "integer",
"minimum": 1,
"example": 128000
},
"resources": {
"type": "array",
"items": {
@@ -1469,7 +1563,14 @@ const docTemplate = `{
"example": 4
},
"startTime": {
"type": "string"
"type": "integer",
"example": 1649723812
},
"statistics": {
"type": "object",
"additionalProperties": {
"$ref": "#/definitions/schema.JobStatistics"
}
},
"subCluster": {
"type": "string",
@@ -1517,147 +1618,6 @@ const docTemplate = `{
}
}
},
"schema.JobMeta": {
"description": "Meta data information of a HPC job.",
"type": "object",
"properties": {
"arrayJobId": {
"type": "integer",
"example": 123000
},
"cluster": {
"type": "string",
"example": "fritz"
},
"concurrentJobs": {
"$ref": "#/definitions/schema.JobLinkResultList"
},
"duration": {
"type": "integer",
"minimum": 1,
"example": 43200
},
"energy": {
"type": "number"
},
"energyFootprint": {
"type": "object",
"additionalProperties": {
"type": "number"
}
},
"exclusive": {
"type": "integer",
"maximum": 2,
"minimum": 0,
"example": 1
},
"footprint": {
"type": "object",
"additionalProperties": {
"type": "number"
}
},
"id": {
"type": "integer"
},
"jobId": {
"type": "integer",
"example": 123000
},
"jobState": {
"enum": [
"completed",
"failed",
"cancelled",
"stopped",
"timeout",
"out_of_memory"
],
"allOf": [
{
"$ref": "#/definitions/schema.JobState"
}
],
"example": "completed"
},
"metaData": {
"type": "object",
"additionalProperties": {
"type": "string"
}
},
"monitoringStatus": {
"type": "integer",
"maximum": 3,
"minimum": 0,
"example": 1
},
"numAcc": {
"type": "integer",
"minimum": 1,
"example": 2
},
"numHwthreads": {
"type": "integer",
"minimum": 1,
"example": 20
},
"numNodes": {
"type": "integer",
"minimum": 1,
"example": 2
},
"partition": {
"type": "string",
"example": "main"
},
"project": {
"type": "string",
"example": "abcd200"
},
"resources": {
"type": "array",
"items": {
"$ref": "#/definitions/schema.Resource"
}
},
"smt": {
"type": "integer",
"example": 4
},
"startTime": {
"type": "integer",
"minimum": 1,
"example": 1649723812
},
"statistics": {
"type": "object",
"additionalProperties": {
"$ref": "#/definitions/schema.JobStatistics"
}
},
"subCluster": {
"type": "string",
"example": "main"
},
"tags": {
"type": "array",
"items": {
"$ref": "#/definitions/schema.Tag"
}
},
"user": {
"type": "string",
"example": "abcd100h"
},
"walltime": {
"type": "integer",
"minimum": 1,
"example": 86400
}
}
},
"schema.JobMetric": {
"type": "object",
"properties": {
@@ -1985,6 +1945,9 @@ const docTemplate = `{
},
"remove": {
"type": "boolean"
},
"unit": {
"$ref": "#/definitions/schema.Unit"
}
}
},

1038
internal/api/job.go Normal file

File diff suppressed because it is too large Load Diff

80
internal/api/node.go Normal file
View File

@@ -0,0 +1,80 @@
// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package api
import (
"fmt"
"net/http"
"strings"
"github.com/ClusterCockpit/cc-backend/internal/repository"
"github.com/ClusterCockpit/cc-lib/schema"
)
type Node struct {
Name string `json:"hostname"`
States []string `json:"states"`
CpusAllocated int `json:"cpusAllocated"`
CpusTotal int `json:"cpusTotal"`
MemoryAllocated int `json:"memoryAllocated"`
MemoryTotal int `json:"memoryTotal"`
GpusAllocated int `json:"gpusAllocated"`
GpusTotal int `json:"gpusTotal"`
}
// updateNodeStatesRequest model
type UpdateNodeStatesRequest struct {
Nodes []Node `json:"nodes"`
Cluster string `json:"cluster" example:"fritz"`
}
// this routine assumes that only one of them exists per node
func determineState(states []string) schema.NodeState {
for _, state := range states {
switch strings.ToLower(state) {
case "allocated":
return schema.NodeStateAllocated
case "reserved":
return schema.NodeStateReserved
case "idle":
return schema.NodeStateIdle
case "down":
return schema.NodeStateDown
case "mixed":
return schema.NodeStateMixed
}
}
return schema.NodeStateUnknown
}
// updateNodeStates godoc
// @summary Deliver updated Slurm node states
// @tags Nodestates
// @description Returns a JSON-encoded list of users.
// @description Required query-parameter defines if all users or only users with additional special roles are returned.
// @produce json
// @param request body UpdateNodeStatesRequest true "Request body containing nodes and their states"
// @success 200 {object} api.DefaultApiResponse "Success message"
// @failure 400 {object} api.ErrorResponse "Bad Request"
// @failure 401 {object} api.ErrorResponse "Unauthorized"
// @failure 403 {object} api.ErrorResponse "Forbidden"
// @failure 500 {object} api.ErrorResponse "Internal Server Error"
// @security ApiKeyAuth
// @router /api/nodestats/ [post]
func (api *RestApi) updateNodeStates(rw http.ResponseWriter, r *http.Request) {
// Parse request body
req := UpdateNodeStatesRequest{}
if err := decode(r.Body, &req); err != nil {
handleError(fmt.Errorf("parsing request body failed: %w", err), http.StatusBadRequest, rw)
return
}
repo := repository.GetNodeRepository()
for _, node := range req.Nodes {
state := determineState(node.States)
repo.UpdateNodeState(node.Name, req.Cluster, &state)
}
}

File diff suppressed because it is too large Load Diff

159
internal/api/user.go Normal file
View File

@@ -0,0 +1,159 @@
// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package api
import (
"encoding/json"
"fmt"
"net/http"
"github.com/ClusterCockpit/cc-backend/internal/repository"
"github.com/ClusterCockpit/cc-lib/schema"
"github.com/gorilla/mux"
)
type ApiReturnedUser struct {
Username string `json:"username"`
Name string `json:"name"`
Roles []string `json:"roles"`
Email string `json:"email"`
Projects []string `json:"projects"`
}
// getUsers godoc
// @summary Returns a list of users
// @tags User
// @description Returns a JSON-encoded list of users.
// @description Required query-parameter defines if all users or only users with additional special roles are returned.
// @produce json
// @param not-just-user query bool true "If returned list should contain all users or only users with additional special roles"
// @success 200 {array} api.ApiReturnedUser "List of users returned successfully"
// @failure 400 {string} string "Bad Request"
// @failure 401 {string} string "Unauthorized"
// @failure 403 {string} string "Forbidden"
// @failure 500 {string} string "Internal Server Error"
// @security ApiKeyAuth
// @router /api/users/ [get]
func (api *RestApi) getUsers(rw http.ResponseWriter, r *http.Request) {
// SecuredCheck() only worked with TokenAuth: Removed
if user := repository.GetUserFromContext(r.Context()); !user.HasRole(schema.RoleAdmin) {
http.Error(rw, "Only admins are allowed to fetch a list of users", http.StatusForbidden)
return
}
users, err := repository.GetUserRepository().ListUsers(r.URL.Query().Get("not-just-user") == "true")
if err != nil {
http.Error(rw, err.Error(), http.StatusInternalServerError)
return
}
json.NewEncoder(rw).Encode(users)
}
func (api *RestApi) updateUser(rw http.ResponseWriter, r *http.Request) {
// SecuredCheck() only worked with TokenAuth: Removed
if user := repository.GetUserFromContext(r.Context()); !user.HasRole(schema.RoleAdmin) {
http.Error(rw, "Only admins are allowed to update a user", http.StatusForbidden)
return
}
// Get Values
newrole := r.FormValue("add-role")
delrole := r.FormValue("remove-role")
newproj := r.FormValue("add-project")
delproj := r.FormValue("remove-project")
// TODO: Handle anything but roles...
if newrole != "" {
if err := repository.GetUserRepository().AddRole(r.Context(), mux.Vars(r)["id"], newrole); err != nil {
http.Error(rw, err.Error(), http.StatusUnprocessableEntity)
return
}
rw.Write([]byte("Add Role Success"))
} else if delrole != "" {
if err := repository.GetUserRepository().RemoveRole(r.Context(), mux.Vars(r)["id"], delrole); err != nil {
http.Error(rw, err.Error(), http.StatusUnprocessableEntity)
return
}
rw.Write([]byte("Remove Role Success"))
} else if newproj != "" {
if err := repository.GetUserRepository().AddProject(r.Context(), mux.Vars(r)["id"], newproj); err != nil {
http.Error(rw, err.Error(), http.StatusUnprocessableEntity)
return
}
rw.Write([]byte("Add Project Success"))
} else if delproj != "" {
if err := repository.GetUserRepository().RemoveProject(r.Context(), mux.Vars(r)["id"], delproj); err != nil {
http.Error(rw, err.Error(), http.StatusUnprocessableEntity)
return
}
rw.Write([]byte("Remove Project Success"))
} else {
http.Error(rw, "Not Add or Del [role|project]?", http.StatusInternalServerError)
}
}
func (api *RestApi) createUser(rw http.ResponseWriter, r *http.Request) {
// SecuredCheck() only worked with TokenAuth: Removed
rw.Header().Set("Content-Type", "text/plain")
me := repository.GetUserFromContext(r.Context())
if !me.HasRole(schema.RoleAdmin) {
http.Error(rw, "Only admins are allowed to create new users", http.StatusForbidden)
return
}
username, password, role, name, email, project := r.FormValue("username"),
r.FormValue("password"), r.FormValue("role"), r.FormValue("name"),
r.FormValue("email"), r.FormValue("project")
if len(password) == 0 && role != schema.GetRoleString(schema.RoleApi) {
http.Error(rw, "Only API users are allowed to have a blank password (login will be impossible)", http.StatusBadRequest)
return
}
if len(project) != 0 && role != schema.GetRoleString(schema.RoleManager) {
http.Error(rw, "only managers require a project (can be changed later)",
http.StatusBadRequest)
return
} else if len(project) == 0 && role == schema.GetRoleString(schema.RoleManager) {
http.Error(rw, "managers require a project to manage (can be changed later)",
http.StatusBadRequest)
return
}
if err := repository.GetUserRepository().AddUser(&schema.User{
Username: username,
Name: name,
Password: password,
Email: email,
Projects: []string{project},
Roles: []string{role},
}); err != nil {
http.Error(rw, err.Error(), http.StatusUnprocessableEntity)
return
}
fmt.Fprintf(rw, "User %v successfully created!\n", username)
}
func (api *RestApi) deleteUser(rw http.ResponseWriter, r *http.Request) {
// SecuredCheck() only worked with TokenAuth: Removed
if user := repository.GetUserFromContext(r.Context()); !user.HasRole(schema.RoleAdmin) {
http.Error(rw, "Only admins are allowed to delete a user", http.StatusForbidden)
return
}
username := r.FormValue("username")
if err := repository.GetUserRepository().DelUser(username); err != nil {
http.Error(rw, err.Error(), http.StatusUnprocessableEntity)
return
}
rw.WriteHeader(http.StatusOK)
}