ClusterCockpit/cc-backend
1
0
Fork 0
mirror of https://github.com/ClusterCockpit/cc-backend synced 2026-06-17 17:07:29 +02:00
Code Issues Packages Releases Activity

Merge branch 'main' into feature/526-average-resample

Browse Source
This commit is contained in:
Jan Eitzinger
2026-06-17 06:30:54 +02:00
parent 0c56591e4b 3bfd3d06ca
commit 5d8d4e228e
69 changed files with 5681 additions and 5268 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
internal/api/api_test.go
View File

@@ -170,6 +170,7 @@ func setup(t *testing.T) *api.RestAPI {
archiver.Start(repository.GetJobRepository(), context.Background())
t.Setenv("SESSION_KEY", "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=")
if cfg := ccconf.GetPackageConfig("auth"); cfg != nil {
auth.Init(&cfg)
} else {

257
internal/api/docs.go
View File

@@ -398,11 +398,6 @@ const docTemplate = `{
},
"/api/jobs/edit_meta/": {
"patch": {
"security": [
{
"ApiKeyAuth": []
}
],
"description": "Edit key value pairs in metadata json of job specified by jobID, StartTime and Cluster\nIf a key already exists its content will be overwritten",
"consumes": [
"application/json"
@@ -413,7 +408,7 @@ const docTemplate = `{
"tags": [
"Job add and modify"
],
"summary": "Edit meta-data json by request",
"summary": "Edit meta-data json of job identified by request",
"parameters": [
{
"description": "Specifies job and payload to add or update",
@@ -456,12 +451,17 @@ const docTemplate = `{
"$ref": "#/definitions/api.ErrorResponse"
}
}
}
},
"security": [
{
"ApiKeyAuth": []
}
]
}
},
"/api/jobs/edit_meta/{id}": {
"patch": {
"description": "Edit key value pairs in job metadata json\nIf a key already exists its content will be overwritten",
"description": "Edit key value pairs in job metadata json of job specified by database id\nIf a key already exists its content will be overwritten",
"consumes": [
"application/json"
],
@@ -471,7 +471,7 @@ const docTemplate = `{
"tags": [
"Job add and modify"
],
"summary": "Edit meta-data json",
"summary": "Edit meta-data json of job identified by database id",
"parameters": [
{
"type": "integer",
@@ -481,7 +481,7 @@ const docTemplate = `{
"required": true
},
{
"description": "Kay value pair to add",
"description": "Metadata Key value pair to add or update",
"name": "request",
"in": "body",
"required": true,
@@ -743,6 +743,64 @@ const docTemplate = `{
]
}
},
"/api/jobs/used_nodes": {
"get": {
"description": "Get a map of cluster names to lists of unique hostnames that are currently in use by running jobs that started before the specified timestamp.",
"produces": [
"application/json"
],
"tags": [
"Job query"
],
"summary": "Lists used nodes by cluster",
"parameters": [
{
"type": "integer",
"description": "Unix timestamp to filter jobs (jobs with start_time \u003c ts)",
"name": "ts",
"in": "query",
"required": true
}
],
"responses": {
"200": {
"description": "Map of cluster names to hostname lists",
"schema": {
"$ref": "#/definitions/api.GetUsedNodesAPIResponse"
}
},
"400": {
"description": "Bad Request",
"schema": {
"$ref": "#/definitions/api.ErrorResponse"
}
},
"401": {
"description": "Unauthorized",
"schema": {
"$ref": "#/definitions/api.ErrorResponse"
}
},
"403": {
"description": "Forbidden",
"schema": {
"$ref": "#/definitions/api.ErrorResponse"
}
},
"500": {
"description": "Internal Server Error",
"schema": {
"$ref": "#/definitions/api.ErrorResponse"
}
}
},
"security": [
{
"ApiKeyAuth": []
}
]
}
},
"/api/jobs/{id}": {
"get": {
"description": "Job to get is specified by database ID\nReturns full job resource information according to 'Job' scheme and all metrics according to 'JobData'.",
@@ -965,8 +1023,11 @@ const docTemplate = `{
"/api/user/{id}": {
"post": {
"description": "Allows admins to add/remove roles and projects for a user",
"consumes": [
"application/json"
],
"produces": [
"text/plain"
"application/json"
],
"tags": [
"User"
@@ -981,35 +1042,26 @@ const docTemplate = `{
"required": true
},
{
"type": "string",
"description": "Role to add",
"name": "add-role",
"in": "formData"
},
{
"type": "string",
"description": "Role to remove",
"name": "remove-role",
"in": "formData"
},
{
"type": "string",
"description": "Project to add",
"name": "add-project",
"in": "formData"
},
{
"type": "string",
"description": "Project to remove",
"name": "remove-project",
"in": "formData"
"description": "Single Field Changes",
"name": "request",
"in": "body",
"required": true,
"schema": {
"$ref": "#/definitions/api.UpdateUserAPIRequest"
}
}
],
"responses": {
"200": {
"description": "Success message",
"description": "OK",
"schema": {
"type": "string"
"$ref": "#/definitions/api.DefaultAPIResponse"
}
},
"400": {
"description": "Bad Request",
"schema": {
"$ref": "#/definitions/api.ErrorResponse"
}
},
"403": {
@@ -1381,63 +1433,6 @@ const docTemplate = `{
]
}
},
"/healthcheck/": {
"get": {
"description": "This endpoint allows the users to check if a node is healthy",
"produces": [
"application/json"
],
"tags": [
"healthcheck"
],
"summary": "HealthCheck endpoint",
"parameters": [
{
"type": "string",
"description": "Selector",
"name": "selector",
"in": "query"
}
],
"responses": {
"200": {
"description": "Debug dump",
"schema": {
"type": "string"
}
},
"400": {
"description": "Bad Request",
"schema": {
"$ref": "#/definitions/api.ErrorResponse"
}
},
"401": {
"description": "Unauthorized",
"schema": {
"$ref": "#/definitions/api.ErrorResponse"
}
},
"403": {
"description": "Forbidden",
"schema": {
"$ref": "#/definitions/api.ErrorResponse"
}
},
"500": {
"description": "Internal Server Error",
"schema": {
"$ref": "#/definitions/api.ErrorResponse"
}
}
},
"security": [
{
"ApiKeyAuth": []
}
]
}
},
"/jobs/tag_job/{id}": {
"delete": {
"description": "Removes tag(s) from a job specified by DB ID. Name and Type of Tag(s) must match.\nTag Scope is required for matching, options: \"global\", \"admin\". Private tags can not be deleted via API.\nIf tagged job is already finished: Tag will be removed from respective archive files.",
@@ -1940,6 +1935,31 @@ const docTemplate = `{
}
}
},
"api.UpdateUserAPIRequest": {
"type": "object",
"properties": {
"add-role": {
"description": "Role to add to user $ID",
"type": "string",
"example": "user"
},
"remove-role": {
"description": "Role to remove from user $ID",
"type": "string",
"example": "user"
},
"add-project": {
"description": "Project to add to user $ID managed array",
"type": "string",
"example": "abcd100"
},
"remove-project": {
"description": "Project to remove from user $ID managed array",
"type": "string",
"example": "abcd100"
}
}
},
"api.DefaultAPIResponse": {
"type": "object",
"properties": {
@@ -2043,6 +2063,52 @@ const docTemplate = `{
}
}
},
"api.GetUsedNodesAPIResponse": {
"type": "object",
"properties": {
"usedNodes": {
"description": "Map of cluster names to lists of used node hostnames",
"type": "object",
"additionalProperties": {
"type": "array",
"items": {
"type": "string"
}
}
}
}
},
"api.JobMetaRequest": {
"type": "object",
"required": [
"jobId"
],
"properties": {
"cluster": {
"description": "Cluster of job",
"type": "string",
"example": "fritz"
},
"jobId": {
"description": "Cluster Job ID of job",
"type": "integer",
"example": 123000
},
"payload": {
"description": "Content to Add to Job Meta_Data",
"allOf": [
{
"$ref": "#/definitions/api.EditMetaRequest"
}
]
},
"startTime": {
"description": "Start Time of job as epoch",
"type": "integer",
"example": 1649723812
}
}
},
"api.JobMetricWithName": {
"type": "object",
"properties": {
@@ -2175,13 +2241,6 @@ const docTemplate = `{
"format": "float64"
}
},
"exclusive": {
"description": "for backwards compatibility",
"type": "integer",
"maximum": 2,
"minimum": 0,
"example": 1
},
"footprint": {
"type": "object",
"additionalProperties": {

8
internal/api/job.go
View File

@@ -166,6 +166,10 @@ func (api *RestAPI) getJobs(rw http.ResponseWriter, r *http.Request) {
handleError(err, http.StatusBadRequest, rw)
return
}
if x < 1 {
handleError(fmt.Errorf("page must be >= 1"), http.StatusBadRequest, rw)
return
}
page.Page = x
case "items-per-page":
x, err := strconv.Atoi(vals[0])
@@ -173,6 +177,10 @@ func (api *RestAPI) getJobs(rw http.ResponseWriter, r *http.Request) {
handleError(err, http.StatusBadRequest, rw)
return
}
if x < 1 {
handleError(fmt.Errorf("items-per-page must be >= 1"), http.StatusBadRequest, rw)
return
}
page.ItemsPerPage = x
case "with-metadata":
withMetadata = true

49
internal/api/nats.go
View File

@@ -78,6 +78,12 @@ type NatsAPI struct {
jobCh chan natsMessage
// nodeCh receives node state messages for processing by worker goroutines.
nodeCh chan natsMessage
// stop signals worker goroutines and subscription callbacks to stop.
// Closing it (via Shutdown) makes workers exit and callbacks drop further
// messages instead of blocking; the channels are never closed so in-flight
// callbacks can never send on a closed channel.
stop chan struct{}
stopOnce sync.Once
}
// NewNatsAPI creates a new NatsAPI instance with channel-based worker pools.
@@ -99,6 +105,7 @@ func NewNatsAPI() *NatsAPI {
JobRepository: repository.GetJobRepository(),
jobCh: make(chan natsMessage, jobConc),
nodeCh: make(chan natsMessage, nodeConc),
stop: make(chan struct{}),
}
// Start worker goroutines
@@ -112,17 +119,36 @@ func NewNatsAPI() *NatsAPI {
return api
}
// Shutdown stops the worker goroutines and tells subscription callbacks to stop
// enqueueing. It is safe to call multiple times. Callers must ensure the NATS
// client is closed first so no new callbacks are invoked.
func (api *NatsAPI) Shutdown() {
api.stopOnce.Do(func() {
close(api.stop)
})
}
// jobWorker processes job event messages from the job channel.
func (api *NatsAPI) jobWorker() {
for msg := range api.jobCh {
api.handleJobEvent(msg.subject, msg.data)
for {
select {
case <-api.stop:
return
case msg := <-api.jobCh:
api.handleJobEvent(msg.subject, msg.data)
}
}
}
// nodeWorker processes node state messages from the node channel.
func (api *NatsAPI) nodeWorker() {
for msg := range api.nodeCh {
api.handleNodeState(msg.subject, msg.data)
for {
select {
case <-api.stop:
return
case msg := <-api.nodeCh:
api.handleNodeState(msg.subject, msg.data)
}
}
}
@@ -140,18 +166,27 @@ func (api *NatsAPI) StartSubscriptions() error {
s := config.Keys.APISubjects
if err := client.Subscribe(s.SubjectJobEvent, func(subject string, data []byte) {
api.jobCh <- natsMessage{subject: subject, data: data}
select {
case api.jobCh <- natsMessage{subject: subject, data: data}:
case <-api.stop:
}
}); err != nil {
return err
}
if err := client.Subscribe(s.SubjectNodeState, func(subject string, data []byte) {
api.nodeCh <- natsMessage{subject: subject, data: data}
select {
case api.nodeCh <- natsMessage{subject: subject, data: data}:
case <-api.stop:
}
}); err != nil {
return err
}
cclog.Info("NATS API subscriptions started")
cclog.Warnf("NATS API subscriptions started on subjects %q and %q — these are UNAUTHENTICATED: "+
"anyone with publish rights on the broker can start/stop jobs and update node state. "+
"Restrict publish ACLs on the NATS broker to trusted producers only.",
s.SubjectJobEvent, s.SubjectNodeState)
}
return nil
}

1
internal/api/nats_test.go
View File

@@ -156,6 +156,7 @@ func setupNatsTest(t *testing.T) *NatsAPI {
archiver.Start(repository.GetJobRepository(), context.Background())
t.Setenv("SESSION_KEY", "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=")
if cfg := ccconf.GetPackageConfig("auth"); cfg != nil {
auth.Init(&cfg)
} else {

4
internal/api/rest.go
View File

@@ -79,6 +79,8 @@ func (api *RestAPI) MountAPIRoutes(r chi.Router) {
// REST API Uses TokenAuth
// User List
r.Get("/users/", api.getUsers)
// User Edit
r.Post("/user/{id}", api.updateUserByRequest)
// Cluster List
r.Get("/clusters/", api.getClusters)
// Slurm node state
@@ -152,7 +154,7 @@ func (api *RestAPI) MountConfigAPIRoutes(r chi.Router) {
r.Put("/config/users/", api.createUser)
r.Get("/config/users/", api.getUsers)
r.Delete("/config/users/", api.deleteUser)
r.Post("/config/user/{id}", api.updateUser)
r.Post("/config/user/{id}", api.updateUserByForm)
r.Post("/config/notice/", api.editNotice)
r.Get("/config/taggers/", api.getTaggers)
r.Post("/config/taggers/run/", api.runTagger)

76
internal/api/user.go
View File

@@ -24,6 +24,14 @@ type APIReturnedUser struct {
Projects []string `json:"projects"`
}
// UpdateUserAPIRequest model
type UpdateUserAPIRequest struct {
NewRole string `json:"add-role" example:"user"` // Role to add to user $ID
DelRole string `json:"remove-role" example:"user"` // Role to remove from user $ID
NewProj string `json:"add-project" example:"abcd100"` // Project to add to user $ID managed array
DelProj string `json:"remove-project" example:"abcd100"` // Project to remove from user $ID managed array
}
// getUsers godoc
// @summary Returns a list of users
// @tags User
@@ -58,22 +66,74 @@ func (api *RestAPI) getUsers(rw http.ResponseWriter, r *http.Request) {
}
}
// updateUser godoc
// updateUserByRequest godoc
// @summary Update user roles and projects
// @tags User
// @description Allows admins to add/remove roles and projects for a user
// @produce plain
// @param id path string true "Username"
// @param add-role formData string false "Role to add"
// @param remove-role formData string false "Role to remove"
// @param add-project formData string false "Project to add"
// @param remove-project formData string false "Project to remove"
// @accept json
// @produce json
// @param id path string true "Username"
// @param request body api.UpdateUserAPIRequest true "Single Field Changes"
// @success 200 {string} string "Success message"
// @failure 403 {object} api.ErrorResponse "Forbidden"
// @failure 422 {object} api.ErrorResponse "Unprocessable Entity"
// @security ApiKeyAuth
// @router /api/user/{id} [post]
func (api *RestAPI) updateUser(rw http.ResponseWriter, r *http.Request) {
func (api *RestAPI) updateUserByRequest(rw http.ResponseWriter, r *http.Request) {
if user := repository.GetUserFromContext(r.Context()); !user.HasRole(schema.RoleAdmin) {
handleError(fmt.Errorf("only admins are allowed to update a user"), http.StatusForbidden, rw)
return
}
// Get Values
var req UpdateUserAPIRequest
if err := decode(r.Body, &req); err != nil {
handleError(fmt.Errorf("decoding request failed: %w", err), http.StatusBadRequest, rw)
return
}
rw.Header().Set("Content-Type", "application/json")
// Handle role updates
if req.NewRole != "" {
if err := repository.GetUserRepository().AddRole(r.Context(), chi.URLParam(r, "id"), req.NewRole); err != nil {
handleError(fmt.Errorf("adding role failed: %w", err), http.StatusUnprocessableEntity, rw)
return
}
if err := json.NewEncoder(rw).Encode(DefaultAPIResponse{Message: fmt.Sprintf("Add Role Success for user %s", chi.URLParam(r, "id"))}); err != nil {
cclog.Errorf("Failed to encode response: %v", err)
}
} else if req.DelRole != "" {
if err := repository.GetUserRepository().RemoveRole(r.Context(), chi.URLParam(r, "id"), req.DelRole); err != nil {
handleError(fmt.Errorf("removing role failed: %w", err), http.StatusUnprocessableEntity, rw)
return
}
if err := json.NewEncoder(rw).Encode(DefaultAPIResponse{Message: fmt.Sprintf("Remove Role Success for user %s", chi.URLParam(r, "id"))}); err != nil {
cclog.Errorf("Failed to encode response: %v", err)
}
} else if req.NewProj != "" {
if err := repository.GetUserRepository().AddProject(r.Context(), chi.URLParam(r, "id"), req.NewProj); err != nil {
handleError(fmt.Errorf("adding project failed: %w", err), http.StatusUnprocessableEntity, rw)
return
}
if err := json.NewEncoder(rw).Encode(DefaultAPIResponse{Message: fmt.Sprintf("Add Project Success for user %s", chi.URLParam(r, "id"))}); err != nil {
cclog.Errorf("Failed to encode response: %v", err)
}
} else if req.DelProj != "" {
if err := repository.GetUserRepository().RemoveProject(r.Context(), chi.URLParam(r, "id"), req.DelProj); err != nil {
handleError(fmt.Errorf("removing project failed: %w", err), http.StatusUnprocessableEntity, rw)
return
}
if err := json.NewEncoder(rw).Encode(DefaultAPIResponse{Message: fmt.Sprintf("Remove Project Success for user %s", chi.URLParam(r, "id"))}); err != nil {
cclog.Errorf("Failed to encode response: %v", err)
}
} else {
handleError(fmt.Errorf("no operation specified: must provide add-role, remove-role, add-project, or remove-project"), http.StatusBadRequest, rw)
}
}
func (api *RestAPI) updateUserByForm(rw http.ResponseWriter, r *http.Request) {
// SecuredCheck() only worked with TokenAuth: Removed
if user := repository.GetUserFromContext(r.Context()); !user.HasRole(schema.RoleAdmin) {