ClusterCockpit/cc-backend
1
0
Fork 0
mirror of https://github.com/ClusterCockpit/cc-backend synced 2025-11-04 01:25:06 +01:00
Code Issues Packages Releases Activity

Merge pull request from ClusterCockpit/feature-62-rest-endpoint-delete-jobs

Browse Source 
Feature 62 rest endpoint delete jobs
Fix
This commit is contained in:
Jan Eitzinger
2022-11-30 14:48:44 +01:00
committed by GitHub
parent 3cc006d4ce a39fc73345
commit c7eaccd3b5
7 changed files with 1102 additions and 219 deletions

321
api/swagger.json

@@ -1,9 +1,8 @@
{ {
"swagger": "2.0", "swagger": "2.0",
"info": { "info": {
"description": "Defines a tag using name and type.", "description": "API for batch job control.",
"title": "ClusterCockpit REST API", "title": "ClusterCockpit REST API",
"termsOfService": "https://monitoring.nhr.fau.de/imprint",
"contact": { "contact": {
"name": "ClusterCockpit Project", "name": "ClusterCockpit Project",
"url": "https://github.com/ClusterCockpit", "url": "https://github.com/ClusterCockpit",
@@ -13,9 +12,9 @@
"name": "MIT License", "name": "MIT License",
"url": "https://opensource.org/licenses/MIT" "url": "https://opensource.org/licenses/MIT"
}, },
"version": "0.1.0" "version": "0.2.0"
}, },
"host": "clustercockpit.localhost:8082", "host": "localhost:8080",
"basePath": "/api", "basePath": "/api",
"paths": { "paths": {
"/jobs/": { "/jobs/": {
@@ -26,12 +25,12 @@
} }
], ],
"description": "Get a list of all jobs. Filters can be applied using query parameters.\nNumber of results can be limited by page. Results are sorted by descending startTime.", "description": "Get a list of all jobs. Filters can be applied using query parameters.\nNumber of results can be limited by page. Results are sorted by descending startTime.",
"consumes": [
"application/json"
],
"produces": [ "produces": [
"application/json" "application/json"
], ],
"tags": [
"query"
],
"summary": "Lists all jobs", "summary": "Lists all jobs",
"parameters": [ "parameters": [
{ {
@@ -62,13 +61,13 @@
}, },
{ {
"type": "integer", "type": "integer",
"description": "Items per page (If empty: No Limit)", "description": "Items per page (Default: 25)",
"name": "items-per-page", "name": "items-per-page",
"in": "query" "in": "query"
}, },
{ {
"type": "integer", "type": "integer",
"description": "Page Number (If empty: No Paging)", "description": "Page Number (Default: 1)",
"name": "page", "name": "page",
"in": "query" "in": "query"
}, },
@@ -110,6 +109,221 @@
} }
} }
}, },
"/jobs/delete_job/": {
"delete": {
"security": [
{
"ApiKeyAuth": []
}
],
"description": "Job to delete is specified by request body. All fields are required in this case.",
"consumes": [
"application/json"
],
"produces": [
"application/json"
],
"tags": [
"remove"
],
"summary": "Remove a job from the sql database",
"parameters": [
{
"description": "All fields required",
"name": "request",
"in": "body",
"required": true,
"schema": {
"$ref": "#/definitions/api.DeleteJobApiRequest"
}
}
],
"responses": {
"200": {
"description": "Success message",
"schema": {
"$ref": "#/definitions/api.DeleteJobApiResponse"
}
},
"400": {
"description": "Bad Request",
"schema": {
"$ref": "#/definitions/api.ErrorResponse"
}
},
"401": {
"description": "Unauthorized",
"schema": {
"$ref": "#/definitions/api.ErrorResponse"
}
},
"403": {
"description": "Forbidden",
"schema": {
"$ref": "#/definitions/api.ErrorResponse"
}
},
"404": {
"description": "Resource not found",
"schema": {
"$ref": "#/definitions/api.ErrorResponse"
}
},
"422": {
"description": "Unprocessable Entity: finding job failed: sql: no rows in result set",
"schema": {
"$ref": "#/definitions/api.ErrorResponse"
}
},
"500": {
"description": "Internal Server Error",
"schema": {
"$ref": "#/definitions/api.ErrorResponse"
}
}
}
}
},
"/jobs/delete_job/{id}": {
"delete": {
"security": [
{
"ApiKeyAuth": []
}
],
"description": "Job to remove is specified by database ID. This will not remove the job from the job archive.",
"produces": [
"application/json"
],
"tags": [
"remove"
],
"summary": "Remove a job from the sql database",
"parameters": [
{
"type": "integer",
"description": "Database ID of Job",
"name": "id",
"in": "path",
"required": true
}
],
"responses": {
"200": {
"description": "Success message",
"schema": {
"$ref": "#/definitions/api.DeleteJobApiResponse"
}
},
"400": {
"description": "Bad Request",
"schema": {
"$ref": "#/definitions/api.ErrorResponse"
}
},
"401": {
"description": "Unauthorized",
"schema": {
"$ref": "#/definitions/api.ErrorResponse"
}
},
"403": {
"description": "Forbidden",
"schema": {
"$ref": "#/definitions/api.ErrorResponse"
}
},
"404": {
"description": "Resource not found",
"schema": {
"$ref": "#/definitions/api.ErrorResponse"
}
},
"422": {
"description": "Unprocessable Entity: finding job failed: sql: no rows in result set",
"schema": {
"$ref": "#/definitions/api.ErrorResponse"
}
},
"500": {
"description": "Internal Server Error",
"schema": {
"$ref": "#/definitions/api.ErrorResponse"
}
}
}
}
},
"/jobs/delete_job_before/{ts}": {
"delete": {
"security": [
{
"ApiKeyAuth": []
}
],
"description": "Remove all jobs with start time before timestamp. The jobs will not be removed from the job archive.",
"produces": [
"application/json"
],
"tags": [
"remove"
],
"summary": "Remove a job from the sql database",
"parameters": [
{
"type": "integer",
"description": "Unix epoch timestamp",
"name": "ts",
"in": "path",
"required": true
}
],
"responses": {
"200": {
"description": "Success message",
"schema": {
"$ref": "#/definitions/api.DeleteJobApiResponse"
}
},
"400": {
"description": "Bad Request",
"schema": {
"$ref": "#/definitions/api.ErrorResponse"
}
},
"401": {
"description": "Unauthorized",
"schema": {
"$ref": "#/definitions/api.ErrorResponse"
}
},
"403": {
"description": "Forbidden",
"schema": {
"$ref": "#/definitions/api.ErrorResponse"
}
},
"404": {
"description": "Resource not found",
"schema": {
"$ref": "#/definitions/api.ErrorResponse"
}
},
"422": {
"description": "Unprocessable Entity: finding job failed: sql: no rows in result set",
"schema": {
"$ref": "#/definitions/api.ErrorResponse"
}
},
"500": {
"description": "Internal Server Error",
"schema": {
"$ref": "#/definitions/api.ErrorResponse"
}
}
}
}
},
"/jobs/start_job/": { "/jobs/start_job/": {
"post": { "post": {
"security": [ "security": [
@@ -124,6 +338,9 @@
"produces": [ "produces": [
"application/json" "application/json"
], ],
"tags": [
"add and modify"
],
"summary": "Adds a new job as \"running\"", "summary": "Adds a new job as \"running\"",
"parameters": [ "parameters": [
{ {
@@ -187,6 +404,9 @@
"produces": [ "produces": [
"application/json" "application/json"
], ],
"tags": [
"add and modify"
],
"summary": "Marks job as completed and triggers archiving", "summary": "Marks job as completed and triggers archiving",
"parameters": [ "parameters": [
{ {
@@ -201,7 +421,7 @@
], ],
"responses": { "responses": {
"200": { "200": {
"description": "Job resource", "description": "Success message",
"schema": { "schema": {
"$ref": "#/definitions/schema.JobMeta" "$ref": "#/definitions/schema.JobMeta"
} }
@@ -259,6 +479,9 @@
"produces": [ "produces": [
"application/json" "application/json"
], ],
"tags": [
"add and modify"
],
"summary": "Marks job as completed and triggers archiving", "summary": "Marks job as completed and triggers archiving",
"parameters": [ "parameters": [
{ {
@@ -338,6 +561,9 @@
"produces": [ "produces": [
"application/json" "application/json"
], ],
"tags": [
"add and modify"
],
"summary": "Adds one or more tags to a job", "summary": "Adds one or more tags to a job",
"parameters": [ "parameters": [
{ {
@@ -355,7 +581,7 @@
"schema": { "schema": {
"type": "array", "type": "array",
"items": { "items": {
"$ref": "#/definitions/api.Tag" "$ref": "#/definitions/api.ApiTag"
} }
} }
} }
@@ -396,8 +622,53 @@
} }
}, },
"definitions": { "definitions": {
"api.ApiTag": {
"type": "object",
"properties": {
"name": {
"description": "Tag Name",
"type": "string",
"example": "Testjob"
},
"type": {
"description": "Tag Type",
"type": "string",
"example": "Debug"
}
}
},
"api.DeleteJobApiRequest": {
"type": "object",
"required": [
"jobId"
],
"properties": {
"cluster": {
"description": "Cluster of job",
"type": "string",
"example": "fritz"
},
"jobId": {
"description": "Cluster Job ID of job",
"type": "integer",
"example": 123000
},
"startTime": {
"description": "Start Time of job as epoch",
"type": "integer",
"example": 1649723812
}
}
},
"api.DeleteJobApiResponse": {
"type": "object",
"properties": {
"msg": {
"type": "string"
}
}
},
"api.ErrorResponse": { "api.ErrorResponse": {
"description": "Error message as returned from backend.",
"type": "object", "type": "object",
"properties": { "properties": {
"error": { "error": {
@@ -411,7 +682,6 @@
} }
}, },
"api.StartJobApiResponse": { "api.StartJobApiResponse": {
"description": "Successful job start response with database id of new job.",
"type": "object", "type": "object",
"properties": { "properties": {
"id": { "id": {
@@ -421,7 +691,6 @@
} }
}, },
"api.StopJobApiRequest": { "api.StopJobApiRequest": {
"description": "Request to stop running job using stoptime and final state. They are only required if no database id was provided with endpoint.",
"type": "object", "type": "object",
"required": [ "required": [
"jobState", "jobState",
@@ -462,22 +731,6 @@
} }
} }
}, },
"api.Tag": {
"description": "Defines a tag using name and type.",
"type": "object",
"properties": {
"name": {
"description": "Tag Name",
"type": "string",
"example": "Testjob"
},
"type": {
"description": "Tag Type",
"type": "string",
"example": "Debug"
}
}
},
"schema.Job": { "schema.Job": {
"description": "Information of a HPC job.", "description": "Information of a HPC job.",
"type": "object", "type": "object",
@@ -831,10 +1084,14 @@
}, },
"securityDefinitions": { "securityDefinitions": {
"ApiKeyAuth": { "ApiKeyAuth": {
"description": "JWT based authentification for general API endpoint use.",
"type": "apiKey", "type": "apiKey",
"name": "X-Auth-Token", "name": "X-Auth-Token",
"in": "header" "in": "header"
} }
} },
"tags": [
{
"name": "Job API"
}
]
} }

223
api/swagger.yaml

@@ -1,7 +1,39 @@
basePath: /api basePath: /api
definitions: definitions:
api.ApiTag:
properties:
name:
description: Tag Name
example: Testjob
type: string
type:
description: Tag Type
example: Debug
type: string
type: object
api.DeleteJobApiRequest:
properties:
cluster:
description: Cluster of job
example: fritz
type: string
jobId:
description: Cluster Job ID of job
example: 123000
type: integer
startTime:
description: Start Time of job as epoch
example: 1649723812
type: integer
required:
- jobId
type: object
api.DeleteJobApiResponse:
properties:
msg:
type: string
type: object
api.ErrorResponse: api.ErrorResponse:
description: Error message as returned from backend.
properties: properties:
error: error:
description: Error Message description: Error Message
@@ -11,15 +43,12 @@ definitions:
type: string type: string
type: object type: object
api.StartJobApiResponse: api.StartJobApiResponse:
description: Successful job start response with database id of new job.
properties: properties:
id: id:
description: Database ID of new job description: Database ID of new job
type: integer type: integer
type: object type: object
api.StopJobApiRequest: api.StopJobApiRequest:
description: Request to stop running job using stoptime and final state. They
are only required if no database id was provided with endpoint.
properties: properties:
cluster: cluster:
description: Cluster of job description: Cluster of job
@@ -51,18 +80,6 @@ definitions:
- jobState - jobState
- stopTime - stopTime
type: object type: object
api.Tag:
description: Defines a tag using name and type.
properties:
name:
description: Tag Name
example: Testjob
type: string
type:
description: Tag Type
example: Debug
type: string
type: object
schema.Job: schema.Job:
description: Information of a HPC job. description: Information of a HPC job.
properties: properties:
@@ -344,24 +361,21 @@ definitions:
example: Debug example: Debug
type: string type: string
type: object type: object
host: clustercockpit.localhost:8082 host: localhost:8080
info: info:
contact: contact:
email: support@clustercockpit.org email: support@clustercockpit.org
name: ClusterCockpit Project name: ClusterCockpit Project
url: https://github.com/ClusterCockpit url: https://github.com/ClusterCockpit
description: Defines a tag using name and type. description: API for batch job control.
license: license:
name: MIT License name: MIT License
url: https://opensource.org/licenses/MIT url: https://opensource.org/licenses/MIT
termsOfService: https://monitoring.nhr.fau.de/imprint
title: ClusterCockpit REST API title: ClusterCockpit REST API
version: 0.1.0 version: 0.2.0
paths: paths:
/jobs/: /jobs/:
get: get:
consumes:
- application/json
description: |- description: |-
Get a list of all jobs. Filters can be applied using query parameters. Get a list of all jobs. Filters can be applied using query parameters.
Number of results can be limited by page. Results are sorted by descending startTime. Number of results can be limited by page. Results are sorted by descending startTime.
@@ -385,11 +399,11 @@ paths:
in: query in: query
name: start-time name: start-time
type: string type: string
- description: 'Items per page (If empty: No Limit)' - description: 'Items per page (Default: 25)'
in: query in: query
name: items-per-page name: items-per-page
type: integer type: integer
- description: 'Page Number (If empty: No Paging)' - description: 'Page Number (Default: 1)'
in: query in: query
name: page name: page
type: integer type: integer
@@ -421,6 +435,152 @@ paths:
security: security:
- ApiKeyAuth: [] - ApiKeyAuth: []
summary: Lists all jobs summary: Lists all jobs
tags:
- query
/jobs/delete_job/:
delete:
consumes:
- application/json
description: Job to delete is specified by request body. All fields are required
in this case.
parameters:
- description: All fields required
in: body
name: request
required: true
schema:
$ref: '#/definitions/api.DeleteJobApiRequest'
produces:
- application/json
responses:
"200":
description: Success message
schema:
$ref: '#/definitions/api.DeleteJobApiResponse'
"400":
description: Bad Request
schema:
$ref: '#/definitions/api.ErrorResponse'
"401":
description: Unauthorized
schema:
$ref: '#/definitions/api.ErrorResponse'
"403":
description: Forbidden
schema:
$ref: '#/definitions/api.ErrorResponse'
"404":
description: Resource not found
schema:
$ref: '#/definitions/api.ErrorResponse'
"422":
description: 'Unprocessable Entity: finding job failed: sql: no rows in
result set'
schema:
$ref: '#/definitions/api.ErrorResponse'
"500":
description: Internal Server Error
schema:
$ref: '#/definitions/api.ErrorResponse'
security:
- ApiKeyAuth: []
summary: Remove a job from the sql database
tags:
- remove
/jobs/delete_job/{id}:
delete:
description: Job to remove is specified by database ID. This will not remove
the job from the job archive.
parameters:
- description: Database ID of Job
in: path
name: id
required: true
type: integer
produces:
- application/json
responses:
"200":
description: Success message
schema:
$ref: '#/definitions/api.DeleteJobApiResponse'
"400":
description: Bad Request
schema:
$ref: '#/definitions/api.ErrorResponse'
"401":
description: Unauthorized
schema:
$ref: '#/definitions/api.ErrorResponse'
"403":
description: Forbidden
schema:
$ref: '#/definitions/api.ErrorResponse'
"404":
description: Resource not found
schema:
$ref: '#/definitions/api.ErrorResponse'
"422":
description: 'Unprocessable Entity: finding job failed: sql: no rows in
result set'
schema:
$ref: '#/definitions/api.ErrorResponse'
"500":
description: Internal Server Error
schema:
$ref: '#/definitions/api.ErrorResponse'
security:
- ApiKeyAuth: []
summary: Remove a job from the sql database
tags:
- remove
/jobs/delete_job_before/{ts}:
delete:
description: Remove all jobs with start time before timestamp. The jobs will
not be removed from the job archive.
parameters:
- description: Unix epoch timestamp
in: path
name: ts
required: true
type: integer
produces:
- application/json
responses:
"200":
description: Success message
schema:
$ref: '#/definitions/api.DeleteJobApiResponse'
"400":
description: Bad Request
schema:
$ref: '#/definitions/api.ErrorResponse'
"401":
description: Unauthorized
schema:
$ref: '#/definitions/api.ErrorResponse'
"403":
description: Forbidden
schema:
$ref: '#/definitions/api.ErrorResponse'
"404":
description: Resource not found
schema:
$ref: '#/definitions/api.ErrorResponse'
"422":
description: 'Unprocessable Entity: finding job failed: sql: no rows in
result set'
schema:
$ref: '#/definitions/api.ErrorResponse'
"500":
description: Internal Server Error
schema:
$ref: '#/definitions/api.ErrorResponse'
security:
- ApiKeyAuth: []
summary: Remove a job from the sql database
tags:
- remove
/jobs/start_job/: /jobs/start_job/:
post: post:
consumes: consumes:
@@ -466,6 +626,8 @@ paths:
security: security:
- ApiKeyAuth: [] - ApiKeyAuth: []
summary: Adds a new job as "running" summary: Adds a new job as "running"
tags:
- add and modify
/jobs/stop_job/: /jobs/stop_job/:
post: post:
description: |- description: |-
@@ -482,7 +644,7 @@ paths:
- application/json - application/json
responses: responses:
"200": "200":
description: Job resource description: Success message
schema: schema:
$ref: '#/definitions/schema.JobMeta' $ref: '#/definitions/schema.JobMeta'
"400": "400":
@@ -513,6 +675,8 @@ paths:
security: security:
- ApiKeyAuth: [] - ApiKeyAuth: []
summary: Marks job as completed and triggers archiving summary: Marks job as completed and triggers archiving
tags:
- add and modify
/jobs/stop_job/{id}: /jobs/stop_job/{id}:
post: post:
consumes: consumes:
@@ -567,6 +731,8 @@ paths:
security: security:
- ApiKeyAuth: [] - ApiKeyAuth: []
summary: Marks job as completed and triggers archiving summary: Marks job as completed and triggers archiving
tags:
- add and modify
/jobs/tag_job/{id}: /jobs/tag_job/{id}:
post: post:
consumes: consumes:
@@ -586,7 +752,7 @@ paths:
required: true required: true
schema: schema:
items: items:
$ref: '#/definitions/api.Tag' $ref: '#/definitions/api.ApiTag'
type: array type: array
produces: produces:
- application/json - application/json
@@ -614,10 +780,13 @@ paths:
security: security:
- ApiKeyAuth: [] - ApiKeyAuth: []
summary: Adds one or more tags to a job summary: Adds one or more tags to a job
tags:
- add and modify
securityDefinitions: securityDefinitions:
ApiKeyAuth: ApiKeyAuth:
description: JWT based authentification for general API endpoint use.
in: header in: header
name: X-Auth-Token name: X-Auth-Token
type: apiKey type: apiKey
swagger: "2.0" swagger: "2.0"
tags:
- name: Job API

324
internal/api/docs.go

@@ -1,6 +1,5 @@
// Package api GENERATED BY SWAG; DO NOT EDIT // Package api GENERATED BY SWAG; DO NOT EDIT
// This file was generated by swaggo/swag at // This file was generated by swaggo/swag
// 2022-09-22 13:31:53.353204065 +0200 CEST m=+0.139444562
package api package api
import "github.com/swaggo/swag" import "github.com/swaggo/swag"
@@ -11,7 +10,6 @@ const docTemplate = `{
"info": { "info": {
"description": "{{escape .Description}}", "description": "{{escape .Description}}",
"title": "{{.Title}}", "title": "{{.Title}}",
"termsOfService": "https://monitoring.nhr.fau.de/imprint",
"contact": { "contact": {
"name": "ClusterCockpit Project", "name": "ClusterCockpit Project",
"url": "https://github.com/ClusterCockpit", "url": "https://github.com/ClusterCockpit",
@@ -34,12 +32,12 @@ const docTemplate = `{
} }
], ],
"description": "Get a list of all jobs. Filters can be applied using query parameters.\nNumber of results can be limited by page. Results are sorted by descending startTime.", "description": "Get a list of all jobs. Filters can be applied using query parameters.\nNumber of results can be limited by page. Results are sorted by descending startTime.",
"consumes": [
"application/json"
],
"produces": [ "produces": [
"application/json" "application/json"
], ],
"tags": [
"query"
],
"summary": "Lists all jobs", "summary": "Lists all jobs",
"parameters": [ "parameters": [
{ {
@@ -70,13 +68,13 @@ const docTemplate = `{
}, },
{ {
"type": "integer", "type": "integer",
"description": "Items per page (If empty: No Limit)", "description": "Items per page (Default: 25)",
"name": "items-per-page", "name": "items-per-page",
"in": "query" "in": "query"
}, },
{ {
"type": "integer", "type": "integer",
"description": "Page Number (If empty: No Paging)", "description": "Page Number (Default: 1)",
"name": "page", "name": "page",
"in": "query" "in": "query"
}, },
@@ -118,6 +116,221 @@ const docTemplate = `{
} }
} }
}, },
"/jobs/delete_job/": {
"delete": {
"security": [
{
"ApiKeyAuth": []
}
],
"description": "Job to delete is specified by request body. All fields are required in this case.",
"consumes": [
"application/json"
],
"produces": [
"application/json"
],
"tags": [
"remove"
],
"summary": "Remove a job from the sql database",
"parameters": [
{
"description": "All fields required",
"name": "request",
"in": "body",
"required": true,
"schema": {
"$ref": "#/definitions/api.DeleteJobApiRequest"
}
}
],
"responses": {
"200": {
"description": "Success message",
"schema": {
"$ref": "#/definitions/api.DeleteJobApiResponse"
}
},
"400": {
"description": "Bad Request",
"schema": {
"$ref": "#/definitions/api.ErrorResponse"
}
},
"401": {
"description": "Unauthorized",
"schema": {
"$ref": "#/definitions/api.ErrorResponse"
}
},
"403": {
"description": "Forbidden",
"schema": {
"$ref": "#/definitions/api.ErrorResponse"
}
},
"404": {
"description": "Resource not found",
"schema": {
"$ref": "#/definitions/api.ErrorResponse"
}
},
"422": {
"description": "Unprocessable Entity: finding job failed: sql: no rows in result set",
"schema": {
"$ref": "#/definitions/api.ErrorResponse"
}
},
"500": {
"description": "Internal Server Error",
"schema": {
"$ref": "#/definitions/api.ErrorResponse"
}
}
}
}
},
"/jobs/delete_job/{id}": {
"delete": {
"security": [
{
"ApiKeyAuth": []
}
],
"description": "Job to remove is specified by database ID. This will not remove the job from the job archive.",
"produces": [
"application/json"
],
"tags": [
"remove"
],
"summary": "Remove a job from the sql database",
"parameters": [
{
"type": "integer",
"description": "Database ID of Job",
"name": "id",
"in": "path",
"required": true
}
],
"responses": {
"200": {
"description": "Success message",
"schema": {
"$ref": "#/definitions/api.DeleteJobApiResponse"
}
},
"400": {
"description": "Bad Request",
"schema": {
"$ref": "#/definitions/api.ErrorResponse"
}
},
"401": {
"description": "Unauthorized",
"schema": {
"$ref": "#/definitions/api.ErrorResponse"
}
},
"403": {
"description": "Forbidden",
"schema": {
"$ref": "#/definitions/api.ErrorResponse"
}
},
"404": {
"description": "Resource not found",
"schema": {
"$ref": "#/definitions/api.ErrorResponse"
}
},
"422": {
"description": "Unprocessable Entity: finding job failed: sql: no rows in result set",
"schema": {
"$ref": "#/definitions/api.ErrorResponse"
}
},
"500": {
"description": "Internal Server Error",
"schema": {
"$ref": "#/definitions/api.ErrorResponse"
}
}
}
}
},
"/jobs/delete_job_before/{ts}": {
"delete": {
"security": [
{
"ApiKeyAuth": []
}
],
"description": "Remove all jobs with start time before timestamp. The jobs will not be removed from the job archive.",
"produces": [
"application/json"
],
"tags": [
"remove"
],
"summary": "Remove a job from the sql database",
"parameters": [
{
"type": "integer",
"description": "Unix epoch timestamp",
"name": "ts",
"in": "path",
"required": true
}
],
"responses": {
"200": {
"description": "Success message",
"schema": {
"$ref": "#/definitions/api.DeleteJobApiResponse"
}
},
"400": {
"description": "Bad Request",
"schema": {
"$ref": "#/definitions/api.ErrorResponse"
}
},
"401": {
"description": "Unauthorized",
"schema": {
"$ref": "#/definitions/api.ErrorResponse"
}
},
"403": {
"description": "Forbidden",
"schema": {
"$ref": "#/definitions/api.ErrorResponse"
}
},
"404": {
"description": "Resource not found",
"schema": {
"$ref": "#/definitions/api.ErrorResponse"
}
},
"422": {
"description": "Unprocessable Entity: finding job failed: sql: no rows in result set",
"schema": {
"$ref": "#/definitions/api.ErrorResponse"
}
},
"500": {
"description": "Internal Server Error",
"schema": {
"$ref": "#/definitions/api.ErrorResponse"
}
}
}
}
},
"/jobs/start_job/": { "/jobs/start_job/": {
"post": { "post": {
"security": [ "security": [
@@ -132,6 +345,9 @@ const docTemplate = `{
"produces": [ "produces": [
"application/json" "application/json"
], ],
"tags": [
"add and modify"
],
"summary": "Adds a new job as \"running\"", "summary": "Adds a new job as \"running\"",
"parameters": [ "parameters": [
{ {
@@ -195,6 +411,9 @@ const docTemplate = `{
"produces": [ "produces": [
"application/json" "application/json"
], ],
"tags": [
"add and modify"
],
"summary": "Marks job as completed and triggers archiving", "summary": "Marks job as completed and triggers archiving",
"parameters": [ "parameters": [
{ {
@@ -209,7 +428,7 @@ const docTemplate = `{
], ],
"responses": { "responses": {
"200": { "200": {
"description": "Job resource", "description": "Success message",
"schema": { "schema": {
"$ref": "#/definitions/schema.JobMeta" "$ref": "#/definitions/schema.JobMeta"
} }
@@ -267,6 +486,9 @@ const docTemplate = `{
"produces": [ "produces": [
"application/json" "application/json"
], ],
"tags": [
"add and modify"
],
"summary": "Marks job as completed and triggers archiving", "summary": "Marks job as completed and triggers archiving",
"parameters": [ "parameters": [
{ {
@@ -346,6 +568,9 @@ const docTemplate = `{
"produces": [ "produces": [
"application/json" "application/json"
], ],
"tags": [
"add and modify"
],
"summary": "Adds one or more tags to a job", "summary": "Adds one or more tags to a job",
"parameters": [ "parameters": [
{ {
@@ -363,7 +588,7 @@ const docTemplate = `{
"schema": { "schema": {
"type": "array", "type": "array",
"items": { "items": {
"$ref": "#/definitions/api.Tag" "$ref": "#/definitions/api.ApiTag"
} }
} }
} }
@@ -404,8 +629,53 @@ const docTemplate = `{
} }
}, },
"definitions": { "definitions": {
"api.ApiTag": {
"type": "object",
"properties": {
"name": {
"description": "Tag Name",
"type": "string",
"example": "Testjob"
},
"type": {
"description": "Tag Type",
"type": "string",
"example": "Debug"
}
}
},
"api.DeleteJobApiRequest": {
"type": "object",
"required": [
"jobId"
],
"properties": {
"cluster": {
"description": "Cluster of job",
"type": "string",
"example": "fritz"
},
"jobId": {
"description": "Cluster Job ID of job",
"type": "integer",
"example": 123000
},
"startTime": {
"description": "Start Time of job as epoch",
"type": "integer",
"example": 1649723812
}
}
},
"api.DeleteJobApiResponse": {
"type": "object",
"properties": {
"msg": {
"type": "string"
}
}
},
"api.ErrorResponse": { "api.ErrorResponse": {
"description": "Error message as returned from backend.",
"type": "object", "type": "object",
"properties": { "properties": {
"error": { "error": {
@@ -419,7 +689,6 @@ const docTemplate = `{
} }
}, },
"api.StartJobApiResponse": { "api.StartJobApiResponse": {
"description": "Successful job start response with database id of new job.",
"type": "object", "type": "object",
"properties": { "properties": {
"id": { "id": {
@@ -429,7 +698,6 @@ const docTemplate = `{
} }
}, },
"api.StopJobApiRequest": { "api.StopJobApiRequest": {
"description": "Request to stop running job using stoptime and final state. They are only required if no database id was provided with endpoint.",
"type": "object", "type": "object",
"required": [ "required": [
"jobState", "jobState",
@@ -470,22 +738,6 @@ const docTemplate = `{
} }
} }
}, },
"api.Tag": {
"description": "Defines a tag using name and type.",
"type": "object",
"properties": {
"name": {
"description": "Tag Name",
"type": "string",
"example": "Testjob"
},
"type": {
"description": "Tag Type",
"type": "string",
"example": "Debug"
}
}
},
"schema.Job": { "schema.Job": {
"description": "Information of a HPC job.", "description": "Information of a HPC job.",
"type": "object", "type": "object",
@@ -839,22 +1091,26 @@ const docTemplate = `{
}, },
"securityDefinitions": { "securityDefinitions": {
"ApiKeyAuth": { "ApiKeyAuth": {
"description": "JWT based authentification for general API endpoint use.",
"type": "apiKey", "type": "apiKey",
"name": "X-Auth-Token", "name": "X-Auth-Token",
"in": "header" "in": "header"
} }
} },
"tags": [
{
"name": "Job API"
}
]
}` }`
// SwaggerInfo holds exported Swagger Info so clients can modify it // SwaggerInfo holds exported Swagger Info so clients can modify it
var SwaggerInfo = &swag.Spec{ var SwaggerInfo = &swag.Spec{
Version: "0.1.0", Version: "0.2.0",
Host: "clustercockpit.localhost:8082", Host: "localhost:8080",
BasePath: "/api", BasePath: "/api",
Schemes: []string{}, Schemes: []string{},
Title: "ClusterCockpit REST API", Title: "ClusterCockpit REST API",
Description: "Defines a tag using name and type.", Description: "API for batch job control.",
InfoInstanceName: "swagger", InfoInstanceName: "swagger",
SwaggerTemplate: docTemplate, SwaggerTemplate: docTemplate,
} }

356
internal/api/rest.go

@@ -32,9 +32,10 @@ import (
) )
// @title ClusterCockpit REST API // @title ClusterCockpit REST API
// @version 0.1.0 // @version 0.2.0
// @description API for batch job control. // @description API for batch job control.
// @termsOfService https://monitoring.nhr.fau.de/imprint
// @tag.name Job API
// @contact.name ClusterCockpit Project // @contact.name ClusterCockpit Project
// @contact.url https://github.com/ClusterCockpit // @contact.url https://github.com/ClusterCockpit
@@ -43,13 +44,12 @@ import (
// @license.name MIT License // @license.name MIT License
// @license.url https://opensource.org/licenses/MIT // @license.url https://opensource.org/licenses/MIT
// @host clustercockpit.localhost:8082 // @host localhost:8080
// @BasePath /api // @basePath /api
// @securityDefinitions.apikey ApiKeyAuth // @securityDefinitions.apikey ApiKeyAuth
// @in header // @in header
// @name X-Auth-Token // @name X-Auth-Token
// @description JWT based authentification for general API endpoint use.
type RestApi struct { type RestApi struct {
JobRepository *repository.JobRepository JobRepository *repository.JobRepository
@@ -73,6 +73,9 @@ func (api *RestApi) MountRoutes(r *mux.Router) {
// r.HandleFunc("/jobs/{id}", api.getJob).Methods(http.MethodGet) // r.HandleFunc("/jobs/{id}", api.getJob).Methods(http.MethodGet)
r.HandleFunc("/jobs/tag_job/{id}", api.tagJob).Methods(http.MethodPost, http.MethodPatch) r.HandleFunc("/jobs/tag_job/{id}", api.tagJob).Methods(http.MethodPost, http.MethodPatch)
r.HandleFunc("/jobs/metrics/{id}", api.getJobMetrics).Methods(http.MethodGet) r.HandleFunc("/jobs/metrics/{id}", api.getJobMetrics).Methods(http.MethodGet)
r.HandleFunc("/jobs/delete_job/", api.deleteJobByRequest).Methods(http.MethodDelete)
r.HandleFunc("/jobs/delete_job/{id}", api.deleteJobById).Methods(http.MethodDelete)
r.HandleFunc("/jobs/delete_job_before/{ts}", api.deleteJobBefore).Methods(http.MethodDelete)
if api.Authentication != nil { if api.Authentication != nil {
r.HandleFunc("/jwt/", api.getJWT).Methods(http.MethodGet) r.HandleFunc("/jwt/", api.getJWT).Methods(http.MethodGet)
@@ -90,15 +93,17 @@ func (api *RestApi) MountRoutes(r *mux.Router) {
} }
// StartJobApiResponse model // StartJobApiResponse model
// @Description Successful job start response with database id of new job.
type StartJobApiResponse struct { type StartJobApiResponse struct {
// Database ID of new job // Database ID of new job
DBID int64 `json:"id"` DBID int64 `json:"id"`
} }
// DeleteJobApiResponse model
type DeleteJobApiResponse struct {
Message string `json:"msg"`
}
// StopJobApiRequest model // StopJobApiRequest model
// @Description Request to stop running job using stoptime and final state.
// @Description They are only required if no database id was provided with endpoint.
type StopJobApiRequest struct { type StopJobApiRequest struct {
// Stop Time of job as epoch // Stop Time of job as epoch
StopTime int64 `json:"stopTime" validate:"required" example:"1649763839"` StopTime int64 `json:"stopTime" validate:"required" example:"1649763839"`
@@ -108,23 +113,28 @@ type StopJobApiRequest struct {
StartTime *int64 `json:"startTime" example:"1649723812"` // Start Time of job as epoch StartTime *int64 `json:"startTime" example:"1649723812"` // Start Time of job as epoch
} }
// DeleteJobApiRequest model
type DeleteJobApiRequest struct {
JobId *int64 `json:"jobId" validate:"required" example:"123000"` // Cluster Job ID of job
Cluster *string `json:"cluster" example:"fritz"` // Cluster of job
StartTime *int64 `json:"startTime" example:"1649723812"` // Start Time of job as epoch
}
// ErrorResponse model // ErrorResponse model
// @Description Error message as returned from backend.
type ErrorResponse struct { type ErrorResponse struct {
// Statustext of Errorcode // Statustext of Errorcode
Status string `json:"status"` Status string `json:"status"`
Error string `json:"error"` // Error Message Error string `json:"error"` // Error Message
} }
// Tag model // ApiTag model
// @Description Defines a tag using name and type. type ApiTag struct {
type Tag struct {
// Tag Type // Tag Type
Type string `json:"type" example:"Debug"` Type string `json:"type" example:"Debug"`
Name string `json:"name" example:"Testjob"` // Tag Name Name string `json:"name" example:"Testjob"` // Tag Name
} }
type TagJobApiRequest []*Tag type TagJobApiRequest []*ApiTag
func handleError(err error, statusCode int, rw http.ResponseWriter) { func handleError(err error, statusCode int, rw http.ResponseWriter) {
log.Warnf("REST API: %s", err.Error()) log.Warnf("REST API: %s", err.Error())
@@ -143,28 +153,34 @@ func decode(r io.Reader, val interface{}) error {
} }
// getJobs godoc // getJobs godoc
// @Summary Lists all jobs // @summary Lists all jobs
// @Description Get a list of all jobs. Filters can be applied using query parameters. // @tags query
// @Description Number of results can be limited by page. Results are sorted by descending startTime. // @description Get a list of all jobs. Filters can be applied using query parameters.
// @Accept json // @description Number of results can be limited by page. Results are sorted by descending startTime.
// @Produce json // @produce json
// @Param state query string false "Job State" Enums(running, completed, failed, cancelled, stopped, timeout) // @param state query string false "Job State" Enums(running, completed, failed, cancelled, stopped, timeout)
// @Param cluster query string false "Job Cluster" // @param cluster query string false "Job Cluster"
// @Param start-time query string false "Syntax: '$from-$to', as unix epoch timestamps in seconds" // @param start-time query string false "Syntax: '$from-$to', as unix epoch timestamps in seconds"
// @Param items-per-page query int false "Items per page (If empty: No Limit)" // @param items-per-page query int false "Items per page (Default: 25)"
// @Param page query int false "Page Number (If empty: No Paging)" // @param page query int false "Page Number (Default: 1)"
// @Param with-metadata query bool false "Include metadata (e.g. jobScript) in response" // @param with-metadata query bool false "Include metadata (e.g. jobScript) in response"
// @Success 200 {array} schema.Job "Array of matching jobs" // @success 200 {array} schema.Job "Array of matching jobs"
// @Failure 400 {object} api.ErrorResponse "Bad Request" // @failure 400 {object} api.ErrorResponse "Bad Request"
// @Failure 401 {object} api.ErrorResponse "Unauthorized" // @failure 401 {object} api.ErrorResponse "Unauthorized"
// @Failure 500 {object} api.ErrorResponse "Internal Server Error" // @failure 500 {object} api.ErrorResponse "Internal Server Error"
// @Security ApiKeyAuth // @security ApiKeyAuth
// @Router /jobs/ [get] // @router /jobs/ [get]
func (api *RestApi) getJobs(rw http.ResponseWriter, r *http.Request) { func (api *RestApi) getJobs(rw http.ResponseWriter, r *http.Request) {
if user := auth.GetUser(r.Context()); user != nil && !user.HasRole(auth.RoleApi) {
handleError(fmt.Errorf("missing role: %#v", auth.RoleApi), http.StatusForbidden, rw)
return
}
withMetadata := false withMetadata := false
filter := &model.JobFilter{} filter := &model.JobFilter{}
page := &model.PageRequest{ItemsPerPage: -1, Page: 1} page := &model.PageRequest{ItemsPerPage: 25, Page: 1}
order := &model.OrderByInput{Field: "startTime", Order: model.SortDirectionEnumDesc} order := &model.OrderByInput{Field: "startTime", Order: model.SortDirectionEnumDesc}
for key, vals := range r.URL.Query() { for key, vals := range r.URL.Query() {
switch key { switch key {
case "state": case "state":
@@ -270,21 +286,27 @@ func (api *RestApi) getJobs(rw http.ResponseWriter, r *http.Request) {
} }
// tagJob godoc // tagJob godoc
// @Summary Adds one or more tags to a job // @summary Adds one or more tags to a job
// @Description Adds tag(s) to a job specified by DB ID. Name and Type of Tag(s) can be chosen freely. // @tags add and modify
// @Description If tagged job is already finished: Tag will be written directly to respective archive files. // @description Adds tag(s) to a job specified by DB ID. Name and Type of Tag(s) can be chosen freely.
// @Accept json // @description If tagged job is already finished: Tag will be written directly to respective archive files.
// @Produce json // @accept json
// @Param id path int true "Job Database ID" // @produce json
// @Param request body api.TagJobApiRequest true "Array of tag-objects to add" // @param id path int true "Job Database ID"
// @Success 200 {object} schema.Job "Updated job resource" // @param request body api.TagJobApiRequest true "Array of tag-objects to add"
// @Failure 400 {object} api.ErrorResponse "Bad Request" // @success 200 {object} schema.Job "Updated job resource"
// @Failure 401 {object} api.ErrorResponse "Unauthorized" // @failure 400 {object} api.ErrorResponse "Bad Request"
// @Failure 404 {object} api.ErrorResponse "Job or tag does not exist" // @failure 401 {object} api.ErrorResponse "Unauthorized"
// @Failure 500 {object} api.ErrorResponse "Internal Server Error" // @failure 404 {object} api.ErrorResponse "Job or tag does not exist"
// @Security ApiKeyAuth // @failure 500 {object} api.ErrorResponse "Internal Server Error"
// @Router /jobs/tag_job/{id} [post] // @security ApiKeyAuth
// @router /jobs/tag_job/{id} [post]
func (api *RestApi) tagJob(rw http.ResponseWriter, r *http.Request) { func (api *RestApi) tagJob(rw http.ResponseWriter, r *http.Request) {
if user := auth.GetUser(r.Context()); user != nil && !user.HasRole(auth.RoleApi) {
handleError(fmt.Errorf("missing role: %#v", auth.RoleApi), http.StatusForbidden, rw)
return
}
iid, err := strconv.ParseInt(mux.Vars(r)["id"], 10, 64) iid, err := strconv.ParseInt(mux.Vars(r)["id"], 10, 64)
if err != nil { if err != nil {
http.Error(rw, err.Error(), http.StatusBadRequest) http.Error(rw, err.Error(), http.StatusBadRequest)
@@ -329,20 +351,21 @@ func (api *RestApi) tagJob(rw http.ResponseWriter, r *http.Request) {
} }
// startJob godoc // startJob godoc
// @Summary Adds a new job as "running" // @summary Adds a new job as "running"
// @Description Job specified in request body will be saved to database as "running" with new DB ID. // @tags add and modify
// @Description Job specifications follow the 'JobMeta' scheme, API will fail to execute if requirements are not met. // @description Job specified in request body will be saved to database as "running" with new DB ID.
// @Accept json // @description Job specifications follow the 'JobMeta' scheme, API will fail to execute if requirements are not met.
// @Produce json // @accept json
// @Param request body schema.JobMeta true "Job to add" // @produce json
// @Success 201 {object} api.StartJobApiResponse "Job added successfully" // @param request body schema.JobMeta true "Job to add"
// @Failure 400 {object} api.ErrorResponse "Bad Request" // @success 201 {object} api.StartJobApiResponse "Job added successfully"
// @Failure 401 {object} api.ErrorResponse "Unauthorized" // @failure 400 {object} api.ErrorResponse "Bad Request"
// @Failure 403 {object} api.ErrorResponse "Forbidden" // @failure 401 {object} api.ErrorResponse "Unauthorized"
// @Failure 422 {object} api.ErrorResponse "Unprocessable Entity: The combination of jobId, clusterId and startTime does already exist" // @failure 403 {object} api.ErrorResponse "Forbidden"
// @Failure 500 {object} api.ErrorResponse "Internal Server Error" // @failure 422 {object} api.ErrorResponse "Unprocessable Entity: The combination of jobId, clusterId and startTime does already exist"
// @Security ApiKeyAuth // @failure 500 {object} api.ErrorResponse "Internal Server Error"
// @Router /jobs/start_job/ [post] // @security ApiKeyAuth
// @router /jobs/start_job/ [post]
func (api *RestApi) startJob(rw http.ResponseWriter, r *http.Request) { func (api *RestApi) startJob(rw http.ResponseWriter, r *http.Request) {
if user := auth.GetUser(r.Context()); user != nil && !user.HasRole(auth.RoleApi) { if user := auth.GetUser(r.Context()); user != nil && !user.HasRole(auth.RoleApi) {
handleError(fmt.Errorf("missing role: %#v", auth.RoleApi), http.StatusForbidden, rw) handleError(fmt.Errorf("missing role: %#v", auth.RoleApi), http.StatusForbidden, rw)
@@ -407,22 +430,23 @@ func (api *RestApi) startJob(rw http.ResponseWriter, r *http.Request) {
} }
// stopJobById godoc // stopJobById godoc
// @Summary Marks job as completed and triggers archiving // @summary Marks job as completed and triggers archiving
// @Description Job to stop is specified by database ID. Only stopTime and final state are required in request body. // @tags add and modify
// @Description Returns full job resource information according to 'JobMeta' scheme. // @description Job to stop is specified by database ID. Only stopTime and final state are required in request body.
// @Accept json // @description Returns full job resource information according to 'JobMeta' scheme.
// @Produce json // @accept json
// @Param id path int true "Database ID of Job" // @produce json
// @Param request body api.StopJobApiRequest true "stopTime and final state in request body" // @param id path int true "Database ID of Job"
// @Success 200 {object} schema.JobMeta "Job resource" // @param request body api.StopJobApiRequest true "stopTime and final state in request body"
// @Failure 400 {object} api.ErrorResponse "Bad Request" // @success 200 {object} schema.JobMeta "Job resource"
// @Failure 401 {object} api.ErrorResponse "Unauthorized" // @failure 400 {object} api.ErrorResponse "Bad Request"
// @Failure 403 {object} api.ErrorResponse "Forbidden" // @failure 401 {object} api.ErrorResponse "Unauthorized"
// @Failure 404 {object} api.ErrorResponse "Resource not found" // @failure 403 {object} api.ErrorResponse "Forbidden"
// @Failure 422 {object} api.ErrorResponse "Unprocessable Entity: finding job failed: sql: no rows in result set" // @failure 404 {object} api.ErrorResponse "Resource not found"
// @Failure 500 {object} api.ErrorResponse "Internal Server Error" // @failure 422 {object} api.ErrorResponse "Unprocessable Entity: finding job failed: sql: no rows in result set"
// @Security ApiKeyAuth // @failure 500 {object} api.ErrorResponse "Internal Server Error"
// @Router /jobs/stop_job/{id} [post] // @security ApiKeyAuth
// @router /jobs/stop_job/{id} [post]
func (api *RestApi) stopJobById(rw http.ResponseWriter, r *http.Request) { func (api *RestApi) stopJobById(rw http.ResponseWriter, r *http.Request) {
if user := auth.GetUser(r.Context()); user != nil && !user.HasRole(auth.RoleApi) { if user := auth.GetUser(r.Context()); user != nil && !user.HasRole(auth.RoleApi) {
handleError(fmt.Errorf("missing role: %#v", auth.RoleApi), http.StatusForbidden, rw) handleError(fmt.Errorf("missing role: %#v", auth.RoleApi), http.StatusForbidden, rw)
@@ -461,20 +485,21 @@ func (api *RestApi) stopJobById(rw http.ResponseWriter, r *http.Request) {
} }
// stopJobByRequest godoc // stopJobByRequest godoc
// @Summary Marks job as completed and triggers archiving // @summary Marks job as completed and triggers archiving
// @Description Job to stop is specified by request body. All fields are required in this case. // @tags add and modify
// @Description Returns full job resource information according to 'JobMeta' scheme. // @description Job to stop is specified by request body. All fields are required in this case.
// @Produce json // @description Returns full job resource information according to 'JobMeta' scheme.
// @Param request body api.StopJobApiRequest true "All fields required" // @produce json
// @Success 200 {object} schema.JobMeta "Job resource" // @param request body api.StopJobApiRequest true "All fields required"
// @Failure 400 {object} api.ErrorResponse "Bad Request" // @success 200 {object} schema.JobMeta "Success message"
// @Failure 401 {object} api.ErrorResponse "Unauthorized" // @failure 400 {object} api.ErrorResponse "Bad Request"
// @Failure 403 {object} api.ErrorResponse "Forbidden" // @failure 401 {object} api.ErrorResponse "Unauthorized"
// @Failure 404 {object} api.ErrorResponse "Resource not found" // @failure 403 {object} api.ErrorResponse "Forbidden"
// @Failure 422 {object} api.ErrorResponse "Unprocessable Entity: finding job failed: sql: no rows in result set" // @failure 404 {object} api.ErrorResponse "Resource not found"
// @Failure 500 {object} api.ErrorResponse "Internal Server Error" // @failure 422 {object} api.ErrorResponse "Unprocessable Entity: finding job failed: sql: no rows in result set"
// @Security ApiKeyAuth // @failure 500 {object} api.ErrorResponse "Internal Server Error"
// @Router /jobs/stop_job/ [post] // @security ApiKeyAuth
// @router /jobs/stop_job/ [post]
func (api *RestApi) stopJobByRequest(rw http.ResponseWriter, r *http.Request) { func (api *RestApi) stopJobByRequest(rw http.ResponseWriter, r *http.Request) {
if user := auth.GetUser(r.Context()); user != nil && !user.HasRole(auth.RoleApi) { if user := auth.GetUser(r.Context()); user != nil && !user.HasRole(auth.RoleApi) {
handleError(fmt.Errorf("missing role: %#v", auth.RoleApi), http.StatusForbidden, rw) handleError(fmt.Errorf("missing role: %#v", auth.RoleApi), http.StatusForbidden, rw)
@@ -506,6 +531,159 @@ func (api *RestApi) stopJobByRequest(rw http.ResponseWriter, r *http.Request) {
api.checkAndHandleStopJob(rw, job, req) api.checkAndHandleStopJob(rw, job, req)
} }
// deleteJobById godoc
// @summary Remove a job from the sql database
// @tags remove
// @description Job to remove is specified by database ID. This will not remove the job from the job archive.
// @produce json
// @param id path int true "Database ID of Job"
// @success 200 {object} api.DeleteJobApiResponse "Success message"
// @failure 400 {object} api.ErrorResponse "Bad Request"
// @failure 401 {object} api.ErrorResponse "Unauthorized"
// @failure 403 {object} api.ErrorResponse "Forbidden"
// @failure 404 {object} api.ErrorResponse "Resource not found"
// @failure 422 {object} api.ErrorResponse "Unprocessable Entity: finding job failed: sql: no rows in result set"
// @failure 500 {object} api.ErrorResponse "Internal Server Error"
// @security ApiKeyAuth
// @router /jobs/delete_job/{id} [delete]
func (api *RestApi) deleteJobById(rw http.ResponseWriter, r *http.Request) {
if user := auth.GetUser(r.Context()); user != nil && !user.HasRole(auth.RoleApi) {
handleError(fmt.Errorf("missing role: %#v", auth.RoleApi), http.StatusForbidden, rw)
return
}
// Fetch job (that will be stopped) from db
id, ok := mux.Vars(r)["id"]
var err error
if ok {
id, e := strconv.ParseInt(id, 10, 64)
if e != nil {
handleError(fmt.Errorf("integer expected in path for id: %w", e), http.StatusBadRequest, rw)
return
}
err = api.JobRepository.DeleteJobById(id)
} else {
handleError(errors.New("the parameter 'id' is required"), http.StatusBadRequest, rw)
return
}
if err != nil {
handleError(fmt.Errorf("deleting job failed: %w", err), http.StatusUnprocessableEntity, rw)
return
}
rw.Header().Add("Content-Type", "application/json")
rw.WriteHeader(http.StatusOK)
json.NewEncoder(rw).Encode(DeleteJobApiResponse{
Message: fmt.Sprintf("Successfully deleted job %s", id),
})
}
// deleteJobByRequest godoc
// @summary Remove a job from the sql database
// @tags remove
// @description Job to delete is specified by request body. All fields are required in this case.
// @accept json
// @produce json
// @param request body api.DeleteJobApiRequest true "All fields required"
// @success 200 {object} api.DeleteJobApiResponse "Success message"
// @failure 400 {object} api.ErrorResponse "Bad Request"
// @failure 401 {object} api.ErrorResponse "Unauthorized"
// @failure 403 {object} api.ErrorResponse "Forbidden"
// @failure 404 {object} api.ErrorResponse "Resource not found"
// @failure 422 {object} api.ErrorResponse "Unprocessable Entity: finding job failed: sql: no rows in result set"
// @failure 500 {object} api.ErrorResponse "Internal Server Error"
// @security ApiKeyAuth
// @router /jobs/delete_job/ [delete]
func (api *RestApi) deleteJobByRequest(rw http.ResponseWriter, r *http.Request) {
if user := auth.GetUser(r.Context()); user != nil && !user.HasRole(auth.RoleApi) {
handleError(fmt.Errorf("missing role: %#v", auth.RoleApi), http.StatusForbidden, rw)
return
}
// Parse request body
req := DeleteJobApiRequest{}
if err := decode(r.Body, &req); err != nil {
handleError(fmt.Errorf("parsing request body failed: %w", err), http.StatusBadRequest, rw)
return
}
// Fetch job (that will be deleted) from db
var job *schema.Job
var err error
if req.JobId == nil {
handleError(errors.New("the field 'jobId' is required"), http.StatusBadRequest, rw)
return
}
job, err = api.JobRepository.Find(req.JobId, req.Cluster, req.StartTime)
if err != nil {
handleError(fmt.Errorf("finding job failed: %w", err), http.StatusUnprocessableEntity, rw)
return
}
err = api.JobRepository.DeleteJobById(job.ID)
if err != nil {
handleError(fmt.Errorf("deleting job failed: %w", err), http.StatusUnprocessableEntity, rw)
return
}
rw.Header().Add("Content-Type", "application/json")
rw.WriteHeader(http.StatusOK)
json.NewEncoder(rw).Encode(DeleteJobApiResponse{
Message: fmt.Sprintf("Successfully deleted job %d", job.ID),
})
}
// deleteJobBefore godoc
// @summary Remove a job from the sql database
// @tags remove
// @description Remove all jobs with start time before timestamp. The jobs will not be removed from the job archive.
// @produce json
// @param ts path int true "Unix epoch timestamp"
// @success 200 {object} api.DeleteJobApiResponse "Success message"
// @failure 400 {object} api.ErrorResponse "Bad Request"
// @failure 401 {object} api.ErrorResponse "Unauthorized"
// @failure 403 {object} api.ErrorResponse "Forbidden"
// @failure 404 {object} api.ErrorResponse "Resource not found"
// @failure 422 {object} api.ErrorResponse "Unprocessable Entity: finding job failed: sql: no rows in result set"
// @failure 500 {object} api.ErrorResponse "Internal Server Error"
// @security ApiKeyAuth
// @router /jobs/delete_job_before/{ts} [delete]
func (api *RestApi) deleteJobBefore(rw http.ResponseWriter, r *http.Request) {
if user := auth.GetUser(r.Context()); user != nil && !user.HasRole(auth.RoleApi) {
handleError(fmt.Errorf("missing role: %#v", auth.RoleApi), http.StatusForbidden, rw)
return
}
var cnt int
// Fetch job (that will be stopped) from db
id, ok := mux.Vars(r)["ts"]
var err error
if ok {
ts, e := strconv.ParseInt(id, 10, 64)
if e != nil {
handleError(fmt.Errorf("integer expected in path for ts: %w", e), http.StatusBadRequest, rw)
return
}
cnt, err = api.JobRepository.DeleteJobsBefore(ts)
} else {
handleError(errors.New("the parameter 'ts' is required"), http.StatusBadRequest, rw)
return
}
if err != nil {
handleError(fmt.Errorf("deleting jobs failed: %w", err), http.StatusUnprocessableEntity, rw)
return
}
rw.Header().Add("Content-Type", "application/json")
rw.WriteHeader(http.StatusOK)
json.NewEncoder(rw).Encode(DeleteJobApiResponse{
Message: fmt.Sprintf("Successfully deleted %d jobs", cnt),
})
}
func (api *RestApi) checkAndHandleStopJob(rw http.ResponseWriter, job *schema.Job, req StopJobApiRequest) { func (api *RestApi) checkAndHandleStopJob(rw http.ResponseWriter, job *schema.Job, req StopJobApiRequest) {
// Sanity checks // Sanity checks

23
internal/repository/job.go

@@ -246,6 +246,29 @@ func (r *JobRepository) Stop(
return return
} }
func (r *JobRepository) DeleteJobsBefore(startTime int64) (int, error) {
var cnt int
qs := fmt.Sprintf("SELECT count(*) FROM job WHERE job.start_time < %d", startTime)
err := r.DB.Get(&cnt, qs) //ignore error as it will also occur in delete statement
_, err = r.DB.Exec(`DELETE FROM job WHERE job.start_time < ?`, startTime)
if err != nil {
log.Warnf(" DeleteJobsBefore(%d): error %v", startTime, err)
} else {
log.Infof("DeleteJobsBefore(%d): Deleted %d jobs", startTime, cnt)
}
return cnt, err
}
func (r *JobRepository) DeleteJobById(id int64) error {
_, err := r.DB.Exec(`DELETE FROM job WHERE job.id = ?`, id)
if err != nil {
log.Warnf("DeleteJobById(%d): error %v", id, err)
} else {
log.Infof("DeleteJobById(%d): Success", id)
}
return err
}
// TODO: Use node hours instead: SELECT job.user, sum(job.num_nodes * (CASE WHEN job.job_state = "running" THEN CAST(strftime('%s', 'now') AS INTEGER) - job.start_time ELSE job.duration END)) as x FROM job GROUP BY user ORDER BY x DESC; // TODO: Use node hours instead: SELECT job.user, sum(job.num_nodes * (CASE WHEN job.job_state = "running" THEN CAST(strftime('%s', 'now') AS INTEGER) - job.start_time ELSE job.duration END)) as x FROM job GROUP BY user ORDER BY x DESC;
func (r *JobRepository) CountGroupedJobs(ctx context.Context, aggreg model.Aggregate, filters []*model.JobFilter, weight *model.Weights, limit *int) (map[string]int, error) { func (r *JobRepository) CountGroupedJobs(ctx context.Context, aggreg model.Aggregate, filters []*model.JobFilter, weight *model.Weights, limit *int) (map[string]int, error) {
if !aggreg.IsValid() { if !aggreg.IsValid() {

70
pkg/schema/job.go

@@ -17,26 +17,26 @@ import (
type BaseJob struct { type BaseJob struct {
// The unique identifier of a job // The unique identifier of a job
JobID int64 `json:"jobId" db:"job_id" example:"123000"` JobID int64 `json:"jobId" db:"job_id" example:"123000"`
User string `json:"user" db:"user" example:"abcd100h"` // The unique identifier of a user User string `json:"user" db:"user" example:"abcd100h"` // The unique identifier of a user
Project string `json:"project" db:"project" example:"abcd200"` // The unique identifier of a project Project string `json:"project" db:"project" example:"abcd200"` // The unique identifier of a project
Cluster string `json:"cluster" db:"cluster" example:"fritz"` // The unique identifier of a cluster Cluster string `json:"cluster" db:"cluster" example:"fritz"` // The unique identifier of a cluster
SubCluster string `json:"subCluster" db:"subcluster" example:"main"` // The unique identifier of a sub cluster SubCluster string `json:"subCluster" db:"subcluster" example:"main"` // The unique identifier of a sub cluster
Partition string `json:"partition" db:"partition" example:"main"` // The Slurm partition to which the job was submitted Partition string `json:"partition" db:"partition" example:"main"` // The Slurm partition to which the job was submitted
ArrayJobId int64 `json:"arrayJobId" db:"array_job_id" example:"123000"` // The unique identifier of an array job ArrayJobId int64 `json:"arrayJobId" db:"array_job_id" example:"123000"` // The unique identifier of an array job
NumNodes int32 `json:"numNodes" db:"num_nodes" example:"2" minimum:"1"` // Number of nodes used (Min > 0) NumNodes int32 `json:"numNodes" db:"num_nodes" example:"2" minimum:"1"` // Number of nodes used (Min > 0)
NumHWThreads int32 `json:"numHwthreads" db:"num_hwthreads" example:"20" minimum:"1"` // Number of HWThreads used (Min > 0) NumHWThreads int32 `json:"numHwthreads" db:"num_hwthreads" example:"20" minimum:"1"` // Number of HWThreads used (Min > 0)
NumAcc int32 `json:"numAcc" db:"num_acc" example:"2" minimum:"1"` // Number of accelerators used (Min > 0) NumAcc int32 `json:"numAcc" db:"num_acc" example:"2" minimum:"1"` // Number of accelerators used (Min > 0)
Exclusive int32 `json:"exclusive" db:"exclusive" example:"1" minimum:"0" maximum:"2"` // Specifies how nodes are shared: 0 - Shared among multiple jobs of multiple users, 1 - Job exclusive (Default), 2 - Shared among multiple jobs of same user Exclusive int32 `json:"exclusive" db:"exclusive" example:"1" minimum:"0" maximum:"2"` // Specifies how nodes are shared: 0 - Shared among multiple jobs of multiple users, 1 - Job exclusive (Default), 2 - Shared among multiple jobs of same user
MonitoringStatus int32 `json:"monitoringStatus" db:"monitoring_status" example:"1" minimum:"0" maximum:"3"` // State of monitoring system during job run: 0 - Disabled, 1 - Running or Archiving (Default), 2 - Archiving Failed, 3 - Archiving Successfull MonitoringStatus int32 `json:"monitoringStatus" db:"monitoring_status" example:"1" minimum:"0" maximum:"3"` // State of monitoring system during job run: 0 - Disabled, 1 - Running or Archiving (Default), 2 - Archiving Failed, 3 - Archiving Successfull
SMT int32 `json:"smt" db:"smt" example:"4"` // SMT threads used by job SMT int32 `json:"smt" db:"smt" example:"4"` // SMT threads used by job
State JobState `json:"jobState" db:"job_state" example:"completed" enums:"completed,failed,cancelled,stopped,timeout,out_of_memory"` // Final state of job State JobState `json:"jobState" db:"job_state" example:"completed" enums:"completed,failed,cancelled,stopped,timeout,out_of_memory"` // Final state of job
Duration int32 `json:"duration" db:"duration" example:"43200" minimum:"1"` // Duration of job in seconds (Min > 0) Duration int32 `json:"duration" db:"duration" example:"43200" minimum:"1"` // Duration of job in seconds (Min > 0)
Walltime int64 `json:"walltime" db:"walltime" example:"86400" minimum:"1"` // Requested walltime of job in seconds (Min > 0) Walltime int64 `json:"walltime" db:"walltime" example:"86400" minimum:"1"` // Requested walltime of job in seconds (Min > 0)
Tags []*Tag `json:"tags"` // List of tags Tags []*Tag `json:"tags"` // List of tags
RawResources []byte `json:"-" db:"resources"` // Resources used by job [As Bytes] RawResources []byte `json:"-" db:"resources"` // Resources used by job [As Bytes]
Resources []*Resource `json:"resources"` // Resources used by job Resources []*Resource `json:"resources"` // Resources used by job
RawMetaData []byte `json:"-" db:"meta_data"` // Additional information about the job [As Bytes] RawMetaData []byte `json:"-" db:"meta_data"` // Additional information about the job [As Bytes]
MetaData map[string]string `json:"metaData"` // Additional information about the job MetaData map[string]string `json:"metaData"` // Additional information about the job
} }
// Non-Swaggered Comment: Job // Non-Swaggered Comment: Job
@@ -49,15 +49,15 @@ type Job struct {
ID int64 `json:"id" db:"id"` ID int64 `json:"id" db:"id"`
BaseJob BaseJob
StartTimeUnix int64 `json:"-" db:"start_time" example:"1649723812"` // Start epoch time stamp in seconds StartTimeUnix int64 `json:"-" db:"start_time" example:"1649723812"` // Start epoch time stamp in seconds
StartTime time.Time `json:"startTime"` // Start time as 'time.Time' data type StartTime time.Time `json:"startTime"` // Start time as 'time.Time' data type
MemUsedMax float64 `json:"-" db:"mem_used_max"` // MemUsedMax as Float64 MemUsedMax float64 `json:"-" db:"mem_used_max"` // MemUsedMax as Float64
FlopsAnyAvg float64 `json:"-" db:"flops_any_avg"` // FlopsAnyAvg as Float64 FlopsAnyAvg float64 `json:"-" db:"flops_any_avg"` // FlopsAnyAvg as Float64
MemBwAvg float64 `json:"-" db:"mem_bw_avg"` // MemBwAvg as Float64 MemBwAvg float64 `json:"-" db:"mem_bw_avg"` // MemBwAvg as Float64
LoadAvg float64 `json:"-" db:"load_avg"` // LoadAvg as Float64 LoadAvg float64 `json:"-" db:"load_avg"` // LoadAvg as Float64
NetBwAvg float64 `json:"-" db:"net_bw_avg"` // NetBwAvg as Float64 NetBwAvg float64 `json:"-" db:"net_bw_avg"` // NetBwAvg as Float64
NetDataVolTotal float64 `json:"-" db:"net_data_vol_total"` // NetDataVolTotal as Float64 NetDataVolTotal float64 `json:"-" db:"net_data_vol_total"` // NetDataVolTotal as Float64
FileBwAvg float64 `json:"-" db:"file_bw_avg"` // FileBwAvg as Float64 FileBwAvg float64 `json:"-" db:"file_bw_avg"` // FileBwAvg as Float64
FileDataVolTotal float64 `json:"-" db:"file_data_vol_total"` // FileDataVolTotal as Float64 FileDataVolTotal float64 `json:"-" db:"file_data_vol_total"` // FileDataVolTotal as Float64
} }
// Non-Swaggered Comment: JobMeta // Non-Swaggered Comment: JobMeta
@@ -70,11 +70,11 @@ type Job struct {
// JobMeta model // JobMeta model
// @Description Meta data information of a HPC job. // @Description Meta data information of a HPC job.
type JobMeta struct { type JobMeta struct {
// The unique identifier of a job in the database // The unique identifier of a job in the database
ID *int64 `json:"id,omitempty"` ID *int64 `json:"id,omitempty"`
BaseJob BaseJob
StartTime int64 `json:"startTime" db:"start_time" example:"1649723812" minimum:"1"` // Start epoch time stamp in seconds (Min > 0) StartTime int64 `json:"startTime" db:"start_time" example:"1649723812" minimum:"1"` // Start epoch time stamp in seconds (Min > 0)
Statistics map[string]JobStatistics `json:"statistics,omitempty"` // Metric statistics of job Statistics map[string]JobStatistics `json:"statistics,omitempty"` // Metric statistics of job
} }
const ( const (
@@ -102,18 +102,18 @@ type JobStatistics struct {
// Tag model // Tag model
// @Description Defines a tag using name and type. // @Description Defines a tag using name and type.
type Tag struct { type Tag struct {
// The unique DB identifier of a tag // The unique DB identifier of a tag
ID int64 `json:"id" db:"id"` ID int64 `json:"id" db:"id"`
Type string `json:"type" db:"tag_type" example:"Debug"` // Tag Type Type string `json:"type" db:"tag_type" example:"Debug"` // Tag Type
Name string `json:"name" db:"tag_name" example:"Testjob"` // Tag Name Name string `json:"name" db:"tag_name" example:"Testjob"` // Tag Name
} }
// Resource model // Resource model
// @Description A resource used by a job // @Description A resource used by a job
type Resource struct { type Resource struct {
Hostname string `json:"hostname"` // Name of the host (= node) Hostname string `json:"hostname"` // Name of the host (= node)
HWThreads []int `json:"hwthreads,omitempty"` // List of OS processor ids HWThreads []int `json:"hwthreads,omitempty"` // List of OS processor ids
Accelerators []string `json:"accelerators,omitempty"` // List of of accelerator device ids Accelerators []string `json:"accelerators,omitempty"` // List of of accelerator device ids
Configuration string `json:"configuration,omitempty"` // The configuration options of the node Configuration string `json:"configuration,omitempty"` // The configuration options of the node
} }

4
test/integration_test.go

@@ -343,7 +343,7 @@ func TestRestApi(t *testing.T) {
restapi.MountRoutes(r) restapi.MountRoutes(r)
const startJobBody string = `{ const startJobBody string = `{
"jobId": 123, "jobId": 123,
"user": "testuser", "user": "testuser",
"project": "testproj", "project": "testproj",
"cluster": "testcluster", "cluster": "testcluster",
@@ -542,7 +542,7 @@ func subtestLetJobFail(t *testing.T, restapi *api.RestApi, r *mux.Router) {
} }
const stopJobBody string = `{ const stopJobBody string = `{
"jobId": 12345, "jobId": 12345,
"cluster": "testcluster", "cluster": "testcluster",
"jobState": "failed", "jobState": "failed",