From 948da8f10b18351488b43ada23ab27ec57d5255a Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Fri, 11 Nov 2022 15:26:27 +0100 Subject: [PATCH 1/3] Review Sagger config. Delete Job endpoints --- api/swagger.json | 323 +++++++++++++++++++++++++++++---- api/swagger.yaml | 225 ++++++++++++++++++++--- internal/api/docs.go | 326 +++++++++++++++++++++++++++++---- internal/api/rest.go | 356 +++++++++++++++++++++++++++---------- internal/repository/job.go | 15 ++ pkg/schema/job.go | 72 ++++---- 6 files changed, 1096 insertions(+), 221 deletions(-) diff --git a/api/swagger.json b/api/swagger.json index e4d4674..8e3f217 100644 --- a/api/swagger.json +++ b/api/swagger.json @@ -1,9 +1,8 @@ { "swagger": "2.0", "info": { - "description": "Defines a tag using name and type.", + "description": "API for batch job control.", "title": "ClusterCockpit REST API", - "termsOfService": "https://monitoring.nhr.fau.de/imprint", "contact": { "name": "ClusterCockpit Project", "url": "https://github.com/ClusterCockpit", @@ -13,9 +12,9 @@ "name": "MIT License", "url": "https://opensource.org/licenses/MIT" }, - "version": "0.1.0" + "version": "0.2.0" }, - "host": "clustercockpit.localhost:8082", + "host": "localhost:8080", "basePath": "/api", "paths": { "/jobs/": { @@ -26,12 +25,12 @@ } ], "description": "Get a list of all jobs. Filters can be applied using query parameters.\nNumber of results can be limited by page. Results are sorted by descending startTime.", - "consumes": [ - "application/json" - ], "produces": [ "application/json" ], + "tags": [ + "query" + ], "summary": "Lists all jobs", "parameters": [ { @@ -62,13 +61,13 @@ }, { "type": "integer", - "description": "Items per page (If empty: No Limit)", + "description": "Items per page (Default: 25)", "name": "items-per-page", "in": "query" }, { "type": "integer", - "description": "Page Number (If empty: No Paging)", + "description": "Page Number (Default: 1)", "name": "page", "in": "query" }, @@ -110,6 +109,221 @@ } } }, + "/jobs/delete_job/": { + "delete": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Job to delete is specified by request body. All fields are required in this case.", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "remove" + ], + "summary": "Remove a job from the sql database", + "parameters": [ + { + "description": "All fields required", + "name": "request", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/api.DeleteJobApiRequest" + } + } + ], + "responses": { + "200": { + "description": "Success message", + "schema": { + "$ref": "#/definitions/api.DeleteJobApiResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/api.ErrorResponse" + } + }, + "401": { + "description": "Unauthorized", + "schema": { + "$ref": "#/definitions/api.ErrorResponse" + } + }, + "403": { + "description": "Forbidden", + "schema": { + "$ref": "#/definitions/api.ErrorResponse" + } + }, + "404": { + "description": "Resource not found", + "schema": { + "$ref": "#/definitions/api.ErrorResponse" + } + }, + "422": { + "description": "Unprocessable Entity: finding job failed: sql: no rows in result set", + "schema": { + "$ref": "#/definitions/api.ErrorResponse" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/api.ErrorResponse" + } + } + } + } + }, + "/jobs/delete_job/{id}": { + "delete": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Job to remove is specified by database ID. This will not remove the job from the job archive.", + "produces": [ + "application/json" + ], + "tags": [ + "remove" + ], + "summary": "Remove a job from the sql database", + "parameters": [ + { + "type": "integer", + "description": "Database ID of Job", + "name": "id", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "Success message", + "schema": { + "$ref": "#/definitions/api.DeleteJobApiResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/api.ErrorResponse" + } + }, + "401": { + "description": "Unauthorized", + "schema": { + "$ref": "#/definitions/api.ErrorResponse" + } + }, + "403": { + "description": "Forbidden", + "schema": { + "$ref": "#/definitions/api.ErrorResponse" + } + }, + "404": { + "description": "Resource not found", + "schema": { + "$ref": "#/definitions/api.ErrorResponse" + } + }, + "422": { + "description": "Unprocessable Entity: finding job failed: sql: no rows in result set", + "schema": { + "$ref": "#/definitions/api.ErrorResponse" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/api.ErrorResponse" + } + } + } + } + }, + "/jobs/delete_job_before/{ts}": { + "delete": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Job to stop is specified by database ID. This will not remove the job from the job archive.", + "produces": [ + "application/json" + ], + "tags": [ + "remove" + ], + "summary": "Remove a job from the sql database", + "parameters": [ + { + "type": "integer", + "description": "Unix epoch timestamp", + "name": "ts", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "Success message", + "schema": { + "$ref": "#/definitions/api.DeleteJobApiResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/api.ErrorResponse" + } + }, + "401": { + "description": "Unauthorized", + "schema": { + "$ref": "#/definitions/api.ErrorResponse" + } + }, + "403": { + "description": "Forbidden", + "schema": { + "$ref": "#/definitions/api.ErrorResponse" + } + }, + "404": { + "description": "Resource not found", + "schema": { + "$ref": "#/definitions/api.ErrorResponse" + } + }, + "422": { + "description": "Unprocessable Entity: finding job failed: sql: no rows in result set", + "schema": { + "$ref": "#/definitions/api.ErrorResponse" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/api.ErrorResponse" + } + } + } + } + }, "/jobs/start_job/": { "post": { "security": [ @@ -124,6 +338,9 @@ "produces": [ "application/json" ], + "tags": [ + "add and modify" + ], "summary": "Adds a new job as \"running\"", "parameters": [ { @@ -187,6 +404,9 @@ "produces": [ "application/json" ], + "tags": [ + "add and modify" + ], "summary": "Marks job as completed and triggers archiving", "parameters": [ { @@ -201,7 +421,7 @@ ], "responses": { "200": { - "description": "Job resource", + "description": "Success message", "schema": { "$ref": "#/definitions/schema.JobMeta" } @@ -259,6 +479,9 @@ "produces": [ "application/json" ], + "tags": [ + "add and modify" + ], "summary": "Marks job as completed and triggers archiving", "parameters": [ { @@ -338,6 +561,9 @@ "produces": [ "application/json" ], + "tags": [ + "add and modify" + ], "summary": "Adds one or more tags to a job", "parameters": [ { @@ -355,7 +581,7 @@ "schema": { "type": "array", "items": { - "$ref": "#/definitions/api.Tag" + "$ref": "#/definitions/api.ApiTag" } } } @@ -396,8 +622,53 @@ } }, "definitions": { + "api.ApiTag": { + "type": "object", + "properties": { + "name": { + "description": "Tag Name", + "type": "string", + "example": "Testjob" + }, + "type": { + "description": "Tag Type", + "type": "string", + "example": "Debug" + } + } + }, + "api.DeleteJobApiRequest": { + "type": "object", + "required": [ + "jobId" + ], + "properties": { + "cluster": { + "description": "Cluster of job", + "type": "string", + "example": "fritz" + }, + "jobId": { + "description": "Cluster Job ID of job", + "type": "integer", + "example": 123000 + }, + "startTime": { + "description": "Start Time of job as epoch", + "type": "integer", + "example": 1649723812 + } + } + }, + "api.DeleteJobApiResponse": { + "type": "object", + "properties": { + "msg": { + "type": "string" + } + } + }, "api.ErrorResponse": { - "description": "Error message as returned from backend.", "type": "object", "properties": { "error": { @@ -411,7 +682,6 @@ } }, "api.StartJobApiResponse": { - "description": "Successful job start response with database id of new job.", "type": "object", "properties": { "id": { @@ -421,7 +691,6 @@ } }, "api.StopJobApiRequest": { - "description": "Request to stop running job using stoptime and final state. They are only required if no database id was provided with endpoint.", "type": "object", "required": [ "jobState", @@ -462,22 +731,6 @@ } } }, - "api.Tag": { - "description": "Defines a tag using name and type.", - "type": "object", - "properties": { - "name": { - "description": "Tag Name", - "type": "string", - "example": "Testjob" - }, - "type": { - "description": "Tag Type", - "type": "string", - "example": "Debug" - } - } - }, "schema.Job": { "description": "Information of a HPC job.", "type": "object", @@ -809,7 +1062,7 @@ } }, "schema.Tag": { - "description": "Defines a tag using name and type.", + "description": "Defines B tag using name and type.", "type": "object", "properties": { "id": { @@ -831,10 +1084,14 @@ }, "securityDefinitions": { "ApiKeyAuth": { - "description": "JWT based authentification for general API endpoint use.", "type": "apiKey", "name": "X-Auth-Token", "in": "header" } - } + }, + "tags": [ + { + "name": "Job API" + } + ] } \ No newline at end of file diff --git a/api/swagger.yaml b/api/swagger.yaml index 3b3679f..67ad683 100644 --- a/api/swagger.yaml +++ b/api/swagger.yaml @@ -1,7 +1,39 @@ basePath: /api definitions: + api.ApiTag: + properties: + name: + description: Tag Name + example: Testjob + type: string + type: + description: Tag Type + example: Debug + type: string + type: object + api.DeleteJobApiRequest: + properties: + cluster: + description: Cluster of job + example: fritz + type: string + jobId: + description: Cluster Job ID of job + example: 123000 + type: integer + startTime: + description: Start Time of job as epoch + example: 1649723812 + type: integer + required: + - jobId + type: object + api.DeleteJobApiResponse: + properties: + msg: + type: string + type: object api.ErrorResponse: - description: Error message as returned from backend. properties: error: description: Error Message @@ -11,15 +43,12 @@ definitions: type: string type: object api.StartJobApiResponse: - description: Successful job start response with database id of new job. properties: id: description: Database ID of new job type: integer type: object api.StopJobApiRequest: - description: Request to stop running job using stoptime and final state. They - are only required if no database id was provided with endpoint. properties: cluster: description: Cluster of job @@ -51,18 +80,6 @@ definitions: - jobState - stopTime type: object - api.Tag: - description: Defines a tag using name and type. - properties: - name: - description: Tag Name - example: Testjob - type: string - type: - description: Tag Type - example: Debug - type: string - type: object schema.Job: description: Information of a HPC job. properties: @@ -330,7 +347,7 @@ definitions: type: array type: object schema.Tag: - description: Defines a tag using name and type. + description: Defines B tag using name and type. properties: id: description: The unique DB identifier of a tag @@ -344,24 +361,21 @@ definitions: example: Debug type: string type: object -host: clustercockpit.localhost:8082 +host: localhost:8080 info: contact: email: support@clustercockpit.org name: ClusterCockpit Project url: https://github.com/ClusterCockpit - description: Defines a tag using name and type. + description: API for batch job control. license: name: MIT License url: https://opensource.org/licenses/MIT - termsOfService: https://monitoring.nhr.fau.de/imprint title: ClusterCockpit REST API - version: 0.1.0 + version: 0.2.0 paths: /jobs/: get: - consumes: - - application/json description: |- Get a list of all jobs. Filters can be applied using query parameters. Number of results can be limited by page. Results are sorted by descending startTime. @@ -385,11 +399,11 @@ paths: in: query name: start-time type: string - - description: 'Items per page (If empty: No Limit)' + - description: 'Items per page (Default: 25)' in: query name: items-per-page type: integer - - description: 'Page Number (If empty: No Paging)' + - description: 'Page Number (Default: 1)' in: query name: page type: integer @@ -421,6 +435,152 @@ paths: security: - ApiKeyAuth: [] summary: Lists all jobs + tags: + - query + /jobs/delete_job/: + delete: + consumes: + - application/json + description: Job to delete is specified by request body. All fields are required + in this case. + parameters: + - description: All fields required + in: body + name: request + required: true + schema: + $ref: '#/definitions/api.DeleteJobApiRequest' + produces: + - application/json + responses: + "200": + description: Success message + schema: + $ref: '#/definitions/api.DeleteJobApiResponse' + "400": + description: Bad Request + schema: + $ref: '#/definitions/api.ErrorResponse' + "401": + description: Unauthorized + schema: + $ref: '#/definitions/api.ErrorResponse' + "403": + description: Forbidden + schema: + $ref: '#/definitions/api.ErrorResponse' + "404": + description: Resource not found + schema: + $ref: '#/definitions/api.ErrorResponse' + "422": + description: 'Unprocessable Entity: finding job failed: sql: no rows in + result set' + schema: + $ref: '#/definitions/api.ErrorResponse' + "500": + description: Internal Server Error + schema: + $ref: '#/definitions/api.ErrorResponse' + security: + - ApiKeyAuth: [] + summary: Remove a job from the sql database + tags: + - remove + /jobs/delete_job/{id}: + delete: + description: Job to remove is specified by database ID. This will not remove + the job from the job archive. + parameters: + - description: Database ID of Job + in: path + name: id + required: true + type: integer + produces: + - application/json + responses: + "200": + description: Success message + schema: + $ref: '#/definitions/api.DeleteJobApiResponse' + "400": + description: Bad Request + schema: + $ref: '#/definitions/api.ErrorResponse' + "401": + description: Unauthorized + schema: + $ref: '#/definitions/api.ErrorResponse' + "403": + description: Forbidden + schema: + $ref: '#/definitions/api.ErrorResponse' + "404": + description: Resource not found + schema: + $ref: '#/definitions/api.ErrorResponse' + "422": + description: 'Unprocessable Entity: finding job failed: sql: no rows in + result set' + schema: + $ref: '#/definitions/api.ErrorResponse' + "500": + description: Internal Server Error + schema: + $ref: '#/definitions/api.ErrorResponse' + security: + - ApiKeyAuth: [] + summary: Remove a job from the sql database + tags: + - remove + /jobs/delete_job_before/{ts}: + delete: + description: Job to stop is specified by database ID. This will not remove the + job from the job archive. + parameters: + - description: Unix epoch timestamp + in: path + name: ts + required: true + type: integer + produces: + - application/json + responses: + "200": + description: Success message + schema: + $ref: '#/definitions/api.DeleteJobApiResponse' + "400": + description: Bad Request + schema: + $ref: '#/definitions/api.ErrorResponse' + "401": + description: Unauthorized + schema: + $ref: '#/definitions/api.ErrorResponse' + "403": + description: Forbidden + schema: + $ref: '#/definitions/api.ErrorResponse' + "404": + description: Resource not found + schema: + $ref: '#/definitions/api.ErrorResponse' + "422": + description: 'Unprocessable Entity: finding job failed: sql: no rows in + result set' + schema: + $ref: '#/definitions/api.ErrorResponse' + "500": + description: Internal Server Error + schema: + $ref: '#/definitions/api.ErrorResponse' + security: + - ApiKeyAuth: [] + summary: Remove a job from the sql database + tags: + - remove /jobs/start_job/: post: consumes: @@ -466,6 +626,8 @@ paths: security: - ApiKeyAuth: [] summary: Adds a new job as "running" + tags: + - add and modify /jobs/stop_job/: post: description: |- @@ -482,7 +644,7 @@ paths: - application/json responses: "200": - description: Job resource + description: Success message schema: $ref: '#/definitions/schema.JobMeta' "400": @@ -513,6 +675,8 @@ paths: security: - ApiKeyAuth: [] summary: Marks job as completed and triggers archiving + tags: + - add and modify /jobs/stop_job/{id}: post: consumes: @@ -567,6 +731,8 @@ paths: security: - ApiKeyAuth: [] summary: Marks job as completed and triggers archiving + tags: + - add and modify /jobs/tag_job/{id}: post: consumes: @@ -586,7 +752,7 @@ paths: required: true schema: items: - $ref: '#/definitions/api.Tag' + $ref: '#/definitions/api.ApiTag' type: array produces: - application/json @@ -614,10 +780,13 @@ paths: security: - ApiKeyAuth: [] summary: Adds one or more tags to a job + tags: + - add and modify securityDefinitions: ApiKeyAuth: - description: JWT based authentification for general API endpoint use. in: header name: X-Auth-Token type: apiKey swagger: "2.0" +tags: +- name: Job API diff --git a/internal/api/docs.go b/internal/api/docs.go index 2a851ae..33c8064 100644 --- a/internal/api/docs.go +++ b/internal/api/docs.go @@ -1,6 +1,5 @@ // Package api GENERATED BY SWAG; DO NOT EDIT -// This file was generated by swaggo/swag at -// 2022-09-22 13:31:53.353204065 +0200 CEST m=+0.139444562 +// This file was generated by swaggo/swag package api import "github.com/swaggo/swag" @@ -11,7 +10,6 @@ const docTemplate = `{ "info": { "description": "{{escape .Description}}", "title": "{{.Title}}", - "termsOfService": "https://monitoring.nhr.fau.de/imprint", "contact": { "name": "ClusterCockpit Project", "url": "https://github.com/ClusterCockpit", @@ -34,12 +32,12 @@ const docTemplate = `{ } ], "description": "Get a list of all jobs. Filters can be applied using query parameters.\nNumber of results can be limited by page. Results are sorted by descending startTime.", - "consumes": [ - "application/json" - ], "produces": [ "application/json" ], + "tags": [ + "query" + ], "summary": "Lists all jobs", "parameters": [ { @@ -70,13 +68,13 @@ const docTemplate = `{ }, { "type": "integer", - "description": "Items per page (If empty: No Limit)", + "description": "Items per page (Default: 25)", "name": "items-per-page", "in": "query" }, { "type": "integer", - "description": "Page Number (If empty: No Paging)", + "description": "Page Number (Default: 1)", "name": "page", "in": "query" }, @@ -118,6 +116,221 @@ const docTemplate = `{ } } }, + "/jobs/delete_job/": { + "delete": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Job to delete is specified by request body. All fields are required in this case.", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "remove" + ], + "summary": "Remove a job from the sql database", + "parameters": [ + { + "description": "All fields required", + "name": "request", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/api.DeleteJobApiRequest" + } + } + ], + "responses": { + "200": { + "description": "Success message", + "schema": { + "$ref": "#/definitions/api.DeleteJobApiResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/api.ErrorResponse" + } + }, + "401": { + "description": "Unauthorized", + "schema": { + "$ref": "#/definitions/api.ErrorResponse" + } + }, + "403": { + "description": "Forbidden", + "schema": { + "$ref": "#/definitions/api.ErrorResponse" + } + }, + "404": { + "description": "Resource not found", + "schema": { + "$ref": "#/definitions/api.ErrorResponse" + } + }, + "422": { + "description": "Unprocessable Entity: finding job failed: sql: no rows in result set", + "schema": { + "$ref": "#/definitions/api.ErrorResponse" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/api.ErrorResponse" + } + } + } + } + }, + "/jobs/delete_job/{id}": { + "delete": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Job to remove is specified by database ID. This will not remove the job from the job archive.", + "produces": [ + "application/json" + ], + "tags": [ + "remove" + ], + "summary": "Remove a job from the sql database", + "parameters": [ + { + "type": "integer", + "description": "Database ID of Job", + "name": "id", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "Success message", + "schema": { + "$ref": "#/definitions/api.DeleteJobApiResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/api.ErrorResponse" + } + }, + "401": { + "description": "Unauthorized", + "schema": { + "$ref": "#/definitions/api.ErrorResponse" + } + }, + "403": { + "description": "Forbidden", + "schema": { + "$ref": "#/definitions/api.ErrorResponse" + } + }, + "404": { + "description": "Resource not found", + "schema": { + "$ref": "#/definitions/api.ErrorResponse" + } + }, + "422": { + "description": "Unprocessable Entity: finding job failed: sql: no rows in result set", + "schema": { + "$ref": "#/definitions/api.ErrorResponse" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/api.ErrorResponse" + } + } + } + } + }, + "/jobs/delete_job_before/{ts}": { + "delete": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Job to stop is specified by database ID. This will not remove the job from the job archive.", + "produces": [ + "application/json" + ], + "tags": [ + "remove" + ], + "summary": "Remove a job from the sql database", + "parameters": [ + { + "type": "integer", + "description": "Unix epoch timestamp", + "name": "ts", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "Success message", + "schema": { + "$ref": "#/definitions/api.DeleteJobApiResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/api.ErrorResponse" + } + }, + "401": { + "description": "Unauthorized", + "schema": { + "$ref": "#/definitions/api.ErrorResponse" + } + }, + "403": { + "description": "Forbidden", + "schema": { + "$ref": "#/definitions/api.ErrorResponse" + } + }, + "404": { + "description": "Resource not found", + "schema": { + "$ref": "#/definitions/api.ErrorResponse" + } + }, + "422": { + "description": "Unprocessable Entity: finding job failed: sql: no rows in result set", + "schema": { + "$ref": "#/definitions/api.ErrorResponse" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/api.ErrorResponse" + } + } + } + } + }, "/jobs/start_job/": { "post": { "security": [ @@ -132,6 +345,9 @@ const docTemplate = `{ "produces": [ "application/json" ], + "tags": [ + "add and modify" + ], "summary": "Adds a new job as \"running\"", "parameters": [ { @@ -195,6 +411,9 @@ const docTemplate = `{ "produces": [ "application/json" ], + "tags": [ + "add and modify" + ], "summary": "Marks job as completed and triggers archiving", "parameters": [ { @@ -209,7 +428,7 @@ const docTemplate = `{ ], "responses": { "200": { - "description": "Job resource", + "description": "Success message", "schema": { "$ref": "#/definitions/schema.JobMeta" } @@ -267,6 +486,9 @@ const docTemplate = `{ "produces": [ "application/json" ], + "tags": [ + "add and modify" + ], "summary": "Marks job as completed and triggers archiving", "parameters": [ { @@ -346,6 +568,9 @@ const docTemplate = `{ "produces": [ "application/json" ], + "tags": [ + "add and modify" + ], "summary": "Adds one or more tags to a job", "parameters": [ { @@ -363,7 +588,7 @@ const docTemplate = `{ "schema": { "type": "array", "items": { - "$ref": "#/definitions/api.Tag" + "$ref": "#/definitions/api.ApiTag" } } } @@ -404,8 +629,53 @@ const docTemplate = `{ } }, "definitions": { + "api.ApiTag": { + "type": "object", + "properties": { + "name": { + "description": "Tag Name", + "type": "string", + "example": "Testjob" + }, + "type": { + "description": "Tag Type", + "type": "string", + "example": "Debug" + } + } + }, + "api.DeleteJobApiRequest": { + "type": "object", + "required": [ + "jobId" + ], + "properties": { + "cluster": { + "description": "Cluster of job", + "type": "string", + "example": "fritz" + }, + "jobId": { + "description": "Cluster Job ID of job", + "type": "integer", + "example": 123000 + }, + "startTime": { + "description": "Start Time of job as epoch", + "type": "integer", + "example": 1649723812 + } + } + }, + "api.DeleteJobApiResponse": { + "type": "object", + "properties": { + "msg": { + "type": "string" + } + } + }, "api.ErrorResponse": { - "description": "Error message as returned from backend.", "type": "object", "properties": { "error": { @@ -419,7 +689,6 @@ const docTemplate = `{ } }, "api.StartJobApiResponse": { - "description": "Successful job start response with database id of new job.", "type": "object", "properties": { "id": { @@ -429,7 +698,6 @@ const docTemplate = `{ } }, "api.StopJobApiRequest": { - "description": "Request to stop running job using stoptime and final state. They are only required if no database id was provided with endpoint.", "type": "object", "required": [ "jobState", @@ -470,22 +738,6 @@ const docTemplate = `{ } } }, - "api.Tag": { - "description": "Defines a tag using name and type.", - "type": "object", - "properties": { - "name": { - "description": "Tag Name", - "type": "string", - "example": "Testjob" - }, - "type": { - "description": "Tag Type", - "type": "string", - "example": "Debug" - } - } - }, "schema.Job": { "description": "Information of a HPC job.", "type": "object", @@ -817,7 +1069,7 @@ const docTemplate = `{ } }, "schema.Tag": { - "description": "Defines a tag using name and type.", + "description": "Defines B tag using name and type.", "type": "object", "properties": { "id": { @@ -839,22 +1091,26 @@ const docTemplate = `{ }, "securityDefinitions": { "ApiKeyAuth": { - "description": "JWT based authentification for general API endpoint use.", "type": "apiKey", "name": "X-Auth-Token", "in": "header" } - } + }, + "tags": [ + { + "name": "Job API" + } + ] }` // SwaggerInfo holds exported Swagger Info so clients can modify it var SwaggerInfo = &swag.Spec{ - Version: "0.1.0", - Host: "clustercockpit.localhost:8082", + Version: "0.2.0", + Host: "localhost:8080", BasePath: "/api", Schemes: []string{}, Title: "ClusterCockpit REST API", - Description: "Defines a tag using name and type.", + Description: "API for batch job control.", InfoInstanceName: "swagger", SwaggerTemplate: docTemplate, } diff --git a/internal/api/rest.go b/internal/api/rest.go index e689f56..97324c7 100644 --- a/internal/api/rest.go +++ b/internal/api/rest.go @@ -32,9 +32,10 @@ import ( ) // @title ClusterCockpit REST API -// @version 0.1.0 +// @version 0.2.0 // @description API for batch job control. -// @termsOfService https://monitoring.nhr.fau.de/imprint + +// @tag.name Job API // @contact.name ClusterCockpit Project // @contact.url https://github.com/ClusterCockpit @@ -43,13 +44,12 @@ import ( // @license.name MIT License // @license.url https://opensource.org/licenses/MIT -// @host clustercockpit.localhost:8082 -// @BasePath /api +// @host localhost:8080 +// @basePath /api // @securityDefinitions.apikey ApiKeyAuth // @in header // @name X-Auth-Token -// @description JWT based authentification for general API endpoint use. type RestApi struct { JobRepository *repository.JobRepository @@ -72,6 +72,9 @@ func (api *RestApi) MountRoutes(r *mux.Router) { // r.HandleFunc("/jobs/{id}", api.getJob).Methods(http.MethodGet) r.HandleFunc("/jobs/tag_job/{id}", api.tagJob).Methods(http.MethodPost, http.MethodPatch) r.HandleFunc("/jobs/metrics/{id}", api.getJobMetrics).Methods(http.MethodGet) + r.HandleFunc("/jobs/delete_job/", api.deleteJobByRequest).Methods(http.MethodDelete) + r.HandleFunc("/jobs/delete_job/{id}", api.deleteJobById).Methods(http.MethodDelete) + r.HandleFunc("/jobs/delete_job_before/{ts}", api.deleteJobBefore).Methods(http.MethodDelete) if api.Authentication != nil { r.HandleFunc("/jwt/", api.getJWT).Methods(http.MethodGet) @@ -89,15 +92,17 @@ func (api *RestApi) MountRoutes(r *mux.Router) { } // StartJobApiResponse model -// @Description Successful job start response with database id of new job. type StartJobApiResponse struct { // Database ID of new job DBID int64 `json:"id"` } +// DeleteJobApiResponse model +type DeleteJobApiResponse struct { + Message string `json:"msg"` +} + // StopJobApiRequest model -// @Description Request to stop running job using stoptime and final state. -// @Description They are only required if no database id was provided with endpoint. type StopJobApiRequest struct { // Stop Time of job as epoch StopTime int64 `json:"stopTime" validate:"required" example:"1649763839"` @@ -107,23 +112,28 @@ type StopJobApiRequest struct { StartTime *int64 `json:"startTime" example:"1649723812"` // Start Time of job as epoch } +// DeleteJobApiRequest model +type DeleteJobApiRequest struct { + JobId *int64 `json:"jobId" validate:"required" example:"123000"` // Cluster Job ID of job + Cluster *string `json:"cluster" example:"fritz"` // Cluster of job + StartTime *int64 `json:"startTime" example:"1649723812"` // Start Time of job as epoch +} + // ErrorResponse model -// @Description Error message as returned from backend. type ErrorResponse struct { // Statustext of Errorcode Status string `json:"status"` Error string `json:"error"` // Error Message } -// Tag model -// @Description Defines a tag using name and type. -type Tag struct { +// ApiTag model +type ApiTag struct { // Tag Type Type string `json:"type" example:"Debug"` Name string `json:"name" example:"Testjob"` // Tag Name } -type TagJobApiRequest []*Tag +type TagJobApiRequest []*ApiTag func handleError(err error, statusCode int, rw http.ResponseWriter) { log.Warnf("REST API: %s", err.Error()) @@ -142,28 +152,34 @@ func decode(r io.Reader, val interface{}) error { } // getJobs godoc -// @Summary Lists all jobs -// @Description Get a list of all jobs. Filters can be applied using query parameters. -// @Description Number of results can be limited by page. Results are sorted by descending startTime. -// @Accept json -// @Produce json -// @Param state query string false "Job State" Enums(running, completed, failed, cancelled, stopped, timeout) -// @Param cluster query string false "Job Cluster" -// @Param start-time query string false "Syntax: '$from-$to', as unix epoch timestamps in seconds" -// @Param items-per-page query int false "Items per page (If empty: No Limit)" -// @Param page query int false "Page Number (If empty: No Paging)" -// @Param with-metadata query bool false "Include metadata (e.g. jobScript) in response" -// @Success 200 {array} schema.Job "Array of matching jobs" -// @Failure 400 {object} api.ErrorResponse "Bad Request" -// @Failure 401 {object} api.ErrorResponse "Unauthorized" -// @Failure 500 {object} api.ErrorResponse "Internal Server Error" -// @Security ApiKeyAuth -// @Router /jobs/ [get] +// @summary Lists all jobs +// @tags query +// @description Get a list of all jobs. Filters can be applied using query parameters. +// @description Number of results can be limited by page. Results are sorted by descending startTime. +// @produce json +// @param state query string false "Job State" Enums(running, completed, failed, cancelled, stopped, timeout) +// @param cluster query string false "Job Cluster" +// @param start-time query string false "Syntax: '$from-$to', as unix epoch timestamps in seconds" +// @param items-per-page query int false "Items per page (Default: 25)" +// @param page query int false "Page Number (Default: 1)" +// @param with-metadata query bool false "Include metadata (e.g. jobScript) in response" +// @success 200 {array} schema.Job "Array of matching jobs" +// @failure 400 {object} api.ErrorResponse "Bad Request" +// @failure 401 {object} api.ErrorResponse "Unauthorized" +// @failure 500 {object} api.ErrorResponse "Internal Server Error" +// @security ApiKeyAuth +// @router /jobs/ [get] func (api *RestApi) getJobs(rw http.ResponseWriter, r *http.Request) { + if user := auth.GetUser(r.Context()); user != nil && !user.HasRole(auth.RoleApi) { + handleError(fmt.Errorf("missing role: %#v", auth.RoleApi), http.StatusForbidden, rw) + return + } + withMetadata := false filter := &model.JobFilter{} - page := &model.PageRequest{ItemsPerPage: -1, Page: 1} + page := &model.PageRequest{ItemsPerPage: 25, Page: 1} order := &model.OrderByInput{Field: "startTime", Order: model.SortDirectionEnumDesc} + for key, vals := range r.URL.Query() { switch key { case "state": @@ -269,21 +285,27 @@ func (api *RestApi) getJobs(rw http.ResponseWriter, r *http.Request) { } // tagJob godoc -// @Summary Adds one or more tags to a job -// @Description Adds tag(s) to a job specified by DB ID. Name and Type of Tag(s) can be chosen freely. -// @Description If tagged job is already finished: Tag will be written directly to respective archive files. -// @Accept json -// @Produce json -// @Param id path int true "Job Database ID" -// @Param request body api.TagJobApiRequest true "Array of tag-objects to add" -// @Success 200 {object} schema.Job "Updated job resource" -// @Failure 400 {object} api.ErrorResponse "Bad Request" -// @Failure 401 {object} api.ErrorResponse "Unauthorized" -// @Failure 404 {object} api.ErrorResponse "Job or tag does not exist" -// @Failure 500 {object} api.ErrorResponse "Internal Server Error" -// @Security ApiKeyAuth -// @Router /jobs/tag_job/{id} [post] +// @summary Adds one or more tags to a job +// @tags add and modify +// @description Adds tag(s) to a job specified by DB ID. Name and Type of Tag(s) can be chosen freely. +// @description If tagged job is already finished: Tag will be written directly to respective archive files. +// @accept json +// @produce json +// @param id path int true "Job Database ID" +// @param request body api.TagJobApiRequest true "Array of tag-objects to add" +// @success 200 {object} schema.Job "Updated job resource" +// @failure 400 {object} api.ErrorResponse "Bad Request" +// @failure 401 {object} api.ErrorResponse "Unauthorized" +// @failure 404 {object} api.ErrorResponse "Job or tag does not exist" +// @failure 500 {object} api.ErrorResponse "Internal Server Error" +// @security ApiKeyAuth +// @router /jobs/tag_job/{id} [post] func (api *RestApi) tagJob(rw http.ResponseWriter, r *http.Request) { + if user := auth.GetUser(r.Context()); user != nil && !user.HasRole(auth.RoleApi) { + handleError(fmt.Errorf("missing role: %#v", auth.RoleApi), http.StatusForbidden, rw) + return + } + iid, err := strconv.ParseInt(mux.Vars(r)["id"], 10, 64) if err != nil { http.Error(rw, err.Error(), http.StatusBadRequest) @@ -328,20 +350,21 @@ func (api *RestApi) tagJob(rw http.ResponseWriter, r *http.Request) { } // startJob godoc -// @Summary Adds a new job as "running" -// @Description Job specified in request body will be saved to database as "running" with new DB ID. -// @Description Job specifications follow the 'JobMeta' scheme, API will fail to execute if requirements are not met. -// @Accept json -// @Produce json -// @Param request body schema.JobMeta true "Job to add" -// @Success 201 {object} api.StartJobApiResponse "Job added successfully" -// @Failure 400 {object} api.ErrorResponse "Bad Request" -// @Failure 401 {object} api.ErrorResponse "Unauthorized" -// @Failure 403 {object} api.ErrorResponse "Forbidden" -// @Failure 422 {object} api.ErrorResponse "Unprocessable Entity: The combination of jobId, clusterId and startTime does already exist" -// @Failure 500 {object} api.ErrorResponse "Internal Server Error" -// @Security ApiKeyAuth -// @Router /jobs/start_job/ [post] +// @summary Adds a new job as "running" +// @tags add and modify +// @description Job specified in request body will be saved to database as "running" with new DB ID. +// @description Job specifications follow the 'JobMeta' scheme, API will fail to execute if requirements are not met. +// @accept json +// @produce json +// @param request body schema.JobMeta true "Job to add" +// @success 201 {object} api.StartJobApiResponse "Job added successfully" +// @failure 400 {object} api.ErrorResponse "Bad Request" +// @failure 401 {object} api.ErrorResponse "Unauthorized" +// @failure 403 {object} api.ErrorResponse "Forbidden" +// @failure 422 {object} api.ErrorResponse "Unprocessable Entity: The combination of jobId, clusterId and startTime does already exist" +// @failure 500 {object} api.ErrorResponse "Internal Server Error" +// @security ApiKeyAuth +// @router /jobs/start_job/ [post] func (api *RestApi) startJob(rw http.ResponseWriter, r *http.Request) { if user := auth.GetUser(r.Context()); user != nil && !user.HasRole(auth.RoleApi) { handleError(fmt.Errorf("missing role: %#v", auth.RoleApi), http.StatusForbidden, rw) @@ -399,22 +422,23 @@ func (api *RestApi) startJob(rw http.ResponseWriter, r *http.Request) { } // stopJobById godoc -// @Summary Marks job as completed and triggers archiving -// @Description Job to stop is specified by database ID. Only stopTime and final state are required in request body. -// @Description Returns full job resource information according to 'JobMeta' scheme. -// @Accept json -// @Produce json -// @Param id path int true "Database ID of Job" -// @Param request body api.StopJobApiRequest true "stopTime and final state in request body" -// @Success 200 {object} schema.JobMeta "Job resource" -// @Failure 400 {object} api.ErrorResponse "Bad Request" -// @Failure 401 {object} api.ErrorResponse "Unauthorized" -// @Failure 403 {object} api.ErrorResponse "Forbidden" -// @Failure 404 {object} api.ErrorResponse "Resource not found" -// @Failure 422 {object} api.ErrorResponse "Unprocessable Entity: finding job failed: sql: no rows in result set" -// @Failure 500 {object} api.ErrorResponse "Internal Server Error" -// @Security ApiKeyAuth -// @Router /jobs/stop_job/{id} [post] +// @summary Marks job as completed and triggers archiving +// @tags add and modify +// @description Job to stop is specified by database ID. Only stopTime and final state are required in request body. +// @description Returns full job resource information according to 'JobMeta' scheme. +// @accept json +// @produce json +// @param id path int true "Database ID of Job" +// @param request body api.StopJobApiRequest true "stopTime and final state in request body" +// @success 200 {object} schema.JobMeta "Job resource" +// @failure 400 {object} api.ErrorResponse "Bad Request" +// @failure 401 {object} api.ErrorResponse "Unauthorized" +// @failure 403 {object} api.ErrorResponse "Forbidden" +// @failure 404 {object} api.ErrorResponse "Resource not found" +// @failure 422 {object} api.ErrorResponse "Unprocessable Entity: finding job failed: sql: no rows in result set" +// @failure 500 {object} api.ErrorResponse "Internal Server Error" +// @security ApiKeyAuth +// @router /jobs/stop_job/{id} [post] func (api *RestApi) stopJobById(rw http.ResponseWriter, r *http.Request) { if user := auth.GetUser(r.Context()); user != nil && !user.HasRole(auth.RoleApi) { handleError(fmt.Errorf("missing role: %#v", auth.RoleApi), http.StatusForbidden, rw) @@ -453,20 +477,21 @@ func (api *RestApi) stopJobById(rw http.ResponseWriter, r *http.Request) { } // stopJobByRequest godoc -// @Summary Marks job as completed and triggers archiving -// @Description Job to stop is specified by request body. All fields are required in this case. -// @Description Returns full job resource information according to 'JobMeta' scheme. -// @Produce json -// @Param request body api.StopJobApiRequest true "All fields required" -// @Success 200 {object} schema.JobMeta "Job resource" -// @Failure 400 {object} api.ErrorResponse "Bad Request" -// @Failure 401 {object} api.ErrorResponse "Unauthorized" -// @Failure 403 {object} api.ErrorResponse "Forbidden" -// @Failure 404 {object} api.ErrorResponse "Resource not found" -// @Failure 422 {object} api.ErrorResponse "Unprocessable Entity: finding job failed: sql: no rows in result set" -// @Failure 500 {object} api.ErrorResponse "Internal Server Error" -// @Security ApiKeyAuth -// @Router /jobs/stop_job/ [post] +// @summary Marks job as completed and triggers archiving +// @tags add and modify +// @description Job to stop is specified by request body. All fields are required in this case. +// @description Returns full job resource information according to 'JobMeta' scheme. +// @produce json +// @param request body api.StopJobApiRequest true "All fields required" +// @success 200 {object} schema.JobMeta "Success message" +// @failure 400 {object} api.ErrorResponse "Bad Request" +// @failure 401 {object} api.ErrorResponse "Unauthorized" +// @failure 403 {object} api.ErrorResponse "Forbidden" +// @failure 404 {object} api.ErrorResponse "Resource not found" +// @failure 422 {object} api.ErrorResponse "Unprocessable Entity: finding job failed: sql: no rows in result set" +// @failure 500 {object} api.ErrorResponse "Internal Server Error" +// @security ApiKeyAuth +// @router /jobs/stop_job/ [post] func (api *RestApi) stopJobByRequest(rw http.ResponseWriter, r *http.Request) { if user := auth.GetUser(r.Context()); user != nil && !user.HasRole(auth.RoleApi) { handleError(fmt.Errorf("missing role: %#v", auth.RoleApi), http.StatusForbidden, rw) @@ -498,6 +523,159 @@ func (api *RestApi) stopJobByRequest(rw http.ResponseWriter, r *http.Request) { api.checkAndHandleStopJob(rw, job, req) } +// deleteJobById godoc +// @summary Remove a job from the sql database +// @tags remove +// @description Job to remove is specified by database ID. This will not remove the job from the job archive. +// @produce json +// @param id path int true "Database ID of Job" +// @success 200 {object} api.DeleteJobApiResponse "Success message" +// @failure 400 {object} api.ErrorResponse "Bad Request" +// @failure 401 {object} api.ErrorResponse "Unauthorized" +// @failure 403 {object} api.ErrorResponse "Forbidden" +// @failure 404 {object} api.ErrorResponse "Resource not found" +// @failure 422 {object} api.ErrorResponse "Unprocessable Entity: finding job failed: sql: no rows in result set" +// @failure 500 {object} api.ErrorResponse "Internal Server Error" +// @security ApiKeyAuth +// @router /jobs/delete_job/{id} [delete] +func (api *RestApi) deleteJobById(rw http.ResponseWriter, r *http.Request) { + if user := auth.GetUser(r.Context()); user != nil && !user.HasRole(auth.RoleApi) { + handleError(fmt.Errorf("missing role: %#v", auth.RoleApi), http.StatusForbidden, rw) + return + } + + // Fetch job (that will be stopped) from db + id, ok := mux.Vars(r)["id"] + var err error + if ok { + id, e := strconv.ParseInt(id, 10, 64) + if e != nil { + handleError(fmt.Errorf("integer expected in path for id: %w", e), http.StatusBadRequest, rw) + return + } + + err = api.JobRepository.DeleteJobById(id) + } else { + handleError(errors.New("the parameter 'id' is required"), http.StatusBadRequest, rw) + return + } + if err != nil { + handleError(fmt.Errorf("deleting job failed: %w", err), http.StatusUnprocessableEntity, rw) + return + } + rw.Header().Add("Content-Type", "application/json") + rw.WriteHeader(http.StatusOK) + json.NewEncoder(rw).Encode(DeleteJobApiResponse{ + Message: fmt.Sprintf("Successfully deleted job %s", id), + }) +} + +// deleteJobByRequest godoc +// @summary Remove a job from the sql database +// @tags remove +// @description Job to delete is specified by request body. All fields are required in this case. +// @accept json +// @produce json +// @param request body api.DeleteJobApiRequest true "All fields required" +// @success 200 {object} api.DeleteJobApiResponse "Success message" +// @failure 400 {object} api.ErrorResponse "Bad Request" +// @failure 401 {object} api.ErrorResponse "Unauthorized" +// @failure 403 {object} api.ErrorResponse "Forbidden" +// @failure 404 {object} api.ErrorResponse "Resource not found" +// @failure 422 {object} api.ErrorResponse "Unprocessable Entity: finding job failed: sql: no rows in result set" +// @failure 500 {object} api.ErrorResponse "Internal Server Error" +// @security ApiKeyAuth +// @router /jobs/delete_job/ [delete] +func (api *RestApi) deleteJobByRequest(rw http.ResponseWriter, r *http.Request) { + if user := auth.GetUser(r.Context()); user != nil && !user.HasRole(auth.RoleApi) { + handleError(fmt.Errorf("missing role: %#v", auth.RoleApi), http.StatusForbidden, rw) + return + } + + // Parse request body + req := DeleteJobApiRequest{} + if err := decode(r.Body, &req); err != nil { + handleError(fmt.Errorf("parsing request body failed: %w", err), http.StatusBadRequest, rw) + return + } + + // Fetch job (that will be deleted) from db + var job *schema.Job + var err error + if req.JobId == nil { + handleError(errors.New("the field 'jobId' is required"), http.StatusBadRequest, rw) + return + } + + job, err = api.JobRepository.Find(req.JobId, req.Cluster, req.StartTime) + + if err != nil { + handleError(fmt.Errorf("finding job failed: %w", err), http.StatusUnprocessableEntity, rw) + return + } + + err = api.JobRepository.DeleteJobById(job.ID) + if err != nil { + handleError(fmt.Errorf("deleting job failed: %w", err), http.StatusUnprocessableEntity, rw) + return + } + + rw.Header().Add("Content-Type", "application/json") + rw.WriteHeader(http.StatusOK) + json.NewEncoder(rw).Encode(DeleteJobApiResponse{ + Message: fmt.Sprintf("Successfully deleted job %d", job.ID), + }) +} + +// deleteJobBefore godoc +// @summary Remove a job from the sql database +// @tags remove +// @description Job to stop is specified by database ID. This will not remove the job from the job archive. +// @produce json +// @param ts path int true "Unix epoch timestamp" +// @success 200 {object} api.DeleteJobApiResponse "Success message" +// @failure 400 {object} api.ErrorResponse "Bad Request" +// @failure 401 {object} api.ErrorResponse "Unauthorized" +// @failure 403 {object} api.ErrorResponse "Forbidden" +// @failure 404 {object} api.ErrorResponse "Resource not found" +// @failure 422 {object} api.ErrorResponse "Unprocessable Entity: finding job failed: sql: no rows in result set" +// @failure 500 {object} api.ErrorResponse "Internal Server Error" +// @security ApiKeyAuth +// @router /jobs/delete_job_before/{ts} [delete] +func (api *RestApi) deleteJobBefore(rw http.ResponseWriter, r *http.Request) { + if user := auth.GetUser(r.Context()); user != nil && !user.HasRole(auth.RoleApi) { + handleError(fmt.Errorf("missing role: %#v", auth.RoleApi), http.StatusForbidden, rw) + return + } + + var cnt int + // Fetch job (that will be stopped) from db + id, ok := mux.Vars(r)["ts"] + var err error + if ok { + ts, e := strconv.ParseInt(id, 10, 64) + if e != nil { + handleError(fmt.Errorf("integer expected in path for ts: %w", e), http.StatusBadRequest, rw) + return + } + + cnt, err = api.JobRepository.DeleteJobsBefore(ts) + } else { + handleError(errors.New("the parameter 'ts' is required"), http.StatusBadRequest, rw) + return + } + if err != nil { + handleError(fmt.Errorf("deleting jobs failed: %w", err), http.StatusUnprocessableEntity, rw) + return + } + + rw.Header().Add("Content-Type", "application/json") + rw.WriteHeader(http.StatusOK) + json.NewEncoder(rw).Encode(DeleteJobApiResponse{ + Message: fmt.Sprintf("Successfully deleted %d jobs", cnt), + }) +} + func (api *RestApi) checkAndHandleStopJob(rw http.ResponseWriter, job *schema.Job, req StopJobApiRequest) { // Sanity checks diff --git a/internal/repository/job.go b/internal/repository/job.go index a568e11..49aa680 100644 --- a/internal/repository/job.go +++ b/internal/repository/job.go @@ -246,6 +246,21 @@ func (r *JobRepository) Stop( return } +func (r *JobRepository) DeleteJobsBefore(startTime int64) (int, error) { + var cnt int + q := sq.Select("COUNT(*)").From("job").Where("job.start_time < ?", startTime) + q.QueryRow().Scan(&cnt) + + _, err := r.DB.Exec(`DELETE FROM job WHERE job.start_time < ?`, startTime) + return cnt, err +} + +func (r *JobRepository) DeleteJobById(id int64) error { + + _, err := r.DB.Exec(`DELETE FROM job WHERE job.id = ?`, id) + return err +} + // TODO: Use node hours instead: SELECT job.user, sum(job.num_nodes * (CASE WHEN job.job_state = "running" THEN CAST(strftime('%s', 'now') AS INTEGER) - job.start_time ELSE job.duration END)) as x FROM job GROUP BY user ORDER BY x DESC; func (r *JobRepository) CountGroupedJobs(ctx context.Context, aggreg model.Aggregate, filters []*model.JobFilter, weight *model.Weights, limit *int) (map[string]int, error) { if !aggreg.IsValid() { diff --git a/pkg/schema/job.go b/pkg/schema/job.go index d2db324..4de82c1 100644 --- a/pkg/schema/job.go +++ b/pkg/schema/job.go @@ -17,26 +17,26 @@ import ( type BaseJob struct { // The unique identifier of a job JobID int64 `json:"jobId" db:"job_id" example:"123000"` - User string `json:"user" db:"user" example:"abcd100h"` // The unique identifier of a user - Project string `json:"project" db:"project" example:"abcd200"` // The unique identifier of a project - Cluster string `json:"cluster" db:"cluster" example:"fritz"` // The unique identifier of a cluster - SubCluster string `json:"subCluster" db:"subcluster" example:"main"` // The unique identifier of a sub cluster - Partition string `json:"partition" db:"partition" example:"main"` // The Slurm partition to which the job was submitted - ArrayJobId int64 `json:"arrayJobId" db:"array_job_id" example:"123000"` // The unique identifier of an array job - NumNodes int32 `json:"numNodes" db:"num_nodes" example:"2" minimum:"1"` // Number of nodes used (Min > 0) - NumHWThreads int32 `json:"numHwthreads" db:"num_hwthreads" example:"20" minimum:"1"` // Number of HWThreads used (Min > 0) - NumAcc int32 `json:"numAcc" db:"num_acc" example:"2" minimum:"1"` // Number of accelerators used (Min > 0) - Exclusive int32 `json:"exclusive" db:"exclusive" example:"1" minimum:"0" maximum:"2"` // Specifies how nodes are shared: 0 - Shared among multiple jobs of multiple users, 1 - Job exclusive (Default), 2 - Shared among multiple jobs of same user - MonitoringStatus int32 `json:"monitoringStatus" db:"monitoring_status" example:"1" minimum:"0" maximum:"3"` // State of monitoring system during job run: 0 - Disabled, 1 - Running or Archiving (Default), 2 - Archiving Failed, 3 - Archiving Successfull - SMT int32 `json:"smt" db:"smt" example:"4"` // SMT threads used by job + User string `json:"user" db:"user" example:"abcd100h"` // The unique identifier of a user + Project string `json:"project" db:"project" example:"abcd200"` // The unique identifier of a project + Cluster string `json:"cluster" db:"cluster" example:"fritz"` // The unique identifier of a cluster + SubCluster string `json:"subCluster" db:"subcluster" example:"main"` // The unique identifier of a sub cluster + Partition string `json:"partition" db:"partition" example:"main"` // The Slurm partition to which the job was submitted + ArrayJobId int64 `json:"arrayJobId" db:"array_job_id" example:"123000"` // The unique identifier of an array job + NumNodes int32 `json:"numNodes" db:"num_nodes" example:"2" minimum:"1"` // Number of nodes used (Min > 0) + NumHWThreads int32 `json:"numHwthreads" db:"num_hwthreads" example:"20" minimum:"1"` // Number of HWThreads used (Min > 0) + NumAcc int32 `json:"numAcc" db:"num_acc" example:"2" minimum:"1"` // Number of accelerators used (Min > 0) + Exclusive int32 `json:"exclusive" db:"exclusive" example:"1" minimum:"0" maximum:"2"` // Specifies how nodes are shared: 0 - Shared among multiple jobs of multiple users, 1 - Job exclusive (Default), 2 - Shared among multiple jobs of same user + MonitoringStatus int32 `json:"monitoringStatus" db:"monitoring_status" example:"1" minimum:"0" maximum:"3"` // State of monitoring system during job run: 0 - Disabled, 1 - Running or Archiving (Default), 2 - Archiving Failed, 3 - Archiving Successfull + SMT int32 `json:"smt" db:"smt" example:"4"` // SMT threads used by job State JobState `json:"jobState" db:"job_state" example:"completed" enums:"completed,failed,cancelled,stopped,timeout,out_of_memory"` // Final state of job - Duration int32 `json:"duration" db:"duration" example:"43200" minimum:"1"` // Duration of job in seconds (Min > 0) - Walltime int64 `json:"walltime" db:"walltime" example:"86400" minimum:"1"` // Requested walltime of job in seconds (Min > 0) - Tags []*Tag `json:"tags"` // List of tags - RawResources []byte `json:"-" db:"resources"` // Resources used by job [As Bytes] - Resources []*Resource `json:"resources"` // Resources used by job - RawMetaData []byte `json:"-" db:"meta_data"` // Additional information about the job [As Bytes] - MetaData map[string]string `json:"metaData"` // Additional information about the job + Duration int32 `json:"duration" db:"duration" example:"43200" minimum:"1"` // Duration of job in seconds (Min > 0) + Walltime int64 `json:"walltime" db:"walltime" example:"86400" minimum:"1"` // Requested walltime of job in seconds (Min > 0) + Tags []*Tag `json:"tags"` // List of tags + RawResources []byte `json:"-" db:"resources"` // Resources used by job [As Bytes] + Resources []*Resource `json:"resources"` // Resources used by job + RawMetaData []byte `json:"-" db:"meta_data"` // Additional information about the job [As Bytes] + MetaData map[string]string `json:"metaData"` // Additional information about the job } // Non-Swaggered Comment: Job @@ -49,15 +49,15 @@ type Job struct { ID int64 `json:"id" db:"id"` BaseJob StartTimeUnix int64 `json:"-" db:"start_time" example:"1649723812"` // Start epoch time stamp in seconds - StartTime time.Time `json:"startTime"` // Start time as 'time.Time' data type - MemUsedMax float64 `json:"-" db:"mem_used_max"` // MemUsedMax as Float64 - FlopsAnyAvg float64 `json:"-" db:"flops_any_avg"` // FlopsAnyAvg as Float64 - MemBwAvg float64 `json:"-" db:"mem_bw_avg"` // MemBwAvg as Float64 - LoadAvg float64 `json:"-" db:"load_avg"` // LoadAvg as Float64 - NetBwAvg float64 `json:"-" db:"net_bw_avg"` // NetBwAvg as Float64 - NetDataVolTotal float64 `json:"-" db:"net_data_vol_total"` // NetDataVolTotal as Float64 - FileBwAvg float64 `json:"-" db:"file_bw_avg"` // FileBwAvg as Float64 - FileDataVolTotal float64 `json:"-" db:"file_data_vol_total"` // FileDataVolTotal as Float64 + StartTime time.Time `json:"startTime"` // Start time as 'time.Time' data type + MemUsedMax float64 `json:"-" db:"mem_used_max"` // MemUsedMax as Float64 + FlopsAnyAvg float64 `json:"-" db:"flops_any_avg"` // FlopsAnyAvg as Float64 + MemBwAvg float64 `json:"-" db:"mem_bw_avg"` // MemBwAvg as Float64 + LoadAvg float64 `json:"-" db:"load_avg"` // LoadAvg as Float64 + NetBwAvg float64 `json:"-" db:"net_bw_avg"` // NetBwAvg as Float64 + NetDataVolTotal float64 `json:"-" db:"net_data_vol_total"` // NetDataVolTotal as Float64 + FileBwAvg float64 `json:"-" db:"file_bw_avg"` // FileBwAvg as Float64 + FileDataVolTotal float64 `json:"-" db:"file_data_vol_total"` // FileDataVolTotal as Float64 } // Non-Swaggered Comment: JobMeta @@ -70,11 +70,11 @@ type Job struct { // JobMeta model // @Description Meta data information of a HPC job. type JobMeta struct { - // The unique identifier of a job in the database + // The unique identifier of a job in the database ID *int64 `json:"id,omitempty"` BaseJob StartTime int64 `json:"startTime" db:"start_time" example:"1649723812" minimum:"1"` // Start epoch time stamp in seconds (Min > 0) - Statistics map[string]JobStatistics `json:"statistics,omitempty"` // Metric statistics of job + Statistics map[string]JobStatistics `json:"statistics,omitempty"` // Metric statistics of job } const ( @@ -100,20 +100,20 @@ type JobStatistics struct { } // Tag model -// @Description Defines a tag using name and type. +// @Description Defines B tag using name and type. type Tag struct { - // The unique DB identifier of a tag + // The unique DB identifier of a tag ID int64 `json:"id" db:"id"` - Type string `json:"type" db:"tag_type" example:"Debug"` // Tag Type + Type string `json:"type" db:"tag_type" example:"Debug"` // Tag Type Name string `json:"name" db:"tag_name" example:"Testjob"` // Tag Name } // Resource model // @Description A resource used by a job type Resource struct { - Hostname string `json:"hostname"` // Name of the host (= node) - HWThreads []int `json:"hwthreads,omitempty"` // List of OS processor ids - Accelerators []string `json:"accelerators,omitempty"` // List of of accelerator device ids + Hostname string `json:"hostname"` // Name of the host (= node) + HWThreads []int `json:"hwthreads,omitempty"` // List of OS processor ids + Accelerators []string `json:"accelerators,omitempty"` // List of of accelerator device ids Configuration string `json:"configuration,omitempty"` // The configuration options of the node } From fd16a1b637498f510a5b993a5f512c3aad6d55b2 Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Fri, 25 Nov 2022 15:15:05 +0100 Subject: [PATCH 2/3] Fix cnt query scan. Cosmetic changes. --- internal/api/rest.go | 2 +- internal/repository/job.go | 18 +++++++++++++----- test/integration_test.go | 4 ++-- 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/internal/api/rest.go b/internal/api/rest.go index 97324c7..985e4ea 100644 --- a/internal/api/rest.go +++ b/internal/api/rest.go @@ -630,7 +630,7 @@ func (api *RestApi) deleteJobByRequest(rw http.ResponseWriter, r *http.Request) // deleteJobBefore godoc // @summary Remove a job from the sql database // @tags remove -// @description Job to stop is specified by database ID. This will not remove the job from the job archive. +// @description Remove all jobs with start time before timestamp. The jobs will not be removed from the job archive. // @produce json // @param ts path int true "Unix epoch timestamp" // @success 200 {object} api.DeleteJobApiResponse "Success message" diff --git a/internal/repository/job.go b/internal/repository/job.go index 49aa680..d1b2af2 100644 --- a/internal/repository/job.go +++ b/internal/repository/job.go @@ -248,16 +248,24 @@ func (r *JobRepository) Stop( func (r *JobRepository) DeleteJobsBefore(startTime int64) (int, error) { var cnt int - q := sq.Select("COUNT(*)").From("job").Where("job.start_time < ?", startTime) - q.QueryRow().Scan(&cnt) - - _, err := r.DB.Exec(`DELETE FROM job WHERE job.start_time < ?`, startTime) + qs := fmt.Sprintf("SELECT count(*) FROM job WHERE job.start_time < %d", startTime) + err := r.DB.Get(&cnt, qs) //ignore error as it will also occur in delete statement + _, err = r.DB.Exec(`DELETE FROM job WHERE job.start_time < ?`, startTime) + if err != nil { + log.Warnf(" DeleteJobsBefore(%d): error %v", startTime, err) + } else { + log.Infof("DeleteJobsBefore(%d): Deleted %d jobs", startTime, cnt) + } return cnt, err } func (r *JobRepository) DeleteJobById(id int64) error { - _, err := r.DB.Exec(`DELETE FROM job WHERE job.id = ?`, id) + if err != nil { + log.Warnf("DeleteJobById(%d): error %v", id, err) + } else { + log.Infof("DeleteJobById(%d): Success", id) + } return err } diff --git a/test/integration_test.go b/test/integration_test.go index a7753fd..d4455ac 100644 --- a/test/integration_test.go +++ b/test/integration_test.go @@ -343,7 +343,7 @@ func TestRestApi(t *testing.T) { restapi.MountRoutes(r) const startJobBody string = `{ -"jobId": 123, + "jobId": 123, "user": "testuser", "project": "testproj", "cluster": "testcluster", @@ -542,7 +542,7 @@ func subtestLetJobFail(t *testing.T, restapi *api.RestApi, r *mux.Router) { } const stopJobBody string = `{ -"jobId": 12345, + "jobId": 12345, "cluster": "testcluster", "jobState": "failed", From a39fc7334514359eb3fcf8fa64b3075859e9d5be Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Wed, 30 Nov 2022 12:40:07 +0100 Subject: [PATCH 3/3] Rerun swagger and fix typo --- api/swagger.json | 4 ++-- api/swagger.yaml | 6 +++--- internal/api/docs.go | 4 ++-- pkg/schema/job.go | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/api/swagger.json b/api/swagger.json index 8e3f217..1981247 100644 --- a/api/swagger.json +++ b/api/swagger.json @@ -261,7 +261,7 @@ "ApiKeyAuth": [] } ], - "description": "Job to stop is specified by database ID. This will not remove the job from the job archive.", + "description": "Remove all jobs with start time before timestamp. The jobs will not be removed from the job archive.", "produces": [ "application/json" ], @@ -1062,7 +1062,7 @@ } }, "schema.Tag": { - "description": "Defines B tag using name and type.", + "description": "Defines a tag using name and type.", "type": "object", "properties": { "id": { diff --git a/api/swagger.yaml b/api/swagger.yaml index 67ad683..b5d738d 100644 --- a/api/swagger.yaml +++ b/api/swagger.yaml @@ -347,7 +347,7 @@ definitions: type: array type: object schema.Tag: - description: Defines B tag using name and type. + description: Defines a tag using name and type. properties: id: description: The unique DB identifier of a tag @@ -536,8 +536,8 @@ paths: - remove /jobs/delete_job_before/{ts}: delete: - description: Job to stop is specified by database ID. This will not remove the - job from the job archive. + description: Remove all jobs with start time before timestamp. The jobs will + not be removed from the job archive. parameters: - description: Unix epoch timestamp in: path diff --git a/internal/api/docs.go b/internal/api/docs.go index 33c8064..3fa6787 100644 --- a/internal/api/docs.go +++ b/internal/api/docs.go @@ -268,7 +268,7 @@ const docTemplate = `{ "ApiKeyAuth": [] } ], - "description": "Job to stop is specified by database ID. This will not remove the job from the job archive.", + "description": "Remove all jobs with start time before timestamp. The jobs will not be removed from the job archive.", "produces": [ "application/json" ], @@ -1069,7 +1069,7 @@ const docTemplate = `{ } }, "schema.Tag": { - "description": "Defines B tag using name and type.", + "description": "Defines a tag using name and type.", "type": "object", "properties": { "id": { diff --git a/pkg/schema/job.go b/pkg/schema/job.go index 4de82c1..c87d906 100644 --- a/pkg/schema/job.go +++ b/pkg/schema/job.go @@ -100,7 +100,7 @@ type JobStatistics struct { } // Tag model -// @Description Defines B tag using name and type. +// @Description Defines a tag using name and type. type Tag struct { // The unique DB identifier of a tag ID int64 `json:"id" db:"id"`