diff --git a/Makefile b/Makefile index 3a03b45..1210850 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ TARGET = ./cc-backend VAR = ./var CFG = config.json .env FRONTEND = ./web/frontend -VERSION = 1 +VERSION = 1.0.0 GIT_HASH := $(shell git rev-parse --short HEAD || echo 'development') CURRENT_TIME = $(shell date +"%Y-%m-%d:T%H:%M:%S") LD_FLAGS = '-s -X main.buildTime=${CURRENT_TIME} -X main.version=${VERSION} -X main.hash=${GIT_HASH}' diff --git a/README.md b/README.md index 0bd3364..273321f 100644 --- a/README.md +++ b/README.md @@ -51,7 +51,7 @@ cd ./cc-backend ./startDemo.sh ``` You can access the web interface at http://localhost:8080. -Credentials for login: `demo:AdminDev`. +Credentials for login: `demo:demo`. Please note that some views do not work without a metric backend (e.g., the Systems and Status view). ## Howto Build and Run diff --git a/ReleaseNotes.md b/ReleaseNotes.md new file mode 100644 index 0000000..97dd8de --- /dev/null +++ b/ReleaseNotes.md @@ -0,0 +1,27 @@ +# `cc-backend` version 1.0.0 + +Supports job archive version 1 and database version 4. + +This is the initial release of `cc-backend`, the API backend and frontend +implementation of ClusterCockpit. + +**Breaking changes** + +The aggregate job statistic core hours is now computed using the job table +column `num_hwthreads`. In a the future release this column will be renamed to +`num_cores`. For correct display of core hours `num_hwthreads` must be correctly +filled on job start. If your existing jobs do not provide the correct value in +this column then you can set this with one SQL INSERT statement. This only applies +if you have exclusive jobs, only. Please be aware that we treat this column as +it is the number of cores. In case you have SMT enabled and `num_hwthreads` +is not the number of cores the core hours will be too high by a factor! + +**Features** +* Supports user roles admin, support, manager, user, and api. +* Unified search bar supports job id, job name, project id, user name, and name +* Performance improvements for sqlite db backend +* Extended REST api supports to query job metrics +* Better support for shared jobs +* More flexible metric list configuration +* Versioning and migration for database and job archive + diff --git a/api/swagger.json b/api/swagger.json index 5c32a2d..87a7de5 100644 --- a/api/swagger.json +++ b/api/swagger.json @@ -12,7 +12,7 @@ "name": "MIT License", "url": "https://opensource.org/licenses/MIT" }, - "version": "0.2.0" + "version": "1" }, "host": "localhost:8080", "basePath": "/api", @@ -622,6 +622,91 @@ } } } + }, + "/jobs/{id}": { + "post": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Job to get is specified by database ID\nReturns full job resource information according to 'JobMeta' scheme and all metrics according to 'JobData'.", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "query" + ], + "summary": "Get complete job meta and metric data", + "parameters": [ + { + "type": "integer", + "description": "Database ID of Job", + "name": "id", + "in": "path", + "required": true + }, + { + "description": "Array of metric names", + "name": "request", + "in": "body", + "required": true, + "schema": { + "type": "array", + "items": { + "type": "string" + } + } + } + ], + "responses": { + "200": { + "description": "Job resource", + "schema": { + "$ref": "#/definitions/api.GetJobApiResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/api.ErrorResponse" + } + }, + "401": { + "description": "Unauthorized", + "schema": { + "$ref": "#/definitions/api.ErrorResponse" + } + }, + "403": { + "description": "Forbidden", + "schema": { + "$ref": "#/definitions/api.ErrorResponse" + } + }, + "404": { + "description": "Resource not found", + "schema": { + "$ref": "#/definitions/api.ErrorResponse" + } + }, + "422": { + "description": "Unprocessable Entity: finding job failed: sql: no rows in result set", + "schema": { + "$ref": "#/definitions/api.ErrorResponse" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/api.ErrorResponse" + } + } + } + } } }, "definitions": { @@ -684,6 +769,20 @@ } } }, + "api.GetJobApiResponse": { + "type": "object", + "properties": { + "data": { + "type": "array", + "items": { + "$ref": "#/definitions/api.JobMetricWithName" + } + }, + "meta": { + "$ref": "#/definitions/schema.Job" + } + } + }, "api.GetJobsApiResponse": { "type": "object", "properties": { @@ -704,6 +803,20 @@ } } }, + "api.JobMetricWithName": { + "type": "object", + "properties": { + "metric": { + "$ref": "#/definitions/schema.JobMetric" + }, + "name": { + "type": "string" + }, + "scope": { + "$ref": "#/definitions/schema.MetricScope" + } + } + }, "api.StartJobApiResponse": { "type": "object", "properties": { @@ -765,6 +878,9 @@ "type": "string", "example": "fritz" }, + "concurrentJobs": { + "$ref": "#/definitions/schema.JobLinkResultList" + }, "duration": { "description": "Duration of job in seconds (Min \u003e 0)", "type": "integer", @@ -789,6 +905,14 @@ }, "jobState": { "description": "Final state of job", + "enum": [ + "completed", + "failed", + "cancelled", + "stopped", + "timeout", + "out_of_memory" + ], "allOf": [ { "$ref": "#/definitions/schema.JobState" @@ -817,7 +941,7 @@ "example": 2 }, "numHwthreads": { - "description": "Number of HWThreads used (Min \u003e 0)", + "description": "NumCores int32 `json:\"numCores\" db:\"num_cores\" example:\"20\" minimum:\"1\"` // Number of HWThreads used (Min \u003e 0)", "type": "integer", "minimum": 1, "example": 20 @@ -879,6 +1003,31 @@ } } }, + "schema.JobLink": { + "type": "object", + "properties": { + "id": { + "type": "integer" + }, + "jobId": { + "type": "integer" + } + } + }, + "schema.JobLinkResultList": { + "type": "object", + "properties": { + "count": { + "type": "integer" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/schema.JobLink" + } + } + } + }, "schema.JobMeta": { "description": "Meta data information of a HPC job.", "type": "object", @@ -893,6 +1042,9 @@ "type": "string", "example": "fritz" }, + "concurrentJobs": { + "$ref": "#/definitions/schema.JobLinkResultList" + }, "duration": { "description": "Duration of job in seconds (Min \u003e 0)", "type": "integer", @@ -917,6 +1069,14 @@ }, "jobState": { "description": "Final state of job", + "enum": [ + "completed", + "failed", + "cancelled", + "stopped", + "timeout", + "out_of_memory" + ], "allOf": [ { "$ref": "#/definitions/schema.JobState" @@ -945,7 +1105,7 @@ "example": 2 }, "numHwthreads": { - "description": "Number of HWThreads used (Min \u003e 0)", + "description": "NumCores int32 `json:\"numCores\" db:\"num_cores\" example:\"20\" minimum:\"1\"` // Number of HWThreads used (Min \u003e 0)", "type": "integer", "minimum": 1, "example": 20 @@ -1016,6 +1176,26 @@ } } }, + "schema.JobMetric": { + "type": "object", + "properties": { + "series": { + "type": "array", + "items": { + "$ref": "#/definitions/schema.Series" + } + }, + "statisticsSeries": { + "$ref": "#/definitions/schema.StatsSeries" + }, + "timestep": { + "type": "integer" + }, + "unit": { + "$ref": "#/definitions/schema.Unit" + } + } + }, "schema.JobState": { "type": "string", "enum": [ @@ -1062,9 +1242,42 @@ "example": 2000 }, "unit": { - "description": "Metric unit (see schema/unit.schema.json)", - "type": "string", - "example": "GHz" + "$ref": "#/definitions/schema.Unit" + } + } + }, + "schema.MetricScope": { + "type": "string", + "enum": [ + "invalid_scope", + "node", + "socket", + "memoryDomain", + "core", + "hwthread", + "accelerator" + ], + "x-enum-varnames": [ + "MetricScopeInvalid", + "MetricScopeNode", + "MetricScopeSocket", + "MetricScopeMemoryDomain", + "MetricScopeCore", + "MetricScopeHWThread", + "MetricScopeAccelerator" + ] + }, + "schema.MetricStatistics": { + "type": "object", + "properties": { + "avg": { + "type": "number" + }, + "max": { + "type": "number" + }, + "min": { + "type": "number" } } }, @@ -1096,12 +1309,64 @@ } } }, + "schema.Series": { + "type": "object", + "properties": { + "data": { + "type": "array", + "items": { + "type": "number" + } + }, + "hostname": { + "type": "string" + }, + "id": { + "type": "string" + }, + "statistics": { + "$ref": "#/definitions/schema.MetricStatistics" + } + } + }, + "schema.StatsSeries": { + "type": "object", + "properties": { + "max": { + "type": "array", + "items": { + "type": "number" + } + }, + "mean": { + "type": "array", + "items": { + "type": "number" + } + }, + "min": { + "type": "array", + "items": { + "type": "number" + } + }, + "percentiles": { + "type": "object", + "additionalProperties": { + "type": "array", + "items": { + "type": "number" + } + } + } + } + }, "schema.Tag": { "description": "Defines a tag using name and type.", "type": "object", "properties": { "id": { - "description": "The unique DB identifier of a tag", + "description": "The unique DB identifier of a tag\nThe unique DB identifier of a tag", "type": "integer" }, "name": { @@ -1115,6 +1380,17 @@ "example": "Debug" } } + }, + "schema.Unit": { + "type": "object", + "properties": { + "base": { + "type": "string" + }, + "prefix": { + "type": "string" + } + } } }, "securityDefinitions": { diff --git a/api/swagger.yaml b/api/swagger.yaml index 7d008b8..093266d 100644 --- a/api/swagger.yaml +++ b/api/swagger.yaml @@ -42,6 +42,15 @@ definitions: description: Statustext of Errorcode type: string type: object + api.GetJobApiResponse: + properties: + data: + items: + $ref: '#/definitions/api.JobMetricWithName' + type: array + meta: + $ref: '#/definitions/schema.Job' + type: object api.GetJobsApiResponse: properties: items: @@ -56,6 +65,15 @@ definitions: description: Page id returned type: integer type: object + api.JobMetricWithName: + properties: + metric: + $ref: '#/definitions/schema.JobMetric' + name: + type: string + scope: + $ref: '#/definitions/schema.MetricScope' + type: object api.StartJobApiResponse: properties: id: @@ -100,6 +118,8 @@ definitions: description: The unique identifier of a cluster example: fritz type: string + concurrentJobs: + $ref: '#/definitions/schema.JobLinkResultList' duration: description: Duration of job in seconds (Min > 0) example: 43200 @@ -124,6 +144,13 @@ definitions: allOf: - $ref: '#/definitions/schema.JobState' description: Final state of job + enum: + - completed + - failed + - cancelled + - stopped + - timeout + - out_of_memory example: completed metaData: additionalProperties: @@ -143,7 +170,9 @@ definitions: minimum: 1 type: integer numHwthreads: - description: Number of HWThreads used (Min > 0) + description: NumCores int32 `json:"numCores" db:"num_cores" + example:"20" minimum:"1"` // + Number of HWThreads used (Min > 0) example: 20 minimum: 1 type: integer @@ -191,6 +220,22 @@ definitions: minimum: 1 type: integer type: object + schema.JobLink: + properties: + id: + type: integer + jobId: + type: integer + type: object + schema.JobLinkResultList: + properties: + count: + type: integer + items: + items: + $ref: '#/definitions/schema.JobLink' + type: array + type: object schema.JobMeta: description: Meta data information of a HPC job. properties: @@ -202,6 +247,8 @@ definitions: description: The unique identifier of a cluster example: fritz type: string + concurrentJobs: + $ref: '#/definitions/schema.JobLinkResultList' duration: description: Duration of job in seconds (Min > 0) example: 43200 @@ -226,6 +273,13 @@ definitions: allOf: - $ref: '#/definitions/schema.JobState' description: Final state of job + enum: + - completed + - failed + - cancelled + - stopped + - timeout + - out_of_memory example: completed metaData: additionalProperties: @@ -245,7 +299,9 @@ definitions: minimum: 1 type: integer numHwthreads: - description: Number of HWThreads used (Min > 0) + description: NumCores int32 `json:"numCores" db:"num_cores" + example:"20" minimum:"1"` // + Number of HWThreads used (Min > 0) example: 20 minimum: 1 type: integer @@ -300,6 +356,19 @@ definitions: minimum: 1 type: integer type: object + schema.JobMetric: + properties: + series: + items: + $ref: '#/definitions/schema.Series' + type: array + statisticsSeries: + $ref: '#/definitions/schema.StatsSeries' + timestep: + type: integer + unit: + $ref: '#/definitions/schema.Unit' + type: object schema.JobState: enum: - running @@ -339,9 +408,34 @@ definitions: minimum: 0 type: number unit: - description: Metric unit (see schema/unit.schema.json) - example: GHz - type: string + $ref: '#/definitions/schema.Unit' + type: object + schema.MetricScope: + enum: + - invalid_scope + - node + - socket + - memoryDomain + - core + - hwthread + - accelerator + type: string + x-enum-varnames: + - MetricScopeInvalid + - MetricScopeNode + - MetricScopeSocket + - MetricScopeMemoryDomain + - MetricScopeCore + - MetricScopeHWThread + - MetricScopeAccelerator + schema.MetricStatistics: + properties: + avg: + type: number + max: + type: number + min: + type: number type: object schema.Resource: description: A resource used by a job @@ -363,11 +457,47 @@ definitions: type: integer type: array type: object + schema.Series: + properties: + data: + items: + type: number + type: array + hostname: + type: string + id: + type: string + statistics: + $ref: '#/definitions/schema.MetricStatistics' + type: object + schema.StatsSeries: + properties: + max: + items: + type: number + type: array + mean: + items: + type: number + type: array + min: + items: + type: number + type: array + percentiles: + additionalProperties: + items: + type: number + type: array + type: object + type: object schema.Tag: description: Defines a tag using name and type. properties: id: - description: The unique DB identifier of a tag + description: |- + The unique DB identifier of a tag + The unique DB identifier of a tag type: integer name: description: Tag Name @@ -378,6 +508,13 @@ definitions: example: Debug type: string type: object + schema.Unit: + properties: + base: + type: string + prefix: + type: string + type: object host: localhost:8080 info: contact: @@ -389,7 +526,7 @@ info: name: MIT License url: https://opensource.org/licenses/MIT title: ClusterCockpit REST API - version: 0.2.0 + version: "1" paths: /jobs/: get: @@ -456,6 +593,64 @@ paths: summary: Lists all jobs tags: - query + /jobs/{id}: + post: + consumes: + - application/json + description: |- + Job to get is specified by database ID + Returns full job resource information according to 'JobMeta' scheme and all metrics according to 'JobData'. + parameters: + - description: Database ID of Job + in: path + name: id + required: true + type: integer + - description: Array of metric names + in: body + name: request + required: true + schema: + items: + type: string + type: array + produces: + - application/json + responses: + "200": + description: Job resource + schema: + $ref: '#/definitions/api.GetJobApiResponse' + "400": + description: Bad Request + schema: + $ref: '#/definitions/api.ErrorResponse' + "401": + description: Unauthorized + schema: + $ref: '#/definitions/api.ErrorResponse' + "403": + description: Forbidden + schema: + $ref: '#/definitions/api.ErrorResponse' + "404": + description: Resource not found + schema: + $ref: '#/definitions/api.ErrorResponse' + "422": + description: 'Unprocessable Entity: finding job failed: sql: no rows in + result set' + schema: + $ref: '#/definitions/api.ErrorResponse' + "500": + description: Internal Server Error + schema: + $ref: '#/definitions/api.ErrorResponse' + security: + - ApiKeyAuth: [] + summary: Get complete job meta and metric data + tags: + - query /jobs/delete_job/: delete: consumes: diff --git a/internal/api/docs.go b/internal/api/docs.go index af6745f..85acc92 100644 --- a/internal/api/docs.go +++ b/internal/api/docs.go @@ -628,6 +628,91 @@ const docTemplate = `{ } } } + }, + "/jobs/{id}": { + "post": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Job to get is specified by database ID\nReturns full job resource information according to 'JobMeta' scheme and all metrics according to 'JobData'.", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "query" + ], + "summary": "Get complete job meta and metric data", + "parameters": [ + { + "type": "integer", + "description": "Database ID of Job", + "name": "id", + "in": "path", + "required": true + }, + { + "description": "Array of metric names", + "name": "request", + "in": "body", + "required": true, + "schema": { + "type": "array", + "items": { + "type": "string" + } + } + } + ], + "responses": { + "200": { + "description": "Job resource", + "schema": { + "$ref": "#/definitions/api.GetJobApiResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/api.ErrorResponse" + } + }, + "401": { + "description": "Unauthorized", + "schema": { + "$ref": "#/definitions/api.ErrorResponse" + } + }, + "403": { + "description": "Forbidden", + "schema": { + "$ref": "#/definitions/api.ErrorResponse" + } + }, + "404": { + "description": "Resource not found", + "schema": { + "$ref": "#/definitions/api.ErrorResponse" + } + }, + "422": { + "description": "Unprocessable Entity: finding job failed: sql: no rows in result set", + "schema": { + "$ref": "#/definitions/api.ErrorResponse" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/api.ErrorResponse" + } + } + } + } } }, "definitions": { @@ -690,6 +775,20 @@ const docTemplate = `{ } } }, + "api.GetJobApiResponse": { + "type": "object", + "properties": { + "data": { + "type": "array", + "items": { + "$ref": "#/definitions/api.JobMetricWithName" + } + }, + "meta": { + "$ref": "#/definitions/schema.Job" + } + } + }, "api.GetJobsApiResponse": { "type": "object", "properties": { @@ -710,6 +809,20 @@ const docTemplate = `{ } } }, + "api.JobMetricWithName": { + "type": "object", + "properties": { + "metric": { + "$ref": "#/definitions/schema.JobMetric" + }, + "name": { + "type": "string" + }, + "scope": { + "$ref": "#/definitions/schema.MetricScope" + } + } + }, "api.StartJobApiResponse": { "type": "object", "properties": { @@ -771,6 +884,9 @@ const docTemplate = `{ "type": "string", "example": "fritz" }, + "concurrentJobs": { + "$ref": "#/definitions/schema.JobLinkResultList" + }, "duration": { "description": "Duration of job in seconds (Min \u003e 0)", "type": "integer", @@ -795,6 +911,14 @@ const docTemplate = `{ }, "jobState": { "description": "Final state of job", + "enum": [ + "completed", + "failed", + "cancelled", + "stopped", + "timeout", + "out_of_memory" + ], "allOf": [ { "$ref": "#/definitions/schema.JobState" @@ -823,7 +947,7 @@ const docTemplate = `{ "example": 2 }, "numHwthreads": { - "description": "Number of HWThreads used (Min \u003e 0)", + "description": "NumCores int32 ` + "`" + `json:\"numCores\" db:\"num_cores\" example:\"20\" minimum:\"1\"` + "`" + ` // Number of HWThreads used (Min \u003e 0)", "type": "integer", "minimum": 1, "example": 20 @@ -885,6 +1009,31 @@ const docTemplate = `{ } } }, + "schema.JobLink": { + "type": "object", + "properties": { + "id": { + "type": "integer" + }, + "jobId": { + "type": "integer" + } + } + }, + "schema.JobLinkResultList": { + "type": "object", + "properties": { + "count": { + "type": "integer" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/schema.JobLink" + } + } + } + }, "schema.JobMeta": { "description": "Meta data information of a HPC job.", "type": "object", @@ -899,6 +1048,9 @@ const docTemplate = `{ "type": "string", "example": "fritz" }, + "concurrentJobs": { + "$ref": "#/definitions/schema.JobLinkResultList" + }, "duration": { "description": "Duration of job in seconds (Min \u003e 0)", "type": "integer", @@ -923,6 +1075,14 @@ const docTemplate = `{ }, "jobState": { "description": "Final state of job", + "enum": [ + "completed", + "failed", + "cancelled", + "stopped", + "timeout", + "out_of_memory" + ], "allOf": [ { "$ref": "#/definitions/schema.JobState" @@ -951,7 +1111,7 @@ const docTemplate = `{ "example": 2 }, "numHwthreads": { - "description": "Number of HWThreads used (Min \u003e 0)", + "description": "NumCores int32 ` + "`" + `json:\"numCores\" db:\"num_cores\" example:\"20\" minimum:\"1\"` + "`" + ` // Number of HWThreads used (Min \u003e 0)", "type": "integer", "minimum": 1, "example": 20 @@ -1022,6 +1182,26 @@ const docTemplate = `{ } } }, + "schema.JobMetric": { + "type": "object", + "properties": { + "series": { + "type": "array", + "items": { + "$ref": "#/definitions/schema.Series" + } + }, + "statisticsSeries": { + "$ref": "#/definitions/schema.StatsSeries" + }, + "timestep": { + "type": "integer" + }, + "unit": { + "$ref": "#/definitions/schema.Unit" + } + } + }, "schema.JobState": { "type": "string", "enum": [ @@ -1068,9 +1248,42 @@ const docTemplate = `{ "example": 2000 }, "unit": { - "description": "Metric unit (see schema/unit.schema.json)", - "type": "string", - "example": "GHz" + "$ref": "#/definitions/schema.Unit" + } + } + }, + "schema.MetricScope": { + "type": "string", + "enum": [ + "invalid_scope", + "node", + "socket", + "memoryDomain", + "core", + "hwthread", + "accelerator" + ], + "x-enum-varnames": [ + "MetricScopeInvalid", + "MetricScopeNode", + "MetricScopeSocket", + "MetricScopeMemoryDomain", + "MetricScopeCore", + "MetricScopeHWThread", + "MetricScopeAccelerator" + ] + }, + "schema.MetricStatistics": { + "type": "object", + "properties": { + "avg": { + "type": "number" + }, + "max": { + "type": "number" + }, + "min": { + "type": "number" } } }, @@ -1102,12 +1315,64 @@ const docTemplate = `{ } } }, + "schema.Series": { + "type": "object", + "properties": { + "data": { + "type": "array", + "items": { + "type": "number" + } + }, + "hostname": { + "type": "string" + }, + "id": { + "type": "string" + }, + "statistics": { + "$ref": "#/definitions/schema.MetricStatistics" + } + } + }, + "schema.StatsSeries": { + "type": "object", + "properties": { + "max": { + "type": "array", + "items": { + "type": "number" + } + }, + "mean": { + "type": "array", + "items": { + "type": "number" + } + }, + "min": { + "type": "array", + "items": { + "type": "number" + } + }, + "percentiles": { + "type": "object", + "additionalProperties": { + "type": "array", + "items": { + "type": "number" + } + } + } + } + }, "schema.Tag": { "description": "Defines a tag using name and type.", "type": "object", "properties": { "id": { - "description": "The unique DB identifier of a tag", + "description": "The unique DB identifier of a tag\nThe unique DB identifier of a tag", "type": "integer" }, "name": { @@ -1121,6 +1386,17 @@ const docTemplate = `{ "example": "Debug" } } + }, + "schema.Unit": { + "type": "object", + "properties": { + "base": { + "type": "string" + }, + "prefix": { + "type": "string" + } + } } }, "securityDefinitions": { @@ -1139,7 +1415,7 @@ const docTemplate = `{ // SwaggerInfo holds exported Swagger Info so clients can modify it var SwaggerInfo = &swag.Spec{ - Version: "0.2.0", + Version: "1", Host: "localhost:8080", BasePath: "/api", Schemes: []string{}, diff --git a/internal/api/rest.go b/internal/api/rest.go index ae9e8e9..b66a561 100644 --- a/internal/api/rest.go +++ b/internal/api/rest.go @@ -23,6 +23,7 @@ import ( "github.com/ClusterCockpit/cc-backend/internal/graph" "github.com/ClusterCockpit/cc-backend/internal/graph/model" "github.com/ClusterCockpit/cc-backend/internal/importer" + "github.com/ClusterCockpit/cc-backend/internal/metricdata" "github.com/ClusterCockpit/cc-backend/internal/repository" "github.com/ClusterCockpit/cc-backend/pkg/archive" "github.com/ClusterCockpit/cc-backend/pkg/log" @@ -31,7 +32,7 @@ import ( ) // @title ClusterCockpit REST API -// @version 1 +// @version 1.0.0 // @description API for batch job control. // @tag.name Job API @@ -68,7 +69,7 @@ func (api *RestApi) MountRoutes(r *mux.Router) { // r.HandleFunc("/jobs/import/", api.importJob).Methods(http.MethodPost, http.MethodPut) r.HandleFunc("/jobs/", api.getJobs).Methods(http.MethodGet) - // r.HandleFunc("/jobs/{id}", api.getJob).Methods(http.MethodGet) + r.HandleFunc("/jobs/{id}", api.getJobById).Methods(http.MethodPost) r.HandleFunc("/jobs/tag_job/{id}", api.tagJob).Methods(http.MethodPost, http.MethodPatch) r.HandleFunc("/jobs/metrics/{id}", api.getJobMetrics).Methods(http.MethodGet) r.HandleFunc("/jobs/delete_job/", api.deleteJobByRequest).Methods(http.MethodDelete) @@ -142,6 +143,19 @@ type ApiTag struct { type TagJobApiRequest []*ApiTag +type GetJobApiRequest []string + +type GetJobApiResponse struct { + Meta *schema.Job + Data []*JobMetricWithName +} + +type JobMetricWithName struct { + Name string `json:"name"` + Scope schema.MetricScope `json:"scope"` + Metric *schema.JobMetric `json:"metric"` +} + func handleError(err error, statusCode int, rw http.ResponseWriter) { log.Warnf("REST ERROR : %s", err.Error()) rw.Header().Add("Content-Type", "application/json") @@ -301,6 +315,99 @@ func (api *RestApi) getJobs(rw http.ResponseWriter, r *http.Request) { } } +// getJobById godoc +// @summary Get complete job meta and metric data +// @tags query +// @description Job to get is specified by database ID +// @description Returns full job resource information according to 'JobMeta' scheme and all metrics according to 'JobData'. +// @accept json +// @produce json +// @param id path int true "Database ID of Job" +// @param request body api.GetJobApiRequest true "Array of metric names" +// @success 200 {object} api.GetJobApiResponse "Job resource" +// @failure 400 {object} api.ErrorResponse "Bad Request" +// @failure 401 {object} api.ErrorResponse "Unauthorized" +// @failure 403 {object} api.ErrorResponse "Forbidden" +// @failure 404 {object} api.ErrorResponse "Resource not found" +// @failure 422 {object} api.ErrorResponse "Unprocessable Entity: finding job failed: sql: no rows in result set" +// @failure 500 {object} api.ErrorResponse "Internal Server Error" +// @security ApiKeyAuth +// @router /jobs/{id} [post] +func (api *RestApi) getJobById(rw http.ResponseWriter, r *http.Request) { + if user := auth.GetUser(r.Context()); user != nil && !user.HasRole(auth.RoleApi) { + handleError(fmt.Errorf("missing role: %v", + auth.GetRoleString(auth.RoleApi)), http.StatusForbidden, rw) + return + } + + // Fetch job from db + id, ok := mux.Vars(r)["id"] + var job *schema.Job + var err error + if ok { + id, e := strconv.ParseInt(id, 10, 64) + if e != nil { + handleError(fmt.Errorf("integer expected in path for id: %w", e), http.StatusBadRequest, rw) + return + } + + job, err = api.JobRepository.FindById(id) + } else { + handleError(errors.New("the parameter 'id' is required"), http.StatusBadRequest, rw) + return + } + if err != nil { + handleError(fmt.Errorf("finding job failed: %w", err), http.StatusUnprocessableEntity, rw) + return + } + + var metrics GetJobApiRequest + if err = decode(r.Body, &metrics); err != nil { + http.Error(rw, err.Error(), http.StatusBadRequest) + return + } + + var scopes []schema.MetricScope + + if job.NumNodes == 1 { + scopes = []schema.MetricScope{"core"} + } else { + scopes = []schema.MetricScope{"node"} + } + + data, err := metricdata.LoadData(job, metrics, scopes, r.Context()) + if err != nil { + log.Warn("Error while loading job data") + return + } + + res := []*JobMetricWithName{} + for name, md := range data { + for scope, metric := range md { + res = append(res, &JobMetricWithName{ + Name: name, + Scope: scope, + Metric: metric, + }) + } + } + + log.Debugf("/api/job/%s: get job %d", id, job.JobID) + rw.Header().Add("Content-Type", "application/json") + bw := bufio.NewWriter(rw) + defer bw.Flush() + + payload := GetJobApiResponse{ + Meta: job, + Data: res, + } + + if err := json.NewEncoder(bw).Encode(payload); err != nil { + handleError(err, http.StatusInternalServerError, rw) + return + } +} + // tagJob godoc // @summary Adds one or more tags to a job // @tags add and modify diff --git a/pkg/schema/job.go b/pkg/schema/job.go index d967dd0..23ae1ef 100644 --- a/pkg/schema/job.go +++ b/pkg/schema/job.go @@ -115,7 +115,7 @@ type Unit struct { // JobStatistics model // @Description Specification for job metric statistics. type JobStatistics struct { - Unit Unit `json:"unit" example:"GHz"` + Unit Unit `json:"unit"` Avg float64 `json:"avg" example:"2500" minimum:"0"` // Job metric average Min float64 `json:"min" example:"2000" minimum:"0"` // Job metric minimum Max float64 `json:"max" example:"3000" minimum:"0"` // Job metric maximum