ClusterCockpit/cc-backend
1
0
Fork 0
mirror of https://github.com/ClusterCockpit/cc-backend synced 2024-11-10 08:57:25 +01:00
Code Issues Packages Releases Activity

Merge branch 'hotfix' of https://github.com/ClusterCockpit/cc-backend into hotfix

Browse Source
This commit is contained in:
Christoph Kluge 2023-06-15 10:10:52 +02:00
commit 7a40877c2c
8 changed files with 907 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Makefile
View File

@ -2,7 +2,7 @@ TARGET = ./cc-backend
VAR = ./var VAR = ./var
CFG = config.json .env CFG = config.json .env
FRONTEND = ./web/frontend FRONTEND = ./web/frontend
VERSION = 1 VERSION = 1.0.0
GIT_HASH := $(shell git rev-parse --short HEAD || echo 'development') GIT_HASH := $(shell git rev-parse --short HEAD || echo 'development')
CURRENT_TIME = $(shell date +"%Y-%m-%d:T%H:%M:%S") CURRENT_TIME = $(shell date +"%Y-%m-%d:T%H:%M:%S")
LD_FLAGS = '-s -X main.buildTime=${CURRENT_TIME} -X main.version=${VERSION} -X main.hash=${GIT_HASH}' LD_FLAGS = '-s -X main.buildTime=${CURRENT_TIME} -X main.version=${VERSION} -X main.hash=${GIT_HASH}'

2
README.md
View File

@ -51,7 +51,7 @@ cd ./cc-backend
./startDemo.sh ./startDemo.sh
``` ```
You can access the web interface at http://localhost:8080. You can access the web interface at http://localhost:8080.
Credentials for login: `demo:AdminDev`. Credentials for login: `demo:demo`.
Please note that some views do not work without a metric backend (e.g., the Systems and Status view). Please note that some views do not work without a metric backend (e.g., the Systems and Status view).
## Howto Build and Run ## Howto Build and Run

27
ReleaseNotes.md Normal file
View File

@ -0,0 +1,27 @@
# `cc-backend` version 1.0.0
Supports job archive version 1 and database version 4.
This is the initial release of `cc-backend`, the API backend and frontend
implementation of ClusterCockpit.
**Breaking changes**
The aggregate job statistic core hours is now computed using the job table
column `num_hwthreads`. In a the future release this column will be renamed to
`num_cores`. For correct display of core hours `num_hwthreads` must be correctly
filled on job start. If your existing jobs do not provide the correct value in
this column then you can set this with one SQL INSERT statement. This only applies
if you have exclusive jobs, only. Please be aware that we treat this column as
it is the number of cores. In case you have SMT enabled and `num_hwthreads`
is not the number of cores the core hours will be too high by a factor!
**Features**
* Supports user roles admin, support, manager, user, and api.
* Unified search bar supports job id, job name, project id, user name, and name
* Performance improvements for sqlite db backend
* Extended REST api supports to query job metrics
* Better support for shared jobs
* More flexible metric list configuration
* Versioning and migration for database and job archive

290
api/swagger.json
View File

@ -12,7 +12,7 @@
"name": "MIT License", "name": "MIT License",
"url": "https://opensource.org/licenses/MIT" "url": "https://opensource.org/licenses/MIT"
}, },
"version": "0.2.0" "version": "1"
}, },
"host": "localhost:8080", "host": "localhost:8080",
"basePath": "/api", "basePath": "/api",
@ -622,6 +622,91 @@
} }
} }
} }
},
"/jobs/{id}": {
"post": {
"security": [
{
"ApiKeyAuth": []
}
],
"description": "Job to get is specified by database ID\nReturns full job resource information according to 'JobMeta' scheme and all metrics according to 'JobData'.",
"consumes": [
"application/json"
],
"produces": [
"application/json"
],
"tags": [
"query"
],
"summary": "Get complete job meta and metric data",
"parameters": [
{
"type": "integer",
"description": "Database ID of Job",
"name": "id",
"in": "path",
"required": true
},
{
"description": "Array of metric names",
"name": "request",
"in": "body",
"required": true,
"schema": {
"type": "array",
"items": {
"type": "string"
}
}
}
],
"responses": {
"200": {
"description": "Job resource",
"schema": {
"$ref": "#/definitions/api.GetJobApiResponse"
}
},
"400": {
"description": "Bad Request",
"schema": {
"$ref": "#/definitions/api.ErrorResponse"
}
},
"401": {
"description": "Unauthorized",
"schema": {
"$ref": "#/definitions/api.ErrorResponse"
}
},
"403": {
"description": "Forbidden",
"schema": {
"$ref": "#/definitions/api.ErrorResponse"
}
},
"404": {
"description": "Resource not found",
"schema": {
"$ref": "#/definitions/api.ErrorResponse"
}
},
"422": {
"description": "Unprocessable Entity: finding job failed: sql: no rows in result set",
"schema": {
"$ref": "#/definitions/api.ErrorResponse"
}
},
"500": {
"description": "Internal Server Error",
"schema": {
"$ref": "#/definitions/api.ErrorResponse"
}
}
}
}
} }
}, },
"definitions": { "definitions": {
@ -684,6 +769,20 @@
} }
} }
}, },
"api.GetJobApiResponse": {
"type": "object",
"properties": {
"data": {
"type": "array",
"items": {
"$ref": "#/definitions/api.JobMetricWithName"
}
},
"meta": {
"$ref": "#/definitions/schema.Job"
}
}
},
"api.GetJobsApiResponse": { "api.GetJobsApiResponse": {
"type": "object", "type": "object",
"properties": { "properties": {
@ -704,6 +803,20 @@
} }
} }
}, },
"api.JobMetricWithName": {
"type": "object",
"properties": {
"metric": {
"$ref": "#/definitions/schema.JobMetric"
},
"name": {
"type": "string"
},
"scope": {
"$ref": "#/definitions/schema.MetricScope"
}
}
},
"api.StartJobApiResponse": { "api.StartJobApiResponse": {
"type": "object", "type": "object",
"properties": { "properties": {
@ -765,6 +878,9 @@
"type": "string", "type": "string",
"example": "fritz" "example": "fritz"
}, },
"concurrentJobs": {
"$ref": "#/definitions/schema.JobLinkResultList"
},
"duration": { "duration": {
"description": "Duration of job in seconds (Min \u003e 0)", "description": "Duration of job in seconds (Min \u003e 0)",
"type": "integer", "type": "integer",
@ -789,6 +905,14 @@
}, },
"jobState": { "jobState": {
"description": "Final state of job", "description": "Final state of job",
"enum": [
"completed",
"failed",
"cancelled",
"stopped",
"timeout",
"out_of_memory"
],
"allOf": [ "allOf": [
{ {
"$ref": "#/definitions/schema.JobState" "$ref": "#/definitions/schema.JobState"
@ -817,7 +941,7 @@
"example": 2 "example": 2
}, },
"numHwthreads": { "numHwthreads": {
"description": "Number of HWThreads used (Min \u003e 0)", "description": "NumCores int32 `json:\"numCores\" db:\"num_cores\" example:\"20\" minimum:\"1\"` // Number of HWThreads used (Min \u003e 0)",
"type": "integer", "type": "integer",
"minimum": 1, "minimum": 1,
"example": 20 "example": 20
@ -879,6 +1003,31 @@
} }
} }
}, },
"schema.JobLink": {
"type": "object",
"properties": {
"id": {
"type": "integer"
},
"jobId": {
"type": "integer"
}
}
},
"schema.JobLinkResultList": {
"type": "object",
"properties": {
"count": {
"type": "integer"
},
"items": {
"type": "array",
"items": {
"$ref": "#/definitions/schema.JobLink"
}
}
}
},
"schema.JobMeta": { "schema.JobMeta": {
"description": "Meta data information of a HPC job.", "description": "Meta data information of a HPC job.",
"type": "object", "type": "object",
@ -893,6 +1042,9 @@
"type": "string", "type": "string",
"example": "fritz" "example": "fritz"
}, },
"concurrentJobs": {
"$ref": "#/definitions/schema.JobLinkResultList"
},
"duration": { "duration": {
"description": "Duration of job in seconds (Min \u003e 0)", "description": "Duration of job in seconds (Min \u003e 0)",
"type": "integer", "type": "integer",
@ -917,6 +1069,14 @@
}, },
"jobState": { "jobState": {
"description": "Final state of job", "description": "Final state of job",
"enum": [
"completed",
"failed",
"cancelled",
"stopped",
"timeout",
"out_of_memory"
],
"allOf": [ "allOf": [
{ {
"$ref": "#/definitions/schema.JobState" "$ref": "#/definitions/schema.JobState"
@ -945,7 +1105,7 @@
"example": 2 "example": 2
}, },
"numHwthreads": { "numHwthreads": {
"description": "Number of HWThreads used (Min \u003e 0)", "description": "NumCores int32 `json:\"numCores\" db:\"num_cores\" example:\"20\" minimum:\"1\"` // Number of HWThreads used (Min \u003e 0)",
"type": "integer", "type": "integer",
"minimum": 1, "minimum": 1,
"example": 20 "example": 20
@ -1016,6 +1176,26 @@
} }
} }
}, },
"schema.JobMetric": {
"type": "object",
"properties": {
"series": {
"type": "array",
"items": {
"$ref": "#/definitions/schema.Series"
}
},
"statisticsSeries": {
"$ref": "#/definitions/schema.StatsSeries"
},
"timestep": {
"type": "integer"
},
"unit": {
"$ref": "#/definitions/schema.Unit"
}
}
},
"schema.JobState": { "schema.JobState": {
"type": "string", "type": "string",
"enum": [ "enum": [
@ -1062,9 +1242,42 @@
"example": 2000 "example": 2000
}, },
"unit": { "unit": {
"description": "Metric unit (see schema/unit.schema.json)", "$ref": "#/definitions/schema.Unit"
"type": "string", }
"example": "GHz" }
},
"schema.MetricScope": {
"type": "string",
"enum": [
"invalid_scope",
"node",
"socket",
"memoryDomain",
"core",
"hwthread",
"accelerator"
],
"x-enum-varnames": [
"MetricScopeInvalid",
"MetricScopeNode",
"MetricScopeSocket",
"MetricScopeMemoryDomain",
"MetricScopeCore",
"MetricScopeHWThread",
"MetricScopeAccelerator"
]
},
"schema.MetricStatistics": {
"type": "object",
"properties": {
"avg": {
"type": "number"
},
"max": {
"type": "number"
},
"min": {
"type": "number"
} }
} }
}, },
@ -1096,12 +1309,64 @@
} }
} }
}, },
"schema.Series": {
"type": "object",
"properties": {
"data": {
"type": "array",
"items": {
"type": "number"
}
},
"hostname": {
"type": "string"
},
"id": {
"type": "string"
},
"statistics": {
"$ref": "#/definitions/schema.MetricStatistics"
}
}
},
"schema.StatsSeries": {
"type": "object",
"properties": {
"max": {
"type": "array",
"items": {
"type": "number"
}
},
"mean": {
"type": "array",
"items": {
"type": "number"
}
},
"min": {
"type": "array",
"items": {
"type": "number"
}
},
"percentiles": {
"type": "object",
"additionalProperties": {
"type": "array",
"items": {
"type": "number"
}
}
}
}
},
"schema.Tag": { "schema.Tag": {
"description": "Defines a tag using name and type.", "description": "Defines a tag using name and type.",
"type": "object", "type": "object",
"properties": { "properties": {
"id": { "id": {
"description": "The unique DB identifier of a tag", "description": "The unique DB identifier of a tag\nThe unique DB identifier of a tag",
"type": "integer" "type": "integer"
}, },
"name": { "name": {
@ -1115,6 +1380,17 @@
"example": "Debug" "example": "Debug"
} }
} }
},
"schema.Unit": {
"type": "object",
"properties": {
"base": {
"type": "string"
},
"prefix": {
"type": "string"
}
}
} }
}, },
"securityDefinitions": { "securityDefinitions": {

209
api/swagger.yaml
View File

@ -42,6 +42,15 @@ definitions:
description: Statustext of Errorcode description: Statustext of Errorcode
type: string type: string
type: object type: object
api.GetJobApiResponse:
properties:
data:
items:
$ref: '#/definitions/api.JobMetricWithName'
type: array
meta:
$ref: '#/definitions/schema.Job'
type: object
api.GetJobsApiResponse: api.GetJobsApiResponse:
properties: properties:
items: items:
@ -56,6 +65,15 @@ definitions:
description: Page id returned description: Page id returned
type: integer type: integer
type: object type: object
api.JobMetricWithName:
properties:
metric:
$ref: '#/definitions/schema.JobMetric'
name:
type: string
scope:
$ref: '#/definitions/schema.MetricScope'
type: object
api.StartJobApiResponse: api.StartJobApiResponse:
properties: properties:
id: id:
@ -100,6 +118,8 @@ definitions:
description: The unique identifier of a cluster description: The unique identifier of a cluster
example: fritz example: fritz
type: string type: string
concurrentJobs:
$ref: '#/definitions/schema.JobLinkResultList'
duration: duration:
description: Duration of job in seconds (Min > 0) description: Duration of job in seconds (Min > 0)
example: 43200 example: 43200
@ -124,6 +144,13 @@ definitions:
allOf: allOf:
- $ref: '#/definitions/schema.JobState' - $ref: '#/definitions/schema.JobState'
description: Final state of job description: Final state of job
enum:
- completed
- failed
- cancelled
- stopped
- timeout
- out_of_memory
example: completed example: completed
metaData: metaData:
additionalProperties: additionalProperties:
@ -143,7 +170,9 @@ definitions:
minimum: 1 minimum: 1
type: integer type: integer
numHwthreads: numHwthreads:
description: Number of HWThreads used (Min > 0) description: NumCores int32 `json:"numCores" db:"num_cores"
example:"20" minimum:"1"` //
Number of HWThreads used (Min > 0)
example: 20 example: 20
minimum: 1 minimum: 1
type: integer type: integer
@ -191,6 +220,22 @@ definitions:
minimum: 1 minimum: 1
type: integer type: integer
type: object type: object
schema.JobLink:
properties:
id:
type: integer
jobId:
type: integer
type: object
schema.JobLinkResultList:
properties:
count:
type: integer
items:
items:
$ref: '#/definitions/schema.JobLink'
type: array
type: object
schema.JobMeta: schema.JobMeta:
description: Meta data information of a HPC job. description: Meta data information of a HPC job.
properties: properties:
@ -202,6 +247,8 @@ definitions:
description: The unique identifier of a cluster description: The unique identifier of a cluster
example: fritz example: fritz
type: string type: string
concurrentJobs:
$ref: '#/definitions/schema.JobLinkResultList'
duration: duration:
description: Duration of job in seconds (Min > 0) description: Duration of job in seconds (Min > 0)
example: 43200 example: 43200
@ -226,6 +273,13 @@ definitions:
allOf: allOf:
- $ref: '#/definitions/schema.JobState' - $ref: '#/definitions/schema.JobState'
description: Final state of job description: Final state of job
enum:
- completed
- failed
- cancelled
- stopped
- timeout
- out_of_memory
example: completed example: completed
metaData: metaData:
additionalProperties: additionalProperties:
@ -245,7 +299,9 @@ definitions:
minimum: 1 minimum: 1
type: integer type: integer
numHwthreads: numHwthreads:
description: Number of HWThreads used (Min > 0) description: NumCores int32 `json:"numCores" db:"num_cores"
example:"20" minimum:"1"` //
Number of HWThreads used (Min > 0)
example: 20 example: 20
minimum: 1 minimum: 1
type: integer type: integer
@ -300,6 +356,19 @@ definitions:
minimum: 1 minimum: 1
type: integer type: integer
type: object type: object
schema.JobMetric:
properties:
series:
items:
$ref: '#/definitions/schema.Series'
type: array
statisticsSeries:
$ref: '#/definitions/schema.StatsSeries'
timestep:
type: integer
unit:
$ref: '#/definitions/schema.Unit'
type: object
schema.JobState: schema.JobState:
enum: enum:
- running - running
@ -339,9 +408,34 @@ definitions:
minimum: 0 minimum: 0
type: number type: number
unit: unit:
description: Metric unit (see schema/unit.schema.json) $ref: '#/definitions/schema.Unit'
example: GHz type: object
type: string schema.MetricScope:
enum:
- invalid_scope
- node
- socket
- memoryDomain
- core
- hwthread
- accelerator
type: string
x-enum-varnames:
- MetricScopeInvalid
- MetricScopeNode
- MetricScopeSocket
- MetricScopeMemoryDomain
- MetricScopeCore
- MetricScopeHWThread
- MetricScopeAccelerator
schema.MetricStatistics:
properties:
avg:
type: number
max:
type: number
min:
type: number
type: object type: object
schema.Resource: schema.Resource:
description: A resource used by a job description: A resource used by a job
@ -363,11 +457,47 @@ definitions:
type: integer type: integer
type: array type: array
type: object type: object
schema.Series:
properties:
data:
items:
type: number
type: array
hostname:
type: string
id:
type: string
statistics:
$ref: '#/definitions/schema.MetricStatistics'
type: object
schema.StatsSeries:
properties:
max:
items:
type: number
type: array
mean:
items:
type: number
type: array
min:
items:
type: number
type: array
percentiles:
additionalProperties:
items:
type: number
type: array
type: object
type: object
schema.Tag: schema.Tag:
description: Defines a tag using name and type. description: Defines a tag using name and type.
properties: properties:
id: id:
description: The unique DB identifier of a tag description: |-
The unique DB identifier of a tag
The unique DB identifier of a tag
type: integer type: integer
name: name:
description: Tag Name description: Tag Name
@ -378,6 +508,13 @@ definitions:
example: Debug example: Debug
type: string type: string
type: object type: object
schema.Unit:
properties:
base:
type: string
prefix:
type: string
type: object
host: localhost:8080 host: localhost:8080
info: info:
contact: contact:
@ -389,7 +526,7 @@ info:
name: MIT License name: MIT License
url: https://opensource.org/licenses/MIT url: https://opensource.org/licenses/MIT
title: ClusterCockpit REST API title: ClusterCockpit REST API
version: 0.2.0 version: "1"
paths: paths:
/jobs/: /jobs/:
get: get:
@ -456,6 +593,64 @@ paths:
summary: Lists all jobs summary: Lists all jobs
tags: tags:
- query - query
/jobs/{id}:
post:
consumes:
- application/json
description: |-
Job to get is specified by database ID
Returns full job resource information according to 'JobMeta' scheme and all metrics according to 'JobData'.
parameters:
- description: Database ID of Job
in: path
name: id
required: true
type: integer
- description: Array of metric names
in: body
name: request
required: true
schema:
items:
type: string
type: array
produces:
- application/json
responses:
"200":
description: Job resource
schema:
$ref: '#/definitions/api.GetJobApiResponse'
"400":
description: Bad Request
schema:
$ref: '#/definitions/api.ErrorResponse'
"401":
description: Unauthorized
schema:
$ref: '#/definitions/api.ErrorResponse'
"403":
description: Forbidden
schema:
$ref: '#/definitions/api.ErrorResponse'
"404":
description: Resource not found
schema:
$ref: '#/definitions/api.ErrorResponse'
"422":
description: 'Unprocessable Entity: finding job failed: sql: no rows in
result set'
schema:
$ref: '#/definitions/api.ErrorResponse'
"500":
description: Internal Server Error
schema:
$ref: '#/definitions/api.ErrorResponse'
security:
- ApiKeyAuth: []
summary: Get complete job meta and metric data
tags:
- query
/jobs/delete_job/: /jobs/delete_job/:
delete: delete:
consumes: consumes:

290
internal/api/docs.go
View File

@ -628,6 +628,91 @@ const docTemplate = `{
} }
} }
} }
},
"/jobs/{id}": {
"post": {
"security": [
{
"ApiKeyAuth": []
}
],
"description": "Job to get is specified by database ID\nReturns full job resource information according to 'JobMeta' scheme and all metrics according to 'JobData'.",
"consumes": [
"application/json"
],
"produces": [
"application/json"
],
"tags": [
"query"
],
"summary": "Get complete job meta and metric data",
"parameters": [
{
"type": "integer",
"description": "Database ID of Job",
"name": "id",
"in": "path",
"required": true
},
{
"description": "Array of metric names",
"name": "request",
"in": "body",
"required": true,
"schema": {
"type": "array",
"items": {
"type": "string"
}
}
}
],
"responses": {
"200": {
"description": "Job resource",
"schema": {
"$ref": "#/definitions/api.GetJobApiResponse"
}
},
"400": {
"description": "Bad Request",
"schema": {
"$ref": "#/definitions/api.ErrorResponse"
}
},
"401": {
"description": "Unauthorized",
"schema": {
"$ref": "#/definitions/api.ErrorResponse"
}
},
"403": {
"description": "Forbidden",
"schema": {
"$ref": "#/definitions/api.ErrorResponse"
}
},
"404": {
"description": "Resource not found",
"schema": {
"$ref": "#/definitions/api.ErrorResponse"
}
},
"422": {
"description": "Unprocessable Entity: finding job failed: sql: no rows in result set",
"schema": {
"$ref": "#/definitions/api.ErrorResponse"
}
},
"500": {
"description": "Internal Server Error",
"schema": {
"$ref": "#/definitions/api.ErrorResponse"
}
}
}
}
} }
}, },
"definitions": { "definitions": {
@ -690,6 +775,20 @@ const docTemplate = `{
} }
} }
}, },
"api.GetJobApiResponse": {
"type": "object",
"properties": {
"data": {
"type": "array",
"items": {
"$ref": "#/definitions/api.JobMetricWithName"
}
},
"meta": {
"$ref": "#/definitions/schema.Job"
}
}
},
"api.GetJobsApiResponse": { "api.GetJobsApiResponse": {
"type": "object", "type": "object",
"properties": { "properties": {
@ -710,6 +809,20 @@ const docTemplate = `{
} }
} }
}, },
"api.JobMetricWithName": {
"type": "object",
"properties": {
"metric": {
"$ref": "#/definitions/schema.JobMetric"
},
"name": {
"type": "string"
},
"scope": {
"$ref": "#/definitions/schema.MetricScope"
}
}
},
"api.StartJobApiResponse": { "api.StartJobApiResponse": {
"type": "object", "type": "object",
"properties": { "properties": {
@ -771,6 +884,9 @@ const docTemplate = `{
"type": "string", "type": "string",
"example": "fritz" "example": "fritz"
}, },
"concurrentJobs": {
"$ref": "#/definitions/schema.JobLinkResultList"
},
"duration": { "duration": {
"description": "Duration of job in seconds (Min \u003e 0)", "description": "Duration of job in seconds (Min \u003e 0)",
"type": "integer", "type": "integer",
@ -795,6 +911,14 @@ const docTemplate = `{
}, },
"jobState": { "jobState": {
"description": "Final state of job", "description": "Final state of job",
"enum": [
"completed",
"failed",
"cancelled",
"stopped",
"timeout",
"out_of_memory"
],
"allOf": [ "allOf": [
{ {
"$ref": "#/definitions/schema.JobState" "$ref": "#/definitions/schema.JobState"
@ -823,7 +947,7 @@ const docTemplate = `{
"example": 2 "example": 2
}, },
"numHwthreads": { "numHwthreads": {
"description": "Number of HWThreads used (Min \u003e 0)", "description": "NumCores int32 ` + "`" + `json:\"numCores\" db:\"num_cores\" example:\"20\" minimum:\"1\"` + "`" + ` // Number of HWThreads used (Min \u003e 0)",
"type": "integer", "type": "integer",
"minimum": 1, "minimum": 1,
"example": 20 "example": 20
@ -885,6 +1009,31 @@ const docTemplate = `{
} }
} }
}, },
"schema.JobLink": {
"type": "object",
"properties": {
"id": {
"type": "integer"
},
"jobId": {
"type": "integer"
}
}
},
"schema.JobLinkResultList": {
"type": "object",
"properties": {
"count": {
"type": "integer"
},
"items": {
"type": "array",
"items": {
"$ref": "#/definitions/schema.JobLink"
}
}
}
},
"schema.JobMeta": { "schema.JobMeta": {
"description": "Meta data information of a HPC job.", "description": "Meta data information of a HPC job.",
"type": "object", "type": "object",
@ -899,6 +1048,9 @@ const docTemplate = `{
"type": "string", "type": "string",
"example": "fritz" "example": "fritz"
}, },
"concurrentJobs": {
"$ref": "#/definitions/schema.JobLinkResultList"
},
"duration": { "duration": {
"description": "Duration of job in seconds (Min \u003e 0)", "description": "Duration of job in seconds (Min \u003e 0)",
"type": "integer", "type": "integer",
@ -923,6 +1075,14 @@ const docTemplate = `{
}, },
"jobState": { "jobState": {
"description": "Final state of job", "description": "Final state of job",
"enum": [
"completed",
"failed",
"cancelled",
"stopped",
"timeout",
"out_of_memory"
],
"allOf": [ "allOf": [
{ {
"$ref": "#/definitions/schema.JobState" "$ref": "#/definitions/schema.JobState"
@ -951,7 +1111,7 @@ const docTemplate = `{
"example": 2 "example": 2
}, },
"numHwthreads": { "numHwthreads": {
"description": "Number of HWThreads used (Min \u003e 0)", "description": "NumCores int32 ` + "`" + `json:\"numCores\" db:\"num_cores\" example:\"20\" minimum:\"1\"` + "`" + ` // Number of HWThreads used (Min \u003e 0)",
"type": "integer", "type": "integer",
"minimum": 1, "minimum": 1,
"example": 20 "example": 20
@ -1022,6 +1182,26 @@ const docTemplate = `{
} }
} }
}, },
"schema.JobMetric": {
"type": "object",
"properties": {
"series": {
"type": "array",
"items": {
"$ref": "#/definitions/schema.Series"
}
},
"statisticsSeries": {
"$ref": "#/definitions/schema.StatsSeries"
},
"timestep": {
"type": "integer"
},
"unit": {
"$ref": "#/definitions/schema.Unit"
}
}
},
"schema.JobState": { "schema.JobState": {
"type": "string", "type": "string",
"enum": [ "enum": [
@ -1068,9 +1248,42 @@ const docTemplate = `{
"example": 2000 "example": 2000
}, },
"unit": { "unit": {
"description": "Metric unit (see schema/unit.schema.json)", "$ref": "#/definitions/schema.Unit"
"type": "string", }
"example": "GHz" }
},
"schema.MetricScope": {
"type": "string",
"enum": [
"invalid_scope",
"node",
"socket",
"memoryDomain",
"core",
"hwthread",
"accelerator"
],
"x-enum-varnames": [
"MetricScopeInvalid",
"MetricScopeNode",
"MetricScopeSocket",
"MetricScopeMemoryDomain",
"MetricScopeCore",
"MetricScopeHWThread",
"MetricScopeAccelerator"
]
},
"schema.MetricStatistics": {
"type": "object",
"properties": {
"avg": {
"type": "number"
},
"max": {
"type": "number"
},
"min": {
"type": "number"
} }
} }
}, },
@ -1102,12 +1315,64 @@ const docTemplate = `{
} }
} }
}, },
"schema.Series": {
"type": "object",
"properties": {
"data": {
"type": "array",
"items": {
"type": "number"
}
},
"hostname": {
"type": "string"
},
"id": {
"type": "string"
},
"statistics": {
"$ref": "#/definitions/schema.MetricStatistics"
}
}
},
"schema.StatsSeries": {
"type": "object",
"properties": {
"max": {
"type": "array",
"items": {
"type": "number"
}
},
"mean": {
"type": "array",
"items": {
"type": "number"
}
},
"min": {
"type": "array",
"items": {
"type": "number"
}
},
"percentiles": {
"type": "object",
"additionalProperties": {
"type": "array",
"items": {
"type": "number"
}
}
}
}
},
"schema.Tag": { "schema.Tag": {
"description": "Defines a tag using name and type.", "description": "Defines a tag using name and type.",
"type": "object", "type": "object",
"properties": { "properties": {
"id": { "id": {
"description": "The unique DB identifier of a tag", "description": "The unique DB identifier of a tag\nThe unique DB identifier of a tag",
"type": "integer" "type": "integer"
}, },
"name": { "name": {
@ -1121,6 +1386,17 @@ const docTemplate = `{
"example": "Debug" "example": "Debug"
} }
} }
},
"schema.Unit": {
"type": "object",
"properties": {
"base": {
"type": "string"
},
"prefix": {
"type": "string"
}
}
} }
}, },
"securityDefinitions": { "securityDefinitions": {
@ -1139,7 +1415,7 @@ const docTemplate = `{
// SwaggerInfo holds exported Swagger Info so clients can modify it // SwaggerInfo holds exported Swagger Info so clients can modify it
var SwaggerInfo = &swag.Spec{ var SwaggerInfo = &swag.Spec{
Version: "0.2.0", Version: "1",
Host: "localhost:8080", Host: "localhost:8080",
BasePath: "/api", BasePath: "/api",
Schemes: []string{}, Schemes: []string{},

111
internal/api/rest.go
View File

@ -23,6 +23,7 @@ import (
"github.com/ClusterCockpit/cc-backend/internal/graph" "github.com/ClusterCockpit/cc-backend/internal/graph"
"github.com/ClusterCockpit/cc-backend/internal/graph/model" "github.com/ClusterCockpit/cc-backend/internal/graph/model"
"github.com/ClusterCockpit/cc-backend/internal/importer" "github.com/ClusterCockpit/cc-backend/internal/importer"
"github.com/ClusterCockpit/cc-backend/internal/metricdata"
"github.com/ClusterCockpit/cc-backend/internal/repository" "github.com/ClusterCockpit/cc-backend/internal/repository"
"github.com/ClusterCockpit/cc-backend/pkg/archive" "github.com/ClusterCockpit/cc-backend/pkg/archive"
"github.com/ClusterCockpit/cc-backend/pkg/log" "github.com/ClusterCockpit/cc-backend/pkg/log"
@ -31,7 +32,7 @@ import (
) )
// @title ClusterCockpit REST API // @title ClusterCockpit REST API
// @version 1 // @version 1.0.0
// @description API for batch job control. // @description API for batch job control.
// @tag.name Job API // @tag.name Job API
@ -68,7 +69,7 @@ func (api *RestApi) MountRoutes(r *mux.Router) {
// r.HandleFunc("/jobs/import/", api.importJob).Methods(http.MethodPost, http.MethodPut) // r.HandleFunc("/jobs/import/", api.importJob).Methods(http.MethodPost, http.MethodPut)
r.HandleFunc("/jobs/", api.getJobs).Methods(http.MethodGet) r.HandleFunc("/jobs/", api.getJobs).Methods(http.MethodGet)
// r.HandleFunc("/jobs/{id}", api.getJob).Methods(http.MethodGet) r.HandleFunc("/jobs/{id}", api.getJobById).Methods(http.MethodPost)
r.HandleFunc("/jobs/tag_job/{id}", api.tagJob).Methods(http.MethodPost, http.MethodPatch) r.HandleFunc("/jobs/tag_job/{id}", api.tagJob).Methods(http.MethodPost, http.MethodPatch)
r.HandleFunc("/jobs/metrics/{id}", api.getJobMetrics).Methods(http.MethodGet) r.HandleFunc("/jobs/metrics/{id}", api.getJobMetrics).Methods(http.MethodGet)
r.HandleFunc("/jobs/delete_job/", api.deleteJobByRequest).Methods(http.MethodDelete) r.HandleFunc("/jobs/delete_job/", api.deleteJobByRequest).Methods(http.MethodDelete)
@ -142,6 +143,19 @@ type ApiTag struct {
type TagJobApiRequest []*ApiTag type TagJobApiRequest []*ApiTag
type GetJobApiRequest []string
type GetJobApiResponse struct {
Meta *schema.Job
Data []*JobMetricWithName
}
type JobMetricWithName struct {
Name string `json:"name"`
Scope schema.MetricScope `json:"scope"`
Metric *schema.JobMetric `json:"metric"`
}
func handleError(err error, statusCode int, rw http.ResponseWriter) { func handleError(err error, statusCode int, rw http.ResponseWriter) {
log.Warnf("REST ERROR : %s", err.Error()) log.Warnf("REST ERROR : %s", err.Error())
rw.Header().Add("Content-Type", "application/json") rw.Header().Add("Content-Type", "application/json")
@ -301,6 +315,99 @@ func (api *RestApi) getJobs(rw http.ResponseWriter, r *http.Request) {
} }
} }
// getJobById godoc
// @summary Get complete job meta and metric data
// @tags query
// @description Job to get is specified by database ID
// @description Returns full job resource information according to 'JobMeta' scheme and all metrics according to 'JobData'.
// @accept json
// @produce json
// @param id path int true "Database ID of Job"
// @param request body api.GetJobApiRequest true "Array of metric names"
// @success 200 {object} api.GetJobApiResponse "Job resource"
// @failure 400 {object} api.ErrorResponse "Bad Request"
// @failure 401 {object} api.ErrorResponse "Unauthorized"
// @failure 403 {object} api.ErrorResponse "Forbidden"
// @failure 404 {object} api.ErrorResponse "Resource not found"
// @failure 422 {object} api.ErrorResponse "Unprocessable Entity: finding job failed: sql: no rows in result set"
// @failure 500 {object} api.ErrorResponse "Internal Server Error"
// @security ApiKeyAuth
// @router /jobs/{id} [post]
func (api *RestApi) getJobById(rw http.ResponseWriter, r *http.Request) {
if user := auth.GetUser(r.Context()); user != nil && !user.HasRole(auth.RoleApi) {
handleError(fmt.Errorf("missing role: %v",
auth.GetRoleString(auth.RoleApi)), http.StatusForbidden, rw)
return
}
// Fetch job from db
id, ok := mux.Vars(r)["id"]
var job *schema.Job
var err error
if ok {
id, e := strconv.ParseInt(id, 10, 64)
if e != nil {
handleError(fmt.Errorf("integer expected in path for id: %w", e), http.StatusBadRequest, rw)
return
}
job, err = api.JobRepository.FindById(id)
} else {
handleError(errors.New("the parameter 'id' is required"), http.StatusBadRequest, rw)
return
}
if err != nil {
handleError(fmt.Errorf("finding job failed: %w", err), http.StatusUnprocessableEntity, rw)
return
}
var metrics GetJobApiRequest
if err = decode(r.Body, &metrics); err != nil {
http.Error(rw, err.Error(), http.StatusBadRequest)
return
}
var scopes []schema.MetricScope
if job.NumNodes == 1 {
scopes = []schema.MetricScope{"core"}
} else {
scopes = []schema.MetricScope{"node"}
}
data, err := metricdata.LoadData(job, metrics, scopes, r.Context())
if err != nil {
log.Warn("Error while loading job data")
return
}
res := []*JobMetricWithName{}
for name, md := range data {
for scope, metric := range md {
res = append(res, &JobMetricWithName{
Name: name,
Scope: scope,
Metric: metric,
})
}
}
log.Debugf("/api/job/%s: get job %d", id, job.JobID)
rw.Header().Add("Content-Type", "application/json")
bw := bufio.NewWriter(rw)
defer bw.Flush()
payload := GetJobApiResponse{
Meta: job,
Data: res,
}
if err := json.NewEncoder(bw).Encode(payload); err != nil {
handleError(err, http.StatusInternalServerError, rw)
return
}
}
// tagJob godoc // tagJob godoc
// @summary Adds one or more tags to a job // @summary Adds one or more tags to a job
// @tags add and modify // @tags add and modify

2
pkg/schema/job.go
View File

@ -115,7 +115,7 @@ type Unit struct {
// JobStatistics model // JobStatistics model
// @Description Specification for job metric statistics. // @Description Specification for job metric statistics.
type JobStatistics struct { type JobStatistics struct {
Unit Unit `json:"unit" example:"GHz"` Unit Unit `json:"unit"`
Avg float64 `json:"avg" example:"2500" minimum:"0"` // Job metric average Avg float64 `json:"avg" example:"2500" minimum:"0"` // Job metric average
Min float64 `json:"min" example:"2000" minimum:"0"` // Job metric minimum Min float64 `json:"min" example:"2000" minimum:"0"` // Job metric minimum
Max float64 `json:"max" example:"3000" minimum:"0"` // Job metric maximum Max float64 `json:"max" example:"3000" minimum:"0"` // Job metric maximum