From 54aa940d3e58659fedd38b9275cc4052c401295b Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Wed, 14 Jun 2023 15:03:01 +0200 Subject: [PATCH] Add job metricdata rest endpoint Fixes #102 --- api/swagger.json | 139 ++++++++++++++++++++++++++++++++++++++++++- api/swagger.yaml | 96 +++++++++++++++++++++++++++++- internal/api/docs.go | 139 ++++++++++++++++++++++++++++++++++++++++++- internal/api/rest.go | 24 +++++--- 4 files changed, 383 insertions(+), 15 deletions(-) diff --git a/api/swagger.json b/api/swagger.json index 2ce3519..87a7de5 100644 --- a/api/swagger.json +++ b/api/swagger.json @@ -624,7 +624,7 @@ } }, "/jobs/{id}": { - "get": { + "post": { "security": [ { "ApiKeyAuth": [] @@ -666,7 +666,7 @@ "200": { "description": "Job resource", "schema": { - "$ref": "#/definitions/schema.JobMeta" + "$ref": "#/definitions/api.GetJobApiResponse" } }, "400": { @@ -769,6 +769,20 @@ } } }, + "api.GetJobApiResponse": { + "type": "object", + "properties": { + "data": { + "type": "array", + "items": { + "$ref": "#/definitions/api.JobMetricWithName" + } + }, + "meta": { + "$ref": "#/definitions/schema.Job" + } + } + }, "api.GetJobsApiResponse": { "type": "object", "properties": { @@ -789,6 +803,20 @@ } } }, + "api.JobMetricWithName": { + "type": "object", + "properties": { + "metric": { + "$ref": "#/definitions/schema.JobMetric" + }, + "name": { + "type": "string" + }, + "scope": { + "$ref": "#/definitions/schema.MetricScope" + } + } + }, "api.StartJobApiResponse": { "type": "object", "properties": { @@ -1148,6 +1176,26 @@ } } }, + "schema.JobMetric": { + "type": "object", + "properties": { + "series": { + "type": "array", + "items": { + "$ref": "#/definitions/schema.Series" + } + }, + "statisticsSeries": { + "$ref": "#/definitions/schema.StatsSeries" + }, + "timestep": { + "type": "integer" + }, + "unit": { + "$ref": "#/definitions/schema.Unit" + } + } + }, "schema.JobState": { "type": "string", "enum": [ @@ -1198,6 +1246,41 @@ } } }, + "schema.MetricScope": { + "type": "string", + "enum": [ + "invalid_scope", + "node", + "socket", + "memoryDomain", + "core", + "hwthread", + "accelerator" + ], + "x-enum-varnames": [ + "MetricScopeInvalid", + "MetricScopeNode", + "MetricScopeSocket", + "MetricScopeMemoryDomain", + "MetricScopeCore", + "MetricScopeHWThread", + "MetricScopeAccelerator" + ] + }, + "schema.MetricStatistics": { + "type": "object", + "properties": { + "avg": { + "type": "number" + }, + "max": { + "type": "number" + }, + "min": { + "type": "number" + } + } + }, "schema.Resource": { "description": "A resource used by a job", "type": "object", @@ -1226,6 +1309,58 @@ } } }, + "schema.Series": { + "type": "object", + "properties": { + "data": { + "type": "array", + "items": { + "type": "number" + } + }, + "hostname": { + "type": "string" + }, + "id": { + "type": "string" + }, + "statistics": { + "$ref": "#/definitions/schema.MetricStatistics" + } + } + }, + "schema.StatsSeries": { + "type": "object", + "properties": { + "max": { + "type": "array", + "items": { + "type": "number" + } + }, + "mean": { + "type": "array", + "items": { + "type": "number" + } + }, + "min": { + "type": "array", + "items": { + "type": "number" + } + }, + "percentiles": { + "type": "object", + "additionalProperties": { + "type": "array", + "items": { + "type": "number" + } + } + } + } + }, "schema.Tag": { "description": "Defines a tag using name and type.", "type": "object", diff --git a/api/swagger.yaml b/api/swagger.yaml index eb54e4f..093266d 100644 --- a/api/swagger.yaml +++ b/api/swagger.yaml @@ -42,6 +42,15 @@ definitions: description: Statustext of Errorcode type: string type: object + api.GetJobApiResponse: + properties: + data: + items: + $ref: '#/definitions/api.JobMetricWithName' + type: array + meta: + $ref: '#/definitions/schema.Job' + type: object api.GetJobsApiResponse: properties: items: @@ -56,6 +65,15 @@ definitions: description: Page id returned type: integer type: object + api.JobMetricWithName: + properties: + metric: + $ref: '#/definitions/schema.JobMetric' + name: + type: string + scope: + $ref: '#/definitions/schema.MetricScope' + type: object api.StartJobApiResponse: properties: id: @@ -338,6 +356,19 @@ definitions: minimum: 1 type: integer type: object + schema.JobMetric: + properties: + series: + items: + $ref: '#/definitions/schema.Series' + type: array + statisticsSeries: + $ref: '#/definitions/schema.StatsSeries' + timestep: + type: integer + unit: + $ref: '#/definitions/schema.Unit' + type: object schema.JobState: enum: - running @@ -379,6 +410,33 @@ definitions: unit: $ref: '#/definitions/schema.Unit' type: object + schema.MetricScope: + enum: + - invalid_scope + - node + - socket + - memoryDomain + - core + - hwthread + - accelerator + type: string + x-enum-varnames: + - MetricScopeInvalid + - MetricScopeNode + - MetricScopeSocket + - MetricScopeMemoryDomain + - MetricScopeCore + - MetricScopeHWThread + - MetricScopeAccelerator + schema.MetricStatistics: + properties: + avg: + type: number + max: + type: number + min: + type: number + type: object schema.Resource: description: A resource used by a job properties: @@ -399,6 +457,40 @@ definitions: type: integer type: array type: object + schema.Series: + properties: + data: + items: + type: number + type: array + hostname: + type: string + id: + type: string + statistics: + $ref: '#/definitions/schema.MetricStatistics' + type: object + schema.StatsSeries: + properties: + max: + items: + type: number + type: array + mean: + items: + type: number + type: array + min: + items: + type: number + type: array + percentiles: + additionalProperties: + items: + type: number + type: array + type: object + type: object schema.Tag: description: Defines a tag using name and type. properties: @@ -502,7 +594,7 @@ paths: tags: - query /jobs/{id}: - get: + post: consumes: - application/json description: |- @@ -528,7 +620,7 @@ paths: "200": description: Job resource schema: - $ref: '#/definitions/schema.JobMeta' + $ref: '#/definitions/api.GetJobApiResponse' "400": description: Bad Request schema: diff --git a/internal/api/docs.go b/internal/api/docs.go index 442993a..85acc92 100644 --- a/internal/api/docs.go +++ b/internal/api/docs.go @@ -630,7 +630,7 @@ const docTemplate = `{ } }, "/jobs/{id}": { - "get": { + "post": { "security": [ { "ApiKeyAuth": [] @@ -672,7 +672,7 @@ const docTemplate = `{ "200": { "description": "Job resource", "schema": { - "$ref": "#/definitions/schema.JobMeta" + "$ref": "#/definitions/api.GetJobApiResponse" } }, "400": { @@ -775,6 +775,20 @@ const docTemplate = `{ } } }, + "api.GetJobApiResponse": { + "type": "object", + "properties": { + "data": { + "type": "array", + "items": { + "$ref": "#/definitions/api.JobMetricWithName" + } + }, + "meta": { + "$ref": "#/definitions/schema.Job" + } + } + }, "api.GetJobsApiResponse": { "type": "object", "properties": { @@ -795,6 +809,20 @@ const docTemplate = `{ } } }, + "api.JobMetricWithName": { + "type": "object", + "properties": { + "metric": { + "$ref": "#/definitions/schema.JobMetric" + }, + "name": { + "type": "string" + }, + "scope": { + "$ref": "#/definitions/schema.MetricScope" + } + } + }, "api.StartJobApiResponse": { "type": "object", "properties": { @@ -1154,6 +1182,26 @@ const docTemplate = `{ } } }, + "schema.JobMetric": { + "type": "object", + "properties": { + "series": { + "type": "array", + "items": { + "$ref": "#/definitions/schema.Series" + } + }, + "statisticsSeries": { + "$ref": "#/definitions/schema.StatsSeries" + }, + "timestep": { + "type": "integer" + }, + "unit": { + "$ref": "#/definitions/schema.Unit" + } + } + }, "schema.JobState": { "type": "string", "enum": [ @@ -1204,6 +1252,41 @@ const docTemplate = `{ } } }, + "schema.MetricScope": { + "type": "string", + "enum": [ + "invalid_scope", + "node", + "socket", + "memoryDomain", + "core", + "hwthread", + "accelerator" + ], + "x-enum-varnames": [ + "MetricScopeInvalid", + "MetricScopeNode", + "MetricScopeSocket", + "MetricScopeMemoryDomain", + "MetricScopeCore", + "MetricScopeHWThread", + "MetricScopeAccelerator" + ] + }, + "schema.MetricStatistics": { + "type": "object", + "properties": { + "avg": { + "type": "number" + }, + "max": { + "type": "number" + }, + "min": { + "type": "number" + } + } + }, "schema.Resource": { "description": "A resource used by a job", "type": "object", @@ -1232,6 +1315,58 @@ const docTemplate = `{ } } }, + "schema.Series": { + "type": "object", + "properties": { + "data": { + "type": "array", + "items": { + "type": "number" + } + }, + "hostname": { + "type": "string" + }, + "id": { + "type": "string" + }, + "statistics": { + "$ref": "#/definitions/schema.MetricStatistics" + } + } + }, + "schema.StatsSeries": { + "type": "object", + "properties": { + "max": { + "type": "array", + "items": { + "type": "number" + } + }, + "mean": { + "type": "array", + "items": { + "type": "number" + } + }, + "min": { + "type": "array", + "items": { + "type": "number" + } + }, + "percentiles": { + "type": "object", + "additionalProperties": { + "type": "array", + "items": { + "type": "number" + } + } + } + } + }, "schema.Tag": { "description": "Defines a tag using name and type.", "type": "object", diff --git a/internal/api/rest.go b/internal/api/rest.go index 0a4d227..55ca14c 100644 --- a/internal/api/rest.go +++ b/internal/api/rest.go @@ -69,7 +69,7 @@ func (api *RestApi) MountRoutes(r *mux.Router) { // r.HandleFunc("/jobs/import/", api.importJob).Methods(http.MethodPost, http.MethodPut) r.HandleFunc("/jobs/", api.getJobs).Methods(http.MethodGet) - r.HandleFunc("/jobs/{id}", api.getJobById).Methods(http.MethodGet) + r.HandleFunc("/jobs/{id}", api.getJobById).Methods(http.MethodPost) r.HandleFunc("/jobs/tag_job/{id}", api.tagJob).Methods(http.MethodPost, http.MethodPatch) r.HandleFunc("/jobs/metrics/{id}", api.getJobMetrics).Methods(http.MethodGet) r.HandleFunc("/jobs/delete_job/", api.deleteJobByRequest).Methods(http.MethodDelete) @@ -147,7 +147,13 @@ type GetJobApiRequest []string type GetJobApiResponse struct { Meta *schema.Job - Data []*model.JobMetricWithName + Data []*JobMetricWithName +} + +type JobMetricWithName struct { + Name string `json:"name"` + Scope schema.MetricScope `json:"scope"` + Metric *schema.JobMetric `json:"metric"` } func handleError(err error, statusCode int, rw http.ResponseWriter) { @@ -317,8 +323,8 @@ func (api *RestApi) getJobs(rw http.ResponseWriter, r *http.Request) { // @accept json // @produce json // @param id path int true "Database ID of Job" -// @param request body api.GetJobApiRequest true "Array of metric names" -// @success 200 {object} schema.JobMeta "Job resource" +// @param request body api.GetJobApiRequest true "Array of metric names" +// @success 200 {object} api.GetJobApiResponse "Job resource" // @failure 400 {object} api.ErrorResponse "Bad Request" // @failure 401 {object} api.ErrorResponse "Unauthorized" // @failure 403 {object} api.ErrorResponse "Forbidden" @@ -326,7 +332,7 @@ func (api *RestApi) getJobs(rw http.ResponseWriter, r *http.Request) { // @failure 422 {object} api.ErrorResponse "Unprocessable Entity: finding job failed: sql: no rows in result set" // @failure 500 {object} api.ErrorResponse "Internal Server Error" // @security ApiKeyAuth -// @router /jobs/{id} [get] +// @router /jobs/{id} [post] func (api *RestApi) getJobById(rw http.ResponseWriter, r *http.Request) { if user := auth.GetUser(r.Context()); user != nil && !user.HasRole(auth.RoleApi) { handleError(fmt.Errorf("missing role: %v", @@ -356,7 +362,7 @@ func (api *RestApi) getJobById(rw http.ResponseWriter, r *http.Request) { } var metrics GetJobApiRequest - if err := decode(r.Body, &metrics); err != nil { + if err = decode(r.Body, &metrics); err != nil { http.Error(rw, err.Error(), http.StatusBadRequest) return } @@ -375,10 +381,10 @@ func (api *RestApi) getJobById(rw http.ResponseWriter, r *http.Request) { return } - res := []*model.JobMetricWithName{} + res := []*JobMetricWithName{} for name, md := range data { for scope, metric := range md { - res = append(res, &model.JobMetricWithName{ + res = append(res, &JobMetricWithName{ Name: name, Scope: scope, Metric: metric, @@ -386,7 +392,7 @@ func (api *RestApi) getJobById(rw http.ResponseWriter, r *http.Request) { } } - log.Debugf("/api/job/%d: get job %d", id, job.JobID) + log.Debugf("/api/job/%s: get job %d", id, job.JobID) rw.Header().Add("Content-Type", "application/json") bw := bufio.NewWriter(rw) defer bw.Flush()