ClusterCockpit/cc-metric-store
1
0
Fork 0
mirror of https://github.com/ClusterCockpit/cc-metric-store.git synced 2025-07-23 21:31:41 +02:00
Code Issues Packages Projects Releases Wiki Activity

Refactor. Add Swagger UI docs.

Browse Source 
Change from Gorilla mux to net/http
This commit is contained in:
Jan Eitzinger
2024-06-25 20:08:25 +02:00
parent 7538570bc5
commit 826658f762
13 changed files with 1639 additions and 172 deletions
Download Patch File Download Diff File Expand all files Collapse all files

206
internal/api/api.go
View File

@@ -1,11 +1,13 @@
// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package api
import (
"bufio"
"context"
"crypto/ed25519"
"encoding/base64"
"encoding/json"
"errors"
"fmt"
"io"
"log"
@@ -13,25 +15,57 @@ import (
"net/http"
"strconv"
"strings"
"time"
"github.com/ClusterCockpit/cc-metric-store/internal/config"
"github.com/ClusterCockpit/cc-metric-store/internal/memorystore"
"github.com/ClusterCockpit/cc-metric-store/internal/util"
"github.com/gorilla/mux"
"github.com/influxdata/line-protocol/v2/lineprotocol"
)
// @title cc-metric-store REST API
// @version 1.0.0
// @description API for cc-metric-store
// @contact.name ClusterCockpit Project
// @contact.url https://clustercockpit.org
// @contact.email support@clustercockpit.org
// @license.name MIT License
// @license.url https://opensource.org/licenses/MIT
// @host localhost:8082
// @basePath /api/
// @securityDefinitions.apikey ApiKeyAuth
// @in header
// @name X-Auth-Token
// ErrorResponse model
type ErrorResponse struct {
// Statustext of Errorcode
Status string `json:"status"`
Error string `json:"error"` // Error Message
}
type ApiMetricData struct {
Error *string `json:"error,omitempty"`
Error *string `json:"error,omitempty"`
Data util.FloatArray `json:"data,omitempty"`
From int64 `json:"from"`
To int64 `json:"to"`
From int64 `json:"from"`
To int64 `json:"to"`
Avg util.Float `json:"avg"`
Min util.Float `json:"min"`
Max util.Float `json:"max"`
}
func handleError(err error, statusCode int, rw http.ResponseWriter) {
// log.Warnf("REST ERROR : %s", err.Error())
rw.Header().Add("Content-Type", "application/json")
rw.WriteHeader(statusCode)
json.NewEncoder(rw).Encode(ErrorResponse{
Status: http.StatusText(statusCode),
Error: err.Error(),
})
}
// TODO: Optimize this, just like the stats endpoint!
func (data *ApiMetricData) AddStats() {
n := 0
@@ -89,16 +123,33 @@ func (data *ApiMetricData) PadDataWithNull(ms *memorystore.MemoryStore, from, to
}
}
// handleFree godoc
// @summary
// @tags free
// @description
// @produce json
// @param to query string false "up to timestamp"
// @success 200 {string} string "ok"
// @failure 400 {object} api.ErrorResponse "Bad Request"
// @failure 401 {object} api.ErrorResponse "Unauthorized"
// @failure 403 {object} api.ErrorResponse "Forbidden"
// @failure 500 {object} api.ErrorResponse "Internal Server Error"
// @security ApiKeyAuth
// @router /free/ [get]
func handleFree(rw http.ResponseWriter, r *http.Request) {
if err := isAuthenticated(r); err != nil {
handleError(err, http.StatusUnauthorized, rw)
return
}
rawTo := r.URL.Query().Get("to")
if rawTo == "" {
http.Error(rw, "'to' is a required query parameter", http.StatusBadRequest)
handleError(errors.New("'to' is a required query parameter"), http.StatusBadRequest, rw)
return
}
to, err := strconv.ParseInt(rawTo, 10, 64)
if err != nil {
http.Error(rw, err.Error(), http.StatusBadRequest)
handleError(err, http.StatusInternalServerError, rw)
return
}
@@ -109,11 +160,6 @@ func handleFree(rw http.ResponseWriter, r *http.Request) {
// freeUpTo = to
// }
if r.Method != http.MethodPost {
http.Error(rw, "Method Not Allowed", http.StatusMethodNotAllowed)
return
}
bodyDec := json.NewDecoder(r.Body)
var selectors [][]string
err = bodyDec.Decode(&selectors)
@@ -127,7 +173,7 @@ func handleFree(rw http.ResponseWriter, r *http.Request) {
for _, sel := range selectors {
bn, err := ms.Free(sel, to)
if err != nil {
http.Error(rw, err.Error(), http.StatusInternalServerError)
handleError(err, http.StatusInternalServerError, rw)
return
}
@@ -138,16 +184,30 @@ func handleFree(rw http.ResponseWriter, r *http.Request) {
fmt.Fprintf(rw, "buffers freed: %d\n", n)
}
// handleWrite godoc
// @summary Receive metrics in line-protocol
// @tags write
// @description Receives metrics in the influx line-protocol using [this format](https://github.com/ClusterCockpit/cc-specifications/blob/master/metrics/lineprotocol_alternative.md)
// @accept plain
// @produce json
// @param cluster query string false "If the lines in the body do not have a cluster tag, use this value instead."
// @success 200 {string} string "ok"
// @failure 400 {object} api.ErrorResponse "Bad Request"
// @failure 401 {object} api.ErrorResponse "Unauthorized"
// @failure 403 {object} api.ErrorResponse "Forbidden"
// @failure 500 {object} api.ErrorResponse "Internal Server Error"
// @security ApiKeyAuth
// @router /write/ [post]
func handleWrite(rw http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost {
http.Error(rw, "Method Not Allowed", http.StatusMethodNotAllowed)
if err := isAuthenticated(r); err != nil {
handleError(err, http.StatusUnauthorized, rw)
return
}
bytes, err := io.ReadAll(r.Body)
rw.Header().Add("Content-Type", "application/json")
if err != nil {
log.Printf("error while reading request body: %s", err.Error())
http.Error(rw, err.Error(), http.StatusInternalServerError)
handleError(err, http.StatusInternalServerError, rw)
return
}
@@ -155,7 +215,7 @@ func handleWrite(rw http.ResponseWriter, r *http.Request) {
dec := lineprotocol.NewDecoderWithBytes(bytes)
if err := decodeLine(dec, ms, r.URL.Query().Get("cluster")); err != nil {
log.Printf("/api/write error: %s", err.Error())
http.Error(rw, err.Error(), http.StatusBadRequest)
handleError(err, http.StatusBadRequest, rw)
return
}
rw.WriteHeader(http.StatusOK)
@@ -180,19 +240,37 @@ type ApiQueryResponse struct {
type ApiQuery struct {
Type *string `json:"type,omitempty"`
SubType *string `json:"subtype,omitempty"`
Metric string `json:"metric"`
Hostname string `json:"host"`
TypeIds []string `json:"type-ids,omitempty"`
SubTypeIds []string `json:"subtype-ids,omitempty"`
Metric string `json:"metric"`
Hostname string `json:"host"`
TypeIds []string `json:"type-ids,omitempty"`
SubTypeIds []string `json:"subtype-ids,omitempty"`
ScaleFactor util.Float `json:"scale-by,omitempty"`
Aggregate bool `json:"aggreg"`
}
// handleQuery godoc
// @summary Query metrics
// @tags query
// @description Query metrics.
// @accept json
// @produce json
// @param request body api.ApiQueryRequest true "API query payload object"
// @success 200 {object} api.ApiQueryResponse "API query response object"
// @failure 400 {object} api.ErrorResponse "Bad Request"
// @failure 401 {object} api.ErrorResponse "Unauthorized"
// @failure 403 {object} api.ErrorResponse "Forbidden"
// @failure 500 {object} api.ErrorResponse "Internal Server Error"
// @security ApiKeyAuth
// @router /query/ [get]
func handleQuery(rw http.ResponseWriter, r *http.Request) {
if err := isAuthenticated(r); err != nil {
handleError(err, http.StatusUnauthorized, rw)
return
}
var err error
req := ApiQueryRequest{WithStats: true, WithData: true, WithPadding: true}
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
http.Error(rw, err.Error(), http.StatusBadRequest)
handleError(err, http.StatusBadRequest, rw)
return
}
@@ -305,64 +383,34 @@ func handleQuery(rw http.ResponseWriter, r *http.Request) {
}
}
// handleDebug godoc
// @summary Debug endpoint
// @tags write
// @description Write metrics to store
// @produce json
// @param selector query string false "Job Cluster"
// @success 200 {string} string "Debug dump"
// @failure 400 {object} api.ErrorResponse "Bad Request"
// @failure 401 {object} api.ErrorResponse "Unauthorized"
// @failure 403 {object} api.ErrorResponse "Forbidden"
// @failure 500 {object} api.ErrorResponse "Internal Server Error"
// @security ApiKeyAuth
// @router /debug/ [post]
func handleDebug(rw http.ResponseWriter, r *http.Request) {
if err := isAuthenticated(r); err != nil {
handleError(err, http.StatusUnauthorized, rw)
return
}
raw := r.URL.Query().Get("selector")
rw.Header().Add("Content-Type", "application/json")
selector := []string{}
if len(raw) != 0 {
selector = strings.Split(raw, ":")
}
}
ms := memorystore.GetMemoryStore()
if err := ms.DebugDump(bufio.NewWriter(rw), selector); err != nil {
rw.WriteHeader(http.StatusBadRequest)
rw.Write([]byte(err.Error()))
}
}
func StartApiServer(ctx context.Context, httpConfig *config.HttpConfig) error {
r := mux.NewRouter()
r.HandleFunc("/api/free", handleFree)
r.HandleFunc("/api/write", handleWrite)
r.HandleFunc("/api/query", handleQuery)
r.HandleFunc("/api/debug", handleDebug)
server := &http.Server{
Handler: r,
Addr: httpConfig.Address,
WriteTimeout: 30 * time.Second,
ReadTimeout: 30 * time.Second,
}
if len(config.Keys.JwtPublicKey) > 0 {
buf, err := base64.StdEncoding.DecodeString(config.Keys.JwtPublicKey)
if err != nil {
return err
}
publicKey := ed25519.PublicKey(buf)
server.Handler = authentication(server.Handler, publicKey)
}
go func() {
if httpConfig.CertFile != "" && httpConfig.KeyFile != "" {
log.Printf("API https endpoint listening on '%s'\n", httpConfig.Address)
err := server.ListenAndServeTLS(httpConfig.CertFile, httpConfig.KeyFile)
if err != nil && err != http.ErrServerClosed {
log.Println(err)
}
} else {
log.Printf("API http endpoint listening on '%s'\n", httpConfig.Address)
err := server.ListenAndServe()
if err != nil && err != http.ErrServerClosed {
log.Println(err)
}
}
}()
for {
<-ctx.Done()
err := server.Shutdown(context.Background())
log.Println("API server shut down")
return err
handleError(err, http.StatusBadRequest, rw)
return
}
}

70
internal/api/authentication.go
View File

@@ -10,47 +10,43 @@ import (
"github.com/golang-jwt/jwt/v4"
)
func authentication(next http.Handler, publicKey ed25519.PublicKey) http.Handler {
var publicKey ed25519.PublicKey
func isAuthenticated(r *http.Request) error {
cacheLock := sync.RWMutex{}
cache := map[string]*jwt.Token{}
return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
authheader := r.Header.Get("Authorization")
if authheader == "" || !strings.HasPrefix(authheader, "Bearer ") {
http.Error(rw, "Use JWT Authentication", http.StatusUnauthorized)
return
authheader := r.Header.Get("Authorization")
if authheader == "" || !strings.HasPrefix(authheader, "Bearer ") {
return errors.New("Use JWT Authentication")
}
rawtoken := authheader[len("Bearer "):]
cacheLock.RLock()
token, ok := cache[rawtoken]
cacheLock.RUnlock()
if ok && token.Claims.Valid() == nil {
return nil
}
// The actual token is ignored for now.
// In case expiration and so on are specified, the Parse function
// already returns an error for expired tokens.
var err error
token, err = jwt.Parse(rawtoken, func(t *jwt.Token) (interface{}, error) {
if t.Method != jwt.SigningMethodEdDSA {
return nil, errors.New("only Ed25519/EdDSA supported")
}
rawtoken := authheader[len("Bearer "):]
cacheLock.RLock()
token, ok := cache[rawtoken]
cacheLock.RUnlock()
if ok && token.Claims.Valid() == nil {
next.ServeHTTP(rw, r)
return
}
// The actual token is ignored for now.
// In case expiration and so on are specified, the Parse function
// already returns an error for expired tokens.
var err error
token, err = jwt.Parse(rawtoken, func(t *jwt.Token) (interface{}, error) {
if t.Method != jwt.SigningMethodEdDSA {
return nil, errors.New("only Ed25519/EdDSA supported")
}
return publicKey, nil
})
if err != nil {
http.Error(rw, err.Error(), http.StatusUnauthorized)
return
}
cacheLock.Lock()
cache[rawtoken] = token
cacheLock.Unlock()
// Let request through...
next.ServeHTTP(rw, r)
return publicKey, nil
})
if err != nil {
return err
}
cacheLock.Lock()
cache[rawtoken] = token
cacheLock.Unlock()
return nil
}

416
internal/api/docs.go Normal file
View File

@@ -0,0 +1,416 @@
// Package api Code generated by swaggo/swag. DO NOT EDIT
package api
import "github.com/swaggo/swag"
const docTemplate = `{
"schemes": {{ marshal .Schemes }},
"swagger": "2.0",
"info": {
"description": "{{escape .Description}}",
"title": "{{.Title}}",
"contact": {
"name": "ClusterCockpit Project",
"url": "https://clustercockpit.org",
"email": "support@clustercockpit.org"
},
"license": {
"name": "MIT License",
"url": "https://opensource.org/licenses/MIT"
},
"version": "{{.Version}}"
},
"host": "{{.Host}}",
"basePath": "{{.BasePath}}",
"paths": {
"/clusters/": {
"get": {
"security": [
{
"ApiKeyAuth": []
}
],
"produces": [
"application/json"
],
"tags": [
"free"
],
"parameters": [
{
"type": "string",
"description": "up to timestamp",
"name": "to",
"in": "query"
}
],
"responses": {
"200": {
"description": "ok",
"schema": {
"type": "string"
}
},
"400": {
"description": "Bad Request",
"schema": {
"$ref": "#/definitions/api.ErrorResponse"
}
},
"401": {
"description": "Unauthorized",
"schema": {
"$ref": "#/definitions/api.ErrorResponse"
}
},
"403": {
"description": "Forbidden",
"schema": {
"$ref": "#/definitions/api.ErrorResponse"
}
},
"500": {
"description": "Internal Server Error",
"schema": {
"$ref": "#/definitions/api.ErrorResponse"
}
}
}
}
},
"/debug/": {
"post": {
"security": [
{
"ApiKeyAuth": []
}
],
"description": "Write metrics to store",
"produces": [
"application/json"
],
"tags": [
"write"
],
"summary": "Debug endpoint",
"parameters": [
{
"type": "string",
"description": "Job Cluster",
"name": "selector",
"in": "query"
}
],
"responses": {
"200": {
"description": "Debug dump",
"schema": {
"type": "string"
}
},
"400": {
"description": "Bad Request",
"schema": {
"$ref": "#/definitions/api.ErrorResponse"
}
},
"401": {
"description": "Unauthorized",
"schema": {
"$ref": "#/definitions/api.ErrorResponse"
}
},
"403": {
"description": "Forbidden",
"schema": {
"$ref": "#/definitions/api.ErrorResponse"
}
},
"500": {
"description": "Internal Server Error",
"schema": {
"$ref": "#/definitions/api.ErrorResponse"
}
}
}
}
},
"/query/": {
"get": {
"security": [
{
"ApiKeyAuth": []
}
],
"description": "Query metrics.",
"consumes": [
"application/json"
],
"produces": [
"application/json"
],
"tags": [
"query"
],
"summary": "Query metrics",
"parameters": [
{
"description": "API query payload object",
"name": "request",
"in": "body",
"required": true,
"schema": {
"$ref": "#/definitions/api.ApiQueryRequest"
}
}
],
"responses": {
"200": {
"description": "API query response object",
"schema": {
"$ref": "#/definitions/api.ApiQueryResponse"
}
},
"400": {
"description": "Bad Request",
"schema": {
"$ref": "#/definitions/api.ErrorResponse"
}
},
"401": {
"description": "Unauthorized",
"schema": {
"$ref": "#/definitions/api.ErrorResponse"
}
},
"403": {
"description": "Forbidden",
"schema": {
"$ref": "#/definitions/api.ErrorResponse"
}
},
"500": {
"description": "Internal Server Error",
"schema": {
"$ref": "#/definitions/api.ErrorResponse"
}
}
}
}
},
"/write/": {
"post": {
"security": [
{
"ApiKeyAuth": []
}
],
"consumes": [
"text/plain"
],
"produces": [
"application/json"
],
"parameters": [
{
"type": "string",
"description": "If the lines in the body do not have a cluster tag, use this value instead.",
"name": "cluster",
"in": "query"
}
],
"responses": {
"200": {
"description": "ok",
"schema": {
"type": "string"
}
},
"400": {
"description": "Bad Request",
"schema": {
"$ref": "#/definitions/api.ErrorResponse"
}
},
"401": {
"description": "Unauthorized",
"schema": {
"$ref": "#/definitions/api.ErrorResponse"
}
},
"403": {
"description": "Forbidden",
"schema": {
"$ref": "#/definitions/api.ErrorResponse"
}
},
"500": {
"description": "Internal Server Error",
"schema": {
"$ref": "#/definitions/api.ErrorResponse"
}
}
}
}
}
},
"definitions": {
"api.ApiMetricData": {
"type": "object",
"properties": {
"avg": {
"type": "number"
},
"data": {
"type": "array",
"items": {
"type": "number"
}
},
"error": {
"type": "string"
},
"from": {
"type": "integer"
},
"max": {
"type": "number"
},
"min": {
"type": "number"
},
"to": {
"type": "integer"
}
}
},
"api.ApiQuery": {
"type": "object",
"properties": {
"aggreg": {
"type": "boolean"
},
"host": {
"type": "string"
},
"metric": {
"type": "string"
},
"scale-by": {
"type": "number"
},
"subtype": {
"type": "string"
},
"subtype-ids": {
"type": "array",
"items": {
"type": "string"
}
},
"type": {
"type": "string"
},
"type-ids": {
"type": "array",
"items": {
"type": "string"
}
}
}
},
"api.ApiQueryRequest": {
"type": "object",
"properties": {
"cluster": {
"type": "string"
},
"for-all-nodes": {
"type": "array",
"items": {
"type": "string"
}
},
"from": {
"type": "integer"
},
"queries": {
"type": "array",
"items": {
"$ref": "#/definitions/api.ApiQuery"
}
},
"to": {
"type": "integer"
},
"with-data": {
"type": "boolean"
},
"with-padding": {
"type": "boolean"
},
"with-stats": {
"type": "boolean"
}
}
},
"api.ApiQueryResponse": {
"type": "object",
"properties": {
"queries": {
"type": "array",
"items": {
"$ref": "#/definitions/api.ApiQuery"
}
},
"results": {
"type": "array",
"items": {
"type": "array",
"items": {
"$ref": "#/definitions/api.ApiMetricData"
}
}
}
}
},
"api.ErrorResponse": {
"type": "object",
"properties": {
"error": {
"description": "Error Message",
"type": "string"
},
"status": {
"description": "Statustext of Errorcode",
"type": "string"
}
}
}
},
"securityDefinitions": {
"ApiKeyAuth": {
"type": "apiKey",
"name": "X-Auth-Token",
"in": "header"
}
}
}`
// SwaggerInfo holds exported Swagger Info so clients can modify it
var SwaggerInfo = &swag.Spec{
Version: "1.0.0",
Host: "localhost:8082",
BasePath: "/api/",
Schemes: []string{},
Title: "cc-metric-store REST API",
Description: "API for cc-metric-store",
InfoInstanceName: "swagger",
SwaggerTemplate: docTemplate,
LeftDelim: "{{",
RightDelim: "}}",
}
func init() {
swag.Register(SwaggerInfo.InstanceName(), SwaggerInfo)
}

29
internal/api/server.go Normal file
View File

@@ -0,0 +1,29 @@
// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package api
import (
"crypto/ed25519"
"encoding/base64"
"log"
"net/http"
"github.com/ClusterCockpit/cc-metric-store/internal/config"
)
func MountRoutes(r *http.ServeMux) {
if len(config.Keys.JwtPublicKey) > 0 {
buf, err := base64.StdEncoding.DecodeString(config.Keys.JwtPublicKey)
if err != nil {
log.Fatalf("starting server failed: %v", err)
}
publicKey = ed25519.PublicKey(buf)
}
r.HandleFunc("POST /api/free/", handleFree)
r.HandleFunc("POST /api/write/", handleWrite)
r.HandleFunc("GET /api/query/", handleQuery)
r.HandleFunc("GET /api/debug/", handleDebug)
}