Use new simpler cc-metric-store API

This commit is contained in:
Lou Knauer 2022-01-20 10:43:46 +01:00
parent c254c689af
commit 6743d94b0e

View File

@ -5,10 +5,8 @@ import (
"bytes" "bytes"
"context" "context"
"encoding/json" "encoding/json"
"errors"
"fmt" "fmt"
"net/http" "net/http"
"strconv"
"time" "time"
"github.com/ClusterCockpit/cc-jobarchive/config" "github.com/ClusterCockpit/cc-jobarchive/config"
@ -16,23 +14,30 @@ import (
) )
type CCMetricStore struct { type CCMetricStore struct {
jwt string jwt string
url string url string
client http.Client queryEndpoint string
client http.Client
} }
type ApiRequestBody struct { type ApiQueryRequest struct {
Metrics []string `json:"metrics"` Cluster string `json:"cluster"`
Selectors [][]string `json:"selectors"` From int64 `json:"from"`
To int64 `json:"to"`
WithStats bool `json:"with-stats"`
WithData bool `json:"with-data"`
Queries []ApiQuery `json:"queries"`
ForAllNodes []string `json:"for-all-nodes"`
} }
type ApiQuery struct { type ApiQuery struct {
Metric string `json:"metric"` Metric string `json:"metric"`
Hostname string `json:"hostname"` Hostname string `json:"host"`
Type *string `json:"type,omitempty"` Aggregate bool `json:"aggreg"`
TypeIds []string `json:"type-ids,omitempty"` Type *string `json:"type,omitempty"`
SubType *string `json:"subtype,omitempty"` TypeIds []int `json:"type-ids,omitempty"`
SubTypeIds []string `json:"subtype-ids,omitempty"` SubType *string `json:"subtype,omitempty"`
SubTypeIds []int `json:"subtype-ids,omitempty"`
} }
type ApiMetricData struct { type ApiMetricData struct {
@ -45,18 +50,9 @@ type ApiMetricData struct {
Max schema.Float `json:"max"` Max schema.Float `json:"max"`
} }
type ApiStatsData struct {
Error *string `json:"error"`
From int64 `json:"from"`
To int64 `json:"to"`
Samples int `json:"samples"`
Avg schema.Float `json:"avg"`
Min schema.Float `json:"min"`
Max schema.Float `json:"max"`
}
func (ccms *CCMetricStore) Init(url, token string) error { func (ccms *CCMetricStore) Init(url, token string) error {
ccms.url = url ccms.url = url
ccms.queryEndpoint = fmt.Sprintf("%s/api/query", url)
ccms.jwt = token ccms.jwt = token
ccms.client = http.Client{ ccms.client = http.Client{
Timeout: 5 * time.Second, Timeout: 5 * time.Second,
@ -64,100 +60,67 @@ func (ccms *CCMetricStore) Init(url, token string) error {
return nil return nil
} }
func (ccms *CCMetricStore) doRequest(job *schema.Job, suffix string, metrics []string, ctx context.Context) (*http.Response, error) { func (ccms *CCMetricStore) doRequest(ctx context.Context, body *ApiQueryRequest) ([][]ApiMetricData, error) {
from, to := job.StartTime.Unix(), job.StartTime.Add(time.Duration(job.Duration)*time.Second).Unix() buf := &bytes.Buffer{}
reqBody := ApiRequestBody{} if err := json.NewEncoder(buf).Encode(body); err != nil {
reqBody.Metrics = metrics
for _, node := range job.Resources {
if node.Accelerators != nil || node.HWThreads != nil {
// TODO/FIXME:
return nil, errors.New("todo: cc-metric-store resources: Accelerator/HWThreads")
}
reqBody.Selectors = append(reqBody.Selectors, []string{job.Cluster, node.Hostname})
}
reqBodyBytes, err := json.Marshal(reqBody)
if err != nil {
return nil, err return nil, err
} }
req, err := http.NewRequestWithContext(ctx, http.MethodPost, fmt.Sprintf("%s/api/%d/%d/%s", ccms.url, from, to, suffix), bytes.NewReader(reqBodyBytes)) req, err := http.NewRequestWithContext(ctx, http.MethodPost, ccms.queryEndpoint, buf)
if err != nil { if err != nil {
return nil, err return nil, err
} }
if ccms.jwt != "" { if ccms.jwt != "" {
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", ccms.jwt)) req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", ccms.jwt))
} }
return ccms.client.Do(req)
res, err := ccms.client.Do(req)
if err != nil {
return nil, err
}
if res.StatusCode != http.StatusOK {
return nil, fmt.Errorf("'%s': HTTP Status: %s", ccms.queryEndpoint, res.Status)
}
var resBody [][]ApiMetricData
if err := json.NewDecoder(bufio.NewReader(res.Body)).Decode(&resBody); err != nil {
return nil, err
}
return resBody, nil
} }
func (ccms *CCMetricStore) LoadData(job *schema.Job, metrics []string, scopes []schema.MetricScope, ctx context.Context) (schema.JobData, error) { func (ccms *CCMetricStore) LoadData(job *schema.Job, metrics []string, scopes []schema.MetricScope, ctx context.Context) (schema.JobData, error) {
type ApiQueryRequest struct {
Cluster string `json:"cluster"`
From int64 `json:"from"`
To int64 `json:"to"`
Queries []ApiQuery `json:"queries"`
}
type ApiQueryResponse struct {
ApiMetricData
Query *ApiQuery `json:"query"`
}
queries, assignedScope, err := ccms.buildQueries(job, metrics, scopes) queries, assignedScope, err := ccms.buildQueries(job, metrics, scopes)
if err != nil { if err != nil {
return nil, err return nil, err
} }
reqBody := ApiQueryRequest{ req := ApiQueryRequest{
Cluster: job.Cluster, Cluster: job.Cluster,
From: job.StartTime.Unix(), From: job.StartTime.Unix(),
To: job.StartTime.Add(time.Duration(job.Duration) * time.Second).Unix(), To: job.StartTime.Add(time.Duration(job.Duration) * time.Second).Unix(),
Queries: queries, Queries: queries,
WithStats: true,
WithData: true,
} }
buf := &bytes.Buffer{} resBody, err := ccms.doRequest(ctx, &req)
if err := json.NewEncoder(buf).Encode(reqBody); err != nil {
return nil, err
}
req, err := http.NewRequestWithContext(ctx, http.MethodPost, ccms.url+"/api/query", buf)
if err != nil { if err != nil {
return nil, err return nil, err
} }
if ccms.jwt != "" {
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", ccms.jwt))
}
res, err := ccms.client.Do(req)
if err != nil {
return nil, err
}
if res.StatusCode != http.StatusOK {
return nil, fmt.Errorf("cc-metric-store replied with: %s", res.Status)
}
var resBody []ApiQueryResponse
if err := json.NewDecoder(bufio.NewReader(res.Body)).Decode(&resBody); err != nil {
return nil, err
}
// log.Printf("response: %#v", resBody)
var jobData schema.JobData = make(schema.JobData) var jobData schema.JobData = make(schema.JobData)
for i, res := range resBody { for i, row := range resBody {
metric := res.Query.Metric query := req.Queries[i]
if _, ok := jobData[metric]; !ok {
jobData[metric] = make(map[schema.MetricScope]*schema.JobMetric)
}
if res.Error != nil {
return nil, fmt.Errorf("cc-metric-store error while fetching %s: %s", metric, *res.Error)
}
scope := assignedScope[i] scope := assignedScope[i]
mc := config.GetMetricConfig(job.Cluster, metric) mc := config.GetMetricConfig(job.Cluster, query.Metric)
jobMetric, ok := jobData[metric][scope] if _, ok := jobData[query.Metric]; !ok {
jobData[query.Metric] = make(map[schema.MetricScope]*schema.JobMetric)
}
jobMetric, ok := jobData[query.Metric][scope]
if !ok { if !ok {
jobMetric = &schema.JobMetric{ jobMetric = &schema.JobMetric{
Unit: mc.Unit, Unit: mc.Unit,
@ -165,41 +128,47 @@ func (ccms *CCMetricStore) LoadData(job *schema.Job, metrics []string, scopes []
Timestep: mc.Timestep, Timestep: mc.Timestep,
Series: make([]schema.Series, 0), Series: make([]schema.Series, 0),
} }
jobData[metric][scope] = jobMetric jobData[query.Metric][scope] = jobMetric
} }
id := (*int)(nil) for _, res := range row {
if res.Query.Type != nil { if res.Error != nil {
id = new(int) return nil, fmt.Errorf("cc-metric-store error while fetching %s: %s", query.Metric, *res.Error)
*id, _ = strconv.Atoi(res.Query.TypeIds[0]) }
}
if res.Avg.IsNaN() || res.Min.IsNaN() || res.Max.IsNaN() { id := (*int)(nil)
// TODO: use schema.Float instead of float64? if query.Type != nil {
// This is done because regular float64 can not be JSONed when NaN. id = new(int)
res.Avg = schema.Float(0) *id = query.TypeIds[0]
res.Min = schema.Float(0) }
res.Max = schema.Float(0)
}
jobMetric.Series = append(jobMetric.Series, schema.Series{ if res.Avg.IsNaN() || res.Min.IsNaN() || res.Max.IsNaN() {
Hostname: res.Query.Hostname, // TODO: use schema.Float instead of float64?
Id: id, // This is done because regular float64 can not be JSONed when NaN.
Statistics: &schema.MetricStatistics{ res.Avg = schema.Float(0)
Avg: float64(res.Avg), res.Min = schema.Float(0)
Min: float64(res.Min), res.Max = schema.Float(0)
Max: float64(res.Max), }
},
Data: res.Data, jobMetric.Series = append(jobMetric.Series, schema.Series{
}) Hostname: query.Hostname,
Id: id,
Statistics: &schema.MetricStatistics{
Avg: float64(res.Avg),
Min: float64(res.Min),
Max: float64(res.Max),
},
Data: res.Data,
})
}
} }
return jobData, nil return jobData, nil
} }
var ( var (
hwthreadString = string("cpu") // TODO/FIXME: inconsistency between cc-metric-collector and ClusterCockpit hwthreadString = string("cpu") // TODO/FIXME: inconsistency between cc-metric-collector and ClusterCockpit
// coreString = string(schema.MetricScopeCore) coreString = string(schema.MetricScopeCore)
socketString = string(schema.MetricScopeSocket) socketString = string(schema.MetricScopeSocket)
acceleratorString = string(schema.MetricScopeAccelerator) acceleratorString = string(schema.MetricScopeAccelerator)
) )
@ -239,15 +208,14 @@ func (ccms *CCMetricStore) buildQueries(job *schema.Job, metrics []string, scope
// Accelerator -> Accelerator (Use "accelerator" scope if requested scope is lower than node) // Accelerator -> Accelerator (Use "accelerator" scope if requested scope is lower than node)
if nativeScope == schema.MetricScopeAccelerator && scope.LT(schema.MetricScopeNode) { if nativeScope == schema.MetricScopeAccelerator && scope.LT(schema.MetricScopeNode) {
for _, accel := range host.Accelerators { queries = append(queries, ApiQuery{
queries = append(queries, ApiQuery{ Metric: metric,
Metric: metric, Hostname: host.Hostname,
Hostname: host.Hostname, Aggregate: false,
Type: &acceleratorString, Type: &acceleratorString,
TypeIds: []string{strconv.Itoa(accel)}, TypeIds: host.Accelerators,
}) })
assignedScope = append(assignedScope, schema.MetricScopeAccelerator) assignedScope = append(assignedScope, schema.MetricScopeAccelerator)
}
continue continue
} }
@ -258,26 +226,26 @@ func (ccms *CCMetricStore) buildQueries(job *schema.Job, metrics []string, scope
} }
queries = append(queries, ApiQuery{ queries = append(queries, ApiQuery{
Metric: metric, Metric: metric,
Hostname: host.Hostname, Hostname: host.Hostname,
Type: &acceleratorString, Aggregate: true,
TypeIds: toStringSlice(host.Accelerators), Type: &acceleratorString,
TypeIds: host.Accelerators,
}) })
assignedScope = append(assignedScope, schema.MetricScopeNode) assignedScope = append(assignedScope, scope)
continue continue
} }
// HWThread -> HWThead // HWThread -> HWThead
if nativeScope == schema.MetricScopeHWThread && scope == schema.MetricScopeHWThread { if nativeScope == schema.MetricScopeHWThread && scope == schema.MetricScopeHWThread {
for _, hwthread := range hwthreads { queries = append(queries, ApiQuery{
queries = append(queries, ApiQuery{ Metric: metric,
Metric: metric, Hostname: host.Hostname,
Hostname: host.Hostname, Aggregate: false,
Type: &hwthreadString, Type: &hwthreadString,
TypeIds: []string{strconv.Itoa(hwthread)}, TypeIds: hwthreads,
}) })
assignedScope = append(assignedScope, schema.MetricScopeHWThread) assignedScope = append(assignedScope, scope)
}
continue continue
} }
@ -286,12 +254,13 @@ func (ccms *CCMetricStore) buildQueries(job *schema.Job, metrics []string, scope
cores, _ := topology.GetCoresFromHWThreads(hwthreads) cores, _ := topology.GetCoresFromHWThreads(hwthreads)
for _, core := range cores { for _, core := range cores {
queries = append(queries, ApiQuery{ queries = append(queries, ApiQuery{
Metric: metric, Metric: metric,
Hostname: host.Hostname, Hostname: host.Hostname,
Type: &hwthreadString, Aggregate: true,
TypeIds: toStringSlice(topology.Core[core]), Type: &hwthreadString,
TypeIds: topology.Core[core],
}) })
assignedScope = append(assignedScope, schema.MetricScopeCore) assignedScope = append(assignedScope, scope)
} }
continue continue
} }
@ -301,12 +270,13 @@ func (ccms *CCMetricStore) buildQueries(job *schema.Job, metrics []string, scope
sockets, _ := topology.GetSocketsFromHWThreads(hwthreads) sockets, _ := topology.GetSocketsFromHWThreads(hwthreads)
for _, socket := range sockets { for _, socket := range sockets {
queries = append(queries, ApiQuery{ queries = append(queries, ApiQuery{
Metric: metric, Metric: metric,
Hostname: host.Hostname, Hostname: host.Hostname,
Type: &hwthreadString, Aggregate: true,
TypeIds: toStringSlice(topology.Socket[socket]), Type: &hwthreadString,
TypeIds: topology.Socket[socket],
}) })
assignedScope = append(assignedScope, schema.MetricScopeSocket) assignedScope = append(assignedScope, scope)
} }
continue continue
} }
@ -314,27 +284,55 @@ func (ccms *CCMetricStore) buildQueries(job *schema.Job, metrics []string, scope
// HWThread -> Node // HWThread -> Node
if nativeScope == schema.MetricScopeHWThread && scope == schema.MetricScopeNode { if nativeScope == schema.MetricScopeHWThread && scope == schema.MetricScopeNode {
queries = append(queries, ApiQuery{ queries = append(queries, ApiQuery{
Metric: metric, Metric: metric,
Hostname: host.Hostname, Hostname: host.Hostname,
Type: &hwthreadString, Aggregate: true,
TypeIds: toStringSlice(hwthreads), Type: &hwthreadString,
TypeIds: hwthreads,
}) })
assignedScope = append(assignedScope, schema.MetricScopeNode) assignedScope = append(assignedScope, scope)
continue
}
// Core -> Core
if nativeScope == schema.MetricScopeCore && scope == schema.MetricScopeCore {
cores, _ := topology.GetCoresFromHWThreads(hwthreads)
queries = append(queries, ApiQuery{
Metric: metric,
Hostname: host.Hostname,
Aggregate: false,
Type: &coreString,
TypeIds: cores,
})
assignedScope = append(assignedScope, scope)
continue
}
// Core -> Node
if nativeScope == schema.MetricScopeCore && scope == schema.MetricScopeNode {
cores, _ := topology.GetCoresFromHWThreads(hwthreads)
queries = append(queries, ApiQuery{
Metric: metric,
Hostname: host.Hostname,
Aggregate: true,
Type: &coreString,
TypeIds: cores,
})
assignedScope = append(assignedScope, scope)
continue continue
} }
// Socket -> Socket // Socket -> Socket
if nativeScope == schema.MetricScopeSocket && scope == schema.MetricScopeSocket { if nativeScope == schema.MetricScopeSocket && scope == schema.MetricScopeSocket {
sockets, _ := topology.GetSocketsFromHWThreads(hwthreads) sockets, _ := topology.GetSocketsFromHWThreads(hwthreads)
for _, socket := range sockets { queries = append(queries, ApiQuery{
queries = append(queries, ApiQuery{ Metric: metric,
Metric: metric, Hostname: host.Hostname,
Hostname: host.Hostname, Aggregate: false,
Type: &acceleratorString, Type: &socketString,
TypeIds: []string{strconv.Itoa(socket)}, TypeIds: sockets,
}) })
assignedScope = append(assignedScope, schema.MetricScopeSocket) assignedScope = append(assignedScope, scope)
}
continue continue
} }
@ -342,12 +340,13 @@ func (ccms *CCMetricStore) buildQueries(job *schema.Job, metrics []string, scope
if nativeScope == schema.MetricScopeSocket && scope == schema.MetricScopeNode { if nativeScope == schema.MetricScopeSocket && scope == schema.MetricScopeNode {
sockets, _ := topology.GetSocketsFromHWThreads(hwthreads) sockets, _ := topology.GetSocketsFromHWThreads(hwthreads)
queries = append(queries, ApiQuery{ queries = append(queries, ApiQuery{
Metric: metric, Metric: metric,
Hostname: host.Hostname, Hostname: host.Hostname,
Type: &socketString, Aggregate: true,
TypeIds: toStringSlice(sockets), Type: &socketString,
TypeIds: sockets,
}) })
assignedScope = append(assignedScope, schema.MetricScopeNode) assignedScope = append(assignedScope, scope)
continue continue
} }
@ -357,7 +356,7 @@ func (ccms *CCMetricStore) buildQueries(job *schema.Job, metrics []string, scope
Metric: metric, Metric: metric,
Hostname: host.Hostname, Hostname: host.Hostname,
}) })
assignedScope = append(assignedScope, schema.MetricScopeNode) assignedScope = append(assignedScope, scope)
continue continue
} }
@ -369,121 +368,96 @@ func (ccms *CCMetricStore) buildQueries(job *schema.Job, metrics []string, scope
return queries, assignedScope, nil return queries, assignedScope, nil
} }
func toStringSlice(s []int) []string {
ret := make([]string, len(s))
for i, val := range s {
ret[i] = strconv.Itoa(val)
}
return ret
}
func (ccms *CCMetricStore) LoadStats(job *schema.Job, metrics []string, ctx context.Context) (map[string]map[string]schema.MetricStatistics, error) { func (ccms *CCMetricStore) LoadStats(job *schema.Job, metrics []string, ctx context.Context) (map[string]map[string]schema.MetricStatistics, error) {
res, err := ccms.doRequest(job, "stats", metrics, ctx) queries, _, err := ccms.buildQueries(job, metrics, []schema.MetricScope{schema.MetricScopeNode})
if err != nil { if err != nil {
return nil, err return nil, err
} }
resdata := make([]map[string]ApiStatsData, 0, len(job.Resources)) req := ApiQueryRequest{
if err := json.NewDecoder(res.Body).Decode(&resdata); err != nil { Cluster: job.Cluster,
From: job.StartTime.Unix(),
To: job.StartTime.Add(time.Duration(job.Duration) * time.Second).Unix(),
Queries: queries,
WithStats: true,
WithData: false,
}
resBody, err := ccms.doRequest(ctx, &req)
if err != nil {
return nil, err return nil, err
} }
stats := map[string]map[string]schema.MetricStatistics{} stats := make(map[string]map[string]schema.MetricStatistics, len(metrics))
for _, metric := range metrics { for i, res := range resBody {
nodestats := map[string]schema.MetricStatistics{} query := req.Queries[i]
for i, node := range job.Resources { data := res[0]
if node.Accelerators != nil || node.HWThreads != nil { if data.Error != nil {
// TODO/FIXME: return nil, fmt.Errorf("fetching %s for node %s failed: %s", query.Metric, query.Hostname, *data.Error)
return nil, errors.New("todo: cc-metric-store resources: Accelerator/HWThreads")
}
data := resdata[i][metric]
if data.Error != nil {
return nil, errors.New(*data.Error)
}
if data.Samples == 0 {
return nil, fmt.Errorf("no data for node '%s' and metric '%s'", node.Hostname, metric)
}
nodestats[node.Hostname] = schema.MetricStatistics{
Avg: float64(data.Avg),
Min: float64(data.Min),
Max: float64(data.Max),
}
} }
stats[metric] = nodestats metricdata, ok := stats[query.Metric]
if !ok {
metricdata = make(map[string]schema.MetricStatistics, job.NumNodes)
stats[query.Metric] = metricdata
}
if data.Avg.IsNaN() || data.Min.IsNaN() || data.Max.IsNaN() {
return nil, fmt.Errorf("fetching %s for node %s failed: %s", query.Metric, query.Hostname, "avg/min/max is NaN")
}
metricdata[query.Hostname] = schema.MetricStatistics{
Avg: float64(data.Avg),
Min: float64(data.Min),
Max: float64(data.Max),
}
} }
return stats, nil return stats, nil
} }
func (ccms *CCMetricStore) LoadNodeData(clusterId string, metrics, nodes []string, from, to int64, ctx context.Context) (map[string]map[string][]schema.Float, error) { func (ccms *CCMetricStore) LoadNodeData(clusterId string, metrics, nodes []string, from, to int64, ctx context.Context) (map[string]map[string][]schema.Float, error) {
reqBody := ApiRequestBody{} req := ApiQueryRequest{
reqBody.Metrics = metrics Cluster: clusterId,
for _, node := range nodes { From: from,
reqBody.Selectors = append(reqBody.Selectors, []string{clusterId, node}) To: to,
WithStats: false,
WithData: true,
} }
reqBodyBytes, err := json.Marshal(reqBody)
if err != nil {
return nil, err
}
var req *http.Request
if nodes == nil { if nodes == nil {
req, err = http.NewRequestWithContext(ctx, http.MethodPost, fmt.Sprintf("%s/api/%s/%d/%d/all-nodes", ccms.url, clusterId, from, to), bytes.NewReader(reqBodyBytes)) req.ForAllNodes = metrics
} else { } else {
req, err = http.NewRequestWithContext(ctx, http.MethodPost, fmt.Sprintf("%s/api/%d/%d/timeseries", ccms.url, from, to), bytes.NewReader(reqBodyBytes)) for _, node := range nodes {
for _, metric := range metrics {
req.Queries = append(req.Queries, ApiQuery{
Hostname: node,
Metric: metric,
})
}
}
} }
if err != nil {
return nil, err resBody, err := ccms.doRequest(ctx, &req)
}
if ccms.jwt != "" {
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", ccms.jwt))
}
res, err := ccms.client.Do(req)
if err != nil { if err != nil {
return nil, err return nil, err
} }
data := map[string]map[string][]schema.Float{} data := make(map[string]map[string][]schema.Float)
if nodes == nil { for i, res := range resBody {
resdata := map[string]map[string]ApiMetricData{} query := req.Queries[i]
if err := json.NewDecoder(res.Body).Decode(&resdata); err != nil { qdata := res[0]
return nil, err if qdata.Error != nil {
return nil, fmt.Errorf("fetching %s for node %s failed: %s", query.Metric, query.Hostname, *qdata.Error)
} }
for node, metrics := range resdata { nodedata, ok := data[query.Hostname]
nodedata := map[string][]schema.Float{} if !ok {
for metric, data := range metrics { nodedata = make(map[string][]schema.Float)
if data.Error != nil { data[query.Hostname] = nodedata
return nil, errors.New(*data.Error)
}
nodedata[metric] = data.Data
}
data[node] = nodedata
}
} else {
resdata := make([]map[string]ApiMetricData, 0, len(nodes))
if err := json.NewDecoder(res.Body).Decode(&resdata); err != nil {
return nil, err
} }
for i, node := range nodes { nodedata[query.Metric] = qdata.Data
metricsData := map[string][]schema.Float{}
for metric, data := range resdata[i] {
if data.Error != nil {
return nil, errors.New(*data.Error)
}
metricsData[metric] = data.Data
}
data[node] = metricsData
}
} }
return data, nil return data, nil