mirror of
https://github.com/ClusterCockpit/cc-backend
synced 2024-12-26 13:29:05 +01:00
Use new simpler cc-metric-store API
This commit is contained in:
parent
c254c689af
commit
6743d94b0e
@ -5,10 +5,8 @@ import (
|
|||||||
"bytes"
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"errors"
|
|
||||||
"fmt"
|
"fmt"
|
||||||
"net/http"
|
"net/http"
|
||||||
"strconv"
|
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/ClusterCockpit/cc-jobarchive/config"
|
"github.com/ClusterCockpit/cc-jobarchive/config"
|
||||||
@ -16,23 +14,30 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
type CCMetricStore struct {
|
type CCMetricStore struct {
|
||||||
jwt string
|
jwt string
|
||||||
url string
|
url string
|
||||||
client http.Client
|
queryEndpoint string
|
||||||
|
client http.Client
|
||||||
}
|
}
|
||||||
|
|
||||||
type ApiRequestBody struct {
|
type ApiQueryRequest struct {
|
||||||
Metrics []string `json:"metrics"`
|
Cluster string `json:"cluster"`
|
||||||
Selectors [][]string `json:"selectors"`
|
From int64 `json:"from"`
|
||||||
|
To int64 `json:"to"`
|
||||||
|
WithStats bool `json:"with-stats"`
|
||||||
|
WithData bool `json:"with-data"`
|
||||||
|
Queries []ApiQuery `json:"queries"`
|
||||||
|
ForAllNodes []string `json:"for-all-nodes"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type ApiQuery struct {
|
type ApiQuery struct {
|
||||||
Metric string `json:"metric"`
|
Metric string `json:"metric"`
|
||||||
Hostname string `json:"hostname"`
|
Hostname string `json:"host"`
|
||||||
Type *string `json:"type,omitempty"`
|
Aggregate bool `json:"aggreg"`
|
||||||
TypeIds []string `json:"type-ids,omitempty"`
|
Type *string `json:"type,omitempty"`
|
||||||
SubType *string `json:"subtype,omitempty"`
|
TypeIds []int `json:"type-ids,omitempty"`
|
||||||
SubTypeIds []string `json:"subtype-ids,omitempty"`
|
SubType *string `json:"subtype,omitempty"`
|
||||||
|
SubTypeIds []int `json:"subtype-ids,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type ApiMetricData struct {
|
type ApiMetricData struct {
|
||||||
@ -45,18 +50,9 @@ type ApiMetricData struct {
|
|||||||
Max schema.Float `json:"max"`
|
Max schema.Float `json:"max"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type ApiStatsData struct {
|
|
||||||
Error *string `json:"error"`
|
|
||||||
From int64 `json:"from"`
|
|
||||||
To int64 `json:"to"`
|
|
||||||
Samples int `json:"samples"`
|
|
||||||
Avg schema.Float `json:"avg"`
|
|
||||||
Min schema.Float `json:"min"`
|
|
||||||
Max schema.Float `json:"max"`
|
|
||||||
}
|
|
||||||
|
|
||||||
func (ccms *CCMetricStore) Init(url, token string) error {
|
func (ccms *CCMetricStore) Init(url, token string) error {
|
||||||
ccms.url = url
|
ccms.url = url
|
||||||
|
ccms.queryEndpoint = fmt.Sprintf("%s/api/query", url)
|
||||||
ccms.jwt = token
|
ccms.jwt = token
|
||||||
ccms.client = http.Client{
|
ccms.client = http.Client{
|
||||||
Timeout: 5 * time.Second,
|
Timeout: 5 * time.Second,
|
||||||
@ -64,100 +60,67 @@ func (ccms *CCMetricStore) Init(url, token string) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (ccms *CCMetricStore) doRequest(job *schema.Job, suffix string, metrics []string, ctx context.Context) (*http.Response, error) {
|
func (ccms *CCMetricStore) doRequest(ctx context.Context, body *ApiQueryRequest) ([][]ApiMetricData, error) {
|
||||||
from, to := job.StartTime.Unix(), job.StartTime.Add(time.Duration(job.Duration)*time.Second).Unix()
|
buf := &bytes.Buffer{}
|
||||||
reqBody := ApiRequestBody{}
|
if err := json.NewEncoder(buf).Encode(body); err != nil {
|
||||||
reqBody.Metrics = metrics
|
|
||||||
for _, node := range job.Resources {
|
|
||||||
if node.Accelerators != nil || node.HWThreads != nil {
|
|
||||||
// TODO/FIXME:
|
|
||||||
return nil, errors.New("todo: cc-metric-store resources: Accelerator/HWThreads")
|
|
||||||
}
|
|
||||||
|
|
||||||
reqBody.Selectors = append(reqBody.Selectors, []string{job.Cluster, node.Hostname})
|
|
||||||
}
|
|
||||||
|
|
||||||
reqBodyBytes, err := json.Marshal(reqBody)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, fmt.Sprintf("%s/api/%d/%d/%s", ccms.url, from, to, suffix), bytes.NewReader(reqBodyBytes))
|
req, err := http.NewRequestWithContext(ctx, http.MethodPost, ccms.queryEndpoint, buf)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
if ccms.jwt != "" {
|
if ccms.jwt != "" {
|
||||||
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", ccms.jwt))
|
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", ccms.jwt))
|
||||||
}
|
}
|
||||||
return ccms.client.Do(req)
|
|
||||||
|
res, err := ccms.client.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
if res.StatusCode != http.StatusOK {
|
||||||
|
return nil, fmt.Errorf("'%s': HTTP Status: %s", ccms.queryEndpoint, res.Status)
|
||||||
|
}
|
||||||
|
|
||||||
|
var resBody [][]ApiMetricData
|
||||||
|
if err := json.NewDecoder(bufio.NewReader(res.Body)).Decode(&resBody); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return resBody, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (ccms *CCMetricStore) LoadData(job *schema.Job, metrics []string, scopes []schema.MetricScope, ctx context.Context) (schema.JobData, error) {
|
func (ccms *CCMetricStore) LoadData(job *schema.Job, metrics []string, scopes []schema.MetricScope, ctx context.Context) (schema.JobData, error) {
|
||||||
type ApiQueryRequest struct {
|
|
||||||
Cluster string `json:"cluster"`
|
|
||||||
From int64 `json:"from"`
|
|
||||||
To int64 `json:"to"`
|
|
||||||
Queries []ApiQuery `json:"queries"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type ApiQueryResponse struct {
|
|
||||||
ApiMetricData
|
|
||||||
Query *ApiQuery `json:"query"`
|
|
||||||
}
|
|
||||||
|
|
||||||
queries, assignedScope, err := ccms.buildQueries(job, metrics, scopes)
|
queries, assignedScope, err := ccms.buildQueries(job, metrics, scopes)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
reqBody := ApiQueryRequest{
|
req := ApiQueryRequest{
|
||||||
Cluster: job.Cluster,
|
Cluster: job.Cluster,
|
||||||
From: job.StartTime.Unix(),
|
From: job.StartTime.Unix(),
|
||||||
To: job.StartTime.Add(time.Duration(job.Duration) * time.Second).Unix(),
|
To: job.StartTime.Add(time.Duration(job.Duration) * time.Second).Unix(),
|
||||||
Queries: queries,
|
Queries: queries,
|
||||||
|
WithStats: true,
|
||||||
|
WithData: true,
|
||||||
}
|
}
|
||||||
|
|
||||||
buf := &bytes.Buffer{}
|
resBody, err := ccms.doRequest(ctx, &req)
|
||||||
if err := json.NewEncoder(buf).Encode(reqBody); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, ccms.url+"/api/query", buf)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
if ccms.jwt != "" {
|
|
||||||
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", ccms.jwt))
|
|
||||||
}
|
|
||||||
res, err := ccms.client.Do(req)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if res.StatusCode != http.StatusOK {
|
|
||||||
return nil, fmt.Errorf("cc-metric-store replied with: %s", res.Status)
|
|
||||||
}
|
|
||||||
|
|
||||||
var resBody []ApiQueryResponse
|
|
||||||
if err := json.NewDecoder(bufio.NewReader(res.Body)).Decode(&resBody); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
// log.Printf("response: %#v", resBody)
|
|
||||||
|
|
||||||
var jobData schema.JobData = make(schema.JobData)
|
var jobData schema.JobData = make(schema.JobData)
|
||||||
for i, res := range resBody {
|
for i, row := range resBody {
|
||||||
metric := res.Query.Metric
|
query := req.Queries[i]
|
||||||
if _, ok := jobData[metric]; !ok {
|
|
||||||
jobData[metric] = make(map[schema.MetricScope]*schema.JobMetric)
|
|
||||||
}
|
|
||||||
|
|
||||||
if res.Error != nil {
|
|
||||||
return nil, fmt.Errorf("cc-metric-store error while fetching %s: %s", metric, *res.Error)
|
|
||||||
}
|
|
||||||
|
|
||||||
scope := assignedScope[i]
|
scope := assignedScope[i]
|
||||||
mc := config.GetMetricConfig(job.Cluster, metric)
|
mc := config.GetMetricConfig(job.Cluster, query.Metric)
|
||||||
jobMetric, ok := jobData[metric][scope]
|
if _, ok := jobData[query.Metric]; !ok {
|
||||||
|
jobData[query.Metric] = make(map[schema.MetricScope]*schema.JobMetric)
|
||||||
|
}
|
||||||
|
|
||||||
|
jobMetric, ok := jobData[query.Metric][scope]
|
||||||
if !ok {
|
if !ok {
|
||||||
jobMetric = &schema.JobMetric{
|
jobMetric = &schema.JobMetric{
|
||||||
Unit: mc.Unit,
|
Unit: mc.Unit,
|
||||||
@ -165,41 +128,47 @@ func (ccms *CCMetricStore) LoadData(job *schema.Job, metrics []string, scopes []
|
|||||||
Timestep: mc.Timestep,
|
Timestep: mc.Timestep,
|
||||||
Series: make([]schema.Series, 0),
|
Series: make([]schema.Series, 0),
|
||||||
}
|
}
|
||||||
jobData[metric][scope] = jobMetric
|
jobData[query.Metric][scope] = jobMetric
|
||||||
}
|
}
|
||||||
|
|
||||||
id := (*int)(nil)
|
for _, res := range row {
|
||||||
if res.Query.Type != nil {
|
if res.Error != nil {
|
||||||
id = new(int)
|
return nil, fmt.Errorf("cc-metric-store error while fetching %s: %s", query.Metric, *res.Error)
|
||||||
*id, _ = strconv.Atoi(res.Query.TypeIds[0])
|
}
|
||||||
}
|
|
||||||
|
|
||||||
if res.Avg.IsNaN() || res.Min.IsNaN() || res.Max.IsNaN() {
|
id := (*int)(nil)
|
||||||
// TODO: use schema.Float instead of float64?
|
if query.Type != nil {
|
||||||
// This is done because regular float64 can not be JSONed when NaN.
|
id = new(int)
|
||||||
res.Avg = schema.Float(0)
|
*id = query.TypeIds[0]
|
||||||
res.Min = schema.Float(0)
|
}
|
||||||
res.Max = schema.Float(0)
|
|
||||||
}
|
|
||||||
|
|
||||||
jobMetric.Series = append(jobMetric.Series, schema.Series{
|
if res.Avg.IsNaN() || res.Min.IsNaN() || res.Max.IsNaN() {
|
||||||
Hostname: res.Query.Hostname,
|
// TODO: use schema.Float instead of float64?
|
||||||
Id: id,
|
// This is done because regular float64 can not be JSONed when NaN.
|
||||||
Statistics: &schema.MetricStatistics{
|
res.Avg = schema.Float(0)
|
||||||
Avg: float64(res.Avg),
|
res.Min = schema.Float(0)
|
||||||
Min: float64(res.Min),
|
res.Max = schema.Float(0)
|
||||||
Max: float64(res.Max),
|
}
|
||||||
},
|
|
||||||
Data: res.Data,
|
jobMetric.Series = append(jobMetric.Series, schema.Series{
|
||||||
})
|
Hostname: query.Hostname,
|
||||||
|
Id: id,
|
||||||
|
Statistics: &schema.MetricStatistics{
|
||||||
|
Avg: float64(res.Avg),
|
||||||
|
Min: float64(res.Min),
|
||||||
|
Max: float64(res.Max),
|
||||||
|
},
|
||||||
|
Data: res.Data,
|
||||||
|
})
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return jobData, nil
|
return jobData, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
var (
|
var (
|
||||||
hwthreadString = string("cpu") // TODO/FIXME: inconsistency between cc-metric-collector and ClusterCockpit
|
hwthreadString = string("cpu") // TODO/FIXME: inconsistency between cc-metric-collector and ClusterCockpit
|
||||||
// coreString = string(schema.MetricScopeCore)
|
coreString = string(schema.MetricScopeCore)
|
||||||
socketString = string(schema.MetricScopeSocket)
|
socketString = string(schema.MetricScopeSocket)
|
||||||
acceleratorString = string(schema.MetricScopeAccelerator)
|
acceleratorString = string(schema.MetricScopeAccelerator)
|
||||||
)
|
)
|
||||||
@ -239,15 +208,14 @@ func (ccms *CCMetricStore) buildQueries(job *schema.Job, metrics []string, scope
|
|||||||
|
|
||||||
// Accelerator -> Accelerator (Use "accelerator" scope if requested scope is lower than node)
|
// Accelerator -> Accelerator (Use "accelerator" scope if requested scope is lower than node)
|
||||||
if nativeScope == schema.MetricScopeAccelerator && scope.LT(schema.MetricScopeNode) {
|
if nativeScope == schema.MetricScopeAccelerator && scope.LT(schema.MetricScopeNode) {
|
||||||
for _, accel := range host.Accelerators {
|
queries = append(queries, ApiQuery{
|
||||||
queries = append(queries, ApiQuery{
|
Metric: metric,
|
||||||
Metric: metric,
|
Hostname: host.Hostname,
|
||||||
Hostname: host.Hostname,
|
Aggregate: false,
|
||||||
Type: &acceleratorString,
|
Type: &acceleratorString,
|
||||||
TypeIds: []string{strconv.Itoa(accel)},
|
TypeIds: host.Accelerators,
|
||||||
})
|
})
|
||||||
assignedScope = append(assignedScope, schema.MetricScopeAccelerator)
|
assignedScope = append(assignedScope, schema.MetricScopeAccelerator)
|
||||||
}
|
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -258,26 +226,26 @@ func (ccms *CCMetricStore) buildQueries(job *schema.Job, metrics []string, scope
|
|||||||
}
|
}
|
||||||
|
|
||||||
queries = append(queries, ApiQuery{
|
queries = append(queries, ApiQuery{
|
||||||
Metric: metric,
|
Metric: metric,
|
||||||
Hostname: host.Hostname,
|
Hostname: host.Hostname,
|
||||||
Type: &acceleratorString,
|
Aggregate: true,
|
||||||
TypeIds: toStringSlice(host.Accelerators),
|
Type: &acceleratorString,
|
||||||
|
TypeIds: host.Accelerators,
|
||||||
})
|
})
|
||||||
assignedScope = append(assignedScope, schema.MetricScopeNode)
|
assignedScope = append(assignedScope, scope)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
// HWThread -> HWThead
|
// HWThread -> HWThead
|
||||||
if nativeScope == schema.MetricScopeHWThread && scope == schema.MetricScopeHWThread {
|
if nativeScope == schema.MetricScopeHWThread && scope == schema.MetricScopeHWThread {
|
||||||
for _, hwthread := range hwthreads {
|
queries = append(queries, ApiQuery{
|
||||||
queries = append(queries, ApiQuery{
|
Metric: metric,
|
||||||
Metric: metric,
|
Hostname: host.Hostname,
|
||||||
Hostname: host.Hostname,
|
Aggregate: false,
|
||||||
Type: &hwthreadString,
|
Type: &hwthreadString,
|
||||||
TypeIds: []string{strconv.Itoa(hwthread)},
|
TypeIds: hwthreads,
|
||||||
})
|
})
|
||||||
assignedScope = append(assignedScope, schema.MetricScopeHWThread)
|
assignedScope = append(assignedScope, scope)
|
||||||
}
|
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -286,12 +254,13 @@ func (ccms *CCMetricStore) buildQueries(job *schema.Job, metrics []string, scope
|
|||||||
cores, _ := topology.GetCoresFromHWThreads(hwthreads)
|
cores, _ := topology.GetCoresFromHWThreads(hwthreads)
|
||||||
for _, core := range cores {
|
for _, core := range cores {
|
||||||
queries = append(queries, ApiQuery{
|
queries = append(queries, ApiQuery{
|
||||||
Metric: metric,
|
Metric: metric,
|
||||||
Hostname: host.Hostname,
|
Hostname: host.Hostname,
|
||||||
Type: &hwthreadString,
|
Aggregate: true,
|
||||||
TypeIds: toStringSlice(topology.Core[core]),
|
Type: &hwthreadString,
|
||||||
|
TypeIds: topology.Core[core],
|
||||||
})
|
})
|
||||||
assignedScope = append(assignedScope, schema.MetricScopeCore)
|
assignedScope = append(assignedScope, scope)
|
||||||
}
|
}
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
@ -301,12 +270,13 @@ func (ccms *CCMetricStore) buildQueries(job *schema.Job, metrics []string, scope
|
|||||||
sockets, _ := topology.GetSocketsFromHWThreads(hwthreads)
|
sockets, _ := topology.GetSocketsFromHWThreads(hwthreads)
|
||||||
for _, socket := range sockets {
|
for _, socket := range sockets {
|
||||||
queries = append(queries, ApiQuery{
|
queries = append(queries, ApiQuery{
|
||||||
Metric: metric,
|
Metric: metric,
|
||||||
Hostname: host.Hostname,
|
Hostname: host.Hostname,
|
||||||
Type: &hwthreadString,
|
Aggregate: true,
|
||||||
TypeIds: toStringSlice(topology.Socket[socket]),
|
Type: &hwthreadString,
|
||||||
|
TypeIds: topology.Socket[socket],
|
||||||
})
|
})
|
||||||
assignedScope = append(assignedScope, schema.MetricScopeSocket)
|
assignedScope = append(assignedScope, scope)
|
||||||
}
|
}
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
@ -314,27 +284,55 @@ func (ccms *CCMetricStore) buildQueries(job *schema.Job, metrics []string, scope
|
|||||||
// HWThread -> Node
|
// HWThread -> Node
|
||||||
if nativeScope == schema.MetricScopeHWThread && scope == schema.MetricScopeNode {
|
if nativeScope == schema.MetricScopeHWThread && scope == schema.MetricScopeNode {
|
||||||
queries = append(queries, ApiQuery{
|
queries = append(queries, ApiQuery{
|
||||||
Metric: metric,
|
Metric: metric,
|
||||||
Hostname: host.Hostname,
|
Hostname: host.Hostname,
|
||||||
Type: &hwthreadString,
|
Aggregate: true,
|
||||||
TypeIds: toStringSlice(hwthreads),
|
Type: &hwthreadString,
|
||||||
|
TypeIds: hwthreads,
|
||||||
})
|
})
|
||||||
assignedScope = append(assignedScope, schema.MetricScopeNode)
|
assignedScope = append(assignedScope, scope)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
// Core -> Core
|
||||||
|
if nativeScope == schema.MetricScopeCore && scope == schema.MetricScopeCore {
|
||||||
|
cores, _ := topology.GetCoresFromHWThreads(hwthreads)
|
||||||
|
queries = append(queries, ApiQuery{
|
||||||
|
Metric: metric,
|
||||||
|
Hostname: host.Hostname,
|
||||||
|
Aggregate: false,
|
||||||
|
Type: &coreString,
|
||||||
|
TypeIds: cores,
|
||||||
|
})
|
||||||
|
assignedScope = append(assignedScope, scope)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
// Core -> Node
|
||||||
|
if nativeScope == schema.MetricScopeCore && scope == schema.MetricScopeNode {
|
||||||
|
cores, _ := topology.GetCoresFromHWThreads(hwthreads)
|
||||||
|
queries = append(queries, ApiQuery{
|
||||||
|
Metric: metric,
|
||||||
|
Hostname: host.Hostname,
|
||||||
|
Aggregate: true,
|
||||||
|
Type: &coreString,
|
||||||
|
TypeIds: cores,
|
||||||
|
})
|
||||||
|
assignedScope = append(assignedScope, scope)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
// Socket -> Socket
|
// Socket -> Socket
|
||||||
if nativeScope == schema.MetricScopeSocket && scope == schema.MetricScopeSocket {
|
if nativeScope == schema.MetricScopeSocket && scope == schema.MetricScopeSocket {
|
||||||
sockets, _ := topology.GetSocketsFromHWThreads(hwthreads)
|
sockets, _ := topology.GetSocketsFromHWThreads(hwthreads)
|
||||||
for _, socket := range sockets {
|
queries = append(queries, ApiQuery{
|
||||||
queries = append(queries, ApiQuery{
|
Metric: metric,
|
||||||
Metric: metric,
|
Hostname: host.Hostname,
|
||||||
Hostname: host.Hostname,
|
Aggregate: false,
|
||||||
Type: &acceleratorString,
|
Type: &socketString,
|
||||||
TypeIds: []string{strconv.Itoa(socket)},
|
TypeIds: sockets,
|
||||||
})
|
})
|
||||||
assignedScope = append(assignedScope, schema.MetricScopeSocket)
|
assignedScope = append(assignedScope, scope)
|
||||||
}
|
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -342,12 +340,13 @@ func (ccms *CCMetricStore) buildQueries(job *schema.Job, metrics []string, scope
|
|||||||
if nativeScope == schema.MetricScopeSocket && scope == schema.MetricScopeNode {
|
if nativeScope == schema.MetricScopeSocket && scope == schema.MetricScopeNode {
|
||||||
sockets, _ := topology.GetSocketsFromHWThreads(hwthreads)
|
sockets, _ := topology.GetSocketsFromHWThreads(hwthreads)
|
||||||
queries = append(queries, ApiQuery{
|
queries = append(queries, ApiQuery{
|
||||||
Metric: metric,
|
Metric: metric,
|
||||||
Hostname: host.Hostname,
|
Hostname: host.Hostname,
|
||||||
Type: &socketString,
|
Aggregate: true,
|
||||||
TypeIds: toStringSlice(sockets),
|
Type: &socketString,
|
||||||
|
TypeIds: sockets,
|
||||||
})
|
})
|
||||||
assignedScope = append(assignedScope, schema.MetricScopeNode)
|
assignedScope = append(assignedScope, scope)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -357,7 +356,7 @@ func (ccms *CCMetricStore) buildQueries(job *schema.Job, metrics []string, scope
|
|||||||
Metric: metric,
|
Metric: metric,
|
||||||
Hostname: host.Hostname,
|
Hostname: host.Hostname,
|
||||||
})
|
})
|
||||||
assignedScope = append(assignedScope, schema.MetricScopeNode)
|
assignedScope = append(assignedScope, scope)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -369,121 +368,96 @@ func (ccms *CCMetricStore) buildQueries(job *schema.Job, metrics []string, scope
|
|||||||
return queries, assignedScope, nil
|
return queries, assignedScope, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func toStringSlice(s []int) []string {
|
|
||||||
ret := make([]string, len(s))
|
|
||||||
for i, val := range s {
|
|
||||||
ret[i] = strconv.Itoa(val)
|
|
||||||
}
|
|
||||||
return ret
|
|
||||||
}
|
|
||||||
|
|
||||||
func (ccms *CCMetricStore) LoadStats(job *schema.Job, metrics []string, ctx context.Context) (map[string]map[string]schema.MetricStatistics, error) {
|
func (ccms *CCMetricStore) LoadStats(job *schema.Job, metrics []string, ctx context.Context) (map[string]map[string]schema.MetricStatistics, error) {
|
||||||
res, err := ccms.doRequest(job, "stats", metrics, ctx)
|
queries, _, err := ccms.buildQueries(job, metrics, []schema.MetricScope{schema.MetricScopeNode})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
resdata := make([]map[string]ApiStatsData, 0, len(job.Resources))
|
req := ApiQueryRequest{
|
||||||
if err := json.NewDecoder(res.Body).Decode(&resdata); err != nil {
|
Cluster: job.Cluster,
|
||||||
|
From: job.StartTime.Unix(),
|
||||||
|
To: job.StartTime.Add(time.Duration(job.Duration) * time.Second).Unix(),
|
||||||
|
Queries: queries,
|
||||||
|
WithStats: true,
|
||||||
|
WithData: false,
|
||||||
|
}
|
||||||
|
|
||||||
|
resBody, err := ccms.doRequest(ctx, &req)
|
||||||
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
stats := map[string]map[string]schema.MetricStatistics{}
|
stats := make(map[string]map[string]schema.MetricStatistics, len(metrics))
|
||||||
for _, metric := range metrics {
|
for i, res := range resBody {
|
||||||
nodestats := map[string]schema.MetricStatistics{}
|
query := req.Queries[i]
|
||||||
for i, node := range job.Resources {
|
data := res[0]
|
||||||
if node.Accelerators != nil || node.HWThreads != nil {
|
if data.Error != nil {
|
||||||
// TODO/FIXME:
|
return nil, fmt.Errorf("fetching %s for node %s failed: %s", query.Metric, query.Hostname, *data.Error)
|
||||||
return nil, errors.New("todo: cc-metric-store resources: Accelerator/HWThreads")
|
|
||||||
}
|
|
||||||
|
|
||||||
data := resdata[i][metric]
|
|
||||||
if data.Error != nil {
|
|
||||||
return nil, errors.New(*data.Error)
|
|
||||||
}
|
|
||||||
|
|
||||||
if data.Samples == 0 {
|
|
||||||
return nil, fmt.Errorf("no data for node '%s' and metric '%s'", node.Hostname, metric)
|
|
||||||
}
|
|
||||||
|
|
||||||
nodestats[node.Hostname] = schema.MetricStatistics{
|
|
||||||
Avg: float64(data.Avg),
|
|
||||||
Min: float64(data.Min),
|
|
||||||
Max: float64(data.Max),
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
stats[metric] = nodestats
|
metricdata, ok := stats[query.Metric]
|
||||||
|
if !ok {
|
||||||
|
metricdata = make(map[string]schema.MetricStatistics, job.NumNodes)
|
||||||
|
stats[query.Metric] = metricdata
|
||||||
|
}
|
||||||
|
|
||||||
|
if data.Avg.IsNaN() || data.Min.IsNaN() || data.Max.IsNaN() {
|
||||||
|
return nil, fmt.Errorf("fetching %s for node %s failed: %s", query.Metric, query.Hostname, "avg/min/max is NaN")
|
||||||
|
}
|
||||||
|
|
||||||
|
metricdata[query.Hostname] = schema.MetricStatistics{
|
||||||
|
Avg: float64(data.Avg),
|
||||||
|
Min: float64(data.Min),
|
||||||
|
Max: float64(data.Max),
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return stats, nil
|
return stats, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (ccms *CCMetricStore) LoadNodeData(clusterId string, metrics, nodes []string, from, to int64, ctx context.Context) (map[string]map[string][]schema.Float, error) {
|
func (ccms *CCMetricStore) LoadNodeData(clusterId string, metrics, nodes []string, from, to int64, ctx context.Context) (map[string]map[string][]schema.Float, error) {
|
||||||
reqBody := ApiRequestBody{}
|
req := ApiQueryRequest{
|
||||||
reqBody.Metrics = metrics
|
Cluster: clusterId,
|
||||||
for _, node := range nodes {
|
From: from,
|
||||||
reqBody.Selectors = append(reqBody.Selectors, []string{clusterId, node})
|
To: to,
|
||||||
|
WithStats: false,
|
||||||
|
WithData: true,
|
||||||
}
|
}
|
||||||
|
|
||||||
reqBodyBytes, err := json.Marshal(reqBody)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
var req *http.Request
|
|
||||||
if nodes == nil {
|
if nodes == nil {
|
||||||
req, err = http.NewRequestWithContext(ctx, http.MethodPost, fmt.Sprintf("%s/api/%s/%d/%d/all-nodes", ccms.url, clusterId, from, to), bytes.NewReader(reqBodyBytes))
|
req.ForAllNodes = metrics
|
||||||
} else {
|
} else {
|
||||||
req, err = http.NewRequestWithContext(ctx, http.MethodPost, fmt.Sprintf("%s/api/%d/%d/timeseries", ccms.url, from, to), bytes.NewReader(reqBodyBytes))
|
for _, node := range nodes {
|
||||||
|
for _, metric := range metrics {
|
||||||
|
req.Queries = append(req.Queries, ApiQuery{
|
||||||
|
Hostname: node,
|
||||||
|
Metric: metric,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
resBody, err := ccms.doRequest(ctx, &req)
|
||||||
}
|
|
||||||
if ccms.jwt != "" {
|
|
||||||
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", ccms.jwt))
|
|
||||||
}
|
|
||||||
res, err := ccms.client.Do(req)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
data := map[string]map[string][]schema.Float{}
|
data := make(map[string]map[string][]schema.Float)
|
||||||
if nodes == nil {
|
for i, res := range resBody {
|
||||||
resdata := map[string]map[string]ApiMetricData{}
|
query := req.Queries[i]
|
||||||
if err := json.NewDecoder(res.Body).Decode(&resdata); err != nil {
|
qdata := res[0]
|
||||||
return nil, err
|
if qdata.Error != nil {
|
||||||
|
return nil, fmt.Errorf("fetching %s for node %s failed: %s", query.Metric, query.Hostname, *qdata.Error)
|
||||||
}
|
}
|
||||||
|
|
||||||
for node, metrics := range resdata {
|
nodedata, ok := data[query.Hostname]
|
||||||
nodedata := map[string][]schema.Float{}
|
if !ok {
|
||||||
for metric, data := range metrics {
|
nodedata = make(map[string][]schema.Float)
|
||||||
if data.Error != nil {
|
data[query.Hostname] = nodedata
|
||||||
return nil, errors.New(*data.Error)
|
|
||||||
}
|
|
||||||
|
|
||||||
nodedata[metric] = data.Data
|
|
||||||
}
|
|
||||||
data[node] = nodedata
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
resdata := make([]map[string]ApiMetricData, 0, len(nodes))
|
|
||||||
if err := json.NewDecoder(res.Body).Decode(&resdata); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
}
|
||||||
|
|
||||||
for i, node := range nodes {
|
nodedata[query.Metric] = qdata.Data
|
||||||
metricsData := map[string][]schema.Float{}
|
|
||||||
for metric, data := range resdata[i] {
|
|
||||||
if data.Error != nil {
|
|
||||||
return nil, errors.New(*data.Error)
|
|
||||||
}
|
|
||||||
|
|
||||||
metricsData[metric] = data.Data
|
|
||||||
}
|
|
||||||
|
|
||||||
data[node] = metricsData
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return data, nil
|
return data, nil
|
||||||
|
Loading…
Reference in New Issue
Block a user