mirror of
https://github.com/ClusterCockpit/cc-metric-store.git
synced 2024-11-10 05:07:25 +01:00
Start working on pre-computed stats
This commit is contained in:
Lou Knauer
parent
5d89d87a2d
commit
878e9d7154
22
api.go
22
api.go
@ -35,19 +35,15 @@ type ApiMetricData struct {
|
||||
From int64 `json:"from"`
|
||||
To int64 `json:"to"`
|
||||
Data []Float `json:"data"`
|
||||
Avg *float64 `json:"avg"`
|
||||
Min *float64 `json:"min"`
|
||||
Max *float64 `json:"max"`
|
||||
Avg Float `json:"avg"`
|
||||
Min Float `json:"min"`
|
||||
Max Float `json:"max"`
|
||||
}
|
||||
|
||||
// TODO: Optimize this, just like the stats endpoint!
|
||||
func (data *ApiMetricData) AddStats() {
|
||||
if len(data.Data) == 0 || data.Error != nil {
|
||||
return
|
||||
}
|
||||
|
||||
n := 0
|
||||
sum, min, max := 0.0, float64(data.Data[0]), float64(data.Data[0])
|
||||
sum, min, max := 0.0, math.MaxFloat64, -math.MaxFloat64
|
||||
for _, x := range data.Data {
|
||||
if x.IsNaN() {
|
||||
continue
|
||||
@ -59,10 +55,14 @@ func (data *ApiMetricData) AddStats() {
|
||||
max = math.Max(max, float64(x))
|
||||
}
|
||||
|
||||
if n > 0 {
|
||||
avg := sum / float64(n)
|
||||
data.Avg = &avg
|
||||
data.Min = &min
|
||||
data.Max = &max
|
||||
data.Avg = Float(avg)
|
||||
data.Min = Float(min)
|
||||
data.Max = Float(max)
|
||||
} else {
|
||||
data.Avg, data.Min, data.Max = NaN, NaN, NaN
|
||||
}
|
||||
}
|
||||
|
||||
type ApiStatsData struct {
|
||||
|
||||
@ -245,6 +245,7 @@ func (l *level) loadFile(cf *CheckpointFile, m *MemoryStore) error {
|
||||
next: nil,
|
||||
archived: true,
|
||||
}
|
||||
b.close()
|
||||
|
||||
minfo, ok := m.metrics[name]
|
||||
if !ok {
|
||||
|
||||
47
config.json
47
config.json
@ -1,30 +1,41 @@
|
||||
{
|
||||
"metrics": {
|
||||
"load_one": { "frequency": 10, "aggregation": null, "scope": "node" },
|
||||
"load_five": { "frequency": 10, "aggregation": null, "scope": "node" },
|
||||
"load_fifteen": { "frequency": 10, "aggregation": null, "scope": "node" },
|
||||
"proc_run": { "frequency": 10, "aggregation": null, "scope": "node" },
|
||||
"proc_total": { "frequency": 10, "aggregation": null, "scope": "node" },
|
||||
"mem_free": { "frequency": 10, "aggregation": null, "scope": "node" },
|
||||
"mem_used": { "frequency": 10, "aggregation": null, "scope": "node" },
|
||||
"power": { "frequency": 10, "aggregation": "sum", "scope": "socket" },
|
||||
"mem_bw": { "frequency": 10, "aggregation": "sum", "scope": "socket" },
|
||||
"flops_sp": { "frequency": 10, "aggregation": "sum", "scope": "cpu" },
|
||||
"flops_dp": { "frequency": 10, "aggregation": "sum", "scope": "cpu" },
|
||||
"flops_any": { "frequency": 10, "aggregation": "sum", "scope": "cpu" },
|
||||
"clock": { "frequency": 10, "aggregation": "avg", "scope": "cpu" },
|
||||
"cpi": { "frequency": 10, "aggregation": "avg", "scope": "cpu" }
|
||||
"load_one": { "frequency": 3, "aggregation": null, "scope": "node" },
|
||||
"load_five": { "frequency": 3, "aggregation": null, "scope": "node" },
|
||||
"load_fifteen": { "frequency": 3, "aggregation": null, "scope": "node" },
|
||||
"proc_run": { "frequency": 3, "aggregation": null, "scope": "node" },
|
||||
"proc_total": { "frequency": 3, "aggregation": null, "scope": "node" },
|
||||
"mem_free": { "frequency": 3, "aggregation": null, "scope": "node" },
|
||||
"mem_cached": { "frequency": 3, "aggregation": null, "scope": "node" },
|
||||
"mem_total": { "frequency": 3, "aggregation": null, "scope": "node" },
|
||||
"swap_total": { "frequency": 3, "aggregation": null, "scope": "node" },
|
||||
"mem_slab": { "frequency": 3, "aggregation": null, "scope": "node" },
|
||||
"mem_buffers": { "frequency": 3, "aggregation": null, "scope": "node" },
|
||||
"mem_sreclaimable": { "frequency": 3, "aggregation": null, "scope": "node" },
|
||||
"mem_available": { "frequency": 3, "aggregation": null, "scope": "node" },
|
||||
"swap_free": { "frequency": 3, "aggregation": null, "scope": "node" },
|
||||
"mem_used": { "frequency": 3, "aggregation": null, "scope": "node" },
|
||||
"cpu_user": { "frequency": 3, "aggregation": "sum", "scope": "cpu" },
|
||||
"cpu_nice": { "frequency": 3, "aggregation": "sum", "scope": "cpu" },
|
||||
"cpu_system": { "frequency": 3, "aggregation": "sum", "scope": "cpu" },
|
||||
"cpu_idle": { "frequency": 3, "aggregation": "sum", "scope": "cpu" },
|
||||
"cpu_iowait": { "frequency": 3, "aggregation": "sum", "scope": "cpu" },
|
||||
"cpu_irq": { "frequency": 3, "aggregation": "sum", "scope": "cpu" },
|
||||
"cpu_softirq": { "frequency": 3, "aggregation": "sum", "scope": "cpu" },
|
||||
"cpu_steal": { "frequency": 3, "aggregation": "sum", "scope": "cpu" },
|
||||
"cpu_guest": { "frequency": 3, "aggregation": "sum", "scope": "cpu" },
|
||||
"cpu_guest_nice": { "frequency": 3, "aggregation": "sum", "scope": "cpu" }
|
||||
},
|
||||
"checkpoints": {
|
||||
"interval": 21600,
|
||||
"interval": 60,
|
||||
"directory": "./var/checkpoints",
|
||||
"restore": 43200
|
||||
"restore": 120
|
||||
},
|
||||
"archive": {
|
||||
"interval": 86400,
|
||||
"interval": 180,
|
||||
"directory": "./var/archive"
|
||||
},
|
||||
"retention-in-memory": 86400,
|
||||
"retention-in-memory": 120,
|
||||
"http-api-address": "0.0.0.0:8081",
|
||||
"nats": null,
|
||||
"jwt-public-key": "kzfYrYy+TzpanWZHJ5qSdMj5uKUWgq74BWhQG6copP0="
|
||||
|
||||
42
memstore.go
42
memstore.go
@ -2,6 +2,7 @@ package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"math"
|
||||
"sync"
|
||||
)
|
||||
|
||||
@ -37,6 +38,14 @@ type buffer struct {
|
||||
data []Float // The slice should never reallocacte as `cap(data)` is respected.
|
||||
prev, next *buffer // `prev` contains older data, `next` newer data.
|
||||
archived bool // If true, this buffer is already archived
|
||||
|
||||
closed bool
|
||||
statisticts struct {
|
||||
samples int
|
||||
min Float
|
||||
max Float
|
||||
avg Float
|
||||
}
|
||||
}
|
||||
|
||||
func newBuffer(ts, freq int64) *buffer {
|
||||
@ -46,6 +55,7 @@ func newBuffer(ts, freq int64) *buffer {
|
||||
b.prev = nil
|
||||
b.next = nil
|
||||
b.archived = false
|
||||
b.closed = false
|
||||
b.data = b.data[:0]
|
||||
return b
|
||||
}
|
||||
@ -70,6 +80,7 @@ func (b *buffer) write(ts int64, value Float) (*buffer, error) {
|
||||
newbuf := newBuffer(ts, b.frequency)
|
||||
newbuf.prev = b
|
||||
b.next = newbuf
|
||||
b.close()
|
||||
b = newbuf
|
||||
idx = 0
|
||||
}
|
||||
@ -93,6 +104,37 @@ func (b *buffer) end() int64 {
|
||||
return b.start + int64(len(b.data))*b.frequency
|
||||
}
|
||||
|
||||
func (b *buffer) close() {
|
||||
if b.closed {
|
||||
return
|
||||
}
|
||||
|
||||
b.closed = true
|
||||
n, sum, min, max := 0, 0., math.MaxFloat64, -math.MaxFloat64
|
||||
for _, x := range b.data {
|
||||
if x.IsNaN() {
|
||||
continue
|
||||
}
|
||||
|
||||
n += 1
|
||||
f := float64(x)
|
||||
sum += f
|
||||
min = math.Min(min, f)
|
||||
max = math.Max(max, f)
|
||||
}
|
||||
|
||||
b.statisticts.samples = n
|
||||
if n > 0 {
|
||||
b.statisticts.avg = Float(sum / float64(n))
|
||||
b.statisticts.min = Float(min)
|
||||
b.statisticts.max = Float(max)
|
||||
} else {
|
||||
b.statisticts.avg = NaN
|
||||
b.statisticts.min = NaN
|
||||
b.statisticts.max = NaN
|
||||
}
|
||||
}
|
||||
|
||||
// func interpolate(idx int, data []Float) Float {
|
||||
// if idx == 0 || idx+1 == len(data) {
|
||||
// return NaN
|
||||
|
||||
@ -80,6 +80,8 @@ func handleLine(dec *lineprotocol.Decoder) error {
|
||||
typeName = string(val)
|
||||
case "type-id":
|
||||
typeId = string(val)
|
||||
case "unit", "group":
|
||||
// Ignore... (Important only for ganglia)
|
||||
default:
|
||||
return fmt.Errorf("unkown tag: '%s' (value: '%s')", string(key), string(val))
|
||||
}
|
||||
|
||||
3
stats.go
3
stats.go
@ -20,6 +20,9 @@ func (b *buffer) stats(from, to int64) (Stats, int64, int64, error) {
|
||||
from = b.start
|
||||
}
|
||||
|
||||
// TODO: Check if b.closed and if so and the full buffer is queried,
|
||||
// use b.statistics instead of iterating over the buffer.
|
||||
|
||||
samples := 0
|
||||
sum, min, max := 0.0, math.MaxFloat32, -math.MaxFloat32
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user