Start working on pre-computed stats

api.go

@@ -31,23 +31,19 @@ type ApiRequestBody struct {
 }
 
 type ApiMetricData struct {
-	Error *string  `json:"error"`
-	From  int64    `json:"from"`
-	To    int64    `json:"to"`
-	Data  []Float  `json:"data"`
-	Avg   *float64 `json:"avg"`
-	Min   *float64 `json:"min"`
-	Max   *float64 `json:"max"`
+	Error *string `json:"error"`
+	From  int64   `json:"from"`
+	To    int64   `json:"to"`
+	Data  []Float `json:"data"`
+	Avg   Float   `json:"avg"`
+	Min   Float   `json:"min"`
+	Max   Float   `json:"max"`
 }
 
 // TODO: Optimize this, just like the stats endpoint!
 func (data *ApiMetricData) AddStats() {
-	if len(data.Data) == 0 || data.Error != nil {
-		return
-	}
-
 	n := 0
-	sum, min, max := 0.0, float64(data.Data[0]), float64(data.Data[0])
+	sum, min, max := 0.0, math.MaxFloat64, -math.MaxFloat64
 	for _, x := range data.Data {
 		if x.IsNaN() {
 			continue
@@ -59,10 +55,14 @@ func (data *ApiMetricData) AddStats() {
 		max = math.Max(max, float64(x))
 	}
 
-	avg := sum / float64(n)
-	data.Avg = &avg
-	data.Min = &min
-	data.Max = &max
+	if n > 0 {
+		avg := sum / float64(n)
+		data.Avg = Float(avg)
+		data.Min = Float(min)
+		data.Max = Float(max)
+	} else {
+		data.Avg, data.Min, data.Max = NaN, NaN, NaN
+	}
 }
 
 type ApiStatsData struct {

archive.go

@@ -245,6 +245,7 @@ func (l *level) loadFile(cf *CheckpointFile, m *MemoryStore) error {
 			next:      nil,
 			archived:  true,
 		}
+		b.close()
 
 		minfo, ok := m.metrics[name]
 		if !ok {

config.json

@@ -1,30 +1,41 @@
 {
     "metrics": {
-        "load_one":     { "frequency": 10, "aggregation": null,  "scope": "node" },
-        "load_five":    { "frequency": 10, "aggregation": null,  "scope": "node" },
-        "load_fifteen": { "frequency": 10, "aggregation": null,  "scope": "node" },
-        "proc_run":     { "frequency": 10, "aggregation": null,  "scope": "node" },
-        "proc_total":   { "frequency": 10, "aggregation": null,  "scope": "node" },
-        "mem_free":     { "frequency": 10, "aggregation": null,  "scope": "node" },
-        "mem_used":     { "frequency": 10, "aggregation": null,  "scope": "node" },
-        "power":        { "frequency": 10, "aggregation": "sum", "scope": "socket" },
-        "mem_bw":       { "frequency": 10, "aggregation": "sum", "scope": "socket" },
-        "flops_sp":     { "frequency": 10, "aggregation": "sum", "scope": "cpu" },
-        "flops_dp":     { "frequency": 10, "aggregation": "sum", "scope": "cpu" },
-        "flops_any":    { "frequency": 10, "aggregation": "sum", "scope": "cpu" },
-        "clock":        { "frequency": 10, "aggregation": "avg", "scope": "cpu" },
-        "cpi":          { "frequency": 10, "aggregation": "avg", "scope": "cpu" }
+        "load_one": { "frequency": 3, "aggregation": null, "scope": "node" },
+        "load_five": { "frequency": 3, "aggregation": null, "scope": "node" },
+        "load_fifteen": { "frequency": 3, "aggregation": null, "scope": "node" },
+        "proc_run": { "frequency": 3, "aggregation": null, "scope": "node" },
+        "proc_total": { "frequency": 3, "aggregation": null, "scope": "node" },
+        "mem_free": { "frequency": 3, "aggregation": null, "scope": "node" },
+        "mem_cached": { "frequency": 3, "aggregation": null, "scope": "node" },
+        "mem_total": { "frequency": 3, "aggregation": null, "scope": "node" },
+        "swap_total": { "frequency": 3, "aggregation": null, "scope": "node" },
+        "mem_slab": { "frequency": 3, "aggregation": null, "scope": "node" },
+        "mem_buffers": { "frequency": 3, "aggregation": null, "scope": "node" },
+        "mem_sreclaimable": { "frequency": 3, "aggregation": null, "scope": "node" },
+        "mem_available": { "frequency": 3, "aggregation": null, "scope": "node" },
+        "swap_free": { "frequency": 3, "aggregation": null, "scope": "node" },
+        "mem_used": { "frequency": 3, "aggregation": null, "scope": "node" },
+        "cpu_user": { "frequency": 3, "aggregation": "sum", "scope": "cpu" },
+        "cpu_nice": { "frequency": 3, "aggregation": "sum", "scope": "cpu" },
+        "cpu_system": { "frequency": 3, "aggregation": "sum", "scope": "cpu" },
+        "cpu_idle": { "frequency": 3, "aggregation": "sum", "scope": "cpu" },
+        "cpu_iowait": { "frequency": 3, "aggregation": "sum", "scope": "cpu" },
+        "cpu_irq": { "frequency": 3, "aggregation": "sum", "scope": "cpu" },
+        "cpu_softirq": { "frequency": 3, "aggregation": "sum", "scope": "cpu" },
+        "cpu_steal": { "frequency": 3, "aggregation": "sum", "scope": "cpu" },
+        "cpu_guest": { "frequency": 3, "aggregation": "sum", "scope": "cpu" },
+        "cpu_guest_nice": { "frequency": 3, "aggregation": "sum", "scope": "cpu" }
     },
     "checkpoints": {
-        "interval": 21600,
+        "interval": 60,
         "directory": "./var/checkpoints",
-        "restore": 43200
+        "restore": 120
     },
     "archive": {
-        "interval": 86400,
+        "interval": 180,
         "directory": "./var/archive"
     },
-    "retention-in-memory": 86400,
+    "retention-in-memory": 120,
     "http-api-address": "0.0.0.0:8081",
     "nats": null,
     "jwt-public-key": "kzfYrYy+TzpanWZHJ5qSdMj5uKUWgq74BWhQG6copP0="

memstore.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"errors"
+	"math"
 	"sync"
 )
 
@@ -37,6 +38,14 @@ type buffer struct {
 	data       []Float // The slice should never reallocacte as `cap(data)` is respected.
 	prev, next *buffer // `prev` contains older data, `next` newer data.
 	archived   bool    // If true, this buffer is already archived
+
+	closed      bool
+	statisticts struct {
+		samples int
+		min     Float
+		max     Float
+		avg     Float
+	}
 }
 
 func newBuffer(ts, freq int64) *buffer {
@@ -46,6 +55,7 @@ func newBuffer(ts, freq int64) *buffer {
 	b.prev = nil
 	b.next = nil
 	b.archived = false
+	b.closed = false
 	b.data = b.data[:0]
 	return b
 }
@@ -70,6 +80,7 @@ func (b *buffer) write(ts int64, value Float) (*buffer, error) {
 		newbuf := newBuffer(ts, b.frequency)
 		newbuf.prev = b
 		b.next = newbuf
+		b.close()
 		b = newbuf
 		idx = 0
 	}
@@ -93,6 +104,37 @@ func (b *buffer) end() int64 {
 	return b.start + int64(len(b.data))*b.frequency
 }
 
+func (b *buffer) close() {
+	if b.closed {
+		return
+	}
+
+	b.closed = true
+	n, sum, min, max := 0, 0., math.MaxFloat64, -math.MaxFloat64
+	for _, x := range b.data {
+		if x.IsNaN() {
+			continue
+		}
+
+		n += 1
+		f := float64(x)
+		sum += f
+		min = math.Min(min, f)
+		max = math.Max(max, f)
+	}
+
+	b.statisticts.samples = n
+	if n > 0 {
+		b.statisticts.avg = Float(sum / float64(n))
+		b.statisticts.min = Float(min)
+		b.statisticts.max = Float(max)
+	} else {
+		b.statisticts.avg = NaN
+		b.statisticts.min = NaN
+		b.statisticts.max = NaN
+	}
+}
+
 // func interpolate(idx int, data []Float) Float {
 // 	if idx == 0 || idx+1 == len(data) {
 // 		return NaN

metric-store.go

@@ -80,6 +80,8 @@ func handleLine(dec *lineprotocol.Decoder) error {
 				typeName = string(val)
 			case "type-id":
 				typeId = string(val)
+			case "unit", "group":
+				// Ignore... (Important only for ganglia)
 			default:
 				return fmt.Errorf("unkown tag: '%s' (value: '%s')", string(key), string(val))
 			}

stats.go

@@ -20,6 +20,9 @@ func (b *buffer) stats(from, to int64) (Stats, int64, int64, error) {
 		from = b.start
 	}
 
+	// TODO: Check if b.closed and if so and the full buffer is queried,
+	// use b.statistics instead of iterating over the buffer.
+
 	samples := 0
 	sum, min, max := 0.0, math.MaxFloat32, -math.MaxFloat32