mirror of
https://github.com/ClusterCockpit/cc-metric-store.git
synced 2025-02-28 12:07:08 +01:00
89 lines
2.0 KiB
Go
89 lines
2.0 KiB
Go
package memorystore
|
|
|
|
import (
|
|
"bufio"
|
|
"fmt"
|
|
"time"
|
|
)
|
|
|
|
// This is a threshold that allows a node to be healthy with certain number of data points missing.
|
|
// Suppose a node does not receive last 5 data points, then healthCheck endpoint will still say a
|
|
// node is healthy. Anything more than 5 missing points in metrics of the node will deem the node unhealthy.
|
|
const MaxMissingDataPoints int64 = 5
|
|
|
|
// This is a threshold which allows upto certain number of metrics in a node to be unhealthly.
|
|
// Works with MaxMissingDataPoints. Say 5 metrics (including submetrics) do not receive the last
|
|
// MaxMissingDataPoints data points, then the node will be deemed healthy. Any more metrics that does
|
|
// not receive data for MaxMissingDataPoints data points will deem the node unhealthy.
|
|
const MaxUnhealthyMetrics int64 = 5
|
|
|
|
func (b *buffer) healthCheck() int64 {
|
|
|
|
// Check if the buffer is empty
|
|
if b.data == nil {
|
|
return 1
|
|
}
|
|
|
|
buffer_end := b.start + b.frequency*int64(len(b.data))
|
|
t := time.Now().Unix()
|
|
|
|
// Check if the buffer is too old
|
|
if t-buffer_end > MaxMissingDataPoints*b.frequency {
|
|
return 1
|
|
}
|
|
|
|
return 0
|
|
}
|
|
|
|
func (l *Level) healthCheck(m *MemoryStore, count int64) (int64, error) {
|
|
l.lock.RLock()
|
|
defer l.lock.RUnlock()
|
|
|
|
for _, mc := range m.Metrics {
|
|
if b := l.metrics[mc.Offset]; b != nil {
|
|
count += b.healthCheck()
|
|
}
|
|
}
|
|
|
|
for _, lvl := range l.children {
|
|
c, err := lvl.healthCheck(m, 0)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
count += c
|
|
}
|
|
|
|
return count, nil
|
|
}
|
|
|
|
func (m *MemoryStore) HealthCheck(w *bufio.Writer, selector []string) error {
|
|
lvl := m.root.findLevel(selector)
|
|
if lvl == nil {
|
|
return fmt.Errorf("not found: %#v", selector)
|
|
}
|
|
|
|
buf := make([]byte, 0, 25)
|
|
// buf = append(buf, "{"...)
|
|
|
|
var count int64 = 0
|
|
|
|
unhealthyMetricsCount, err := lvl.healthCheck(m, count)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if unhealthyMetricsCount < MaxUnhealthyMetrics {
|
|
buf = append(buf, "Healthy"...)
|
|
} else {
|
|
buf = append(buf, "Unhealthy"...)
|
|
}
|
|
|
|
// buf = append(buf, "}\n"...)
|
|
|
|
if _, err = w.Write(buf); err != nil {
|
|
return err
|
|
}
|
|
|
|
return w.Flush()
|
|
}
|