cc-metric-store/internal/memorystore/healthcheck.go

package memorystore

import (
	"bufio"
	"fmt"
	"time"
)

// This is a threshold that allows a node to be healthy with certain number of data points missing.
// Suppose a node does not receive last 5 data points, then healthCheck endpoint will still say a
// node is healthy. Anything more than 5 missing points in metrics of the node will deem the node unhealthy.
const MaxMissingDataPoints int64 = 5

// This is a threshold which allows upto certain number of metrics in a node to be unhealthly.
// Works with MaxMissingDataPoints. Say 5 metrics (including submetrics) do not receive the last
// MaxMissingDataPoints data points, then the node will be deemed healthy. Any more metrics that does
// not receive data for MaxMissingDataPoints data points will deem the node unhealthy.
const MaxUnhealthyMetrics int64 = 5

func (b *buffer) healthCheck() int64 {

	// Check if the buffer is empty
	if b.data == nil {
		return 1
	}

	buffer_end := b.start + b.frequency*int64(len(b.data))
	t := time.Now().Unix()

	// Check if the buffer is too old
	if t-buffer_end > MaxMissingDataPoints*b.frequency {
		return 1
	}

	return 0
}

func (l *Level) healthCheck(m *MemoryStore, count int64) (int64, error) {
	l.lock.RLock()
	defer l.lock.RUnlock()

	for _, mc := range m.Metrics {
		if b := l.metrics[mc.Offset]; b != nil {
			count += b.healthCheck()
		}
	}

	for _, lvl := range l.children {
		c, err := lvl.healthCheck(m, 0)
		if err != nil {
			return 0, err
		}
		count += c
	}

	return count, nil
}

func (m *MemoryStore) HealthCheck(w *bufio.Writer, selector []string) error {
	lvl := m.root.findLevel(selector)
	if lvl == nil {
		return fmt.Errorf("not found: %#v", selector)
	}

	buf := make([]byte, 0, 25)
	// buf = append(buf, "{"...)

	var count int64 = 0

	unhealthyMetricsCount, err := lvl.healthCheck(m, count)
	if err != nil {
		return err
	}

	if unhealthyMetricsCount < MaxUnhealthyMetrics {
		buf = append(buf, "Healthy"...)
	} else {
		buf = append(buf, "Unhealthy"...)
	}

	// buf = append(buf, "}\n"...)

	if _, err = w.Write(buf); err != nil {
		return err
	}

	return w.Flush()
}