Start a new api package

internal/memstore/memstore.go

@@ -1,29 +1,33 @@
 package memstore
 
-import "sync"
+import (
+	"sync"
+
+	"github.com/ClusterCockpit/cc-metric-store/internal/types"
+)
 
 // Could also be called "node" as this forms a node in a tree structure.
 // Called level because "node" might be confusing here.
 // Can be both a leaf or a inner node. In this tree structue, inner nodes can
 // also hold data (in `metrics`).
-type level struct {
+type Level struct {
 	lock      sync.RWMutex
 	metrics   []*chunk          // Every level can store metrics.
-	sublevels map[string]*level // Lower levels.
+	sublevels map[string]*Level // Lower levels.
 }
 
 // Find the correct level for the given selector, creating it if
 // it does not exist. Example selector in the context of the
 // ClusterCockpit could be: []string{ "emmy", "host123", "cpu0" }.
 // This function would probably benefit a lot from `level.children` beeing a `sync.Map`?
-func (l *level) findLevelOrCreate(selector []string, nMetrics int) *level {
+func (l *Level) findLevelOrCreate(selector []string, nMetrics int) *Level {
 	if len(selector) == 0 {
 		return l
 	}
 
 	// Allow concurrent reads:
 	l.lock.RLock()
-	var child *level
+	var child *Level
 	var ok bool
 	if l.sublevels == nil {
 		// sublevels map needs to be created...
@@ -48,7 +52,7 @@ func (l *level) findLevelOrCreate(selector []string, nMetrics int) *level {
 		}
 	}
 
-	child = &level{
+	child = &Level{
 		metrics:   make([]*chunk, nMetrics),
 		sublevels: nil,
 	}
@@ -56,13 +60,13 @@ func (l *level) findLevelOrCreate(selector []string, nMetrics int) *level {
 	if l.sublevels != nil {
 		l.sublevels[selector[0]] = child
 	} else {
-		l.sublevels = map[string]*level{selector[0]: child}
+		l.sublevels = map[string]*Level{selector[0]: child}
 	}
 	l.lock.Unlock()
 	return child.findLevelOrCreate(selector[1:], nMetrics)
 }
 
-func (l *level) free(t int64) (delme bool, n int) {
+func (l *Level) free(t int64) (delme bool, n int) {
 	l.lock.Lock()
 	defer l.lock.Unlock()
 
@@ -89,14 +93,41 @@ func (l *level) free(t int64) (delme bool, n int) {
 }
 
 type MemoryStore struct {
-	root level // root of the tree structure
+	root Level // root of the tree structure
 	// TODO...
 
-	metrics map[string]int // TODO...
+	metrics map[string]types.MetricConfig // TODO...
 }
 
-func (ms *MemoryStore) GetOffset(metric string) int {
-	return -1 // TODO!
+// Return a new, initialized instance of a MemoryStore.
+// Will panic if values in the metric configurations are invalid.
+func NewMemoryStore(metrics map[string]types.MetricConfig) *MemoryStore {
+	offset := 0
+	for key, config := range metrics {
+		if config.Frequency == 0 {
+			panic("invalid frequency")
+		}
+
+		metrics[key] = types.MetricConfig{
+			Frequency:   config.Frequency,
+			Aggregation: config.Aggregation,
+			Offset:      offset,
+		}
+		offset += 1
+	}
+
+	return &MemoryStore{
+		root: Level{
+			metrics:   make([]*chunk, len(metrics)),
+			sublevels: make(map[string]*Level),
+		},
+		metrics: metrics,
+	}
+}
+
+func (ms *MemoryStore) GetMetricConf(metric string) (types.MetricConfig, bool) {
+	conf, ok := ms.metrics[metric]
+	return conf, ok
 }
 
 func (ms *MemoryStore) GetMetricForOffset(offset int) string {
@@ -106,3 +137,37 @@ func (ms *MemoryStore) GetMetricForOffset(offset int) string {
 func (ms *MemoryStore) MinFrequency() int64 {
 	return 10 // TODO
 }
+
+func (m *MemoryStore) GetLevel(selector []string) *Level {
+	return m.root.findLevelOrCreate(selector, len(m.metrics))
+}
+
+func (m *MemoryStore) WriteToLevel(l *Level, selector []string, ts int64, metrics []types.Metric) error {
+	l = l.findLevelOrCreate(selector, len(m.metrics))
+	l.lock.Lock()
+	defer l.lock.Unlock()
+
+	for _, metric := range metrics {
+		if metric.Conf.Frequency == 0 {
+			continue
+		}
+
+		c := l.metrics[metric.Conf.Offset]
+		if c == nil {
+			// First write to this metric and level
+			c = newChunk(ts, metric.Conf.Frequency)
+			l.metrics[metric.Conf.Offset] = c
+		}
+
+		nc, err := c.write(ts, metric.Value)
+		if err != nil {
+			return err
+		}
+
+		// Last write started a new chunk...
+		if c != nc {
+			l.metrics[metric.Conf.Offset] = nc
+		}
+	}
+	return nil
+}

internal/memstore/streaming-checkpoint.go

@@ -31,7 +31,7 @@ func (ms *MemoryStore) SaveCheckpoint(from, to int64, w io.Writer) error {
 	return nil
 }
 
-func (l *level) saveCheckpoint(ms *MemoryStore, from, to int64, w io.Writer, buf []byte, metricsbuf []types.Float) ([]byte, error) {
+func (l *Level) saveCheckpoint(ms *MemoryStore, from, to int64, w io.Writer, buf []byte, metricsbuf []types.Float) ([]byte, error) {
 	var err error
 	l.lock.RLock()
 	defer l.lock.RUnlock()
@@ -112,7 +112,7 @@ func (ms *MemoryStore) LoadCheckpoint(r io.Reader) error {
 }
 
 // Blocks all other accesses for this level and all its sublevels!
-func (l *level) loadCheckpoint(ms *MemoryStore, r io.Reader, buf []byte) error {
+func (l *Level) loadCheckpoint(ms *MemoryStore, r io.Reader, buf []byte) error {
 	l.lock.Lock()
 	defer l.lock.Unlock()
 
@@ -158,8 +158,8 @@ func (l *level) loadCheckpoint(ms *MemoryStore, r io.Reader, buf []byte) error {
 			return fmt.Errorf("loading metric %#v: %w", key, err)
 		}
 
-		offset := ms.GetOffset(key)
-		if offset == -1 {
+		metricConf, ok := ms.GetMetricConf(key)
+		if !ok {
 			// Skip unkown metrics
 			ReleaseBytes(bytes)
 			continue
@@ -175,7 +175,7 @@ func (l *level) loadCheckpoint(ms *MemoryStore, r io.Reader, buf []byte) error {
 			checkpointed: true,
 		}
 
-		if prevchunk := l.metrics[offset]; prevchunk != nil {
+		if prevchunk := l.metrics[metricConf.Offset]; prevchunk != nil {
 			if prevchunk.end() > chunk.start {
 				return fmt.Errorf(
 					"loading metric %#v: loaded checkpoint overlaps with other chunks or is not loaded in correct order (%d - %d)",
@@ -183,9 +183,9 @@ func (l *level) loadCheckpoint(ms *MemoryStore, r io.Reader, buf []byte) error {
 			}
 			prevchunk.next = chunk
 			chunk.prev = prevchunk
-			l.metrics[offset] = chunk
+			l.metrics[metricConf.Offset] = chunk
 		} else {
-			l.metrics[offset] = chunk
+			l.metrics[metricConf.Offset] = chunk
 		}
 	}
 
@@ -198,11 +198,11 @@ func (l *level) loadCheckpoint(ms *MemoryStore, r io.Reader, buf []byte) error {
 			return err
 		}
 		if l.sublevels == nil {
-			l.sublevels = make(map[string]*level, n)
+			l.sublevels = make(map[string]*Level, n)
 		}
 		sublevel, ok := l.sublevels[key]
 		if !ok {
-			sublevel = &level{}
+			sublevel = &Level{}
 		}
 
 		if err = sublevel.loadCheckpoint(ms, r, buf); err != nil {