     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→// This file implements ingestion of InfluxDB line-protocol metric data received
     7→// over NATS. Each line encodes one metric sample with the following structure:
     8→//
     9→//	<measurement>[,cluster=<c>][,hostname=<h>][,type=<t>][,type-id=<id>][,subtype=<s>][,stype-id=<id>] value=<v> [<timestamp>]
    10→//
    11→// The measurement name identifies the metric (e.g. "cpu_load"). Tags provide
    12→// routing information (cluster, host) and optional sub-device selectors (type,
    13→// subtype). Only one field is expected per line: "value".
    14→//
    15→// After decoding, each sample is:
    16→//  1. Written to the in-memory store via ms.WriteToLevel.
    17→//  2. If the checkpoint format is "wal", also forwarded to the WAL staging
    18→//     goroutine via the WALMessages channel for durable write-ahead logging.
    19→package metricstore
    20→
    21→import (
    22→	"bytes"
    23→	"context"
    24→	"fmt"
    25→	"sync"
    26→	"time"
    27→
    28→	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
    29→	"github.com/ClusterCockpit/cc-lib/v2/nats"
    30→	"github.com/ClusterCockpit/cc-lib/v2/schema"
    31→	"github.com/ClusterCockpit/cc-line-protocol/v2/lineprotocol"
    32→)
    33→
    34→// ReceiveNats subscribes to all configured NATS subjects and feeds incoming
    35→// line-protocol messages into the MemoryStore.
    36→//
    37→// When workers > 1 a pool of goroutines drains a shared channel so that
    38→// multiple messages can be decoded in parallel. With workers == 1 the NATS
    39→// callback decodes inline (no channel overhead, lower latency).
    40→//
    41→// The function blocks until ctx is cancelled and all worker goroutines have
    42→// finished. It returns nil when the NATS client is not configured; callers
    43→// should treat that as a no-op rather than an error.
    44→func ReceiveNats(ms *MemoryStore,
    45→	workers int,
    46→	ctx context.Context,
    47→) error {
    48→	nc := nats.GetClient()
    49→
    50→	if nc == nil {
    51→		cclog.Warn("NATS client not initialized")
    52→		return nil
    53→	}
    54→
    55→	var wg sync.WaitGroup
    56→	msgs := make(chan []byte, workers*2)
    57→
    58→	for _, sc := range *Keys.Subscriptions {
    59→		clusterTag := sc.ClusterTag
    60→		if workers > 1 {
    61→			wg.Add(workers)
    62→
    63→			for range workers {
    64→				go func() {
    65→					defer wg.Done()
    66→					for m := range msgs {
    67→						dec := lineprotocol.NewDecoderWithBytes(m)
    68→						if err := DecodeLine(dec, ms, clusterTag); err != nil {
    69→							cclog.Errorf("error: %s", err.Error())
    70→						}
    71→					}
    72→				}()
    73→			}
    74→
    75→			nc.Subscribe(sc.SubscribeTo, func(subject string, data []byte) {
    76→				select {
    77→				case msgs <- data:
    78→				case <-ctx.Done():
    79→				}
    80→			})
    81→		} else {
    82→			nc.Subscribe(sc.SubscribeTo, func(subject string, data []byte) {
    83→				dec := lineprotocol.NewDecoderWithBytes(data)
    84→				if err := DecodeLine(dec, ms, clusterTag); err != nil {
    85→					cclog.Errorf("error: %s", err.Error())
    86→				}
    87→			})
    88→		}
    89→		cclog.Infof("NATS subscription to '%s' established", sc.SubscribeTo)
    90→	}
    91→
    92→	go func() {
    93→		<-ctx.Done()
    94→		close(msgs)
    95→	}()
    96→
    97→	wg.Wait()
    98→
    99→	return nil
   100→}
   101→
   102→// reorder prepends prefix to buf in-place when buf has enough spare capacity,
   103→// avoiding an allocation. Falls back to a regular append otherwise.
   104→//
   105→// It is used to assemble the "type<type-id>" and "subtype<stype-id>" selector
   106→// strings when the type tag arrives before the type-id tag in the line, so the
   107→// two byte slices need to be concatenated in tag-declaration order regardless
   108→// of wire order.
   109→func reorder(buf, prefix []byte) []byte {
   110→	n := len(prefix)
   111→	m := len(buf)
   112→	if cap(buf) < m+n {
   113→		return append(prefix[:n:n], buf...)
   114→	} else {
   115→		buf = buf[:n+m]
   116→		for i := m - 1; i >= 0; i-- {
   117→			buf[i+n] = buf[i]
   118→		}
   119→		for i := range n {
   120→			buf[i] = prefix[i]
   121→		}
   122→		return buf
   123→	}
   124→}
   125→
   126→// decodeState holds the per-call scratch buffers used by DecodeLine.
   127→// Instances are recycled via decodeStatePool to avoid repeated allocations
   128→// during high-throughput ingestion.
   129→type decodeState struct {
   130→	// metricBuf holds a copy of the current measurement name (line-protocol
   131→	// measurement field). Copied because dec.Measurement() returns a slice
   132→	// that is invalidated by the next decoder call.
   133→	metricBuf []byte
   134→
   135→	// selector is the sub-device path passed to WriteToLevel and WALMessage
   136→	// (e.g. ["socket0"] or ["socket0", "memctrl1"]). Reused across lines.
   137→	selector []string
   138→
   139→	// typeBuf accumulates the concatenated "type"+"type-id" tag value for the
   140→	// current line. Reset at the start of each line's tag-decode loop.
   141→	typeBuf []byte
   142→
   143→	// subTypeBuf accumulates the concatenated "subtype"+"stype-id" tag value.
   144→	// Reset at the start of each line's tag-decode loop.
   145→	subTypeBuf []byte
   146→
   147→	// prevTypeBytes / prevTypeStr cache the last seen typeBuf content and its
   148→	// string conversion. Because consecutive lines in a batch typically address
   149→	// the same sub-device, the cache hit rate is very high and avoids
   150→	// repeated []byte→string allocations.
   151→	prevTypeBytes []byte
   152→	prevTypeStr   string
   153→
   154→	// prevSubTypeBytes / prevSubTypeStr are the same cache for the subtype.
   155→	prevSubTypeBytes []byte
   156→	prevSubTypeStr   string
   157→}
   158→
   159→// decodeStatePool recycles decodeState values across DecodeLine calls to
   160→// reduce GC pressure during sustained metric ingestion.
   161→var decodeStatePool = sync.Pool{
   162→	New: func() any {
   163→		return &decodeState{
   164→			metricBuf:  make([]byte, 0, 16),
   165→			selector:   make([]string, 0, 4),
   166→			typeBuf:    make([]byte, 0, 16),
   167→			subTypeBuf: make([]byte, 0, 16),
   168→		}
   169→	},
   170→}
   171→
   172→// DecodeLine reads all lines from dec (InfluxDB line-protocol) and writes each
   173→// decoded metric sample into ms.
   174→//
   175→// clusterDefault is used as the cluster name for lines that do not carry a
   176→// "cluster" tag. Callers typically supply the ClusterTag value from the NATS
   177→// subscription configuration.
   178→//
   179→// Performance notes:
   180→//   - A decodeState is obtained from decodeStatePool to reuse scratch buffers.
   181→//   - The Level pointer (host-level node in the metric tree) is cached across
   182→//     consecutive lines that share the same cluster+host pair to avoid
   183→//     repeated lock acquisitions on the root and cluster levels.
   184→//   - []byte→string conversions for type/subtype selectors are cached via
   185→//     prevType*/prevSubType* fields because batches typically repeat the same
   186→//     sub-device identifiers.
   187→//   - Timestamp parsing tries Second precision first; if that fails it retries
   188→//     Millisecond, Microsecond, and Nanosecond in turn. A missing timestamp
   189→//     falls back to time.Now().
   190→//
   191→// When the checkpoint format is "wal" each successfully decoded sample is also
   192→// sent to WALMessages so the WAL staging goroutine can persist it durably
   193→// before the next binary snapshot.
   194→func DecodeLine(dec *lineprotocol.Decoder,
   195→	ms *MemoryStore,
   196→	clusterDefault string,
   197→) error {
   198→	// Reduce allocations in loop:
   199→	t := time.Now()
   200→	metric := Metric{}
   201→	st := decodeStatePool.Get().(*decodeState)
   202→	defer decodeStatePool.Put(st)
   203→
   204→	// Optimize for the case where all lines in a "batch" are about the same
   205→	// cluster and host. By using `WriteToLevel` (level = host), we do not need
   206→	// to take the root- and cluster-level lock as often.
   207→	var lvl *Level = nil
   208→	prevCluster, prevHost := "", ""
   209→
   210→	var ok bool
   211→	for dec.Next() {
   212→		rawmeasurement, err := dec.Measurement()
   213→		if err != nil {
   214→			return err
   215→		}
   216→
   217→		// Needs to be copied because another call to dec.* would
   218→		// invalidate the returned slice.
   219→		st.metricBuf = append(st.metricBuf[:0], rawmeasurement...)
   220→
   221→		// The go compiler optimizes map[string(byteslice)] lookups:
   222→		metric.MetricConfig, ok = ms.Metrics[string(rawmeasurement)]
   223→		if !ok {
   224→			continue
   225→		}
   226→
   227→		st.typeBuf, st.subTypeBuf = st.typeBuf[:0], st.subTypeBuf[:0]
   228→		cluster, host := clusterDefault, ""
   229→		for {
   230→			key, val, err := dec.NextTag()
   231→			if err != nil {
   232→				return err
   233→			}
   234→			if key == nil {
   235→				break
   236→			}
   237→
   238→			// The go compiler optimizes string([]byte{...}) == "...":
   239→			switch string(key) {
   240→			case "cluster":
   241→				if string(val) == prevCluster {
   242→					cluster = prevCluster
   243→				} else {
   244→					cluster = string(val)
   245→					lvl = nil
   246→				}
   247→			case "hostname", "host":
   248→				if string(val) == prevHost {
   249→					host = prevHost
   250→				} else {
   251→					host = string(val)
   252→					lvl = nil
   253→				}
   254→			case "type":
   255→				if string(val) == "node" {
   256→					break
   257→				}
   258→
   259→				// We cannot be sure that the "type" tag comes before the "type-id" tag:
   260→				if len(st.typeBuf) == 0 {
   261→					st.typeBuf = append(st.typeBuf, val...)
   262→				} else {
   263→					st.typeBuf = reorder(st.typeBuf, val)
   264→				}
   265→			case "type-id":
   266→				st.typeBuf = append(st.typeBuf, val...)
   267→			case "subtype":
   268→				// We cannot be sure that the "subtype" tag comes before the "stype-id" tag:
   269→				if len(st.subTypeBuf) == 0 {
   270→					st.subTypeBuf = append(st.subTypeBuf, val...)
   271→				} else {
   272→					st.subTypeBuf = reorder(st.subTypeBuf, val)
   273→				}
   274→			case "stype-id":
   275→				st.subTypeBuf = append(st.subTypeBuf, val...)
   276→			default:
   277→			}
   278→		}
   279→
   280→		// If the cluster or host changed, the lvl was set to nil
   281→		if lvl == nil {
   282→			st.selector = st.selector[:2]
   283→			st.selector[0], st.selector[1] = cluster, host
   284→			lvl = ms.GetLevel(st.selector)
   285→			prevCluster, prevHost = cluster, host
   286→		}
   287→
   288→		// subtypes: cache []byte→string conversions; messages in a batch typically
   289→		// share the same type/subtype so the hit rate is very high.
   290→		st.selector = st.selector[:0]
   291→		if len(st.typeBuf) > 0 {
   292→			if !bytes.Equal(st.typeBuf, st.prevTypeBytes) {
   293→				st.prevTypeBytes = append(st.prevTypeBytes[:0], st.typeBuf...)
   294→				st.prevTypeStr = string(st.typeBuf)
   295→			}
   296→			st.selector = append(st.selector, st.prevTypeStr)
   297→			if len(st.subTypeBuf) > 0 {
   298→				if !bytes.Equal(st.subTypeBuf, st.prevSubTypeBytes) {
   299→					st.prevSubTypeBytes = append(st.prevSubTypeBytes[:0], st.subTypeBuf...)
   300→					st.prevSubTypeStr = string(st.subTypeBuf)
   301→				}
   302→				st.selector = append(st.selector, st.prevSubTypeStr)
   303→			}
   304→		}
   305→
   306→		for {
   307→			key, val, err := dec.NextField()
   308→			if err != nil {
   309→				return err
   310→			}
   311→
   312→			if key == nil {
   313→				break
   314→			}
   315→
   316→			if string(key) != "value" {
   317→				return fmt.Errorf("host %s: unknown field: '%s' (value: %#v)", host, string(key), val)
   318→			}
   319→
   320→			if val.Kind() == lineprotocol.Float {
   321→				metric.Value = schema.Float(val.FloatV())
   322→			} else if val.Kind() == lineprotocol.Int {
   323→				metric.Value = schema.Float(val.IntV())
   324→			} else if val.Kind() == lineprotocol.Uint {
   325→				metric.Value = schema.Float(val.UintV())
   326→			} else {
   327→				return fmt.Errorf("host %s: unsupported value type in message: %s", host, val.Kind().String())
   328→			}
   329→		}
   330→
   331→		if t, err = dec.Time(lineprotocol.Second, t); err != nil {
   332→			t = time.Now()
   333→			if t, err = dec.Time(lineprotocol.Millisecond, t); err != nil {
   334→				t = time.Now()
   335→				if t, err = dec.Time(lineprotocol.Microsecond, t); err != nil {
   336→					t = time.Now()
   337→					if t, err = dec.Time(lineprotocol.Nanosecond, t); err != nil {
   338→						return fmt.Errorf("host %s: timestamp : %#v with error : %#v", host, t, err.Error())
   339→					}
   340→				}
   341→			}
   342→		}
   343→
   344→		if err != nil {
   345→			return fmt.Errorf("host %s: timestamp : %#v with error : %#v", host, t, err.Error())
   346→		}
   347→
   348→		time := t.Unix()
   349→
   350→		if Keys.Checkpoints.FileFormat == "wal" {
   351→			WALMessages <- &WALMessage{
   352→				MetricName: string(st.metricBuf),
   353→				Cluster:    cluster,
   354→				Node:       host,
   355→				Selector:   append([]string{}, st.selector...),
   356→				Value:      metric.Value,
   357→				Timestamp:  time,
   358→			}
   359→		}
   360→
   361→		if err := ms.WriteToLevel(lvl, st.selector, time, []Metric{metric}); err != nil {
   362→			return err
   363→		}
   364→	}
   365→	return nil
   366→}
   367→

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

// This file implements ingestion of InfluxDB line-protocol metric data received
// over NATS. Each line encodes one metric sample with the following structure:
//
//	<measurement>[,cluster=<c>][,hostname=<h>][,type=<t>][,type-id=<id>][,subtype=<s>][,stype-id=<id>] value=<v> [<timestamp>]
//
// The measurement name identifies the metric (e.g. "cpu_load"). Tags provide
// routing information (cluster, host) and optional sub-device selectors (type,
// subtype). Only one field is expected per line: "value".
//
// After decoding, each sample is:
//  1. Written to the in-memory store via ms.WriteToLevel.
//  2. If the checkpoint format is "wal", also forwarded to the WAL staging
//     goroutine via the WALMessages channel for durable write-ahead logging.
package metricstore

import (
	"bytes"
	"context"
	"fmt"
	"sync"
	"time"

	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
	"github.com/ClusterCockpit/cc-lib/v2/nats"
	"github.com/ClusterCockpit/cc-lib/v2/schema"
	"github.com/ClusterCockpit/cc-line-protocol/v2/lineprotocol"
)

// ReceiveNats subscribes to all configured NATS subjects and feeds incoming
// line-protocol messages into the MemoryStore.
//
// When workers > 1 a pool of goroutines drains a shared channel so that
// multiple messages can be decoded in parallel. With workers == 1 the NATS
// callback decodes inline (no channel overhead, lower latency).
//
// The function blocks until ctx is cancelled and all worker goroutines have
// finished. It returns nil when the NATS client is not configured; callers
// should treat that as a no-op rather than an error.
func ReceiveNats(ms *MemoryStore,
	workers int,
	ctx context.Context,
) error {
	nc := nats.GetClient()

	if nc == nil {
		cclog.Warn("NATS client not initialized")
		return nil
	}

	var wg sync.WaitGroup
	msgs := make(chan []byte, workers*2)

	for _, sc := range *Keys.Subscriptions {
		clusterTag := sc.ClusterTag
		if workers > 1 {
			wg.Add(workers)

			for range workers {
				go func() {
					defer wg.Done()
					for m := range msgs {
						dec := lineprotocol.NewDecoderWithBytes(m)
						if err := DecodeLine(dec, ms, clusterTag); err != nil {
							cclog.Errorf("error: %s", err.Error())
						}
					}
				}()
			}

			nc.Subscribe(sc.SubscribeTo, func(subject string, data []byte) {
				select {
				case msgs <- data:
				case <-ctx.Done():
				}
			})
		} else {
			nc.Subscribe(sc.SubscribeTo, func(subject string, data []byte) {
				dec := lineprotocol.NewDecoderWithBytes(data)
				if err := DecodeLine(dec, ms, clusterTag); err != nil {
					cclog.Errorf("error: %s", err.Error())
				}
			})
		}
		cclog.Infof("NATS subscription to '%s' established", sc.SubscribeTo)
	}

	go func() {
		<-ctx.Done()
		close(msgs)
	}()

	wg.Wait()

	return nil
}

// reorder prepends prefix to buf in-place when buf has enough spare capacity,
// avoiding an allocation. Falls back to a regular append otherwise.
//
// It is used to assemble the "type<type-id>" and "subtype<stype-id>" selector
// strings when the type tag arrives before the type-id tag in the line, so the
// two byte slices need to be concatenated in tag-declaration order regardless
// of wire order.
func reorder(buf, prefix []byte) []byte {
	n := len(prefix)
	m := len(buf)
	if cap(buf) < m+n {
		return append(prefix[:n:n], buf...)
	} else {
		buf = buf[:n+m]
		for i := m - 1; i >= 0; i-- {
			buf[i+n] = buf[i]
		}
		for i := range n {
			buf[i] = prefix[i]
		}
		return buf
	}
}

// decodeState holds the per-call scratch buffers used by DecodeLine.
// Instances are recycled via decodeStatePool to avoid repeated allocations
// during high-throughput ingestion.
type decodeState struct {
	// metricBuf holds a copy of the current measurement name (line-protocol
	// measurement field). Copied because dec.Measurement() returns a slice
	// that is invalidated by the next decoder call.
	metricBuf []byte

	// selector is the sub-device path passed to WriteToLevel and WALMessage
	// (e.g. ["socket0"] or ["socket0", "memctrl1"]). Reused across lines.
	selector []string

	// typeBuf accumulates the concatenated "type"+"type-id" tag value for the
	// current line. Reset at the start of each line's tag-decode loop.
	typeBuf []byte

	// subTypeBuf accumulates the concatenated "subtype"+"stype-id" tag value.
	// Reset at the start of each line's tag-decode loop.
	subTypeBuf []byte

	// prevTypeBytes / prevTypeStr cache the last seen typeBuf content and its
	// string conversion. Because consecutive lines in a batch typically address
	// the same sub-device, the cache hit rate is very high and avoids
	// repeated []byte→string allocations.
	prevTypeBytes []byte
	prevTypeStr   string

	// prevSubTypeBytes / prevSubTypeStr are the same cache for the subtype.
	prevSubTypeBytes []byte
	prevSubTypeStr   string
}

// decodeStatePool recycles decodeState values across DecodeLine calls to
// reduce GC pressure during sustained metric ingestion.
var decodeStatePool = sync.Pool{
	New: func() any {
		return &decodeState{
			metricBuf:  make([]byte, 0, 16),
			selector:   make([]string, 0, 4),
			typeBuf:    make([]byte, 0, 16),
			subTypeBuf: make([]byte, 0, 16),
		}
	},
}

// DecodeLine reads all lines from dec (InfluxDB line-protocol) and writes each
// decoded metric sample into ms.
//
// clusterDefault is used as the cluster name for lines that do not carry a
// "cluster" tag. Callers typically supply the ClusterTag value from the NATS
// subscription configuration.
//
// Performance notes:
//   - A decodeState is obtained from decodeStatePool to reuse scratch buffers.
//   - The Level pointer (host-level node in the metric tree) is cached across
//     consecutive lines that share the same cluster+host pair to avoid
//     repeated lock acquisitions on the root and cluster levels.
//   - []byte→string conversions for type/subtype selectors are cached via
//     prevType*/prevSubType* fields because batches typically repeat the same
//     sub-device identifiers.
//   - Timestamp parsing tries Second precision first; if that fails it retries
//     Millisecond, Microsecond, and Nanosecond in turn. A missing timestamp
//     falls back to time.Now().
//
// When the checkpoint format is "wal" each successfully decoded sample is also
// sent to WALMessages so the WAL staging goroutine can persist it durably
// before the next binary snapshot.
func DecodeLine(dec *lineprotocol.Decoder,
	ms *MemoryStore,
	clusterDefault string,
) error {
	// Reduce allocations in loop:
	t := time.Now()
	metric := Metric{}
	st := decodeStatePool.Get().(*decodeState)
	defer decodeStatePool.Put(st)

	// Optimize for the case where all lines in a "batch" are about the same
	// cluster and host. By using `WriteToLevel` (level = host), we do not need
	// to take the root- and cluster-level lock as often.
	var lvl *Level = nil
	prevCluster, prevHost := "", ""

	var ok bool
	for dec.Next() {
		rawmeasurement, err := dec.Measurement()
		if err != nil {
			return err
		}

		// Needs to be copied because another call to dec.* would
		// invalidate the returned slice.
		st.metricBuf = append(st.metricBuf[:0], rawmeasurement...)

		// The go compiler optimizes map[string(byteslice)] lookups:
		metric.MetricConfig, ok = ms.Metrics[string(rawmeasurement)]
		if !ok {
			continue
		}

		st.typeBuf, st.subTypeBuf = st.typeBuf[:0], st.subTypeBuf[:0]
		cluster, host := clusterDefault, ""
		for {
			key, val, err := dec.NextTag()
			if err != nil {
				return err
			}
			if key == nil {
				break
			}

			// The go compiler optimizes string([]byte{...}) == "...":
			switch string(key) {
			case "cluster":
				if string(val) == prevCluster {
					cluster = prevCluster
				} else {
					cluster = string(val)
					lvl = nil
				}
			case "hostname", "host":
				if string(val) == prevHost {
					host = prevHost
				} else {
					host = string(val)
					lvl = nil
				}
			case "type":
				if string(val) == "node" {
					break
				}

				// We cannot be sure that the "type" tag comes before the "type-id" tag:
				if len(st.typeBuf) == 0 {
					st.typeBuf = append(st.typeBuf, val...)
				} else {
					st.typeBuf = reorder(st.typeBuf, val)
				}
			case "type-id":
				st.typeBuf = append(st.typeBuf, val...)
			case "subtype":
				// We cannot be sure that the "subtype" tag comes before the "stype-id" tag:
				if len(st.subTypeBuf) == 0 {
					st.subTypeBuf = append(st.subTypeBuf, val...)
				} else {
					st.subTypeBuf = reorder(st.subTypeBuf, val)
				}
			case "stype-id":
				st.subTypeBuf = append(st.subTypeBuf, val...)
			default:
			}
		}

		// If the cluster or host changed, the lvl was set to nil
		if lvl == nil {
			st.selector = st.selector[:2]
			st.selector[0], st.selector[1] = cluster, host
			lvl = ms.GetLevel(st.selector)
			prevCluster, prevHost = cluster, host
		}

		// subtypes: cache []byte→string conversions; messages in a batch typically
		// share the same type/subtype so the hit rate is very high.
		st.selector = st.selector[:0]
		if len(st.typeBuf) > 0 {
			if !bytes.Equal(st.typeBuf, st.prevTypeBytes) {
				st.prevTypeBytes = append(st.prevTypeBytes[:0], st.typeBuf...)
				st.prevTypeStr = string(st.typeBuf)
			}
			st.selector = append(st.selector, st.prevTypeStr)
			if len(st.subTypeBuf) > 0 {
				if !bytes.Equal(st.subTypeBuf, st.prevSubTypeBytes) {
					st.prevSubTypeBytes = append(st.prevSubTypeBytes[:0], st.subTypeBuf...)
					st.prevSubTypeStr = string(st.subTypeBuf)
				}
				st.selector = append(st.selector, st.prevSubTypeStr)
			}
		}

		for {
			key, val, err := dec.NextField()
			if err != nil {
				return err
			}

			if key == nil {
				break
			}

			if string(key) != "value" {
				return fmt.Errorf("host %s: unknown field: '%s' (value: %#v)", host, string(key), val)
			}

			if val.Kind() == lineprotocol.Float {
				metric.Value = schema.Float(val.FloatV())
			} else if val.Kind() == lineprotocol.Int {
				metric.Value = schema.Float(val.IntV())
			} else if val.Kind() == lineprotocol.Uint {
				metric.Value = schema.Float(val.UintV())
			} else {
				return fmt.Errorf("host %s: unsupported value type in message: %s", host, val.Kind().String())
			}
		}

		if t, err = dec.Time(lineprotocol.Second, t); err != nil {
			t = time.Now()
			if t, err = dec.Time(lineprotocol.Millisecond, t); err != nil {
				t = time.Now()
				if t, err = dec.Time(lineprotocol.Microsecond, t); err != nil {
					t = time.Now()
					if t, err = dec.Time(lineprotocol.Nanosecond, t); err != nil {
						return fmt.Errorf("host %s: timestamp : %#v with error : %#v", host, t, err.Error())
					}
				}
			}
		}

		if err != nil {
			return fmt.Errorf("host %s: timestamp : %#v with error : %#v", host, t, err.Error())
		}

		time := t.Unix()

		if Keys.Checkpoints.FileFormat == "wal" {
			WALMessages <- &WALMessage{
				MetricName: string(st.metricBuf),
				Cluster:    cluster,
				Node:       host,
				Selector:   append([]string{}, st.selector...),
				Value:      metric.Value,
				Timestamp:  time,
			}
		}

		if err := ms.WriteToLevel(lvl, st.selector, time, []Metric{metric}); err != nil {
			return err
		}
	}
	return nil
}


     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→// Package metricstore provides level.go: Hierarchical tree structure for metric storage.
     7→//
     8→// # Level Architecture
     9→//
    10→// The Level type forms a tree structure where each node represents a level in the
    11→// ClusterCockpit hierarchy: cluster → host → socket → core → hwthread, with special
    12→// nodes for memory domains and accelerators.
    13→//
    14→// Structure:
    15→//
    16→//	Root Level (cluster="emmy")
    17→//	├─ Level (host="node001")
    18→//	│  ├─ Level (socket="0")
    19→//	│  │  ├─ Level (core="0") [stores cpu0 metrics]
    20→//	│  │  └─ Level (core="1") [stores cpu1 metrics]
    21→//	│  └─ Level (socket="1")
    22→//	│     └─ ...
    23→//	└─ Level (host="node002")
    24→//	   └─ ...
    25→//
    26→// Each Level can:
    27→//   - Hold data (metrics slice of buffer pointers)
    28→//   - Have child nodes (children map[string]*Level)
    29→//   - Both simultaneously (inner nodes can store aggregated metrics)
    30→//
    31→// # Selector Paths
    32→//
    33→// Selectors are hierarchical paths: []string{"cluster", "host", "component"}.
    34→// Example: []string{"emmy", "node001", "cpu0"} navigates to the cpu0 core level.
    35→//
    36→// # Concurrency
    37→//
    38→// RWMutex protects children map and metrics slice. Read-heavy workload (metric reads)
    39→// uses RLock. Writes (new levels, buffer updates) use Lock. Double-checked locking
    40→// prevents races during level creation.
    41→package metricstore
    42→
    43→import (
    44→	"sync"
    45→	"time"
    46→	"unsafe"
    47→
    48→	"github.com/ClusterCockpit/cc-lib/v2/schema"
    49→	"github.com/ClusterCockpit/cc-lib/v2/util"
    50→)
    51→
    52→// Level represents a node in the hierarchical metric storage tree.
    53→//
    54→// Can be both a leaf or inner node. Inner nodes hold data in 'metrics' for aggregated
    55→// values (e.g., socket-level metrics derived from core-level data). Named "Level"
    56→// instead of "node" to avoid confusion with cluster nodes (hosts).
    57→//
    58→// Fields:
    59→//   - children: Map of child level names to Level pointers (e.g., "cpu0" → Level)
    60→//   - metrics:  Slice of buffer pointers (one per metric, indexed by MetricConfig.offset)
    61→//   - lock:     RWMutex for concurrent access (read-heavy, write-rare)
    62→type Level struct {
    63→	children map[string]*Level
    64→	metrics  []*buffer
    65→	lock     sync.RWMutex
    66→}
    67→
    68→// findLevelOrCreate navigates to or creates the level specified by selector.
    69→//
    70→// Recursively descends the tree, creating missing levels as needed. Uses double-checked
    71→// locking: RLock first (fast path), then Lock if creation needed (slow path), then
    72→// re-check after acquiring Lock to handle races.
    73→//
    74→// Example selector: []string{"emmy", "node001", "cpu0"}
    75→// Navigates: root → emmy → node001 → cpu0, creating levels as needed.
    76→//
    77→// Parameters:
    78→//   - selector: Hierarchical path (consumed recursively, decreasing depth)
    79→//   - nMetrics: Number of metric slots to allocate in new levels
    80→//
    81→// Returns:
    82→//   - *Level: The target level (existing or newly created)
    83→//
    84→// Note: sync.Map may improve performance for high-concurrency writes, but current
    85→// approach suffices for read-heavy workload.
    86→func (l *Level) findLevelOrCreate(selector []string, nMetrics int) *Level {
    87→	if len(selector) == 0 {
    88→		return l
    89→	}
    90→
    91→	// Allow concurrent reads:
    92→	l.lock.RLock()
    93→	var child *Level
    94→	var ok bool
    95→	if l.children == nil {
    96→		// Children map needs to be created...
    97→		l.lock.RUnlock()
    98→	} else {
    99→		child, ok = l.children[selector[0]]
   100→		l.lock.RUnlock()
   101→		if ok {
   102→			return child.findLevelOrCreate(selector[1:], nMetrics)
   103→		}
   104→	}
   105→
   106→	// The level does not exist, take write lock for unique access:
   107→	l.lock.Lock()
   108→	// While this thread waited for the write lock, another thread
   109→	// could have created the child node.
   110→	if l.children != nil {
   111→		child, ok = l.children[selector[0]]
   112→		if ok {
   113→			l.lock.Unlock()
   114→			return child.findLevelOrCreate(selector[1:], nMetrics)
   115→		}
   116→	}
   117→
   118→	child = &Level{
   119→		metrics:  make([]*buffer, nMetrics),
   120→		children: nil,
   121→	}
   122→
   123→	if l.children != nil {
   124→		l.children[selector[0]] = child
   125→	} else {
   126→		l.children = map[string]*Level{selector[0]: child}
   127→	}
   128→	l.lock.Unlock()
   129→	return child.findLevelOrCreate(selector[1:], nMetrics)
   130→}
   131→
   132→// collectPaths gathers all selector paths at the specified depth in the tree.
   133→//
   134→// Recursively traverses children, collecting paths when currentDepth+1 == targetDepth.
   135→// Each path is a selector that can be used with findLevel() or findBuffers().
   136→//
   137→// Explicitly copies slices to avoid shared underlying arrays between siblings, preventing
   138→// unintended mutations.
   139→//
   140→// Parameters:
   141→//   - currentDepth: Depth of current level (0 = root)
   142→//   - targetDepth:  Depth to collect paths from
   143→//   - currentPath:  Path accumulated so far
   144→//   - results:      Output slice (appended to)
   145→//
   146→// Example: collectPaths(0, 2, []string{}, &results) collects all 2-level paths
   147→// like []string{"emmy", "node001"}, []string{"emmy", "node002"}, etc.
   148→func (l *Level) collectPaths(currentDepth, targetDepth int, currentPath []string, results *[][]string) {
   149→	l.lock.RLock()
   150→	defer l.lock.RUnlock()
   151→
   152→	for key, child := range l.children {
   153→		if child == nil {
   154→			continue
   155→		}
   156→
   157→		// We explicitly make a new slice and copy data to avoid sharing underlying arrays between siblings
   158→		newPath := make([]string, len(currentPath))
   159→		copy(newPath, currentPath)
   160→		newPath = append(newPath, key)
   161→
   162→		// Check depth, and just return if depth reached
   163→		if currentDepth+1 == targetDepth {
   164→			*results = append(*results, newPath)
   165→		} else {
   166→			child.collectPaths(currentDepth+1, targetDepth, newPath, results)
   167→		}
   168→	}
   169→}
   170→
   171→// free removes buffers older than the retention threshold from the entire subtree.
   172→//
   173→// Recursively frees buffers in this level's metrics and all child levels. Buffers
   174→// with standard capacity (BufferCap) are returned to the pool. Called by the
   175→// retention worker to enforce retention policies.
   176→//
   177→// Parameters:
   178→//   - t: Retention threshold timestamp (Unix seconds)
   179→//
   180→// Returns:
   181→//   - int:   Total number of buffers freed in this subtree
   182→//   - error: Non-nil on failure (propagated from children)
   183→func (l *Level) free(t int64) (int, error) {
   184→	l.lock.Lock()
   185→	defer l.lock.Unlock()
   186→
   187→	n := 0
   188→	for i, b := range l.metrics {
   189→		if b != nil {
   190→			delme, m := b.free(t)
   191→			n += m
   192→			if delme {
   193→				if cap(b.data) != BufferCap {
   194→					b.data = make([]schema.Float, 0, BufferCap)
   195→				}
   196→				b.lastUsed = time.Now().Unix()
   197→				bufferPool.Put(b)
   198→				l.metrics[i] = nil
   199→			}
   200→		}
   201→	}
   202→
   203→	for _, l := range l.children {
   204→		m, err := l.free(t)
   205→		n += m
   206→		if err != nil {
   207→			return n, err
   208→		}
   209→	}
   210→
   211→	return n, nil
   212→}
   213→
   214→// forceFree removes the oldest buffer from each metric chain in the subtree.
   215→//
   216→// Unlike free(), which removes based on time threshold, this unconditionally removes
   217→// the oldest buffer in each chain. Used by MemoryUsageTracker when memory cap is
   218→// exceeded and time-based retention is insufficient.
   219→//
   220→// Recursively processes current level's metrics and all child levels.
   221→//
   222→// Returns:
   223→//   - int:   Total number of buffers freed in this subtree
   224→//   - error: Non-nil on failure (propagated from children)
   225→func (l *Level) forceFree() (int, error) {
   226→	l.lock.Lock()
   227→	defer l.lock.Unlock()
   228→
   229→	n := 0
   230→
   231→	// Iterate over metrics in the current level
   232→	for i, b := range l.metrics {
   233→		if b != nil {
   234→			// Attempt to free the oldest buffer in this chain
   235→			delme, freedCount := b.forceFreeOldest()
   236→			n += freedCount
   237→
   238→			// If delme is true, it means 'b' itself (the head) was the oldest
   239→			// and needs to be removed from the slice.
   240→			if delme {
   241→				b.next = nil
   242→				b.prev = nil
   243→				if cap(b.data) != BufferCap {
   244→					b.data = make([]schema.Float, 0, BufferCap)
   245→				}
   246→				b.lastUsed = time.Now().Unix()
   247→				bufferPool.Put(b)
   248→				l.metrics[i] = nil
   249→			}
   250→		}
   251→	}
   252→
   253→	// Recursively traverse children
   254→	for _, child := range l.children {
   255→		m, err := child.forceFree()
   256→		n += m
   257→		if err != nil {
   258→			return n, err
   259→		}
   260→	}
   261→
   262→	return n, nil
   263→}
   264→
   265→// sizeInBytes calculates the total memory usage of all buffers in the subtree.
   266→//
   267→// Recursively sums buffer data sizes (count of Float values × sizeof(Float)) across
   268→// this level's metrics and all child levels. Used by MemoryUsageTracker to enforce
   269→// memory cap limits.
   270→//
   271→// Returns:
   272→//   - int64: Total bytes used by buffer data in this subtree
   273→func (l *Level) sizeInBytes() int64 {
   274→	l.lock.RLock()
   275→	defer l.lock.RUnlock()
   276→	size := int64(0)
   277→
   278→	for _, b := range l.metrics {
   279→		if b != nil {
   280→			size += b.count() * int64(unsafe.Sizeof(schema.Float(0)))
   281→		}
   282→	}
   283→
   284→	for _, child := range l.children {
   285→		size += child.sizeInBytes()
   286→	}
   287→
   288→	return size
   289→}
   290→
   291→// findLevel navigates to the level specified by selector, returning nil if not found.
   292→//
   293→// Read-only variant of findLevelOrCreate. Does not create missing levels.
   294→// Recursively descends the tree following the selector path.
   295→//
   296→// Parameters:
   297→//   - selector: Hierarchical path (e.g., []string{"emmy", "node001", "cpu0"})
   298→//
   299→// Returns:
   300→//   - *Level: The target level, or nil if any component in the path does not exist
   301→func (l *Level) findLevel(selector []string) *Level {
   302→	if len(selector) == 0 {
   303→		return l
   304→	}
   305→
   306→	l.lock.RLock()
   307→	defer l.lock.RUnlock()
   308→
   309→	lvl := l.children[selector[0]]
   310→	if lvl == nil {
   311→		return nil
   312→	}
   313→
   314→	return lvl.findLevel(selector[1:])
   315→}
   316→
   317→// findBuffers invokes callback on all buffers matching the selector pattern.
   318→//
   319→// Supports flexible selector patterns (from cc-lib/util.Selector):
   320→//   - Exact match: Selector element with String set (e.g., "node001")
   321→//   - Group match: Selector element with Group set (e.g., ["cpu0", "cpu2", "cpu4"])
   322→//   - Wildcard:    Selector element with Any=true (matches all children)
   323→//
   324→// Empty selector (len==0) matches current level's buffer at 'offset' and recursively
   325→// all descendant buffers at the same offset (used for aggregation queries).
   326→//
   327→// Parameters:
   328→//   - selector: Pattern to match (consumed recursively)
   329→//   - offset:   Metric index in metrics slice (from MetricConfig.offset)
   330→//   - f:        Callback invoked on each matching buffer
   331→//
   332→// Returns:
   333→//   - error: First error returned by callback, or nil if all succeeded
   334→//
   335→// Example:
   336→//
   337→//	// Find all cpu0 buffers across all hosts:
   338→//	findBuffers([]Selector{{Any: true}, {String: "cpu0"}}, metricOffset, callback)
   339→func (l *Level) findBuffers(selector util.Selector, offset int, f func(b *buffer) error) error {
   340→	l.lock.RLock()
   341→	defer l.lock.RUnlock()
   342→
   343→	if len(selector) == 0 {
   344→		b := l.metrics[offset]
   345→		if b != nil {
   346→			return f(b)
   347→		}
   348→
   349→		for _, lvl := range l.children {
   350→			err := lvl.findBuffers(nil, offset, f)
   351→			if err != nil {
   352→				return err
   353→			}
   354→		}
   355→		return nil
   356→	}
   357→
   358→	sel := selector[0]
   359→	if len(sel.String) != 0 && l.children != nil {
   360→		lvl, ok := l.children[sel.String]
   361→		if ok {
   362→			err := lvl.findBuffers(selector[1:], offset, f)
   363→			if err != nil {
   364→				return err
   365→			}
   366→		}
   367→		return nil
   368→	}
   369→
   370→	if sel.Group != nil && l.children != nil {
   371→		for _, key := range sel.Group {
   372→			lvl, ok := l.children[key]
   373→			if ok {
   374→				err := lvl.findBuffers(selector[1:], offset, f)
   375→				if err != nil {
   376→					return err
   377→				}
   378→			}
   379→		}
   380→		return nil
   381→	}
   382→
   383→	if sel.Any && l.children != nil {
   384→		for _, lvl := range l.children {
   385→			if err := lvl.findBuffers(selector[1:], offset, f); err != nil {
   386→				return err
   387→			}
   388→		}
   389→		return nil
   390→	}
   391→
   392→	return nil
   393→}
   394→

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

// Package metricstore provides level.go: Hierarchical tree structure for metric storage.
//
// # Level Architecture
//
// The Level type forms a tree structure where each node represents a level in the
// ClusterCockpit hierarchy: cluster → host → socket → core → hwthread, with special
// nodes for memory domains and accelerators.
//
// Structure:
//
//	Root Level (cluster="emmy")
//	├─ Level (host="node001")
//	│  ├─ Level (socket="0")
//	│  │  ├─ Level (core="0") [stores cpu0 metrics]
//	│  │  └─ Level (core="1") [stores cpu1 metrics]
//	│  └─ Level (socket="1")
//	│     └─ ...
//	└─ Level (host="node002")
//	   └─ ...
//
// Each Level can:
//   - Hold data (metrics slice of buffer pointers)
//   - Have child nodes (children map[string]*Level)
//   - Both simultaneously (inner nodes can store aggregated metrics)
//
// # Selector Paths
//
// Selectors are hierarchical paths: []string{"cluster", "host", "component"}.
// Example: []string{"emmy", "node001", "cpu0"} navigates to the cpu0 core level.
//
// # Concurrency
//
// RWMutex protects children map and metrics slice. Read-heavy workload (metric reads)
// uses RLock. Writes (new levels, buffer updates) use Lock. Double-checked locking
// prevents races during level creation.
package metricstore

import (
	"sync"
	"time"
	"unsafe"

	"github.com/ClusterCockpit/cc-lib/v2/schema"
	"github.com/ClusterCockpit/cc-lib/v2/util"
)

// Level represents a node in the hierarchical metric storage tree.
//
// Can be both a leaf or inner node. Inner nodes hold data in 'metrics' for aggregated
// values (e.g., socket-level metrics derived from core-level data). Named "Level"
// instead of "node" to avoid confusion with cluster nodes (hosts).
//
// Fields:
//   - children: Map of child level names to Level pointers (e.g., "cpu0" → Level)
//   - metrics:  Slice of buffer pointers (one per metric, indexed by MetricConfig.offset)
//   - lock:     RWMutex for concurrent access (read-heavy, write-rare)
type Level struct {
	children map[string]*Level
	metrics  []*buffer
	lock     sync.RWMutex
}

// findLevelOrCreate navigates to or creates the level specified by selector.
//
// Recursively descends the tree, creating missing levels as needed. Uses double-checked
// locking: RLock first (fast path), then Lock if creation needed (slow path), then
// re-check after acquiring Lock to handle races.
//
// Example selector: []string{"emmy", "node001", "cpu0"}
// Navigates: root → emmy → node001 → cpu0, creating levels as needed.
//
// Parameters:
//   - selector: Hierarchical path (consumed recursively, decreasing depth)
//   - nMetrics: Number of metric slots to allocate in new levels
//
// Returns:
//   - *Level: The target level (existing or newly created)
//
// Note: sync.Map may improve performance for high-concurrency writes, but current
// approach suffices for read-heavy workload.
func (l *Level) findLevelOrCreate(selector []string, nMetrics int) *Level {
	if len(selector) == 0 {
		return l
	}

	// Allow concurrent reads:
	l.lock.RLock()
	var child *Level
	var ok bool
	if l.children == nil {
		// Children map needs to be created...
		l.lock.RUnlock()
	} else {
		child, ok = l.children[selector[0]]
		l.lock.RUnlock()
		if ok {
			return child.findLevelOrCreate(selector[1:], nMetrics)
		}
	}

	// The level does not exist, take write lock for unique access:
	l.lock.Lock()
	// While this thread waited for the write lock, another thread
	// could have created the child node.
	if l.children != nil {
		child, ok = l.children[selector[0]]
		if ok {
			l.lock.Unlock()
			return child.findLevelOrCreate(selector[1:], nMetrics)
		}
	}

	child = &Level{
		metrics:  make([]*buffer, nMetrics),
		children: nil,
	}

	if l.children != nil {
		l.children[selector[0]] = child
	} else {
		l.children = map[string]*Level{selector[0]: child}
	}
	l.lock.Unlock()
	return child.findLevelOrCreate(selector[1:], nMetrics)
}

// collectPaths gathers all selector paths at the specified depth in the tree.
//
// Recursively traverses children, collecting paths when currentDepth+1 == targetDepth.
// Each path is a selector that can be used with findLevel() or findBuffers().
//
// Explicitly copies slices to avoid shared underlying arrays between siblings, preventing
// unintended mutations.
//
// Parameters:
//   - currentDepth: Depth of current level (0 = root)
//   - targetDepth:  Depth to collect paths from
//   - currentPath:  Path accumulated so far
//   - results:      Output slice (appended to)
//
// Example: collectPaths(0, 2, []string{}, &results) collects all 2-level paths
// like []string{"emmy", "node001"}, []string{"emmy", "node002"}, etc.
func (l *Level) collectPaths(currentDepth, targetDepth int, currentPath []string, results *[][]string) {
	l.lock.RLock()
	defer l.lock.RUnlock()

	for key, child := range l.children {
		if child == nil {
			continue
		}

		// We explicitly make a new slice and copy data to avoid sharing underlying arrays between siblings
		newPath := make([]string, len(currentPath))
		copy(newPath, currentPath)
		newPath = append(newPath, key)

		// Check depth, and just return if depth reached
		if currentDepth+1 == targetDepth {
			*results = append(*results, newPath)
		} else {
			child.collectPaths(currentDepth+1, targetDepth, newPath, results)
		}
	}
}

// free removes buffers older than the retention threshold from the entire subtree.
//
// Recursively frees buffers in this level's metrics and all child levels. Buffers
// with standard capacity (BufferCap) are returned to the pool. Called by the
// retention worker to enforce retention policies.
//
// Parameters:
//   - t: Retention threshold timestamp (Unix seconds)
//
// Returns:
//   - int:   Total number of buffers freed in this subtree
//   - error: Non-nil on failure (propagated from children)
func (l *Level) free(t int64) (int, error) {
	l.lock.Lock()
	defer l.lock.Unlock()

	n := 0
	for i, b := range l.metrics {
		if b != nil {
			delme, m := b.free(t)
			n += m
			if delme {
				if cap(b.data) != BufferCap {
					b.data = make([]schema.Float, 0, BufferCap)
				}
				b.lastUsed = time.Now().Unix()
				bufferPool.Put(b)
				l.metrics[i] = nil
			}
		}
	}

	for _, l := range l.children {
		m, err := l.free(t)
		n += m
		if err != nil {
			return n, err
		}
	}

	return n, nil
}

// forceFree removes the oldest buffer from each metric chain in the subtree.
//
// Unlike free(), which removes based on time threshold, this unconditionally removes
// the oldest buffer in each chain. Used by MemoryUsageTracker when memory cap is
// exceeded and time-based retention is insufficient.
//
// Recursively processes current level's metrics and all child levels.
//
// Returns:
//   - int:   Total number of buffers freed in this subtree
//   - error: Non-nil on failure (propagated from children)
func (l *Level) forceFree() (int, error) {
	l.lock.Lock()
	defer l.lock.Unlock()

	n := 0

	// Iterate over metrics in the current level
	for i, b := range l.metrics {
		if b != nil {
			// Attempt to free the oldest buffer in this chain
			delme, freedCount := b.forceFreeOldest()
			n += freedCount

			// If delme is true, it means 'b' itself (the head) was the oldest
			// and needs to be removed from the slice.
			if delme {
				b.next = nil
				b.prev = nil
				if cap(b.data) != BufferCap {
					b.data = make([]schema.Float, 0, BufferCap)
				}
				b.lastUsed = time.Now().Unix()
				bufferPool.Put(b)
				l.metrics[i] = nil
			}
		}
	}

	// Recursively traverse children
	for _, child := range l.children {
		m, err := child.forceFree()
		n += m
		if err != nil {
			return n, err
		}
	}

	return n, nil
}

// sizeInBytes calculates the total memory usage of all buffers in the subtree.
//
// Recursively sums buffer data sizes (count of Float values × sizeof(Float)) across
// this level's metrics and all child levels. Used by MemoryUsageTracker to enforce
// memory cap limits.
//
// Returns:
//   - int64: Total bytes used by buffer data in this subtree
func (l *Level) sizeInBytes() int64 {
	l.lock.RLock()
	defer l.lock.RUnlock()
	size := int64(0)

	for _, b := range l.metrics {
		if b != nil {
			size += b.count() * int64(unsafe.Sizeof(schema.Float(0)))
		}
	}

	for _, child := range l.children {
		size += child.sizeInBytes()
	}

	return size
}

// findLevel navigates to the level specified by selector, returning nil if not found.
//
// Read-only variant of findLevelOrCreate. Does not create missing levels.
// Recursively descends the tree following the selector path.
//
// Parameters:
//   - selector: Hierarchical path (e.g., []string{"emmy", "node001", "cpu0"})
//
// Returns:
//   - *Level: The target level, or nil if any component in the path does not exist
func (l *Level) findLevel(selector []string) *Level {
	if len(selector) == 0 {
		return l
	}

	l.lock.RLock()
	defer l.lock.RUnlock()

	lvl := l.children[selector[0]]
	if lvl == nil {
		return nil
	}

	return lvl.findLevel(selector[1:])
}

// findBuffers invokes callback on all buffers matching the selector pattern.
//
// Supports flexible selector patterns (from cc-lib/util.Selector):
//   - Exact match: Selector element with String set (e.g., "node001")
//   - Group match: Selector element with Group set (e.g., ["cpu0", "cpu2", "cpu4"])
//   - Wildcard:    Selector element with Any=true (matches all children)
//
// Empty selector (len==0) matches current level's buffer at 'offset' and recursively
// all descendant buffers at the same offset (used for aggregation queries).
//
// Parameters:
//   - selector: Pattern to match (consumed recursively)
//   - offset:   Metric index in metrics slice (from MetricConfig.offset)
//   - f:        Callback invoked on each matching buffer
//
// Returns:
//   - error: First error returned by callback, or nil if all succeeded
//
// Example:
//
//	// Find all cpu0 buffers across all hosts:
//	findBuffers([]Selector{{Any: true}, {String: "cpu0"}}, metricOffset, callback)
func (l *Level) findBuffers(selector util.Selector, offset int, f func(b *buffer) error) error {
	l.lock.RLock()
	defer l.lock.RUnlock()

	if len(selector) == 0 {
		b := l.metrics[offset]
		if b != nil {
			return f(b)
		}

		for _, lvl := range l.children {
			err := lvl.findBuffers(nil, offset, f)
			if err != nil {
				return err
			}
		}
		return nil
	}

	sel := selector[0]
	if len(sel.String) != 0 && l.children != nil {
		lvl, ok := l.children[sel.String]
		if ok {
			err := lvl.findBuffers(selector[1:], offset, f)
			if err != nil {
				return err
			}
		}
		return nil
	}

	if sel.Group != nil && l.children != nil {
		for _, key := range sel.Group {
			lvl, ok := l.children[key]
			if ok {
				err := lvl.findBuffers(selector[1:], offset, f)
				if err != nil {
					return err
				}
			}
		}
		return nil
	}

	if sel.Any && l.children != nil {
		for _, lvl := range l.children {
			if err := lvl.findBuffers(selector[1:], offset, f); err != nil {
				return err
			}
		}
		return nil
	}

	return nil
}


     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→// Package metricstore provides buffer.go: Time-series data buffer implementation.
     7→//
     8→// # Buffer Architecture
     9→//
    10→// Each metric at each hierarchical level (cluster/host/cpu/etc.) uses a linked-list
    11→// chain of fixed-size buffers to store time-series data. This design:
    12→//
    13→//   - Avoids reallocation/copying when growing (new links added instead)
    14→//   - Enables efficient pooling (buffers returned to sync.Pool)
    15→//   - Supports traversal back in time (via prev pointers)
    16→//   - Maintains temporal ordering (newer data in later buffers)
    17→//
    18→// # Buffer Chain Example
    19→//
    20→//	[oldest buffer] <- prev -- [older] <- prev -- [newest buffer (head)]
    21→//	  start=1000               start=1512           start=2024
    22→//	  data=[v0...v511]         data=[v0...v511]     data=[v0...v42]
    23→//
    24→// When the head buffer reaches capacity (BufferCap = 512), a new buffer becomes
    25→// the new head and the old head is linked via prev.
    26→//
    27→// # Pooling Strategy
    28→//
    29→// sync.Pool reduces GC pressure for the common case (BufferCap-sized allocations).
    30→// Non-standard capacity buffers are not pooled (e.g., from checkpoint deserialization).
    31→//
    32→// # Time Alignment
    33→//
    34→// Timestamps are aligned to measurement frequency intervals:
    35→//
    36→//	index = (timestamp - buffer.start) / buffer.frequency
    37→//	actualTime = buffer.start + (frequency / 2) + (index * frequency)
    38→//
    39→// Missing data points are represented as NaN values. The read() function performs
    40→// linear interpolation where possible.
    41→package metricstore
    42→
    43→import (
    44→	"errors"
    45→	"sync"
    46→	"time"
    47→
    48→	"github.com/ClusterCockpit/cc-lib/v2/schema"
    49→)
    50→
    51→// BufferCap is the default buffer capacity.
    52→// buffer.data will only ever grow up to its capacity and a new link
    53→// in the buffer chain will be created if needed so that no copying
    54→// of data or reallocation needs to happen on writes.
    55→const BufferCap int = DefaultBufferCapacity
    56→
    57→// BufferPool is the global instance.
    58→// It is initialized immediately when the package loads.
    59→var bufferPool = NewPersistentBufferPool()
    60→
    61→type PersistentBufferPool struct {
    62→	pool []*buffer
    63→	mu   sync.Mutex
    64→}
    65→
    66→// NewPersistentBufferPool creates a dynamic pool for buffers.
    67→func NewPersistentBufferPool() *PersistentBufferPool {
    68→	return &PersistentBufferPool{
    69→		pool: make([]*buffer, 0),
    70→	}
    71→}
    72→
    73→func (p *PersistentBufferPool) Get() *buffer {
    74→	p.mu.Lock()
    75→	defer p.mu.Unlock()
    76→
    77→	n := len(p.pool)
    78→	if n == 0 {
    79→		// Pool is empty, allocate a new one
    80→		return &buffer{
    81→			data: make([]schema.Float, 0, BufferCap),
    82→		}
    83→	}
    84→
    85→	// Reuse existing buffer from the pool
    86→	b := p.pool[n-1]
    87→	p.pool[n-1] = nil // Avoid memory leak
    88→	p.pool = p.pool[:n-1]
    89→	return b
    90→}
    91→
    92→// Put returns b to the pool. The caller must set b.lastUsed = time.Now().Unix()
    93→// before calling Put so that Clean() can evict idle entries correctly.
    94→func (p *PersistentBufferPool) Put(b *buffer) {
    95→	// Reset the buffer before putting it back
    96→	b.data = b.data[:0]
    97→
    98→	p.mu.Lock()
    99→	defer p.mu.Unlock()
   100→
   101→	p.pool = append(p.pool, b)
   102→}
   103→
   104→// GetSize returns the exact number of buffers currently sitting in the pool.
   105→func (p *PersistentBufferPool) GetSize() int {
   106→	p.mu.Lock()
   107→	defer p.mu.Unlock()
   108→	return len(p.pool)
   109→}
   110→
   111→// Clear drains all buffers currently in the pool, allowing the GC to collect them.
   112→func (p *PersistentBufferPool) Clear() {
   113→	p.mu.Lock()
   114→	defer p.mu.Unlock()
   115→	for i := range p.pool {
   116→		p.pool[i] = nil
   117→	}
   118→	p.pool = p.pool[:0]
   119→}
   120→
   121→// Clean removes buffers from the pool that haven't been used in the given duration.
   122→// It uses a simple LRU approach based on the lastUsed timestamp.
   123→func (p *PersistentBufferPool) Clean(threshold int64) {
   124→	p.mu.Lock()
   125→	defer p.mu.Unlock()
   126→
   127→	// Filter in place, retaining only buffers returned to the pool recently enough.
   128→	active := p.pool[:0]
   129→	for _, b := range p.pool {
   130→		if b.lastUsed >= threshold {
   131→			active = append(active, b)
   132→		}
   133→	}
   134→
   135→	// Nullify the rest to prevent memory leaks
   136→	for i := len(active); i < len(p.pool); i++ {
   137→		p.pool[i] = nil
   138→	}
   139→
   140→	p.pool = active
   141→}
   142→
   143→var (
   144→	// ErrNoData indicates no time-series data exists for the requested metric/level.
   145→	ErrNoData error = errors.New("[METRICSTORE]> no data for this metric/level")
   146→
   147→	// ErrDataDoesNotAlign indicates that aggregated data from child scopes
   148→	// does not align with the parent scope's expected timestamps/intervals.
   149→	ErrDataDoesNotAlign error = errors.New("[METRICSTORE]> data from lower granularities does not align")
   150→)
   151→
   152→// buffer stores time-series data for a single metric at a specific hierarchical level.
   153→//
   154→// Buffers form doubly-linked chains ordered by time. When capacity is reached,
   155→// a new buffer becomes the head and the old head is linked via prev/next.
   156→//
   157→// Fields:
   158→//   - prev:      Link to older buffer in the chain (nil if this is oldest)
   159→//   - next:      Link to newer buffer in the chain (nil if this is newest/head)
   160→//   - data:      Time-series values (schema.Float supports NaN for missing data)
   161→//   - frequency: Measurement interval in seconds
   162→//   - start:     Start timestamp (adjusted by -frequency/2 for alignment)
   163→//   - archived:  True if data has been persisted to disk archive
   164→//   - closed:    True if buffer is no longer accepting writes
   165→//
   166→// Index calculation: index = (timestamp - start) / frequency
   167→// Actual data timestamp: start + (frequency / 2) + (index * frequency)
   168→type buffer struct {
   169→	prev      *buffer
   170→	next      *buffer
   171→	data      []schema.Float
   172→	frequency int64
   173→	start     int64
   174→	archived  bool
   175→	closed    bool
   176→	lastUsed  int64
   177→}
   178→
   179→func newBuffer(ts, freq int64) *buffer {
   180→	b := bufferPool.Get()
   181→	b.frequency = freq
   182→	b.start = ts - (freq / 2)
   183→	b.prev = nil
   184→	b.next = nil
   185→	b.archived = false
   186→	b.closed = false
   187→	b.data = b.data[:0]
   188→	return b
   189→}
   190→
   191→// write appends a timestamped value to the buffer chain.
   192→//
   193→// Returns the head buffer (which may be newly created if capacity was reached).
   194→// Timestamps older than the buffer's start are rejected. If the calculated index
   195→// exceeds capacity, a new buffer is allocated and linked as the new head.
   196→//
   197→// Missing timestamps are automatically filled with NaN values to maintain alignment.
   198→// Overwrites are allowed if the index is already within the existing data slice.
   199→//
   200→// Parameters:
   201→//   - ts:    Unix timestamp in seconds
   202→//   - value: Metric value (can be schema.NaN for missing data)
   203→//
   204→// Returns:
   205→//   - *buffer: The new head buffer (same as b if no new buffer created)
   206→//   - error:   Non-nil if timestamp is before buffer start
   207→func (b *buffer) write(ts int64, value schema.Float) (*buffer, error) {
   208→	if ts < b.start {
   209→		return nil, errors.New("[METRICSTORE]> cannot write value to buffer from past")
   210→	}
   211→
   212→	// idx := int((ts - b.start + (b.frequency / 3)) / b.frequency)
   213→	idx := int((ts - b.start) / b.frequency)
   214→	if idx >= cap(b.data) {
   215→		newbuf := newBuffer(ts, b.frequency)
   216→		newbuf.prev = b
   217→		b.next = newbuf
   218→		b = newbuf
   219→		idx = 0
   220→	}
   221→
   222→	// Overwriting value or writing value from past
   223→	if idx < len(b.data) {
   224→		b.data[idx] = value
   225→		return b, nil
   226→	}
   227→
   228→	// Fill up unwritten slots with NaN
   229→	for i := len(b.data); i < idx; i++ {
   230→		b.data = append(b.data, schema.NaN)
   231→	}
   232→
   233→	b.data = append(b.data, value)
   234→	return b, nil
   235→}
   236→
   237→func (b *buffer) end() int64 {
   238→	return b.firstWrite() + int64(len(b.data))*b.frequency
   239→}
   240→
   241→func (b *buffer) firstWrite() int64 {
   242→	return b.start + (b.frequency / 2)
   243→}
   244→
   245→// read retrieves time-series data from the buffer chain for the specified time range.
   246→//
   247→// Traverses the buffer chain backwards (via prev links) if 'from' precedes the current
   248→// buffer's start. Missing data points are represented as NaN. Values are accumulated
   249→// into the provided 'data' slice (using +=, so caller must zero-initialize if needed).
   250→//
   251→// The function adjusts the actual time range returned if data is unavailable at the
   252→// boundaries (returned via adjusted from/to timestamps).
   253→//
   254→// Parameters:
   255→//   - from: Start timestamp (Unix seconds)
   256→//   - to:   End timestamp (Unix seconds, exclusive)
   257→//   - data: Pre-allocated slice to accumulate results (must be large enough)
   258→//
   259→// Returns:
   260→//   - []schema.Float: Slice of data (may be shorter than input 'data' slice)
   261→//   - int64:          Actual start timestamp with available data
   262→//   - int64:          Actual end timestamp (exclusive)
   263→//   - error:          Non-nil on failure
   264→//
   265→// Panics if 'data' slice is too small to hold all values in [from, to).
   266→func (b *buffer) read(from, to int64, data []schema.Float) ([]schema.Float, int64, int64, error) {
   267→	// Walk back to the buffer that covers 'from', adjusting if we hit the oldest.
   268→	for from < b.firstWrite() {
   269→		if b.prev == nil {
   270→			from = b.firstWrite()
   271→			break
   272→		}
   273→		b = b.prev
   274→	}
   275→
   276→	i := 0
   277→	t := from
   278→	for ; t < to; t += b.frequency {
   279→		idx := int((t - b.start) / b.frequency)
   280→		if idx >= cap(b.data) {
   281→			if b.next == nil {
   282→				break
   283→			}
   284→			b = b.next
   285→			// Recalculate idx in the new buffer; a gap between buffers may exist.
   286→			idx = int((t - b.start) / b.frequency)
   287→		}
   288→
   289→		if idx >= len(b.data) {
   290→			if b.next == nil || to <= b.next.start {
   291→				break
   292→			}
   293→			data[i] += schema.NaN // NaN + anything = NaN; propagates missing data
   294→		} else if t < b.start {
   295→			data[i] += schema.NaN // gap before this buffer's first write
   296→		} else {
   297→			data[i] += b.data[idx]
   298→		}
   299→		i++
   300→	}
   301→
   302→	return data[:i], from, t, nil
   303→}
   304→
   305→// free removes buffers older than the specified timestamp from the chain.
   306→//
   307→// Recursively traverses backwards (via prev) and unlinks buffers whose end time
   308→// is before the retention threshold. Freed buffers are returned to the pool if
   309→// they have the standard capacity (BufferCap).
   310→//
   311→// Parameters:
   312→//   - t: Retention threshold timestamp (Unix seconds)
   313→//
   314→// Returns:
   315→//   - delme: True if the current buffer itself should be deleted by caller
   316→//   - n:     Number of buffers freed in this subtree
   317→func (b *buffer) free(t int64) (delme bool, n int) {
   318→	if b.prev != nil {
   319→		delme, m := b.prev.free(t)
   320→		n += m
   321→		if delme {
   322→			b.prev.next = nil
   323→			if cap(b.prev.data) != BufferCap {
   324→				b.prev.data = make([]schema.Float, 0, BufferCap)
   325→			}
   326→			b.prev.lastUsed = time.Now().Unix()
   327→			bufferPool.Put(b.prev)
   328→			b.prev = nil
   329→		}
   330→	}
   331→
   332→	end := b.end()
   333→	if end < t {
   334→		return true, n + 1
   335→	}
   336→
   337→	return false, n
   338→}
   339→
   340→// forceFreeOldest recursively finds the end of the linked list (the oldest buffer)
   341→// and removes it.
   342→// Returns:
   343→//
   344→//	delme: true if 'b' itself is the oldest and should be removed by the caller
   345→//	n:     the number of buffers freed (will be 1 or 0)
   346→func (b *buffer) forceFreeOldest() (delme bool, n int) {
   347→	// If there is a previous buffer, recurse down to find the oldest
   348→	if b.prev != nil {
   349→		delPrev, freed := b.prev.forceFreeOldest()
   350→
   351→		// If the previous buffer signals it should be deleted:
   352→		if delPrev {
   353→			b.prev.next = nil
   354→			if cap(b.prev.data) != BufferCap {
   355→				b.prev.data = make([]schema.Float, 0, BufferCap)
   356→			}
   357→			b.prev.lastUsed = time.Now().Unix()
   358→			bufferPool.Put(b.prev)
   359→			b.prev = nil
   360→		}
   361→		return false, freed
   362→	}
   363→
   364→	// If b.prev is nil, THIS buffer is the oldest.
   365→	// We return true so the parent (or the Level loop) knows to delete reference to 'b'.
   366→	return true, 1
   367→}
   368→
   369→// iterFromTo invokes callback on every buffer in the chain that overlaps [from, to].
   370→//
   371→// Traverses backwards (via prev) first, then processes current buffer if it overlaps
   372→// the time range. Used for checkpoint/archive operations that need to serialize buffers
   373→// within a specific time window.
   374→//
   375→// Parameters:
   376→//   - from:     Start timestamp (Unix seconds, inclusive)
   377→//   - to:       End timestamp (Unix seconds, inclusive)
   378→//   - callback: Function to invoke on each overlapping buffer
   379→//
   380→// Returns:
   381→//   - error: First error returned by callback, or nil if all succeeded
   382→func (b *buffer) iterFromTo(from, to int64, callback func(b *buffer) error) error {
   383→	if b == nil {
   384→		return nil
   385→	}
   386→
   387→	// Collect overlapping buffers walking backwards (newest → oldest).
   388→	var matching []*buffer
   389→	for cur := b; cur != nil; cur = cur.prev {
   390→		if from <= cur.end() && cur.start <= to {
   391→			matching = append(matching, cur)
   392→		}
   393→	}
   394→
   395→	// Invoke callback in chronological order (oldest → newest).
   396→	for i := len(matching) - 1; i >= 0; i-- {
   397→		if err := callback(matching[i]); err != nil {
   398→			return err
   399→		}
   400→	}
   401→	return nil
   402→}
   403→
   404→func (b *buffer) count() int64 {
   405→	var res int64
   406→	for ; b != nil; b = b.prev {
   407→		res += int64(len(b.data))
   408→	}
   409→	return res
   410→}
   411→

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

// Package metricstore provides buffer.go: Time-series data buffer implementation.
//
// # Buffer Architecture
//
// Each metric at each hierarchical level (cluster/host/cpu/etc.) uses a linked-list
// chain of fixed-size buffers to store time-series data. This design:
//
//   - Avoids reallocation/copying when growing (new links added instead)
//   - Enables efficient pooling (buffers returned to sync.Pool)
//   - Supports traversal back in time (via prev pointers)
//   - Maintains temporal ordering (newer data in later buffers)
//
// # Buffer Chain Example
//
//	[oldest buffer] <- prev -- [older] <- prev -- [newest buffer (head)]
//	  start=1000               start=1512           start=2024
//	  data=[v0...v511]         data=[v0...v511]     data=[v0...v42]
//
// When the head buffer reaches capacity (BufferCap = 512), a new buffer becomes
// the new head and the old head is linked via prev.
//
// # Pooling Strategy
//
// sync.Pool reduces GC pressure for the common case (BufferCap-sized allocations).
// Non-standard capacity buffers are not pooled (e.g., from checkpoint deserialization).
//
// # Time Alignment
//
// Timestamps are aligned to measurement frequency intervals:
//
//	index = (timestamp - buffer.start) / buffer.frequency
//	actualTime = buffer.start + (frequency / 2) + (index * frequency)
//
// Missing data points are represented as NaN values. The read() function performs
// linear interpolation where possible.
package metricstore

import (
	"errors"
	"sync"
	"time"

	"github.com/ClusterCockpit/cc-lib/v2/schema"
)

// BufferCap is the default buffer capacity.
// buffer.data will only ever grow up to its capacity and a new link
// in the buffer chain will be created if needed so that no copying
// of data or reallocation needs to happen on writes.
const BufferCap int = DefaultBufferCapacity

// BufferPool is the global instance.
// It is initialized immediately when the package loads.
var bufferPool = NewPersistentBufferPool()

type PersistentBufferPool struct {
	pool []*buffer
	mu   sync.Mutex
}

// NewPersistentBufferPool creates a dynamic pool for buffers.
func NewPersistentBufferPool() *PersistentBufferPool {
	return &PersistentBufferPool{
		pool: make([]*buffer, 0),
	}
}

func (p *PersistentBufferPool) Get() *buffer {
	p.mu.Lock()
	defer p.mu.Unlock()

	n := len(p.pool)
	if n == 0 {
		// Pool is empty, allocate a new one
		return &buffer{
			data: make([]schema.Float, 0, BufferCap),
		}
	}

	// Reuse existing buffer from the pool
	b := p.pool[n-1]
	p.pool[n-1] = nil // Avoid memory leak
	p.pool = p.pool[:n-1]
	return b
}

// Put returns b to the pool. The caller must set b.lastUsed = time.Now().Unix()
// before calling Put so that Clean() can evict idle entries correctly.
func (p *PersistentBufferPool) Put(b *buffer) {
	// Reset the buffer before putting it back
	b.data = b.data[:0]

	p.mu.Lock()
	defer p.mu.Unlock()

	p.pool = append(p.pool, b)
}

// GetSize returns the exact number of buffers currently sitting in the pool.
func (p *PersistentBufferPool) GetSize() int {
	p.mu.Lock()
	defer p.mu.Unlock()
	return len(p.pool)
}

// Clear drains all buffers currently in the pool, allowing the GC to collect them.
func (p *PersistentBufferPool) Clear() {
	p.mu.Lock()
	defer p.mu.Unlock()
	for i := range p.pool {
		p.pool[i] = nil
	}
	p.pool = p.pool[:0]
}

// Clean removes buffers from the pool that haven't been used in the given duration.
// It uses a simple LRU approach based on the lastUsed timestamp.
func (p *PersistentBufferPool) Clean(threshold int64) {
	p.mu.Lock()
	defer p.mu.Unlock()

	// Filter in place, retaining only buffers returned to the pool recently enough.
	active := p.pool[:0]
	for _, b := range p.pool {
		if b.lastUsed >= threshold {
			active = append(active, b)
		}
	}

	// Nullify the rest to prevent memory leaks
	for i := len(active); i < len(p.pool); i++ {
		p.pool[i] = nil
	}

	p.pool = active
}

var (
	// ErrNoData indicates no time-series data exists for the requested metric/level.
	ErrNoData error = errors.New("[METRICSTORE]> no data for this metric/level")

	// ErrDataDoesNotAlign indicates that aggregated data from child scopes
	// does not align with the parent scope's expected timestamps/intervals.
	ErrDataDoesNotAlign error = errors.New("[METRICSTORE]> data from lower granularities does not align")
)

// buffer stores time-series data for a single metric at a specific hierarchical level.
//
// Buffers form doubly-linked chains ordered by time. When capacity is reached,
// a new buffer becomes the head and the old head is linked via prev/next.
//
// Fields:
//   - prev:      Link to older buffer in the chain (nil if this is oldest)
//   - next:      Link to newer buffer in the chain (nil if this is newest/head)
//   - data:      Time-series values (schema.Float supports NaN for missing data)
//   - frequency: Measurement interval in seconds
//   - start:     Start timestamp (adjusted by -frequency/2 for alignment)
//   - archived:  True if data has been persisted to disk archive
//   - closed:    True if buffer is no longer accepting writes
//
// Index calculation: index = (timestamp - start) / frequency
// Actual data timestamp: start + (frequency / 2) + (index * frequency)
type buffer struct {
	prev      *buffer
	next      *buffer
	data      []schema.Float
	frequency int64
	start     int64
	archived  bool
	closed    bool
	lastUsed  int64
}

func newBuffer(ts, freq int64) *buffer {
	b := bufferPool.Get()
	b.frequency = freq
	b.start = ts - (freq / 2)
	b.prev = nil
	b.next = nil
	b.archived = false
	b.closed = false
	b.data = b.data[:0]
	return b
}

// write appends a timestamped value to the buffer chain.
//
// Returns the head buffer (which may be newly created if capacity was reached).
// Timestamps older than the buffer's start are rejected. If the calculated index
// exceeds capacity, a new buffer is allocated and linked as the new head.
//
// Missing timestamps are automatically filled with NaN values to maintain alignment.
// Overwrites are allowed if the index is already within the existing data slice.
//
// Parameters:
//   - ts:    Unix timestamp in seconds
//   - value: Metric value (can be schema.NaN for missing data)
//
// Returns:
//   - *buffer: The new head buffer (same as b if no new buffer created)
//   - error:   Non-nil if timestamp is before buffer start
func (b *buffer) write(ts int64, value schema.Float) (*buffer, error) {
	if ts < b.start {
		return nil, errors.New("[METRICSTORE]> cannot write value to buffer from past")
	}

	// idx := int((ts - b.start + (b.frequency / 3)) / b.frequency)
	idx := int((ts - b.start) / b.frequency)
	if idx >= cap(b.data) {
		newbuf := newBuffer(ts, b.frequency)
		newbuf.prev = b
		b.next = newbuf
		b = newbuf
		idx = 0
	}

	// Overwriting value or writing value from past
	if idx < len(b.data) {
		b.data[idx] = value
		return b, nil
	}

	// Fill up unwritten slots with NaN
	for i := len(b.data); i < idx; i++ {
		b.data = append(b.data, schema.NaN)
	}

	b.data = append(b.data, value)
	return b, nil
}

func (b *buffer) end() int64 {
	return b.firstWrite() + int64(len(b.data))*b.frequency
}

func (b *buffer) firstWrite() int64 {
	return b.start + (b.frequency / 2)
}

// read retrieves time-series data from the buffer chain for the specified time range.
//
// Traverses the buffer chain backwards (via prev links) if 'from' precedes the current
// buffer's start. Missing data points are represented as NaN. Values are accumulated
// into the provided 'data' slice (using +=, so caller must zero-initialize if needed).
//
// The function adjusts the actual time range returned if data is unavailable at the
// boundaries (returned via adjusted from/to timestamps).
//
// Parameters:
//   - from: Start timestamp (Unix seconds)
//   - to:   End timestamp (Unix seconds, exclusive)
//   - data: Pre-allocated slice to accumulate results (must be large enough)
//
// Returns:
//   - []schema.Float: Slice of data (may be shorter than input 'data' slice)
//   - int64:          Actual start timestamp with available data
//   - int64:          Actual end timestamp (exclusive)
//   - error:          Non-nil on failure
//
// Panics if 'data' slice is too small to hold all values in [from, to).
func (b *buffer) read(from, to int64, data []schema.Float) ([]schema.Float, int64, int64, error) {
	// Walk back to the buffer that covers 'from', adjusting if we hit the oldest.
	for from < b.firstWrite() {
		if b.prev == nil {
			from = b.firstWrite()
			break
		}
		b = b.prev
	}

	i := 0
	t := from
	for ; t < to; t += b.frequency {
		idx := int((t - b.start) / b.frequency)
		if idx >= cap(b.data) {
			if b.next == nil {
				break
			}
			b = b.next
			// Recalculate idx in the new buffer; a gap between buffers may exist.
			idx = int((t - b.start) / b.frequency)
		}

		if idx >= len(b.data) {
			if b.next == nil || to <= b.next.start {
				break
			}
			data[i] += schema.NaN // NaN + anything = NaN; propagates missing data
		} else if t < b.start {
			data[i] += schema.NaN // gap before this buffer's first write
		} else {
			data[i] += b.data[idx]
		}
		i++
	}

	return data[:i], from, t, nil
}

// free removes buffers older than the specified timestamp from the chain.
//
// Recursively traverses backwards (via prev) and unlinks buffers whose end time
// is before the retention threshold. Freed buffers are returned to the pool if
// they have the standard capacity (BufferCap).
//
// Parameters:
//   - t: Retention threshold timestamp (Unix seconds)
//
// Returns:
//   - delme: True if the current buffer itself should be deleted by caller
//   - n:     Number of buffers freed in this subtree
func (b *buffer) free(t int64) (delme bool, n int) {
	if b.prev != nil {
		delme, m := b.prev.free(t)
		n += m
		if delme {
			b.prev.next = nil
			if cap(b.prev.data) != BufferCap {
				b.prev.data = make([]schema.Float, 0, BufferCap)
			}
			b.prev.lastUsed = time.Now().Unix()
			bufferPool.Put(b.prev)
			b.prev = nil
		}
	}

	end := b.end()
	if end < t {
		return true, n + 1
	}

	return false, n
}

// forceFreeOldest recursively finds the end of the linked list (the oldest buffer)
// and removes it.
// Returns:
//
//	delme: true if 'b' itself is the oldest and should be removed by the caller
//	n:     the number of buffers freed (will be 1 or 0)
func (b *buffer) forceFreeOldest() (delme bool, n int) {
	// If there is a previous buffer, recurse down to find the oldest
	if b.prev != nil {
		delPrev, freed := b.prev.forceFreeOldest()

		// If the previous buffer signals it should be deleted:
		if delPrev {
			b.prev.next = nil
			if cap(b.prev.data) != BufferCap {
				b.prev.data = make([]schema.Float, 0, BufferCap)
			}
			b.prev.lastUsed = time.Now().Unix()
			bufferPool.Put(b.prev)
			b.prev = nil
		}
		return false, freed
	}

	// If b.prev is nil, THIS buffer is the oldest.
	// We return true so the parent (or the Level loop) knows to delete reference to 'b'.
	return true, 1
}

// iterFromTo invokes callback on every buffer in the chain that overlaps [from, to].
//
// Traverses backwards (via prev) first, then processes current buffer if it overlaps
// the time range. Used for checkpoint/archive operations that need to serialize buffers
// within a specific time window.
//
// Parameters:
//   - from:     Start timestamp (Unix seconds, inclusive)
//   - to:       End timestamp (Unix seconds, inclusive)
//   - callback: Function to invoke on each overlapping buffer
//
// Returns:
//   - error: First error returned by callback, or nil if all succeeded
func (b *buffer) iterFromTo(from, to int64, callback func(b *buffer) error) error {
	if b == nil {
		return nil
	}

	// Collect overlapping buffers walking backwards (newest → oldest).
	var matching []*buffer
	for cur := b; cur != nil; cur = cur.prev {
		if from <= cur.end() && cur.start <= to {
			matching = append(matching, cur)
		}
	}

	// Invoke callback in chronological order (oldest → newest).
	for i := len(matching) - 1; i >= 0; i-- {
		if err := callback(matching[i]); err != nil {
			return err
		}
	}
	return nil
}

func (b *buffer) count() int64 {
	var res int64
	for ; b != nil; b = b.prev {
		res += int64(len(b.data))
	}
	return res
}


Now I have a clear picture. Let me implement all changes.

**1. config.go** — Reduce default check interval:


Edit

DefaultMemoryUsageTrackerInterval = 1 * time.Hour

DefaultMemoryUsageTrackerInterval = 5 * time.Minute


The file /Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/config.go has been updated successfully.

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

// Package metricstore provides config.go: Configuration structures and metric management.
//
// # Configuration Hierarchy
//
// The metricstore package uses nested configuration structures:
//
//	MetricStoreConfig (Keys)
//	├─ NumWorkers: Parallel checkpoint/archive workers
//	├─ RetentionInMemory: How long to keep data in RAM (also used as cleanup interval)
//	├─ MemoryCap: Memory limit in bytes (triggers forceFree)
//	├─ Checkpoints: Persistence configuration
//	│  ├─ FileFormat: "json" or "wal" (default: "wal")
//	│  └─ RootDir: Checkpoint storage path
//	├─ Cleanup: Long-term storage configuration (interval = RetentionInMemory)
//	│  ├─ RootDir: Archive storage path (archive mode only)
//	│  └─ Mode: "delete" or "archive"
//	├─ Debug: Development/debugging options
//	└─ Subscriptions: NATS topic subscriptions for metric ingestion
//
// # Metric Configuration
//
// Each metric (e.g., "cpu_load", "mem_used") has a MetricConfig entry in the global
// Metrics map, defining:
//
//   - Frequency: Measurement interval in seconds
//   - Aggregation: How to combine values (sum/avg/none) when transforming scopes
//   - offset: Internal index into Level.metrics slice (assigned during Init)
//
// # AggregationStrategy
//
// Determines how to combine metric values when aggregating from finer to coarser scopes:
//
//   - NoAggregation: Do not combine (incompatible scopes)
//   - SumAggregation: Add values (e.g., power consumption: core→socket)
//   - AvgAggregation: Average values (e.g., temperature: core→socket)
package metricstore

import (
	"fmt"
	"time"

	"github.com/ClusterCockpit/cc-backend/pkg/archive"
	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
	"github.com/ClusterCockpit/cc-lib/v2/schema"
)

const (
	DefaultMaxWorkers                 = 10
	DefaultBufferCapacity             = 512
	DefaultGCTriggerInterval          = 100
	DefaultMemoryUsageTrackerInterval = 1 * time.Hour
)

// Checkpoints configures periodic persistence of in-memory metric data.
//
// Fields:
//   - FileFormat: "json" (human-readable, periodic) or "wal" (binary snapshot + WAL, crash-safe); default is "wal"
//   - RootDir:    Filesystem path for checkpoint files (created if missing)
type Checkpoints struct {
	FileFormat string `json:"file-format"`
	RootDir    string `json:"directory"`
}

// Debug provides development and profiling options.
//
// Fields:
//   - DumpToFile: Path to dump checkpoint data for inspection (empty = disabled)
//   - EnableGops: Enable gops agent for live runtime debugging (https://github.com/google/gops)
type Debug struct {
	DumpToFile string `json:"dump-to-file"`
	EnableGops bool   `json:"gops"`
}

// Cleanup configures long-term storage of old metric data.
//
// Data older than RetentionInMemory is archived to disk or deleted.
// The cleanup interval is always RetentionInMemory.
//
// Fields:
//   - RootDir: Filesystem path for archived data (used in "archive" mode)
//   - Mode:    "delete" (discard old data) or "archive" (write to RootDir)
type Cleanup struct {
	RootDir string `json:"directory"`
	Mode    string `json:"mode"`
}

// Subscriptions defines NATS topics to subscribe to for metric ingestion.
//
// Each subscription receives metrics via NATS messaging, enabling real-time
// data collection from compute nodes.
//
// Fields:
//   - SubscribeTo: NATS subject/channel name (e.g., "metrics.compute.*")
//   - ClusterTag:  Default cluster name for metrics without cluster tag (optional)
type Subscriptions []struct {
	// Channel name
	SubscribeTo string `json:"subscribe-to"`

	// Allow lines without a cluster tag, use this as default, optional
	ClusterTag string `json:"cluster-tag"`
}

// MetricStoreConfig defines the main configuration for the metricstore.
//
// Loaded from cc-backend's config.json "metricstore" section. Controls memory usage,
// persistence, archiving, and metric ingestion.
//
// Fields:
//   - NumWorkers:        Parallel workers for checkpoint/archive (0 = auto: min(NumCPU/2+1, 10))
//   - RetentionInMemory: Duration string (e.g., "48h") for in-memory data retention
//   - MemoryCap:         Max bytes for buffer data (0 = unlimited); triggers forceFree when exceeded
//   - Checkpoints:       Periodic persistence configuration
//   - Debug:             Development/profiling options (nil = disabled)
//   - Archive:           Long-term storage configuration (nil = disabled)
//   - Subscriptions:     NATS topics for metric ingestion (nil = polling only)
type MetricStoreConfig struct {
	// Number of concurrent workers for checkpoint and archive operations.
	// If not set or 0, defaults to min(runtime.NumCPU()/2+1, 10)
	NumWorkers        int            `json:"num-workers"`
	RetentionInMemory string         `json:"retention-in-memory"`
	MemoryCap         int            `json:"memory-cap"`
	Checkpoints       Checkpoints    `json:"checkpoints"`
	Debug             *Debug         `json:"debug"`
	Cleanup           *Cleanup       `json:"cleanup"`
	Subscriptions     *Subscriptions `json:"nats-subscriptions"`
}

// Keys is the global metricstore configuration instance.
//
// Initialized with defaults, then overwritten by cc-backend's config.json.
// Accessed by Init(), Checkpointing(), and other lifecycle functions.
var Keys MetricStoreConfig = MetricStoreConfig{
	Checkpoints: Checkpoints{
		FileFormat: "wal",
		RootDir:    "./var/checkpoints",
	},
	Cleanup: &Cleanup{
		Mode: "delete",
	},
}

// AggregationStrategy defines how to combine metric values across hierarchy levels.
//
// Used when transforming data from finer-grained scopes (e.g., core) to coarser scopes
// (e.g., socket). This is SPATIAL aggregation, not TEMPORAL (time-based) aggregation.
//
// Values:
//   - NoAggregation:  Do not aggregate (incompatible scopes or non-aggregatable metrics)
//   - SumAggregation: Add values (e.g., power: sum core power → socket power)
//   - AvgAggregation: Average values (e.g., temperature: average core temps → socket temp)
type AggregationStrategy int

const (
	NoAggregation  AggregationStrategy = iota // Do not aggregate
	SumAggregation                            // Sum values (e.g., power, energy)
	AvgAggregation                            // Average values (e.g., temperature, utilization)
)

// AssignAggregationStrategy parses a string into an AggregationStrategy value.
//
// Used when loading metric configurations from JSON/YAML files.
//
// Parameters:
//   - str: "sum", "avg", or "" (empty string for NoAggregation)
//
// Returns:
//   - AggregationStrategy: Parsed value
//   - error:               Non-nil if str is unrecognized
func AssignAggregationStrategy(str string) (AggregationStrategy, error) {
	switch str {
	case "":
		return NoAggregation, nil
	case "sum":
		return SumAggregation, nil
	case "avg":
		return AvgAggregation, nil
	default:
		return NoAggregation, fmt.Errorf("[METRICSTORE]> unknown aggregation strategy: %s", str)
	}
}

// MetricConfig defines configuration for a single metric type.
//
// Stored in the global Metrics map, keyed by metric name (e.g., "cpu_load").
//
// Fields:
//   - Frequency:   Measurement interval in seconds (e.g., 60 for 1-minute granularity)
//   - Aggregation: How to combine values across hierarchy levels (sum/avg/none)
//   - offset:      Internal index into Level.metrics slice (assigned during Init)
type MetricConfig struct {
	// Interval in seconds at which measurements are stored
	Frequency int64

	// Can be 'sum', 'avg' or null. Describes how to aggregate metrics from the same timestep over the hierarchy.
	Aggregation AggregationStrategy

	// Private, used internally...
	offset int
}

func BuildMetricList() map[string]MetricConfig {
	var metrics map[string]MetricConfig = make(map[string]MetricConfig)

	addMetric := func(name string, metric MetricConfig) error {
		if metrics == nil {
			metrics = make(map[string]MetricConfig, 0)
		}

		if existingMetric, ok := metrics[name]; ok {
			if existingMetric.Frequency != metric.Frequency {
				if existingMetric.Frequency < metric.Frequency {
					existingMetric.Frequency = metric.Frequency
					metrics[name] = existingMetric
				}
			}
		} else {
			metrics[name] = metric
		}

		return nil
	}

	// Helper function to add metric configuration
	addMetricConfig := func(mc *schema.MetricConfig) {
		agg, err := AssignAggregationStrategy(mc.Aggregation)
		if err != nil {
			cclog.Warnf("Could not find aggregation strategy for metric config '%s': %s", mc.Name, err.Error())
		}

		addMetric(mc.Name, MetricConfig{
			Frequency:   int64(mc.Timestep),
			Aggregation: agg,
		})
	}
	for _, c := range archive.Clusters {
		for _, mc := range c.MetricConfig {
			addMetricConfig(mc)
		}

		for _, sc := range c.SubClusters {
			for _, mc := range sc.MetricConfig {
				addMetricConfig(mc)
			}
		}
	}

	return metrics
}

-  DefaultMemoryUsageTrackerInterval = 1 * time.Hour

+  DefaultMemoryUsageTrackerInterval = 5 * time.Minute


**2. lineprotocol.go** — Don't abort batch on write error, log and continue:


		if err := ms.WriteToLevel(lvl, st.selector, time, []Metric{metric}); err != nil {
			return err
		}

		if err := ms.WriteToLevel(lvl, st.selector, time, []Metric{metric}); err != nil {
			cclog.Warnf("write error for host %s metric %s at ts %d: %s", host, string(st.metricBuf), time, err.Error())
			continue
		}


The file /Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/lineprotocol.go has been updated successfully.

		if err := ms.WriteToLevel(lvl, st.selector, time, []Metric{metric}); err != nil {
			cclog.Warnf("write error for host %s metric %s at ts %d: %s", host, string(st.metricBuf), time, err.Error())
			continue
		}

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

// This file implements ingestion of InfluxDB line-protocol metric data received
// over NATS. Each line encodes one metric sample with the following structure:
//
//	<measurement>[,cluster=<c>][,hostname=<h>][,type=<t>][,type-id=<id>][,subtype=<s>][,stype-id=<id>] value=<v> [<timestamp>]
//
// The measurement name identifies the metric (e.g. "cpu_load"). Tags provide
// routing information (cluster, host) and optional sub-device selectors (type,
// subtype). Only one field is expected per line: "value".
//
// After decoding, each sample is:
//  1. Written to the in-memory store via ms.WriteToLevel.
//  2. If the checkpoint format is "wal", also forwarded to the WAL staging
//     goroutine via the WALMessages channel for durable write-ahead logging.
package metricstore

import (
	"bytes"
	"context"
	"fmt"
	"sync"
	"time"

	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
	"github.com/ClusterCockpit/cc-lib/v2/nats"
	"github.com/ClusterCockpit/cc-lib/v2/schema"
	"github.com/ClusterCockpit/cc-line-protocol/v2/lineprotocol"
)

// ReceiveNats subscribes to all configured NATS subjects and feeds incoming
// line-protocol messages into the MemoryStore.
//
// When workers > 1 a pool of goroutines drains a shared channel so that
// multiple messages can be decoded in parallel. With workers == 1 the NATS
// callback decodes inline (no channel overhead, lower latency).
//
// The function blocks until ctx is cancelled and all worker goroutines have
// finished. It returns nil when the NATS client is not configured; callers
// should treat that as a no-op rather than an error.
func ReceiveNats(ms *MemoryStore,
	workers int,
	ctx context.Context,
) error {
	nc := nats.GetClient()

	if nc == nil {
		cclog.Warn("NATS client not initialized")
		return nil
	}

	var wg sync.WaitGroup
	msgs := make(chan []byte, workers*2)

	for _, sc := range *Keys.Subscriptions {
		clusterTag := sc.ClusterTag
		if workers > 1 {
			wg.Add(workers)

			for range workers {
				go func() {
					defer wg.Done()
					for m := range msgs {
						dec := lineprotocol.NewDecoderWithBytes(m)
						if err := DecodeLine(dec, ms, clusterTag); err != nil {
							cclog.Errorf("error: %s", err.Error())
						}
					}
				}()
			}

			nc.Subscribe(sc.SubscribeTo, func(subject string, data []byte) {
				select {
				case msgs <- data:
				case <-ctx.Done():
				}
			})
		} else {
			nc.Subscribe(sc.SubscribeTo, func(subject string, data []byte) {
				dec := lineprotocol.NewDecoderWithBytes(data)
				if err := DecodeLine(dec, ms, clusterTag); err != nil {
					cclog.Errorf("error: %s", err.Error())
				}
			})
		}
		cclog.Infof("NATS subscription to '%s' established", sc.SubscribeTo)
	}

	go func() {
		<-ctx.Done()
		close(msgs)
	}()

	wg.Wait()

	return nil
}

// reorder prepends prefix to buf in-place when buf has enough spare capacity,
// avoiding an allocation. Falls back to a regular append otherwise.
//
// It is used to assemble the "type<type-id>" and "subtype<stype-id>" selector
// strings when the type tag arrives before the type-id tag in the line, so the
// two byte slices need to be concatenated in tag-declaration order regardless
// of wire order.
func reorder(buf, prefix []byte) []byte {
	n := len(prefix)
	m := len(buf)
	if cap(buf) < m+n {
		return append(prefix[:n:n], buf...)
	} else {
		buf = buf[:n+m]
		for i := m - 1; i >= 0; i-- {
			buf[i+n] = buf[i]
		}
		for i := range n {
			buf[i] = prefix[i]
		}
		return buf
	}
}

// decodeState holds the per-call scratch buffers used by DecodeLine.
// Instances are recycled via decodeStatePool to avoid repeated allocations
// during high-throughput ingestion.
type decodeState struct {
	// metricBuf holds a copy of the current measurement name (line-protocol
	// measurement field). Copied because dec.Measurement() returns a slice
	// that is invalidated by the next decoder call.
	metricBuf []byte

	// selector is the sub-device path passed to WriteToLevel and WALMessage
	// (e.g. ["socket0"] or ["socket0", "memctrl1"]). Reused across lines.
	selector []string

	// typeBuf accumulates the concatenated "type"+"type-id" tag value for the
	// current line. Reset at the start of each line's tag-decode loop.
	typeBuf []byte

	// subTypeBuf accumulates the concatenated "subtype"+"stype-id" tag value.
	// Reset at the start of each line's tag-decode loop.
	subTypeBuf []byte

	// prevTypeBytes / prevTypeStr cache the last seen typeBuf content and its
	// string conversion. Because consecutive lines in a batch typically address
	// the same sub-device, the cache hit rate is very high and avoids
	// repeated []byte→string allocations.
	prevTypeBytes []byte
	prevTypeStr   string

	// prevSubTypeBytes / prevSubTypeStr are the same cache for the subtype.
	prevSubTypeBytes []byte
	prevSubTypeStr   string
}

// decodeStatePool recycles decodeState values across DecodeLine calls to
// reduce GC pressure during sustained metric ingestion.
var decodeStatePool = sync.Pool{
	New: func() any {
		return &decodeState{
			metricBuf:  make([]byte, 0, 16),
			selector:   make([]string, 0, 4),
			typeBuf:    make([]byte, 0, 16),
			subTypeBuf: make([]byte, 0, 16),
		}
	},
}

// DecodeLine reads all lines from dec (InfluxDB line-protocol) and writes each
// decoded metric sample into ms.
//
// clusterDefault is used as the cluster name for lines that do not carry a
// "cluster" tag. Callers typically supply the ClusterTag value from the NATS
// subscription configuration.
//
// Performance notes:
//   - A decodeState is obtained from decodeStatePool to reuse scratch buffers.
//   - The Level pointer (host-level node in the metric tree) is cached across
//     consecutive lines that share the same cluster+host pair to avoid
//     repeated lock acquisitions on the root and cluster levels.
//   - []byte→string conversions for type/subtype selectors are cached via
//     prevType*/prevSubType* fields because batches typically repeat the same
//     sub-device identifiers.
//   - Timestamp parsing tries Second precision first; if that fails it retries
//     Millisecond, Microsecond, and Nanosecond in turn. A missing timestamp
//     falls back to time.Now().
//
// When the checkpoint format is "wal" each successfully decoded sample is also
// sent to WALMessages so the WAL staging goroutine can persist it durably
// before the next binary snapshot.
func DecodeLine(dec *lineprotocol.Decoder,
	ms *MemoryStore,
	clusterDefault string,
) error {
	// Reduce allocations in loop:
	t := time.Now()
	metric := Metric{}
	st := decodeStatePool.Get().(*decodeState)
	defer decodeStatePool.Put(st)

	// Optimize for the case where all lines in a "batch" are about the same
	// cluster and host. By using `WriteToLevel` (level = host), we do not need
	// to take the root- and cluster-level lock as often.
	var lvl *Level = nil
	prevCluster, prevHost := "", ""

	var ok bool
	for dec.Next() {
		rawmeasurement, err := dec.Measurement()
		if err != nil {
			return err
		}

		// Needs to be copied because another call to dec.* would
		// invalidate the returned slice.
		st.metricBuf = append(st.metricBuf[:0], rawmeasurement...)

		// The go compiler optimizes map[string(byteslice)] lookups:
		metric.MetricConfig, ok = ms.Metrics[string(rawmeasurement)]
		if !ok {
			continue
		}

		st.typeBuf, st.subTypeBuf = st.typeBuf[:0], st.subTypeBuf[:0]
		cluster, host := clusterDefault, ""
		for {
			key, val, err := dec.NextTag()
			if err != nil {
				return err
			}
			if key == nil {
				break
			}

			// The go compiler optimizes string([]byte{...}) == "...":
			switch string(key) {
			case "cluster":
				if string(val) == prevCluster {
					cluster = prevCluster
				} else {
					cluster = string(val)
					lvl = nil
				}
			case "hostname", "host":
				if string(val) == prevHost {
					host = prevHost
				} else {
					host = string(val)
					lvl = nil
				}
			case "type":
				if string(val) == "node" {
					break
				}

				// We cannot be sure that the "type" tag comes before the "type-id" tag:
				if len(st.typeBuf) == 0 {
					st.typeBuf = append(st.typeBuf, val...)
				} else {
					st.typeBuf = reorder(st.typeBuf, val)
				}
			case "type-id":
				st.typeBuf = append(st.typeBuf, val...)
			case "subtype":
				// We cannot be sure that the "subtype" tag comes before the "stype-id" tag:
				if len(st.subTypeBuf) == 0 {
					st.subTypeBuf = append(st.subTypeBuf, val...)
				} else {
					st.subTypeBuf = reorder(st.subTypeBuf, val)
				}
			case "stype-id":
				st.subTypeBuf = append(st.subTypeBuf, val...)
			default:
			}
		}

		// If the cluster or host changed, the lvl was set to nil
		if lvl == nil {
			st.selector = st.selector[:2]
			st.selector[0], st.selector[1] = cluster, host
			lvl = ms.GetLevel(st.selector)
			prevCluster, prevHost = cluster, host
		}

		// subtypes: cache []byte→string conversions; messages in a batch typically
		// share the same type/subtype so the hit rate is very high.
		st.selector = st.selector[:0]
		if len(st.typeBuf) > 0 {
			if !bytes.Equal(st.typeBuf, st.prevTypeBytes) {
				st.prevTypeBytes = append(st.prevTypeBytes[:0], st.typeBuf...)
				st.prevTypeStr = string(st.typeBuf)
			}
			st.selector = append(st.selector, st.prevTypeStr)
			if len(st.subTypeBuf) > 0 {
				if !bytes.Equal(st.subTypeBuf, st.prevSubTypeBytes) {
					st.prevSubTypeBytes = append(st.prevSubTypeBytes[:0], st.subTypeBuf...)
					st.prevSubTypeStr = string(st.subTypeBuf)
				}
				st.selector = append(st.selector, st.prevSubTypeStr)
			}
		}

		for {
			key, val, err := dec.NextField()
			if err != nil {
				return err
			}

			if key == nil {
				break
			}

			if string(key) != "value" {
				return fmt.Errorf("host %s: unknown field: '%s' (value: %#v)", host, string(key), val)
			}

			if val.Kind() == lineprotocol.Float {
				metric.Value = schema.Float(val.FloatV())
			} else if val.Kind() == lineprotocol.Int {
				metric.Value = schema.Float(val.IntV())
			} else if val.Kind() == lineprotocol.Uint {
				metric.Value = schema.Float(val.UintV())
			} else {
				return fmt.Errorf("host %s: unsupported value type in message: %s", host, val.Kind().String())
			}
		}

		if t, err = dec.Time(lineprotocol.Second, t); err != nil {
			t = time.Now()
			if t, err = dec.Time(lineprotocol.Millisecond, t); err != nil {
				t = time.Now()
				if t, err = dec.Time(lineprotocol.Microsecond, t); err != nil {
					t = time.Now()
					if t, err = dec.Time(lineprotocol.Nanosecond, t); err != nil {
						return fmt.Errorf("host %s: timestamp : %#v with error : %#v", host, t, err.Error())
					}
				}
			}
		}

		if err != nil {
			return fmt.Errorf("host %s: timestamp : %#v with error : %#v", host, t, err.Error())
		}

		time := t.Unix()

		if Keys.Checkpoints.FileFormat == "wal" {
			WALMessages <- &WALMessage{
				MetricName: string(st.metricBuf),
				Cluster:    cluster,
				Node:       host,
				Selector:   append([]string{}, st.selector...),
				Value:      metric.Value,
				Timestamp:  time,
			}
		}

		if err := ms.WriteToLevel(lvl, st.selector, time, []Metric{metric}); err != nil {
			return err
		}
	}
	return nil
}

+      cclog.Warnf("write error for host %s metric %s at ts %d: %s", host, string(st.metricBuf), time, err.Error())


**3. buffer.go** — Add `bufferCount()` helper:


func (b *buffer) count() int64 {
	var res int64
	for ; b != nil; b = b.prev {
		res += int64(len(b.data))
	}
	return res
}

func (b *buffer) count() int64 {
	var res int64
	for ; b != nil; b = b.prev {
		res += int64(len(b.data))
	}
	return res
}

// bufferCount returns the number of buffer nodes in the linked list.
func (b *buffer) bufferCount() int64 {
	var n int64
	for ; b != nil; b = b.prev {
		n++
	}
	return n
}