cc-metric-store/internal/avro/avroHelper.go

package avro

import (
	"context"
	"fmt"
	"strconv"
	"sync"

	"github.com/ClusterCockpit/cc-metric-store/internal/config"
)

func DataStaging(wg *sync.WaitGroup, ctx context.Context) {

	if config.Keys.Checkpoints.FileFormat == "json" {
		wg.Done()
	}

	// AvroPool is a pool of Avro writers.
	go func() {
		defer wg.Done()

		var avroLevel *AvroLevel
		oldSelector := make([]string, 0)

		for {
			select {
			case <-ctx.Done():
				return
			case val := <-LineProtocolMessages:
				//Fetch the frequency of the metric from the global configuration
				freq, err := config.Keys.GetMetricFrequency(val.MetricName)
				if err != nil {
					fmt.Printf("Error fetching metric frequency: %s\n", err)
					continue
				}

				metricName := ""

				for i, name := range val.Selector {
					if i == 0 {
						metricName += name
					} else {
						metricName += "_" + name
					}
				}

				metricName += val.MetricName

				// Create a new selector for the Avro level
				// The selector is a slice of strings that represents the path to the
				// Avro level. It is created by appending the cluster, node, and metric
				// name to the selector.
				var selector []string
				selector = append(selector, val.Cluster, val.Node, strconv.FormatInt(freq, 10))

				if !testEq(oldSelector, selector) {
					// Get the Avro level for the metric
					avroLevel = avroStore.root.findAvroLevelOrCreate(selector)

					// If the Avro level is nil, create a new one
					if avroLevel == nil {
						fmt.Printf("Error creating or finding the level with cluster : %s, node : %s, metric : %s\n", val.Cluster, val.Node, val.MetricName)
					}
					copy(oldSelector, selector)
				}

				avroLevel.addMetric(metricName, val.Value, val.Timestamp, int(freq))
			}
		}
	}()
}

func testEq(a, b []string) bool {
	if len(a) != len(b) {
		return false
	}
	for i := range a {
		if a[i] != b[i] {
			return false
		}
	}
	return true
}