// Copyright (C) NHR@FAU, University Erlangen-Nuremberg. // All rights reserved. This file is part of cc-backend. // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. package metricstore import ( "cmp" "fmt" "slices" "time" cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger" "github.com/ClusterCockpit/cc-lib/v2/schema" ) // HealthCheckResponse represents the result of a health check operation. // // Status indicates the monitoring state (Full, Partial, Failed). // Error contains any error encountered during the health check. type HealthCheckResponse struct { Status schema.MonitoringState Error error } // MaxMissingDataPoints is a threshold that allows a node to be healthy with certain number of data points missing. // Suppose a node does not receive last 5 data points, then healthCheck endpoint will still say a // node is healthy. Anything more than 5 missing points in metrics of the node will deem the node unhealthy. const MaxMissingDataPoints int64 = 5 // isBufferHealthy checks if a buffer has received data for the last MaxMissingDataPoints. // // Returns true if the buffer is healthy (recent data within threshold), false otherwise. // A nil buffer or empty buffer is considered unhealthy. func (b *buffer) bufferExists() bool { // Check if the buffer is empty if b == nil || b.data == nil || len(b.data) == 0 { return false } return true } // isBufferHealthy checks if a buffer has received data for the last MaxMissingDataPoints. // // Returns true if the buffer is healthy (recent data within threshold), false otherwise. // A nil buffer or empty buffer is considered unhealthy. func (b *buffer) isBufferHealthy() bool { // Get the last endtime of the buffer bufferEnd := b.start + b.frequency*int64(len(b.data)) t := time.Now().Unix() // Check if the buffer has recent data (within MaxMissingDataPoints threshold) if t-bufferEnd > MaxMissingDataPoints*b.frequency { return false } return true } // MergeUniqueSorted merges two lists, sorts them, and removes duplicates. // Requires 'cmp.Ordered' because we need to sort the data. func mergeList[string cmp.Ordered](list1, list2 []string) []string { // 1. Combine both lists result := append(list1, list2...) // 2. Sort the combined list slices.Sort(result) // 3. Compact removes consecutive duplicates (standard in Go 1.21+) // e.g. [1, 1, 2, 3, 3] -> [1, 2, 3] result = slices.Compact(result) return result } // getHealthyMetrics recursively collects healthy and degraded metrics at this level and below. // // A metric is considered: // - Healthy: buffer has recent data within MaxMissingDataPoints threshold AND has few/no NaN values // - Degraded: buffer exists and has recent data, but contains more than MaxMissingDataPoints NaN values // // This routine walks the entire subtree starting from the current level. // // Parameters: // - m: MemoryStore containing the global metric configuration // // Returns: // - []string: Flat list of healthy metric names from this level and all children // - []string: Flat list of degraded metric names (exist but have too many missing values) // - error: Non-nil only for internal errors during recursion // // The routine mirrors healthCheck() but provides more granular classification: // - healthCheck() finds problems (stale/missing) // - getHealthyMetrics() separates healthy from degraded metrics func (l *Level) getHealthyMetrics(m *MemoryStore, expectedMetrics []string) ([]string, []string, error) { l.lock.RLock() defer l.lock.RUnlock() globalMetrics := m.Metrics missingList := make([]string, 0) degradedList := make([]string, 0) // Phase 1: Check metrics at this level for _, metricName := range expectedMetrics { offset := globalMetrics[metricName].offset b := l.metrics[offset] if !b.bufferExists() { missingList = append(missingList, metricName) } else if !b.isBufferHealthy() { degradedList = append(degradedList, metricName) } } // Phase 2: Recursively check child levels for _, lvl := range l.children { childMissing, childDegraded, err := lvl.getHealthyMetrics(m, expectedMetrics) if err != nil { return nil, nil, err } missingList = mergeList(missingList, childMissing) degradedList = mergeList(degradedList, childDegraded) } return missingList, degradedList, nil } // GetHealthyMetrics returns healthy and degraded metrics for a specific node as flat lists. // // This routine walks the metric tree starting from the specified node selector // and collects all metrics that have received data within the last MaxMissingDataPoints // (default: 5 data points). Metrics are classified into two categories: // // - Healthy: Buffer has recent data AND contains few/no NaN (missing) values // - Degraded: Buffer has recent data BUT contains more than MaxMissingDataPoints NaN values // // The returned lists include both node-level metrics (e.g., "load", "mem_used") and // hardware-level metrics (e.g., "cpu_user", "gpu_temp") in flat slices. // // Parameters: // - selector: Hierarchical path to the target node, typically []string{cluster, hostname}. // Example: []string{"emmy", "node001"} navigates to the "node001" host in the "emmy" cluster. // The selector must match the hierarchy used during metric ingestion. // // Returns: // - []string: Flat list of healthy metric names (recent data, few missing values) // - []string: Flat list of degraded metric names (recent data, many missing values) // - error: Non-nil if the node is not found or internal errors occur // // Example usage: // // selector := []string{"emmy", "node001"} // healthyMetrics, degradedMetrics, err := ms.GetHealthyMetrics(selector) // if err != nil { // // Node not found or internal error // return err // } // fmt.Printf("Healthy metrics: %v\n", healthyMetrics) // // Output: ["load", "mem_used", "cpu_user", ...] // fmt.Printf("Degraded metrics: %v\n", degradedMetrics) // // Output: ["gpu_temp", "network_rx", ...] (metrics with many NaN values) // // Note: This routine provides more granular classification than HealthCheck: // - HealthCheck reports stale/missing metrics (problems) // - GetHealthyMetrics separates fully healthy from degraded metrics (quality levels) func (m *MemoryStore) GetHealthyMetrics(selector []string, expectedMetrics []string) ([]string, []string, error) { lvl := m.root.findLevel(selector) if lvl == nil { return nil, nil, fmt.Errorf("[METRICSTORE]> error while GetHealthyMetrics, host not found: %#v", selector) } missingList, degradedList, err := lvl.getHealthyMetrics(m, expectedMetrics) if err != nil { return nil, nil, err } return missingList, degradedList, nil } // HealthCheck performs health checks on multiple nodes and returns their monitoring states. // // This routine provides a batch health check interface that evaluates multiple nodes // against a specific set of expected metrics. For each node, it determines the overall // monitoring state based on which metrics are healthy, degraded, or missing. // // Health Status Classification: // - MonitoringStateFull: All expected metrics are healthy (recent data, few missing values) // - MonitoringStatePartial: Some metrics are degraded (many missing values) or missing // - MonitoringStateFailed: Node not found or all expected metrics are missing/stale // // Parameters: // - cluster: Cluster name (first element of selector path) // - nodes: List of node hostnames to check // - expectedMetrics: List of metric names that should be present on each node // // Returns: // - map[string]schema.MonitoringState: Map keyed by hostname containing monitoring state for each node // - error: Non-nil only for internal errors (individual node failures are captured as MonitoringStateFailed) // // Example usage: // // cluster := "emmy" // nodes := []string{"node001", "node002", "node003"} // expectedMetrics := []string{"load", "mem_used", "cpu_user", "cpu_system"} // healthStates, err := ms.HealthCheck(cluster, nodes, expectedMetrics) // if err != nil { // return err // } // for hostname, state := range healthStates { // fmt.Printf("Node %s: %s\n", hostname, state) // } // // Note: This routine is optimized for batch operations where you need to check // the same set of metrics across multiple nodes. func (m *MemoryStore) HealthCheck(cluster string, nodes []string, expectedMetrics []string, ) (map[string]schema.MonitoringState, error) { results := make(map[string]schema.MonitoringState, len(nodes)) // Create a set of expected metrics for fast lookup expectedSet := make(map[string]bool, len(expectedMetrics)) for _, metric := range expectedMetrics { expectedSet[metric] = true } // Check each node for _, hostname := range nodes { selector := []string{cluster, hostname} status := schema.MonitoringStateFull healthyCount := 0 degradedCount := 0 missingCount := 0 // Get healthy and degraded metrics for this node missingList, degradedList, err := m.GetHealthyMetrics(selector, expectedMetrics) if err != nil { // Node not found or internal error results[hostname] = schema.MonitoringStateFailed continue } missingCount = len(missingList) degradedCount = len(degradedList) uniqueList := mergeList(missingList, degradedList) healthyCount = len(expectedMetrics) - len(uniqueList) // Debug log missing and degraded metrics if missingCount > 0 { cclog.ComponentDebug("metricstore", "HealthCheck: node", hostname, "missing metrics:", missingList) } if degradedCount > 0 { cclog.ComponentDebug("metricstore", "HealthCheck: node", hostname, "degraded metrics:", degradedList) } // Determine overall health status if missingCount > 0 || degradedCount > 0 { if healthyCount == 0 { // No healthy metrics at all status = schema.MonitoringStateFailed } else { // Some healthy, some degraded/missing status = schema.MonitoringStatePartial } } // else: all metrics healthy, status remains MonitoringStateFull results[hostname] = status } return results, nil }