ClusterCockpit/cc-backend
1
0
Fork 0
mirror of https://github.com/ClusterCockpit/cc-backend synced 2026-01-28 23:11:46 +01:00
Code Issues Packages Releases Activity

Take into account the real allocated heap memory in MemoryUsageTracker

Browse Source
This commit is contained in:
Jan Eitzinger
2026-01-27 18:23:09 +01:00
parent bbde91a1f9
commit 9d15a87c88
1 changed files with 26 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42
pkg/metricstore/metricstore.go
View File

@@ -358,9 +358,13 @@ func Retention(wg *sync.WaitGroup, ctx context.Context) {
// MemoryUsageTracker starts a background goroutine that monitors memory usage. // MemoryUsageTracker starts a background goroutine that monitors memory usage.
// //
// This worker checks memory usage periodically and force-frees buffers if memory // This worker checks actual process memory usage (via runtime.MemStats) periodically
// exceeds the configured cap. It uses FreeOSMemory() to return memory to the OS // and force-frees buffers if memory exceeds the configured cap. It uses FreeOSMemory()
// after freeing buffers, avoiding aggressive GC that causes performance issues. // to return memory to the OS after freeing buffers, avoiding aggressive GC that causes
// performance issues.
//
// The tracker logs both actual memory usage (heap allocated) and metric data size for
// visibility into memory overhead from Go runtime structures and allocations.
// //
// Parameters: // Parameters:
// - wg: WaitGroup to signal completion when context is cancelled // - wg: WaitGroup to signal completion when context is cancelled
@@ -387,8 +391,11 @@ func MemoryUsageTracker(wg *sync.WaitGroup, ctx context.Context) {
case <-ctx.Done(): case <-ctx.Done():
return return
case <-ticker.C: case <-ticker.C:
memoryUsageGB := ms.SizeInGB() var mem runtime.MemStats
cclog.Infof("[METRICSTORE]> current memory usage: %.2f GB", memoryUsageGB) runtime.ReadMemStats(&mem)
actualMemoryGB := float64(mem.Alloc) / 1e9
metricDataGB := ms.SizeInGB()
cclog.Infof("[METRICSTORE]> memory usage: %.2f GB actual (%.2f GB metric data)", actualMemoryGB, metricDataGB)
freedExcluded := 0 freedExcluded := 0
freedEmergency := 0 freedEmergency := 0
@@ -411,15 +418,16 @@ func MemoryUsageTracker(wg *sync.WaitGroup, ctx context.Context) {
} }
} }
memoryUsageGB = ms.SizeInGB() runtime.ReadMemStats(&mem)
actualMemoryGB = float64(mem.Alloc) / 1e9
if memoryUsageGB > float64(Keys.MemoryCap) { if actualMemoryGB > float64(Keys.MemoryCap) {
cclog.Warnf("[METRICSTORE]> memory usage %.2f GB exceeds cap %d GB, starting emergency buffer freeing", memoryUsageGB, Keys.MemoryCap) cclog.Warnf("[METRICSTORE]> memory usage %.2f GB exceeds cap %d GB, starting emergency buffer freeing", actualMemoryGB, Keys.MemoryCap)
const maxIterations = 100 const maxIterations = 100
for i := 0; i < maxIterations; i++ { for i := range maxIterations {
if memoryUsageGB < float64(Keys.MemoryCap) { if actualMemoryGB < float64(Keys.MemoryCap) {
break break
} }
@@ -428,13 +436,14 @@ func MemoryUsageTracker(wg *sync.WaitGroup, ctx context.Context) {
cclog.Errorf("[METRICSTORE]> error while force-freeing buffers: %s", err) cclog.Errorf("[METRICSTORE]> error while force-freeing buffers: %s", err)
} }
if freed == 0 { if freed == 0 {
cclog.Errorf("[METRICSTORE]> no more buffers to free after %d emergency frees, memory usage %.2f GB still exceeds cap %d GB", freedEmergency, memoryUsageGB, Keys.MemoryCap) cclog.Errorf("[METRICSTORE]> no more buffers to free after %d emergency frees, memory usage %.2f GB still exceeds cap %d GB", freedEmergency, actualMemoryGB, Keys.MemoryCap)
break break
} }
freedEmergency += freed freedEmergency += freed
if i%10 == 0 && freedEmergency > 0 { if i%10 == 0 && freedEmergency > 0 {
memoryUsageGB = ms.SizeInGB() runtime.ReadMemStats(&mem)
actualMemoryGB = float64(mem.Alloc) / 1e9
} }
} }
@@ -442,12 +451,13 @@ func MemoryUsageTracker(wg *sync.WaitGroup, ctx context.Context) {
debug.FreeOSMemory() debug.FreeOSMemory()
} }
memoryUsageGB = ms.SizeInGB() runtime.ReadMemStats(&mem)
actualMemoryGB = float64(mem.Alloc) / 1e9
if memoryUsageGB >= float64(Keys.MemoryCap) { if actualMemoryGB >= float64(Keys.MemoryCap) {
cclog.Errorf("[METRICSTORE]> after %d emergency frees, memory usage %.2f GB still at/above cap %d GB", freedEmergency, memoryUsageGB, Keys.MemoryCap) cclog.Errorf("[METRICSTORE]> after %d emergency frees, memory usage %.2f GB still at/above cap %d GB", freedEmergency, actualMemoryGB, Keys.MemoryCap)
} else { } else {
cclog.Infof("[METRICSTORE]> emergency freeing complete: %d buffers freed, memory now %.2f GB", freedEmergency, memoryUsageGB) cclog.Infof("[METRICSTORE]> emergency freeing complete: %d buffers freed, memory now %.2f GB", freedEmergency, actualMemoryGB)
} }
} }