Merge pull request #507 from ClusterCockpit/dev

Dev

configs/config-demo.json

@@ -20,11 +20,7 @@
     }
   ],
   "metric-store": {
-    "checkpoints": {
-      "file-format": "wal",
-      "interval": "12h"
-    },
-    "retention-in-memory": "48h",
+    "retention-in-memory": "24h",
     "memory-cap": 100
   }
 }

configs/config.json

@@ -74,14 +74,12 @@
   ],
   "metric-store": {
     "checkpoints": {
-      "interval": "12h",
       "directory": "./var/checkpoints"
     },
     "memory-cap": 100,
-    "retention-in-memory": "48h",
+    "retention-in-memory": "24h",
     "cleanup": {
       "mode": "archive",
-      "interval": "48h",
       "directory": "./var/archive"
     },
     "nats-subscriptions": [

pkg/metricstore/archive.go

@@ -24,19 +24,15 @@ func CleanUp(wg *sync.WaitGroup, ctx context.Context) {
 	if Keys.Cleanup.Mode == "archive" {
 		// Run as Archiver
 		cleanUpWorker(wg, ctx,
-			Keys.Cleanup.Interval,
+			Keys.RetentionInMemory,
 			"archiving",
 			Keys.Cleanup.RootDir,
 			false,
 		)
 	} else {
-		if Keys.Cleanup.Interval == "" {
-			Keys.Cleanup.Interval = Keys.RetentionInMemory
-		}
-
 		// Run as Deleter
 		cleanUpWorker(wg, ctx,
-			Keys.Cleanup.Interval,
+			Keys.RetentionInMemory,
 			"deleting",
 			"",
 			true,

pkg/metricstore/checkpoint.go

@@ -86,9 +86,11 @@ var (
 
 // Checkpointing starts a background worker that periodically saves metric data to disk.
 //
+// Checkpoints are written every 12 hours (hardcoded).
+//
 // Format behaviour:
-//   - "json": Periodic checkpointing based on Keys.Checkpoints.Interval
-//   - "wal":  Periodic binary snapshots + WAL rotation at Keys.Checkpoints.Interval
+//   - "json": Periodic checkpointing every checkpointInterval
+//   - "wal":  Periodic binary snapshots + WAL rotation every checkpointInterval
 func Checkpointing(wg *sync.WaitGroup, ctx context.Context) {
 	lastCheckpointMu.Lock()
 	lastCheckpoint = time.Now()
@@ -98,14 +100,8 @@ func Checkpointing(wg *sync.WaitGroup, ctx context.Context) {
 
 	wg.Go(func() {
 
-		d, err := time.ParseDuration(Keys.Checkpoints.Interval)
-		if err != nil {
-			cclog.Fatalf("[METRICSTORE]> invalid checkpoint interval '%s': %s", Keys.Checkpoints.Interval, err.Error())
-		}
-		if d <= 0 {
-			cclog.Warnf("[METRICSTORE]> checkpoint interval is zero or negative (%s), checkpointing disabled", d)
-			return
-		}
+		const checkpointInterval = 12 * time.Hour
+		d := checkpointInterval
 
 		ticker := time.NewTicker(d)
 		defer ticker.Stop()

pkg/metricstore/config.go

@@ -11,15 +11,13 @@
 //
 //	MetricStoreConfig (Keys)
 //	├─ NumWorkers: Parallel checkpoint/archive workers
-//	├─ RetentionInMemory: How long to keep data in RAM
+//	├─ RetentionInMemory: How long to keep data in RAM (also used as cleanup interval)
 //	├─ MemoryCap: Memory limit in bytes (triggers forceFree)
 //	├─ Checkpoints: Persistence configuration
-//	│  ├─ FileFormat: "json" or "wal"
-//	│  ├─ Interval: How often to save (e.g., "1h")
+//	│  ├─ FileFormat: "json" or "wal" (default: "wal")
 //	│  └─ RootDir: Checkpoint storage path
-//	├─ Cleanup: Long-term storage configuration
-//	│  ├─ Interval: How often to delete/archive
-//	│  ├─ RootDir: Archive storage path
+//	├─ Cleanup: Long-term storage configuration (interval = RetentionInMemory)
+//	│  ├─ RootDir: Archive storage path (archive mode only)
 //	│  └─ Mode: "delete" or "archive"
 //	├─ Debug: Development/debugging options
 //	└─ Subscriptions: NATS topic subscriptions for metric ingestion
@@ -61,12 +59,10 @@ const (
 // Checkpoints configures periodic persistence of in-memory metric data.
 //
 // Fields:
-//   - FileFormat: "json" (human-readable, periodic) or "wal" (binary snapshot + WAL, crash-safe)
-//   - Interval:   Duration string (e.g., "1h", "30m") between checkpoint saves
+//   - FileFormat: "json" (human-readable, periodic) or "wal" (binary snapshot + WAL, crash-safe); default is "wal"
 //   - RootDir:    Filesystem path for checkpoint files (created if missing)
 type Checkpoints struct {
 	FileFormat string `json:"file-format"`
-	Interval   string `json:"interval"`
 	RootDir    string `json:"directory"`
 }
 
@@ -80,18 +76,17 @@ type Debug struct {
 	EnableGops bool   `json:"gops"`
 }
 
-// Archive configures long-term storage of old metric data.
+// Cleanup configures long-term storage of old metric data.
 //
 // Data older than RetentionInMemory is archived to disk or deleted.
+// The cleanup interval is always RetentionInMemory.
 //
 // Fields:
-//   - ArchiveInterval: Duration string (e.g., "24h") between archive operations
-//   - RootDir:         Filesystem path for archived data (created if missing)
-//   - DeleteInstead:   If true, delete old data instead of archiving (saves disk space)
+//   - RootDir: Filesystem path for archived data (used in "archive" mode)
+//   - Mode:    "delete" (discard old data) or "archive" (write to RootDir)
 type Cleanup struct {
-	Interval string `json:"interval"`
-	RootDir  string `json:"directory"`
-	Mode     string `json:"mode"`
+	RootDir string `json:"directory"`
+	Mode    string `json:"mode"`
 }
 
 // Subscriptions defines NATS topics to subscribe to for metric ingestion.
@@ -141,7 +136,7 @@ type MetricStoreConfig struct {
 // Accessed by Init(), Checkpointing(), and other lifecycle functions.
 var Keys MetricStoreConfig = MetricStoreConfig{
 	Checkpoints: Checkpoints{
-		FileFormat: "json",
+		FileFormat: "wal",
 		RootDir:    "./var/checkpoints",
 	},
 	Cleanup: &Cleanup{

pkg/metricstore/configSchema.go

@@ -18,35 +18,26 @@ const configSchema = `{
       "type": "object",
       "properties": {
         "file-format": {
-          "description": "Specify the format for checkpoint files. Two variants: 'json' (human-readable, periodic) and 'wal' (binary snapshot + Write-Ahead Log, crash-safe). Default is 'json'.",
-          "type": "string"
-        },
-        "interval": {
-          "description": "Interval at which the metrics should be checkpointed.",
+          "description": "Specify the format for checkpoint files. Two variants: 'json' (human-readable, periodic) and 'wal' (binary snapshot + Write-Ahead Log, crash-safe). Default is 'wal'.",
           "type": "string"
         },
         "directory": {
           "description": "Path in which the checkpointed files should be placed.",
           "type": "string"
         }
-      },
-      "required": ["interval"]
+      }
     },
     "cleanup": {
-      "description": "Configuration for the cleanup process.",
+      "description": "Configuration for the cleanup process. The cleanup interval is always 'retention-in-memory'.",
       "type": "object",
       "properties": {
         "mode": {
           "description": "The operation mode (e.g., 'archive' or 'delete').",
           "type": "string",
-          "enum": ["archive", "delete"] 
-        },
-        "interval": {
-          "description": "Interval at which the cleanup runs.",
-          "type": "string"
+          "enum": ["archive", "delete"]
         },
         "directory": {
-          "description": "Target directory for operations.",
+          "description": "Target directory for archive operations.",
           "type": "string"
         }
       },
@@ -56,7 +47,7 @@ const configSchema = `{
         }
       },
       "then": {
-        "required": ["interval", "directory"]
+        "required": ["directory"]
       }
     },
     "retention-in-memory": {
@@ -86,5 +77,5 @@ const configSchema = `{
       }
     }
   },
-  "required": ["checkpoints", "retention-in-memory", "memory-cap"]
+  "required": ["retention-in-memory", "memory-cap"]
 }`