From 39635ea123f9b2d2ae9771e3e02c129a7a532477 Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Wed, 4 Mar 2026 10:37:43 +0100 Subject: [PATCH 1/2] Cleanup metricstore options Entire-Checkpoint: 2f9a4e1c2e87 --- configs/config-demo.json | 2 -- configs/config.json | 2 -- pkg/metricstore/archive.go | 8 ++------ pkg/metricstore/checkpoint.go | 16 ++++++---------- pkg/metricstore/config.go | 29 ++++++++++++----------------- pkg/metricstore/configSchema.go | 23 +++++++---------------- 6 files changed, 27 insertions(+), 53 deletions(-) diff --git a/configs/config-demo.json b/configs/config-demo.json index 50dff298..4a69ecd2 100644 --- a/configs/config-demo.json +++ b/configs/config-demo.json @@ -21,8 +21,6 @@ ], "metric-store": { "checkpoints": { - "file-format": "wal", - "interval": "12h" }, "retention-in-memory": "48h", "memory-cap": 100 diff --git a/configs/config.json b/configs/config.json index c2361a1c..f880b8d8 100644 --- a/configs/config.json +++ b/configs/config.json @@ -74,14 +74,12 @@ ], "metric-store": { "checkpoints": { - "interval": "12h", "directory": "./var/checkpoints" }, "memory-cap": 100, "retention-in-memory": "48h", "cleanup": { "mode": "archive", - "interval": "48h", "directory": "./var/archive" }, "nats-subscriptions": [ diff --git a/pkg/metricstore/archive.go b/pkg/metricstore/archive.go index 77f4264a..916736d0 100644 --- a/pkg/metricstore/archive.go +++ b/pkg/metricstore/archive.go @@ -24,19 +24,15 @@ func CleanUp(wg *sync.WaitGroup, ctx context.Context) { if Keys.Cleanup.Mode == "archive" { // Run as Archiver cleanUpWorker(wg, ctx, - Keys.Cleanup.Interval, + Keys.RetentionInMemory, "archiving", Keys.Cleanup.RootDir, false, ) } else { - if Keys.Cleanup.Interval == "" { - Keys.Cleanup.Interval = Keys.RetentionInMemory - } - // Run as Deleter cleanUpWorker(wg, ctx, - Keys.Cleanup.Interval, + Keys.RetentionInMemory, "deleting", "", true, diff --git a/pkg/metricstore/checkpoint.go b/pkg/metricstore/checkpoint.go index 45b2bc2a..ba1f7ba0 100644 --- a/pkg/metricstore/checkpoint.go +++ b/pkg/metricstore/checkpoint.go @@ -86,9 +86,11 @@ var ( // Checkpointing starts a background worker that periodically saves metric data to disk. // +// Checkpoints are written every 12 hours (hardcoded). +// // Format behaviour: -// - "json": Periodic checkpointing based on Keys.Checkpoints.Interval -// - "wal": Periodic binary snapshots + WAL rotation at Keys.Checkpoints.Interval +// - "json": Periodic checkpointing every checkpointInterval +// - "wal": Periodic binary snapshots + WAL rotation every checkpointInterval func Checkpointing(wg *sync.WaitGroup, ctx context.Context) { lastCheckpointMu.Lock() lastCheckpoint = time.Now() @@ -98,14 +100,8 @@ func Checkpointing(wg *sync.WaitGroup, ctx context.Context) { wg.Go(func() { - d, err := time.ParseDuration(Keys.Checkpoints.Interval) - if err != nil { - cclog.Fatalf("[METRICSTORE]> invalid checkpoint interval '%s': %s", Keys.Checkpoints.Interval, err.Error()) - } - if d <= 0 { - cclog.Warnf("[METRICSTORE]> checkpoint interval is zero or negative (%s), checkpointing disabled", d) - return - } + const checkpointInterval = 12 * time.Hour + d := checkpointInterval ticker := time.NewTicker(d) defer ticker.Stop() diff --git a/pkg/metricstore/config.go b/pkg/metricstore/config.go index 53716967..3b6be529 100644 --- a/pkg/metricstore/config.go +++ b/pkg/metricstore/config.go @@ -11,15 +11,13 @@ // // MetricStoreConfig (Keys) // ├─ NumWorkers: Parallel checkpoint/archive workers -// ├─ RetentionInMemory: How long to keep data in RAM +// ├─ RetentionInMemory: How long to keep data in RAM (also used as cleanup interval) // ├─ MemoryCap: Memory limit in bytes (triggers forceFree) // ├─ Checkpoints: Persistence configuration -// │ ├─ FileFormat: "json" or "wal" -// │ ├─ Interval: How often to save (e.g., "1h") +// │ ├─ FileFormat: "json" or "wal" (default: "wal") // │ └─ RootDir: Checkpoint storage path -// ├─ Cleanup: Long-term storage configuration -// │ ├─ Interval: How often to delete/archive -// │ ├─ RootDir: Archive storage path +// ├─ Cleanup: Long-term storage configuration (interval = RetentionInMemory) +// │ ├─ RootDir: Archive storage path (archive mode only) // │ └─ Mode: "delete" or "archive" // ├─ Debug: Development/debugging options // └─ Subscriptions: NATS topic subscriptions for metric ingestion @@ -61,12 +59,10 @@ const ( // Checkpoints configures periodic persistence of in-memory metric data. // // Fields: -// - FileFormat: "json" (human-readable, periodic) or "wal" (binary snapshot + WAL, crash-safe) -// - Interval: Duration string (e.g., "1h", "30m") between checkpoint saves +// - FileFormat: "json" (human-readable, periodic) or "wal" (binary snapshot + WAL, crash-safe); default is "wal" // - RootDir: Filesystem path for checkpoint files (created if missing) type Checkpoints struct { FileFormat string `json:"file-format"` - Interval string `json:"interval"` RootDir string `json:"directory"` } @@ -80,18 +76,17 @@ type Debug struct { EnableGops bool `json:"gops"` } -// Archive configures long-term storage of old metric data. +// Cleanup configures long-term storage of old metric data. // // Data older than RetentionInMemory is archived to disk or deleted. +// The cleanup interval is always RetentionInMemory. // // Fields: -// - ArchiveInterval: Duration string (e.g., "24h") between archive operations -// - RootDir: Filesystem path for archived data (created if missing) -// - DeleteInstead: If true, delete old data instead of archiving (saves disk space) +// - RootDir: Filesystem path for archived data (used in "archive" mode) +// - Mode: "delete" (discard old data) or "archive" (write to RootDir) type Cleanup struct { - Interval string `json:"interval"` - RootDir string `json:"directory"` - Mode string `json:"mode"` + RootDir string `json:"directory"` + Mode string `json:"mode"` } // Subscriptions defines NATS topics to subscribe to for metric ingestion. @@ -141,7 +136,7 @@ type MetricStoreConfig struct { // Accessed by Init(), Checkpointing(), and other lifecycle functions. var Keys MetricStoreConfig = MetricStoreConfig{ Checkpoints: Checkpoints{ - FileFormat: "json", + FileFormat: "wal", RootDir: "./var/checkpoints", }, Cleanup: &Cleanup{ diff --git a/pkg/metricstore/configSchema.go b/pkg/metricstore/configSchema.go index 67f30976..ed9bccaa 100644 --- a/pkg/metricstore/configSchema.go +++ b/pkg/metricstore/configSchema.go @@ -18,35 +18,26 @@ const configSchema = `{ "type": "object", "properties": { "file-format": { - "description": "Specify the format for checkpoint files. Two variants: 'json' (human-readable, periodic) and 'wal' (binary snapshot + Write-Ahead Log, crash-safe). Default is 'json'.", - "type": "string" - }, - "interval": { - "description": "Interval at which the metrics should be checkpointed.", + "description": "Specify the format for checkpoint files. Two variants: 'json' (human-readable, periodic) and 'wal' (binary snapshot + Write-Ahead Log, crash-safe). Default is 'wal'.", "type": "string" }, "directory": { "description": "Path in which the checkpointed files should be placed.", "type": "string" } - }, - "required": ["interval"] + } }, "cleanup": { - "description": "Configuration for the cleanup process.", + "description": "Configuration for the cleanup process. The cleanup interval is always 'retention-in-memory'.", "type": "object", "properties": { "mode": { "description": "The operation mode (e.g., 'archive' or 'delete').", "type": "string", - "enum": ["archive", "delete"] - }, - "interval": { - "description": "Interval at which the cleanup runs.", - "type": "string" + "enum": ["archive", "delete"] }, "directory": { - "description": "Target directory for operations.", + "description": "Target directory for archive operations.", "type": "string" } }, @@ -56,7 +47,7 @@ const configSchema = `{ } }, "then": { - "required": ["interval", "directory"] + "required": ["directory"] } }, "retention-in-memory": { @@ -86,5 +77,5 @@ const configSchema = `{ } } }, - "required": ["checkpoints", "retention-in-memory", "memory-cap"] + "required": ["retention-in-memory", "memory-cap"] }` From 87425c0b09bbbe9321fef0c4050f18624c140946 Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Wed, 4 Mar 2026 10:41:52 +0100 Subject: [PATCH 2/2] Cleanup and update example config files --- configs/config-demo.json | 4 +--- configs/config.json | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/configs/config-demo.json b/configs/config-demo.json index 4a69ecd2..8c72e37f 100644 --- a/configs/config-demo.json +++ b/configs/config-demo.json @@ -20,9 +20,7 @@ } ], "metric-store": { - "checkpoints": { - }, - "retention-in-memory": "48h", + "retention-in-memory": "24h", "memory-cap": 100 } } diff --git a/configs/config.json b/configs/config.json index f880b8d8..6b654e4f 100644 --- a/configs/config.json +++ b/configs/config.json @@ -77,7 +77,7 @@ "directory": "./var/checkpoints" }, "memory-cap": 100, - "retention-in-memory": "48h", + "retention-in-memory": "24h", "cleanup": { "mode": "archive", "directory": "./var/archive"