Merge pull request #60 from ClusterCockpit/dev

Cleanup README

.claude/settings.json

@@ -0,0 +1,84 @@
+{
+  "hooks": {
+    "PostToolUse": [
+      {
+        "matcher": "Task",
+        "hooks": [
+          {
+            "type": "command",
+            "command": "entire hooks claude-code post-task"
+          }
+        ]
+      },
+      {
+        "matcher": "TodoWrite",
+        "hooks": [
+          {
+            "type": "command",
+            "command": "entire hooks claude-code post-todo"
+          }
+        ]
+      }
+    ],
+    "PreToolUse": [
+      {
+        "matcher": "Task",
+        "hooks": [
+          {
+            "type": "command",
+            "command": "entire hooks claude-code pre-task"
+          }
+        ]
+      }
+    ],
+    "SessionEnd": [
+      {
+        "matcher": "",
+        "hooks": [
+          {
+            "type": "command",
+            "command": "entire hooks claude-code session-end"
+          }
+        ]
+      }
+    ],
+    "SessionStart": [
+      {
+        "matcher": "",
+        "hooks": [
+          {
+            "type": "command",
+            "command": "entire hooks claude-code session-start"
+          }
+        ]
+      }
+    ],
+    "Stop": [
+      {
+        "matcher": "",
+        "hooks": [
+          {
+            "type": "command",
+            "command": "entire hooks claude-code stop"
+          }
+        ]
+      }
+    ],
+    "UserPromptSubmit": [
+      {
+        "matcher": "",
+        "hooks": [
+          {
+            "type": "command",
+            "command": "entire hooks claude-code user-prompt-submit"
+          }
+        ]
+      }
+    ]
+  },
+  "permissions": {
+    "deny": [
+      "Read(./.entire/metadata/**)"
+    ]
+  }
+}

.entire/.gitignore

@@ -0,0 +1,4 @@
+tmp/
+settings.local.json
+metadata/
+logs/

.entire/settings.json

@@ -0,0 +1,4 @@
+{
+  "enabled": true,
+  "telemetry": true
+}

.github/dependabot.yml

@@ -0,0 +1,11 @@
+# To get started with Dependabot version updates, you'll need to specify which
+# package ecosystems to update and where the package manifests are located.
+# Please see the documentation for all configuration options:
+# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file
+
+version: 2
+updates:
+  - package-ecosystem: "gomod"
+    directory: "/"
+    schedule:
+      interval: "weekly"

.gitignore

@@ -26,3 +26,5 @@ migrateTimestamps.pl
 sample.txt
 sample_alex.txt
 sample_fritz.txt
+
+test_ccms_write_api.sh

.goreleaser.yaml

@@ -1,3 +1,4 @@
+version: 2
 before:
   hooks:
     - go mod tidy
@@ -19,7 +20,7 @@ builds:
     tags:
       - static_build
 archives:
-  - format: tar.gz
+  - formats: tar.gz
     # this name template makes the OS and Arch compatible with the results of uname.
     name_template: >-
       {{ .ProjectName }}_
@@ -30,7 +31,7 @@ archives:
 checksum:
   name_template: "checksums.txt"
 snapshot:
-  name_template: "{{ incpatch .Version }}-next"
+  version_template: "{{ incpatch .Version }}-next"
 changelog:
   sort: asc
   filters:

CLAUDE.md

@@ -0,0 +1,77 @@
+# CLAUDE.md
+
+This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
+
+## Project Overview
+
+cc-metric-store is an in-memory time-series database for HPC cluster metrics, part of the ClusterCockpit monitoring suite. Data is indexed by a hierarchical tree (cluster → host → socket/cpu/gpu) and accessed via selectors. The core storage engine lives in `cc-backend/pkg/metricstore`; this repo provides the HTTP API wrapper.
+
+## Build Commands
+
+```bash
+make                 # Build binary, copy config template, create checkpoint dirs
+make clean           # Clean build cache and binary
+make distclean       # Also remove ./var and config.json
+make swagger         # Regenerate Swagger from source comments
+make test            # Run go build, go vet, go test
+```
+
+## Testing
+
+```bash
+go test -v ./...              # Run tests
+go test -bench=. -race -v ./...  # With benchmarks and race detector
+```
+
+Integration test scripts in `/endpoint-test-scripts/` for manual API testing.
+
+## Running
+
+```bash
+./cc-metric-store                           # Uses ./config.json
+./cc-metric-store -config /path/to/config.json
+./cc-metric-store -dev                      # Enable Swagger UI at /swagger/
+./cc-metric-store -loglevel debug           # debug|info|warn|err|crit
+```
+
+## Architecture
+
+**Entry point:** `cmd/cc-metric-store/main.go`
+- `run()` → parse flags, init logging/config, connect NATS
+- `runServer()` → init metricstore from cc-backend, start HTTP server
+
+**Key packages:**
+- `internal/api/` - REST endpoints (query, write, free, debug, healthcheck) and JWT auth (Ed25519)
+- `internal/config/` - Config loading and JSON schema validation
+- External: `cc-backend/pkg/metricstore` - actual time-series storage engine
+
+**API endpoints** (all support optional JWT auth):
+- `GET /api/query/` - Query metrics with selectors
+- `POST /api/write/` - Write metrics (InfluxDB line protocol)
+- `POST /api/free/` - Free buffers up to timestamp
+- `GET /api/debug/` - Dump internal state
+- `GET /api/healthcheck/` - Node health status
+
+## Selectors
+
+Data is accessed via hierarchical selectors:
+```
+["cluster1", "host1", "cpu0"]           # Specific CPU
+["cluster1", "host1", ["cpu4", "cpu5"]] # Multiple CPUs
+["cluster1", "host1"]                   # Entire node (all CPUs implied)
+```
+
+## Configuration
+
+Config file structure (see `configs/config.json`):
+- `main` - Server address, TLS certs, JWT public key, user/group for privilege drop
+- `metrics` - Per-metric frequency and aggregation strategy (sum/avg/null)
+- `metric-store` - Checkpoints, memory cap, retention, cleanup mode, NATS subscriptions
+- `nats` - Optional NATS connection for receiving metrics
+
+## Test JWT
+
+For testing with JWT auth enabled:
+```
+eyJ0eXAiOiJKV1QiLCJhbGciOiJFZERTQSJ9.eyJ1c2VyIjoiYWRtaW4iLCJyb2xlcyI6WyJST0xFX0FETUlOIiwiUk9MRV9BTkFMWVNUIiwiUk9MRV9VU0VSIl19.d-3_3FZTsadPjDEdsWrrQ7nS0edMAR4zjl-eK7rJU3HziNBfI9PDHDIpJVHTNN5E5SlLGLFXctWyKAkwhXL-Dw
+```

Makefile

@@ -1,6 +1,6 @@
 TARGET = ./cc-metric-store
 VAR = ./var/checkpoints/
-VERSION = 0.1.1
+VERSION = 1.5.0
 GIT_HASH := $(shell git rev-parse --short HEAD || echo 'development')
 CURRENT_TIME = $(shell date +"%Y-%m-%d:T%H:%M:%S")
 LD_FLAGS = '-s -X main.date=${CURRENT_TIME} -X main.version=${VERSION} -X main.commit=${GIT_HASH}'
@@ -21,7 +21,7 @@ $(VAR):
 
 swagger:
 	$(info ===>  GENERATE swagger)
-	@go run github.com/swaggo/swag/cmd/swag init -d ./internal/api,./internal/util -g api.go -o ./api
+	@go run github.com/swaggo/swag/cmd/swag init -d ./internal/api -g api.go -o ./api
 	@mv ./api/docs.go ./internal/api/docs.go
 
 clean:

README.md

@@ -5,18 +5,15 @@
 The cc-metric-store provides a simple in-memory time series database for storing
 metrics of cluster nodes at preconfigured intervals. It is meant to be used as
 part of the [ClusterCockpit suite](https://github.com/ClusterCockpit). As all
-data is kept in-memory (but written to disk as compressed JSON for long term
-storage), accessing it is very fast. It also provides topology aware
+data is kept in-memory, accessing it is very fast. It also provides topology aware
 aggregations over time _and_ nodes/sockets/cpus.
 
-There are major limitations: Data only gets written to disk at periodic
-checkpoints, not as soon as it is received. Also only the fixed configured
-duration is stored and available.
+The storage engine is provided by the
+[cc-backend](https://github.com/ClusterCockpit/cc-backend) package
+(`cc-backend/pkg/metricstore`). This repository provides the HTTP API wrapper.
 
-Go look at the [GitHub
-Issues](https://github.com/ClusterCockpit/cc-metric-store/issues) for a progress
-overview. The [NATS.io](https://nats.io/) based writing endpoint consumes messages in [this
-format of the InfluxDB line
+The [NATS.io](https://nats.io/) based writing endpoint and the HTTP write
+endpoint both consume messages in [this format of the InfluxDB line
 protocol](https://github.com/ClusterCockpit/cc-specifications/blob/master/metrics/lineprotocol_alternative.md).
 
 ## Building
@@ -24,22 +21,47 @@ protocol](https://github.com/ClusterCockpit/cc-specifications/blob/master/metric
 `cc-metric-store` can be built using the provided `Makefile`.
 It supports the following targets:
 
-- `make`: Build the application, copy a example configuration file and generate
+- `make`: Build the application, copy an example configuration file and generate
   checkpoint folders if required.
 - `make clean`: Clean the golang build cache and application binary
 - `make distclean`: In addition to the clean target also remove the `./var`
-  folder
+  folder and `config.json`
 - `make swagger`: Regenerate the Swagger files from the source comments.
-- `make test`: Run test and basic checks.
+- `make test`: Run tests and basic checks (`go build`, `go vet`, `go test`).
+
+## Running
+
+```sh
+./cc-metric-store                              # Uses ./config.json
+./cc-metric-store -config /path/to/config.json
+./cc-metric-store -dev                         # Enable Swagger UI at /swagger/
+./cc-metric-store -loglevel debug              # debug|info|warn (default)|err|crit
+./cc-metric-store -logdate                     # Add date and time to log messages
+./cc-metric-store -version                     # Show version information and exit
+./cc-metric-store -gops                        # Enable gops agent for debugging
+```
 
 ## REST API Endpoints
 
 The REST API is documented in [swagger.json](./api/swagger.json). You can
 explore and try the REST API using the integrated [SwaggerUI web
-interface](http://localhost:8082/swagger).
+interface](http://localhost:8082/swagger/) (requires the `-dev` flag).
 
 For more information on the `cc-metric-store` REST API have a look at the
-ClusterCockpit documentation [website](https://clustercockpit.org/docs/reference/cc-metric-store/ccms-rest-api/)
+ClusterCockpit documentation [website](https://clustercockpit.org/docs/reference/cc-metric-store/ccms-rest-api/).
+
+All endpoints support both trailing-slash and non-trailing-slash variants:
+
+| Method | Path                | Description                            |
+| ------ | ------------------- | -------------------------------------- |
+| `GET`  | `/api/query/`       | Query metrics with selectors           |
+| `POST` | `/api/write/`       | Write metrics (InfluxDB line protocol) |
+| `POST` | `/api/free/`        | Free buffers up to a timestamp         |
+| `GET`  | `/api/debug/`       | Dump internal state                    |
+| `GET`  | `/api/healthcheck/` | Check node health status               |
+
+If `jwt-public-key` is set in `config.json`, all endpoints require JWT
+authentication using an Ed25519 key (`Authorization: Bearer <token>`).
 
 ## Run tests
 
@@ -60,11 +82,11 @@ go test -bench=. -race -v ./...
 
 The cc-metric-store works as a time-series database and uses the InfluxDB line
 protocol as input format. Unlike InfluxDB, the data is indexed by one single
-strictly hierarchical tree structure. A selector is build out of the tags in the
+strictly hierarchical tree structure. A selector is built out of the tags in the
 InfluxDB line protocol, and can be used to select a node (not in the sense of a
 compute node, can also be a socket, cpu, ...) in that tree. The implementation
 calls those nodes `level` to avoid confusion. It is impossible to access data
-only by knowing the _socket_ or _cpu_ tag, all higher up levels have to be
+only by knowing the _socket_ or _cpu_ tag — all higher up levels have to be
 specified as well.
 
 This is what the hierarchy currently looks like:
@@ -90,18 +112,154 @@ Example selectors:
 
 1. `["cluster1", "host1", "cpu0"]`: Select only the cpu0 of host1 in cluster1
 2. `["cluster1", "host1", ["cpu4", "cpu5", "cpu6", "cpu7"]]`: Select only CPUs 4-7 of host1 in cluster1
-3. `["cluster1", "host1"]`: Select the complete node. If querying for a CPU-specific metric such as floats, all CPUs are implied
+3. `["cluster1", "host1"]`: Select the complete node. If querying for a CPU-specific metric such as flops, all CPUs are implied
 
 ## Config file
 
-You find the configuration options on the ClusterCockpit [website](https://clustercockpit.org/docs/reference/cc-metric-store/ccms-configuration/).
+The config file is a JSON document with four top-level sections.
+
+### `main`
+
+```json
+"main": {
+  "addr": "0.0.0.0:8082",
+  "https-cert-file": "",
+  "https-key-file": "",
+  "jwt-public-key": "<base64-encoded Ed25519 public key>",
+  "user": "",
+  "group": "",
+  "backend-url": ""
+}
+```
+
+- `addr`: Address and port to listen on (default: `0.0.0.0:8082`)
+- `https-cert-file` / `https-key-file`: Paths to TLS certificate/key for HTTPS
+- `jwt-public-key`: Base64-encoded Ed25519 public key for JWT authentication. If empty, no auth is required.
+- `user` / `group`: Drop privileges to this user/group after startup
+- `backend-url`: Optional URL of a cc-backend instance used as node provider
+
+### `metrics`
+
+Per-metric configuration. Each key is the metric name:
+
+```json
+"metrics": {
+  "cpu_load": { "frequency": 60, "aggregation": null },
+  "flops_any": { "frequency": 60, "aggregation": "sum" },
+  "cpu_user":  { "frequency": 60, "aggregation": "avg" }
+}
+```
+
+- `frequency`: Sampling interval in seconds
+- `aggregation`: How to aggregate sub-level data: `"sum"`, `"avg"`, or `null` (no aggregation)
+
+### `metric-store`
+
+```json
+"metric-store": {
+  "checkpoints": {
+    "file-format": "wal",
+    "directory": "./var/checkpoints"
+  },
+  "memory-cap": 100,
+  "retention-in-memory": "24h",
+  "num-workers": 0,
+  "cleanup": {
+    "mode": "archive",
+    "directory": "./var/archive"
+  },
+  "nats-subscriptions": [
+    { "subscribe-to": "hpc-nats", "cluster-tag": "fritz" }
+  ]
+}
+```
+
+- `checkpoints.file-format`: Checkpoint format: `"json"` (default, human-readable) or `"wal"` (binary WAL, crash-safe). See [Checkpoint formats](#checkpoint-formats) below.
+- `checkpoints.directory`: Root directory for checkpoint files (organized as `<dir>/<cluster>/<host>/`)
+- `memory-cap`: Approximate memory cap in MB for metric buffers
+- `retention-in-memory`: How long to keep data in memory (e.g. `"48h"`)
+- `num-workers`: Number of parallel workers for checkpoint/archive I/O (0 = auto, capped at 10)
+- `cleanup.mode`: What to do with data older than `retention-in-memory`: `"archive"` (write Parquet) or `"delete"`
+- `cleanup.directory`: Root directory for Parquet archive files (required when `mode` is `"archive"`)
+- `nats-subscriptions`: List of NATS subjects to subscribe to, with associated cluster tag
+
+### Checkpoint formats
+
+The `checkpoints.file-format` field controls how in-memory data is persisted to disk.
+
+**`"json"` (default)** — human-readable JSON snapshots written periodically. Each
+snapshot is stored as `<dir>/<cluster>/<host>/<timestamp>.json` and contains the
+full metric hierarchy. Easy to inspect and recover manually, but larger on disk
+and slower to write.
+
+**`"wal"`** — binary Write-Ahead Log format designed for crash safety. Two file
+types are used per host:
+
+- `current.wal` — append-only binary log. Every incoming data point is appended
+  immediately (magic `0xCC1DA7A1`, 4-byte CRC32 per record). Truncated trailing
+  records from unclean shutdowns are silently skipped on restart.
+- `<timestamp>.bin` — binary snapshot written at each checkpoint interval
+  (magic `0xCC5B0001`). Contains the complete hierarchical metric state
+  column-by-column. Written atomically via a `.tmp` rename.
+
+On startup the most recent `.bin` snapshot is loaded, then any remaining WAL
+entries are replayed on top. The WAL is rotated (old file deleted, new one
+started) after each successful snapshot.
+
+The `"wal"` option is the default and will be the only supported option in the
+future. The `"json"` checkpoint format is still provided to migrate from
+previous cc-metric-store version.
+
+### Parquet archive
+
+When `cleanup.mode` is `"archive"`, data that ages out of the in-memory
+retention window is written to [Apache Parquet](https://parquet.apache.org/)
+files before being freed. Files are organized as:
+
+```
+<cleanup.directory>/
+  <cluster>/
+    <timestamp>.parquet
+```
+
+One Parquet file is produced per cluster per cleanup run, consolidating all
+hosts. Rows use a long (tidy) schema:
+
+| Column      | Type    | Description                                                             |
+| ----------- | ------- | ----------------------------------------------------------------------- |
+| `cluster`   | string  | Cluster name                                                            |
+| `hostname`  | string  | Host name                                                               |
+| `metric`    | string  | Metric name                                                             |
+| `scope`     | string  | Hardware scope (`node`, `socket`, `core`, `hwthread`, `accelerator`, …) |
+| `scope_id`  | string  | Numeric ID within the scope (e.g. `"0"`)                                |
+| `timestamp` | int64   | Unix timestamp (seconds)                                                |
+| `frequency` | int64   | Sampling interval in seconds                                            |
+| `value`     | float32 | Metric value                                                            |
+
+Files are compressed with Zstandard and sorted by `(cluster, hostname, metric,
+timestamp)` for efficient columnar reads. The `cpu` prefix in the tree is
+treated as an alias for `hwthread` scope.
+
+### `nats`
+
+```json
+"nats": {
+  "address": "nats://0.0.0.0:4222",
+  "username": "root",
+  "password": "root"
+}
+```
+
+NATS connection is optional. If not configured, only the HTTP write endpoint is available.
+
+For more information see the ClusterCockpit documentation [website](https://clustercockpit.org/docs/reference/cc-metric-store/ccms-configuration/).
 
 ## Test the complete setup (excluding cc-backend itself)
 
 There are two ways for sending data to the cc-metric-store, both of which are
 supported by the
 [cc-metric-collector](https://github.com/ClusterCockpit/cc-metric-collector).
-This example uses NATS, the alternative is to use HTTP.
+This example uses NATS; the alternative is to use HTTP.
 
 ```sh
 # Only needed once, downloads the docker image
@@ -142,22 +300,25 @@ for testing:
 ```sh
 JWT="eyJ0eXAiOiJKV1QiLCJhbGciOiJFZERTQSJ9.eyJ1c2VyIjoiYWRtaW4iLCJyb2xlcyI6WyJST0xFX0FETUlOIiwiUk9MRV9BTkFMWVNUIiwiUk9MRV9VU0VSIl19.d-3_3FZTsadPjDEdsWrrQ7nS0edMAR4zjl-eK7rJU3HziNBfI9PDHDIpJVHTNN5E5SlLGLFXctWyKAkwhXL-Dw"
 
-# If the collector and store and nats-server have been running for at least 60 seconds on the same host, you may run:
-curl -H "Authorization: Bearer $JWT" -D - "http://localhost:8080/api/query" -d "{ \"cluster\": \"testcluster\", \"from\": $(expr $(date +%s) - 60), \"to\": $(date +%s), \"queries\": [{
-  \"metric\": \"load_one\",
-  \"host\": \"$(hostname)\"
-}] }"
-
-# ...
+# If the collector and store and nats-server have been running for at least 60 seconds on the same host:
+curl -H "Authorization: Bearer $JWT" \
+     "http://localhost:8082/api/query/" \
+     -d '{
+       "cluster": "testcluster",
+       "from": '"$(expr $(date +%s) - 60)"',
+       "to": '"$(date +%s)"',
+       "queries": [{ "metric": "cpu_load", "host": "'"$(hostname)"'" }]
+     }'
 ```
 
-For debugging there is a debug endpoint to dump the current content to stdout:
+For debugging, the debug endpoint dumps the current content to stdout:
 
 ```sh
 JWT="eyJ0eXAiOiJKV1QiLCJhbGciOiJFZERTQSJ9.eyJ1c2VyIjoiYWRtaW4iLCJyb2xlcyI6WyJST0xFX0FETUlOIiwiUk9MRV9BTkFMWVNUIiwiUk9MRV9VU0VSIl19.d-3_3FZTsadPjDEdsWrrQ7nS0edMAR4zjl-eK7rJU3HziNBfI9PDHDIpJVHTNN5E5SlLGLFXctWyKAkwhXL-Dw"
 
-# If the collector and store and nats-server have been running for at least 60 seconds on the same host, you may run:
-curl -H "Authorization: Bearer $JWT" -D - "http://localhost:8080/api/debug"
+# Dump everything
+curl -H "Authorization: Bearer $JWT" "http://localhost:8082/api/debug/"
 
-# ...
+# Dump a specific selector (colon-separated path)
+curl -H "Authorization: Bearer $JWT" "http://localhost:8082/api/debug/?selector=testcluster:host1"
 ```

ReleaseNotes.md

@@ -1,19 +1,35 @@
-# `cc-metric-store` version 0.1.1
+# `cc-metric-store` version 1.5.0
 
-This is a bugfix release of `cc-metric-store`, the metric timeseries cache
-implementation of ClusterCockpit.
+This is a major release of `cc-metric-store`, the metric timeseries cache
+implementation of ClusterCockpit. Since the storage engine is now part of
+`cc-backend` we will follow the version number of `cc-backend`.
 For release specific notes visit the [ClusterCockpit Documentation](https://clusterockpit.org/docs/release/).
 
-Notable changes in version 0.1.0:
-
-- Cleanup of code and restructuring
-- Document REST API with Swagger
-- Introduce REST API versioning
-- Provide Swagger UI test web-frontend
-- Introduce re-sampling of metric data
-- Support also ms, ns in line protocol
-- Support NATS credentials
-
 ## Breaking changes
 
-None
+- The internal `memorystore`, `avro`, `resampler`, and `util` packages have been
+  removed. The storage engine is now provided by the
+  [`cc-backend`](https://github.com/ClusterCockpit/cc-backend) package
+  (`cc-backend/pkg/metricstore`). This repository is now the HTTP API wrapper
+  only.
+- The configuration schema has changed. Refer to `configs/config.json` for the
+  updated structure.
+
+## Notable changes
+
+- **Storage engine extracted to `cc-backend` library**: The entire in-memory
+  time-series storage engine was moved to `cc-backend/pkg/metricstore`. This
+  reduces duplication in the ClusterCockpit suite and enables shared maintenance
+  of the storage layer.
+- **HealthCheck API endpoint**: New `GET /api/healthcheck/` endpoint reports the
+  health status of cluster nodes.
+- **Dynamic memory management**: Memory limits can now be adjusted at runtime via
+  a callback from the `cc-backend` library.
+- **Configuration schema validation**: The config and metric config JSON schemas
+  have been updated and are now validated against the structs they describe.
+- **Startup refactored**: Application startup has been split into `cli.go` and
+  `server.go` for clearer separation of concerns.
+- **`go fix` applied**: Codebase updated to current Go idioms.
+- **Dependency upgrades**: `nats.go` bumped from 1.36.0 to 1.47.0;
+  `cc-lib` updated to v2.8.0; `cc-backend` updated to v1.5.0; various other
+  module upgrades.

api/swagger.json

@@ -19,11 +19,6 @@
     "paths": {
         "/debug/": {
             "post": {
-                "security": [
-                    {
-                        "ApiKeyAuth": []
-                    }
-                ],
                 "description": "This endpoint allows the users to print the content of",
                 "produces": [
                     "application/json"
@@ -71,16 +66,16 @@
                             "$ref": "#/definitions/api.ErrorResponse"
                         }
                     }
-                }
-            }
                 },
-        "/free/": {
-            "post": {
                 "security": [
                     {
                         "ApiKeyAuth": []
                     }
-                ],
+                ]
+            }
+        },
+        "/free/": {
+            "post": {
                 "description": "This endpoint allows the users to free the Buffers from the",
                 "produces": [
                     "application/json"
@@ -127,16 +122,16 @@
                             "$ref": "#/definitions/api.ErrorResponse"
                         }
                     }
-                }
-            }
                 },
-        "/healthcheck/": {
-            "get": {
                 "security": [
                     {
                         "ApiKeyAuth": []
                     }
-                ],
+                ]
+            }
+        },
+        "/healthcheck/": {
+            "get": {
                 "description": "This endpoint allows the users to check if a node is healthy",
                 "produces": [
                     "application/json"
@@ -184,16 +179,16 @@
                             "$ref": "#/definitions/api.ErrorResponse"
                         }
                     }
-                }
-            }
                 },
-        "/query/": {
-            "get": {
                 "security": [
                     {
                         "ApiKeyAuth": []
                     }
-                ],
+                ]
+            }
+        },
+        "/query/": {
+            "get": {
                 "description": "This endpoint allows the users to retrieve data from the",
                 "consumes": [
                     "application/json"
@@ -212,7 +207,7 @@
                         "in": "body",
                         "required": true,
                         "schema": {
-                            "$ref": "#/definitions/api.ApiQueryRequest"
+                            "$ref": "#/definitions/api.APIQueryRequest"
                         }
                     }
                 ],
@@ -220,7 +215,7 @@
                     "200": {
                         "description": "API query response object",
                         "schema": {
-                            "$ref": "#/definitions/api.ApiQueryResponse"
+                            "$ref": "#/definitions/api.APIQueryResponse"
                         }
                     },
                     "400": {
@@ -247,16 +242,16 @@
                             "$ref": "#/definitions/api.ErrorResponse"
                         }
                     }
-                }
-            }
                 },
-        "/write/": {
-            "post": {
                 "security": [
                     {
                         "ApiKeyAuth": []
                     }
-                ],
+                ]
+            }
+        },
+        "/write/": {
+            "post": {
                 "consumes": [
                     "text/plain"
                 ],
@@ -302,12 +297,17 @@
                             "$ref": "#/definitions/api.ErrorResponse"
                         }
                     }
+                },
+                "security": [
+                    {
+                        "ApiKeyAuth": []
                     }
+                ]
             }
         }
     },
     "definitions": {
-        "api.ApiMetricData": {
+        "api.APIMetricData": {
             "type": "object",
             "properties": {
                 "avg": {
@@ -339,7 +339,7 @@
                 }
             }
         },
-        "api.ApiQuery": {
+        "api.APIQuery": {
             "type": "object",
             "properties": {
                 "aggreg": {
@@ -377,7 +377,7 @@
                 }
             }
         },
-        "api.ApiQueryRequest": {
+        "api.APIQueryRequest": {
             "type": "object",
             "properties": {
                 "cluster": {
@@ -395,7 +395,7 @@
                 "queries": {
                     "type": "array",
                     "items": {
-                        "$ref": "#/definitions/api.ApiQuery"
+                        "$ref": "#/definitions/api.APIQuery"
                     }
                 },
                 "to": {
@@ -412,13 +412,13 @@
                 }
             }
         },
-        "api.ApiQueryResponse": {
+        "api.APIQueryResponse": {
             "type": "object",
             "properties": {
                 "queries": {
                     "type": "array",
                     "items": {
-                        "$ref": "#/definitions/api.ApiQuery"
+                        "$ref": "#/definitions/api.APIQuery"
                     }
                 },
                 "results": {
@@ -426,7 +426,7 @@
                     "items": {
                         "type": "array",
                         "items": {
-                            "$ref": "#/definitions/api.ApiMetricData"
+                            "$ref": "#/definitions/api.APIMetricData"
                         }
                     }
                 }

api/swagger.yaml

@@ -1,6 +1,6 @@
 basePath: /api/
 definitions:
-  api.ApiMetricData:
+  api.APIMetricData:
     properties:
       avg:
         type: number
@@ -21,7 +21,7 @@ definitions:
       to:
         type: integer
     type: object
-  api.ApiQuery:
+  api.APIQuery:
     properties:
       aggreg:
         type: boolean
@@ -46,7 +46,7 @@ definitions:
           type: string
         type: array
     type: object
-  api.ApiQueryRequest:
+  api.APIQueryRequest:
     properties:
       cluster:
         type: string
@@ -58,7 +58,7 @@ definitions:
         type: integer
       queries:
         items:
-          $ref: '#/definitions/api.ApiQuery'
+          $ref: '#/definitions/api.APIQuery'
         type: array
       to:
         type: integer
@@ -69,16 +69,16 @@ definitions:
       with-stats:
         type: boolean
     type: object
-  api.ApiQueryResponse:
+  api.APIQueryResponse:
     properties:
       queries:
         items:
-          $ref: '#/definitions/api.ApiQuery'
+          $ref: '#/definitions/api.APIQuery'
         type: array
       results:
         items:
           items:
-            $ref: '#/definitions/api.ApiMetricData'
+            $ref: '#/definitions/api.APIMetricData'
           type: array
         type: array
     type: object
@@ -222,14 +222,14 @@ paths:
         name: request
         required: true
         schema:
-          $ref: '#/definitions/api.ApiQueryRequest'
+          $ref: '#/definitions/api.APIQueryRequest'
       produces:
       - application/json
       responses:
         "200":
           description: API query response object
           schema:
-            $ref: '#/definitions/api.ApiQueryResponse'
+            $ref: '#/definitions/api.APIQueryResponse'
         "400":
           description: Bad Request
           schema:

cmd/cc-metric-store/main.go

@@ -1,32 +1,32 @@
 // Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved.
+// All rights reserved. This file is part of cc-metric-store.
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
+
 package main
 
 import (
 	"context"
-	"crypto/tls"
 	"flag"
 	"fmt"
-	"log"
-	"net"
-	"net/http"
 	"os"
 	"os/signal"
-	"runtime"
 	"runtime/debug"
 	"sync"
 	"syscall"
-	"time"
 
+	"github.com/ClusterCockpit/cc-backend/pkg/metricstore"
+	ccconf "github.com/ClusterCockpit/cc-lib/v2/ccConfig"
+	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
+	"github.com/ClusterCockpit/cc-lib/v2/nats"
+	"github.com/ClusterCockpit/cc-lib/v2/runtime"
 	"github.com/ClusterCockpit/cc-metric-store/internal/api"
-	"github.com/ClusterCockpit/cc-metric-store/internal/avro"
 	"github.com/ClusterCockpit/cc-metric-store/internal/config"
-	"github.com/ClusterCockpit/cc-metric-store/internal/memorystore"
-	"github.com/ClusterCockpit/cc-metric-store/internal/runtimeEnv"
 	"github.com/google/gops/agent"
-	httpSwagger "github.com/swaggo/http-swagger"
+)
+
+const (
+	envGOGC = "GOGC"
 )
 
 var (
@@ -35,147 +35,139 @@ var (
 	version string
 )
 
-func main() {
-	var configFile string
-	var enableGopsAgent, flagVersion, flagDev bool
-	flag.StringVar(&configFile, "config", "./config.json", "configuration file")
-	flag.BoolVar(&enableGopsAgent, "gops", false, "Listen via github.com/google/gops/agent")
-	flag.BoolVar(&flagDev, "dev", false, "Enable development Swagger UI component")
-	flag.BoolVar(&flagVersion, "version", false, "Show version information and exit")
-	flag.Parse()
+var (
+	flagGops, flagVersion, flagDev, flagLogDateTime bool
+	flagConfigFile, flagLogLevel                    string
+)
 
-	if flagVersion {
+func printVersion() {
 	fmt.Printf("Version:\t%s\n", version)
 	fmt.Printf("Git hash:\t%s\n", commit)
 	fmt.Printf("Build time:\t%s\n", date)
-		os.Exit(0)
-	}
-
-	startupTime := time.Now()
-	config.Init(configFile)
-	memorystore.Init(config.Keys.Metrics)
-	ms := memorystore.GetMemoryStore()
-
-	if enableGopsAgent || config.Keys.Debug.EnableGops {
-		if err := agent.Listen(agent.Options{}); err != nil {
-			log.Fatal(err)
-		}
-	}
-
-	d, err := time.ParseDuration(config.Keys.Checkpoints.Restore)
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	restoreFrom := startupTime.Add(-d)
-	log.Printf("Loading checkpoints newer than %s\n", restoreFrom.Format(time.RFC3339))
-	files, err := ms.FromCheckpointFiles(config.Keys.Checkpoints.RootDir, restoreFrom.Unix())
-	loadedData := ms.SizeInBytes() / 1024 / 1024 // In MB
-	if err != nil {
-		log.Fatalf("Loading checkpoints failed: %s\n", err.Error())
-	} else {
-		log.Printf("Checkpoints loaded (%d files, %d MB, that took %fs)\n", files, loadedData, time.Since(startupTime).Seconds())
-	}
-
-	// Try to use less memory by forcing a GC run here and then
-	// lowering the target percentage. The default of 100 means
-	// that only once the ratio of new allocations execeds the
-	// previously active heap, a GC is triggered.
-	// Forcing a GC here will set the "previously active heap"
-	// to a minumum.
-	runtime.GC()
-	if loadedData > 1000 && os.Getenv("GOGC") == "" {
-		debug.SetGCPercent(10)
-	}
-
-	ctx, shutdown := context.WithCancel(context.Background())
+}
 
+func runServer(ctx context.Context) error {
 	var wg sync.WaitGroup
-	wg.Add(4)
 
-	memorystore.Retention(&wg, ctx)
-	memorystore.Checkpointing(&wg, ctx)
-	memorystore.Archiving(&wg, ctx)
-	avro.DataStaging(&wg, ctx)
+	mscfg := ccconf.GetPackageConfig("metrics")
+	if mscfg == nil {
+		return fmt.Errorf("missing metrics configuration")
+	}
+	config.InitMetrics(mscfg)
 
-	r := http.NewServeMux()
-	api.MountRoutes(r)
-
-	if flagDev {
-		log.Print("Enable Swagger UI!")
-		r.HandleFunc("GET /swagger/", httpSwagger.Handler(
-			httpSwagger.URL("http://"+config.Keys.HttpConfig.Address+"/swagger/doc.json")))
+	mscfg = ccconf.GetPackageConfig("metric-store")
+	if mscfg == nil {
+		return fmt.Errorf("missing metricstore configuration")
 	}
 
-	server := &http.Server{
-		Handler:      r,
-		Addr:         config.Keys.HttpConfig.Address,
-		WriteTimeout: 30 * time.Second,
-		ReadTimeout:  30 * time.Second,
+	metricstore.Init(mscfg, config.GetMetrics(), &wg)
+
+	// Set GC percent if not configured
+	if os.Getenv(envGOGC) == "" {
+		debug.SetGCPercent(15)
 	}
 
-	// Start http or https server
-	listener, err := net.Listen("tcp", config.Keys.HttpConfig.Address)
+	if config.Keys.BackendURL != "" {
+		ms := metricstore.GetMemoryStore()
+		ms.SetNodeProvider(api.NewBackendNodeProvider(config.Keys.BackendURL))
+		cclog.Infof("Node provider configured with backend URL: %s", config.Keys.BackendURL)
+	}
+
+	// Initialize HTTP server
+	srv, err := NewServer(version, commit, date)
 	if err != nil {
-		log.Fatalf("starting http listener failed: %v", err)
+		return fmt.Errorf("creating server: %w", err)
 	}
 
-	if config.Keys.HttpConfig.CertFile != "" && config.Keys.HttpConfig.KeyFile != "" {
-		cert, err := tls.LoadX509KeyPair(config.Keys.HttpConfig.CertFile, config.Keys.HttpConfig.KeyFile)
-		if err != nil {
-			log.Fatalf("loading X509 keypair failed: %v", err)
+	// Channel to collect errors from server
+	errChan := make(chan error, 1)
+
+	// Start HTTP server
+	wg.Go(func() {
+		if err := srv.Start(ctx); err != nil {
+			errChan <- err
 		}
-		listener = tls.NewListener(listener, &tls.Config{
-			Certificates: []tls.Certificate{cert},
-			CipherSuites: []uint16{
-				tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
-				tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
-			},
-			MinVersion:               tls.VersionTLS12,
-			PreferServerCipherSuites: true,
 	})
-		fmt.Printf("HTTPS server listening at %s...\n", config.Keys.HttpConfig.Address)
-	} else {
-		fmt.Printf("HTTP server listening at %s...\n", config.Keys.HttpConfig.Address)
-	}
-
-	wg.Add(1)
-	go func() {
-		defer wg.Done()
-		if err = server.Serve(listener); err != nil && err != http.ErrServerClosed {
-			log.Fatalf("starting server failed: %v", err)
-		}
-	}()
 
+	// Handle shutdown signals
 	wg.Add(1)
 	sigs := make(chan os.Signal, 1)
 	signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
 	go func() {
 		defer wg.Done()
-		<-sigs
-		runtimeEnv.SystemdNotifiy(false, "Shutting down ...")
-		server.Shutdown(context.Background())
-		shutdown()
-		memorystore.Shutdown()
+		select {
+		case <-sigs:
+			cclog.Info("Shutdown signal received")
+		case <-ctx.Done():
+		}
+
+		runtime.SystemdNotify(false, "Shutting down ...")
+		srv.Shutdown(ctx)
 	}()
 
-	if config.Keys.Nats != nil {
-		for _, natsConf := range config.Keys.Nats {
-			// TODO: When multiple nats configs share a URL, do a single connect.
-			wg.Add(1)
-			nc := natsConf
+	runtime.SystemdNotify(true, "running")
+
+	// Wait for completion or errors
 	go func() {
-				// err := ReceiveNats(conf.Nats, decodeLine, runtime.NumCPU()-1, ctx)
-				err := api.ReceiveNats(nc, ms, 1, ctx)
-				if err != nil {
-					log.Fatal(err)
-				}
-				wg.Done()
+		wg.Wait()
+		close(errChan)
 	}()
+
+	// Wait for either server startup error or shutdown completion
+	if err := <-errChan; err != nil {
+		return err
+	}
+
+	cclog.Print("Graceful shutdown completed!")
+	return nil
+}
+
+func run() error {
+	flag.BoolVar(&flagGops, "gops", false, "Listen via github.com/google/gops/agent (for debugging)")
+	flag.BoolVar(&flagDev, "dev", false, "Enable development component: Swagger UI")
+	flag.BoolVar(&flagVersion, "version", false, "Show version information and exit")
+	flag.BoolVar(&flagLogDateTime, "logdate", false, "Set this flag to add date and time to log messages")
+	flag.StringVar(&flagConfigFile, "config", "./config.json", "Specify alternative path to `config.json`")
+	flag.StringVar(&flagLogLevel, "loglevel", "warn", "Sets the logging level: `[debug, info, warn (default), err, crit]`")
+	flag.Parse()
+
+	if flagVersion {
+		printVersion()
+		return nil
+	}
+
+	cclog.Init(flagLogLevel, flagLogDateTime)
+
+	if flagGops || config.Keys.Debug.EnableGops {
+		if err := agent.Listen(agent.Options{}); err != nil {
+			return fmt.Errorf("starting gops agent: %w", err)
 		}
 	}
 
-	runtimeEnv.SystemdNotifiy(true, "running")
-	wg.Wait()
-	log.Print("Graceful shutdown completed!")
+	ccconf.Init(flagConfigFile)
+
+	cfg := ccconf.GetPackageConfig("main")
+	if cfg == nil {
+		return fmt.Errorf("main configuration must be present")
+	}
+
+	config.Init(cfg)
+
+	natsConfig := ccconf.GetPackageConfig("nats")
+	if err := nats.Init(natsConfig); err != nil {
+		cclog.Warnf("initializing (optional) nats client: %s", err.Error())
+	}
+	nats.Connect()
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	return runServer(ctx)
+}
+
+func main() {
+	if err := run(); err != nil {
+		cclog.Error(err.Error())
+		os.Exit(1)
+	}
 }

cmd/cc-metric-store/server.go

@@ -0,0 +1,142 @@
+// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
+// All rights reserved. This file is part of cc-metric-store.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+// Package main provides the entry point for the ClusterCockpit metric store server.
+// This file contains HTTP server setup, routing configuration.
+package main
+
+import (
+	"context"
+	"crypto/tls"
+	"fmt"
+	"net"
+	"net/http"
+	"time"
+
+	"github.com/ClusterCockpit/cc-backend/pkg/metricstore"
+	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
+	"github.com/ClusterCockpit/cc-lib/v2/nats"
+	"github.com/ClusterCockpit/cc-lib/v2/runtime"
+	"github.com/ClusterCockpit/cc-metric-store/internal/api"
+	"github.com/ClusterCockpit/cc-metric-store/internal/config"
+	httpSwagger "github.com/swaggo/http-swagger/v2"
+)
+
+// Server encapsulates the HTTP server state and dependencies
+type Server struct {
+	router *http.ServeMux
+	server *http.Server
+}
+
+// NewServer creates and initializes a new Server instance
+func NewServer(version, commit, buildDate string) (*Server, error) {
+	s := &Server{
+		router: http.NewServeMux(),
+	}
+
+	if err := s.init(); err != nil {
+		return nil, err
+	}
+
+	return s, nil
+}
+
+func (s *Server) init() error {
+	api.MountRoutes(s.router)
+
+	if flagDev {
+		cclog.Print("Enable Swagger UI!")
+		s.router.HandleFunc("GET /swagger/", httpSwagger.Handler(
+			httpSwagger.URL("http://"+config.Keys.Address+"/swagger/doc.json")))
+	}
+
+	return nil
+}
+
+// Server timeout defaults (in seconds)
+const (
+	defaultReadTimeout  = 30
+	defaultWriteTimeout = 30
+)
+
+func (s *Server) Start(ctx context.Context) error {
+	// Use configurable timeouts with defaults
+	readTimeout := time.Duration(defaultReadTimeout) * time.Second
+	writeTimeout := time.Duration(defaultWriteTimeout) * time.Second
+
+	s.server = &http.Server{
+		ReadTimeout:  readTimeout,
+		WriteTimeout: writeTimeout,
+		Handler:      s.router,
+		Addr:         config.Keys.Address,
+	}
+
+	// Start http or https server
+	listener, err := net.Listen("tcp", config.Keys.Address)
+	if err != nil {
+		return fmt.Errorf("starting listener on '%s': %w", config.Keys.Address, err)
+	}
+
+	if config.Keys.CertFile != "" && config.Keys.KeyFile != "" {
+		cert, err := tls.LoadX509KeyPair(
+			config.Keys.CertFile, config.Keys.KeyFile)
+		if err != nil {
+			return fmt.Errorf("loading X509 keypair (check 'https-cert-file' and 'https-key-file' in config.json): %w", err)
+		}
+		listener = tls.NewListener(listener, &tls.Config{
+			Certificates: []tls.Certificate{cert},
+			CipherSuites: []uint16{
+				tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
+				tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
+			},
+			MinVersion:               tls.VersionTLS12,
+			PreferServerCipherSuites: true,
+		})
+		cclog.Infof("HTTPS server listening at %s...", config.Keys.Address)
+	} else {
+		cclog.Infof("HTTP server listening at %s...", config.Keys.Address)
+	}
+
+	// Because this program will want to bind to a privileged port (like 80), the listener must
+	// be established first, then the user can be changed, and after that,
+	// the actual http server can be started.
+	if err := runtime.DropPrivileges(config.Keys.Group, config.Keys.User); err != nil {
+		return fmt.Errorf("dropping privileges: %w", err)
+	}
+
+	// Handle context cancellation for graceful shutdown
+	go func() {
+		<-ctx.Done()
+		shutdownCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+		defer cancel()
+		if err := s.server.Shutdown(shutdownCtx); err != nil {
+			cclog.Errorf("Server shutdown error: %v", err)
+		}
+	}()
+
+	if err = s.server.Serve(listener); err != nil && err != http.ErrServerClosed {
+		return fmt.Errorf("server failed: %w", err)
+	}
+	return nil
+}
+
+func (s *Server) Shutdown(ctx context.Context) {
+	// Create a shutdown context with timeout
+	shutdownCtx, cancel := context.WithTimeout(ctx, 30*time.Second)
+	defer cancel()
+
+	nc := nats.GetClient()
+	if nc != nil {
+		nc.Close()
+	}
+
+	// First shut down the server gracefully (waiting for all ongoing requests)
+	if err := s.server.Shutdown(shutdownCtx); err != nil {
+		cclog.Errorf("Server shutdown error: %v", err)
+	}
+
+	// Archive all the metric store data
+	metricstore.Shutdown()
+}

configs/config.json

@@ -1,4 +1,38 @@
 {
+  "main": {
+    "addr": "0.0.0.0:8082",
+    "https-cert-file": "",
+    "https-key-file": "",
+    "jwt-public-key": "kzfYrYy+TzpanWZHJ5qSdMj5uKUWgq74BWhQG6copP0="
+  },
+  "nats": {
+    "address": "nats://0.0.0.0:4222",
+    "username": "root",
+    "password": "root"
+  },
+  "metric-store": {
+    "checkpoints": {
+      "interval": "12h",
+      "directory": "./var/checkpoints"
+    },
+    "memory-cap": 100,
+    "retention-in-memory": "48h",
+    "cleanup": {
+      "mode": "archive",
+      "interval": "48h",
+      "directory": "./var/archive"
+    },
+    "nats-subscriptions": [
+      {
+        "subscribe-to": "hpc-nats",
+        "cluster-tag": "fritz"
+      },
+      {
+        "subscribe-to": "hpc-nats",
+        "cluster-tag": "alex"
+      }
+    ]
+  },
   "metrics": {
     "debug_metric": {
       "frequency": 60,
@@ -164,22 +198,5 @@
       "frequency": 60,
       "aggregation": "avg"
     }
-    },
-    "checkpoints": {
-        "interval": "12h",
-        "directory": "./var/checkpoints",
-        "restore": "48h"
-    },
-    "archive": {
-        "interval": "50h",
-        "directory": "./var/archive"
-    },
-    "http-api": {
-        "address": "localhost:8082",
-        "https-cert-file": null,
-        "https-key-file": null
-    },
-    "retention-in-memory": "48h",
-    "nats": null,
-    "jwt-public-key": "kzfYrYy+TzpanWZHJ5qSdMj5uKUWgq74BWhQG6copP0="
+  }
 }

go.mod

@@ -1,39 +1,76 @@
 module github.com/ClusterCockpit/cc-metric-store
 
-go 1.22
+go 1.25.0
 
 require (
-	github.com/golang-jwt/jwt/v4 v4.5.0
-	github.com/google/gops v0.3.28
-	github.com/influxdata/line-protocol/v2 v2.2.1
-	github.com/nats-io/nats.go v1.36.0
-	github.com/swaggo/http-swagger v1.3.4
-	github.com/swaggo/swag v1.16.3
+	github.com/ClusterCockpit/cc-backend v1.5.0
+	github.com/ClusterCockpit/cc-lib/v2 v2.8.0
+	github.com/ClusterCockpit/cc-line-protocol/v2 v2.4.0
+	github.com/golang-jwt/jwt/v4 v4.5.2
+	github.com/google/gops v0.3.29
+	github.com/santhosh-tekuri/jsonschema/v5 v5.3.1
+	github.com/swaggo/http-swagger/v2 v2.0.2
+	github.com/swaggo/swag v1.16.6
 )
 
 require (
 	github.com/KyleBanks/depth v1.2.1 // indirect
-	github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect
-	github.com/go-openapi/jsonpointer v0.21.0 // indirect
-	github.com/go-openapi/jsonreference v0.21.0 // indirect
-	github.com/go-openapi/spec v0.21.0 // indirect
-	github.com/go-openapi/swag v0.23.0 // indirect
-	github.com/golang/snappy v0.0.1 // indirect
-	github.com/josharian/intern v1.0.0 // indirect
-	github.com/klauspost/compress v1.17.9 // indirect
-	github.com/linkedin/goavro/v2 v2.13.1 // indirect
-	github.com/mailru/easyjson v0.7.7 // indirect
-	github.com/nats-io/nkeys v0.4.7 // indirect
+	github.com/andybalholm/brotli v1.2.0 // indirect
+	github.com/aws/aws-sdk-go-v2 v1.41.3 // indirect
+	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.6 // indirect
+	github.com/aws/aws-sdk-go-v2/config v1.32.11 // indirect
+	github.com/aws/aws-sdk-go-v2/credentials v1.19.11 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.19 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.19 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.19 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.5 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.20 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.6 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.11 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.19 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.19 // indirect
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.96.4 // indirect
+	github.com/aws/aws-sdk-go-v2/service/signin v1.0.7 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.30.12 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.16 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.41.8 // indirect
+	github.com/aws/smithy-go v1.24.2 // indirect
+	github.com/cpuguy83/go-md2man/v2 v2.0.7 // indirect
+	github.com/fsnotify/fsnotify v1.9.0 // indirect
+	github.com/go-openapi/jsonpointer v0.22.5 // indirect
+	github.com/go-openapi/jsonreference v0.21.5 // indirect
+	github.com/go-openapi/spec v0.22.4 // indirect
+	github.com/go-openapi/swag/conv v0.25.5 // indirect
+	github.com/go-openapi/swag/jsonname v0.25.5 // indirect
+	github.com/go-openapi/swag/jsonutils v0.25.5 // indirect
+	github.com/go-openapi/swag/loading v0.25.5 // indirect
+	github.com/go-openapi/swag/stringutils v0.25.5 // indirect
+	github.com/go-openapi/swag/typeutils v0.25.5 // indirect
+	github.com/go-openapi/swag/yamlutils v0.25.5 // indirect
+	github.com/google/uuid v1.6.0 // indirect
+	github.com/klauspost/compress v1.18.4 // indirect
+	github.com/mattn/go-sqlite3 v1.14.34 // indirect
+	github.com/nats-io/nats.go v1.49.0 // indirect
+	github.com/nats-io/nkeys v0.4.15 // indirect
 	github.com/nats-io/nuid v1.0.1 // indirect
+	github.com/parquet-go/bitpack v1.0.0 // indirect
+	github.com/parquet-go/jsonlite v1.4.0 // indirect
+	github.com/parquet-go/parquet-go v0.28.0 // indirect
+	github.com/pierrec/lz4/v4 v4.1.26 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
-	github.com/swaggo/files v1.0.1 // indirect
-	github.com/urfave/cli/v2 v2.27.1 // indirect
-	github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913 // indirect
-	golang.org/x/crypto v0.24.0 // indirect
-	golang.org/x/net v0.26.0 // indirect
-	golang.org/x/sys v0.21.0 // indirect
-	golang.org/x/text v0.16.0 // indirect
-	golang.org/x/tools v0.22.0 // indirect
-	gopkg.in/yaml.v3 v3.0.1 // indirect
-	sigs.k8s.io/yaml v1.4.0 // indirect
+	github.com/swaggo/files/v2 v2.0.2 // indirect
+	github.com/twpayne/go-geom v1.6.1 // indirect
+	github.com/urfave/cli/v2 v2.27.7 // indirect
+	github.com/xrash/smetrics v0.0.0-20250705151800-55b8f293f342 // indirect
+	go.yaml.in/yaml/v2 v2.4.3 // indirect
+	go.yaml.in/yaml/v3 v3.0.4 // indirect
+	golang.org/x/crypto v0.48.0 // indirect
+	golang.org/x/mod v0.33.0 // indirect
+	golang.org/x/sync v0.19.0 // indirect
+	golang.org/x/sys v0.41.0 // indirect
+	golang.org/x/text v0.34.0 // indirect
+	golang.org/x/tools v0.42.0 // indirect
+	google.golang.org/protobuf v1.36.11 // indirect
+	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
+	sigs.k8s.io/yaml v1.6.0 // indirect
 )

go.sum

@@ -1,45 +1,122 @@
+github.com/ClusterCockpit/cc-backend v1.5.0 h1:KGAxcb6wB9h2ICX1im3ldHJvmkslzV980aweUtAXCKM=
+github.com/ClusterCockpit/cc-backend v1.5.0/go.mod h1:ASdOqR16C0b23KSvzh0BafydIvgA3RzZocQ/hQ1hhN8=
+github.com/ClusterCockpit/cc-lib/v2 v2.8.0 h1:ROduRzRuusi+6kLB991AAu3Pp2AHOasQJFJc7JU/n/E=
+github.com/ClusterCockpit/cc-lib/v2 v2.8.0/go.mod h1:FwD8vnTIbBM3ngeLNKmCvp9FoSjQZm7xnuaVxEKR23o=
+github.com/ClusterCockpit/cc-line-protocol/v2 v2.4.0 h1:hIzxgTBWcmCIHtoDKDkSCsKCOCOwUC34sFsbD2wcW0Q=
+github.com/ClusterCockpit/cc-line-protocol/v2 v2.4.0/go.mod h1:y42qUu+YFmu5fdNuUAS4VbbIKxVjxCvbVqFdpdh8ahY=
+github.com/DATA-DOG/go-sqlmock v1.5.2 h1:OcvFkGmslmlZibjAjaHm3L//6LiuBgolP7OputlJIzU=
+github.com/DATA-DOG/go-sqlmock v1.5.2/go.mod h1:88MAG/4G7SMwSE3CeA0ZKzrT5CiOU3OJ+JlNzwDqpNU=
 github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc=
 github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE=
-github.com/cpuguy83/go-md2man/v2 v2.0.4 h1:wfIWP927BUkWJb2NmU/kNDYIBTh/ziUX91+lVfRxZq4=
-github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
-github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
-github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
-github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/frankban/quicktest v1.11.0/go.mod h1:K+q6oSqb0W0Ininfk863uOk1lMy69l/P6txr3mVT54s=
-github.com/frankban/quicktest v1.11.2/go.mod h1:K+q6oSqb0W0Ininfk863uOk1lMy69l/P6txr3mVT54s=
+github.com/NVIDIA/go-nvml v0.13.0-1 h1:OLX8Jq3dONuPOQPC7rndB6+iDmDakw0XTYgzMxObkEw=
+github.com/NVIDIA/go-nvml v0.13.0-1/go.mod h1:+KNA7c7gIBH7SKSJ1ntlwkfN80zdx8ovl4hrK3LmPt4=
+github.com/alecthomas/assert/v2 v2.10.0 h1:jjRCHsj6hBJhkmhznrCzoNpbA3zqy0fYiUcYZP/GkPY=
+github.com/alecthomas/assert/v2 v2.10.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k=
+github.com/alecthomas/repr v0.4.0 h1:GhI2A8MACjfegCPVq9f1FLvIBS+DrQ2KQBFZP1iFzXc=
+github.com/alecthomas/repr v0.4.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4=
+github.com/andybalholm/brotli v1.2.0 h1:ukwgCxwYrmACq68yiUqwIWnGY0cTPox/M94sVwToPjQ=
+github.com/andybalholm/brotli v1.2.0/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY=
+github.com/apapsch/go-jsonmerge/v2 v2.0.0 h1:axGnT1gRIfimI7gJifB699GoE/oq+F2MU7Dml6nw9rQ=
+github.com/apapsch/go-jsonmerge/v2 v2.0.0/go.mod h1:lvDnEdqiQrp0O42VQGgmlKpxL1AP2+08jFMw88y4klk=
+github.com/aws/aws-sdk-go-v2 v1.41.3 h1:4kQ/fa22KjDt13QCy1+bYADvdgcxpfH18f0zP542kZA=
+github.com/aws/aws-sdk-go-v2 v1.41.3/go.mod h1:mwsPRE8ceUUpiTgF7QmQIJ7lgsKUPQOUl3o72QBrE1o=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.6 h1:N4lRUXZpZ1KVEUn6hxtco/1d2lgYhNn1fHkkl8WhlyQ=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.6/go.mod h1:lyw7GFp3qENLh7kwzf7iMzAxDn+NzjXEAGjKS2UOKqI=
+github.com/aws/aws-sdk-go-v2/config v1.32.11 h1:ftxI5sgz8jZkckuUHXfC/wMUc8u3fG1vQS0plr2F2Zs=
+github.com/aws/aws-sdk-go-v2/config v1.32.11/go.mod h1:twF11+6ps9aNRKEDimksp923o44w/Thk9+8YIlzWMmo=
+github.com/aws/aws-sdk-go-v2/credentials v1.19.11 h1:NdV8cwCcAXrCWyxArt58BrvZJ9pZ9Fhf9w6Uh5W3Uyc=
+github.com/aws/aws-sdk-go-v2/credentials v1.19.11/go.mod h1:30yY2zqkMPdrvxBqzI9xQCM+WrlrZKSOpSJEsylVU+8=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.19 h1:INUvJxmhdEbVulJYHI061k4TVuS3jzzthNvjqvVvTKM=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.19/go.mod h1:FpZN2QISLdEBWkayloda+sZjVJL+e9Gl0k1SyTgcswU=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.19 h1:/sECfyq2JTifMI2JPyZ4bdRN77zJmr6SrS1eL3augIA=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.19/go.mod h1:dMf8A5oAqr9/oxOfLkC/c2LU/uMcALP0Rgn2BD5LWn0=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.19 h1:AWeJMk33GTBf6J20XJe6qZoRSJo0WfUhsMdUKhoODXE=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.19/go.mod h1:+GWrYoaAsV7/4pNHpwh1kiNLXkKaSoppxQq9lbH8Ejw=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.8.5 h1:clHU5fm//kWS1C2HgtgWxfQbFbx4b6rx+5jzhgX9HrI=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.8.5/go.mod h1:O3h0IK87yXci+kg6flUKzJnWeziQUKciKrLjcatSNcY=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.20 h1:qi3e/dmpdONhj1RyIZdi6DKKpDXS5Lb8ftr3p7cyHJc=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.20/go.mod h1:V1K+TeJVD5JOk3D9e5tsX2KUdL7BlB+FV6cBhdobN8c=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.6 h1:XAq62tBTJP/85lFD5oqOOe7YYgWxY9LvWq8plyDvDVg=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.6/go.mod h1:x0nZssQ3qZSnIcePWLvcoFisRXJzcTVvYpAAdYX8+GI=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.11 h1:BYf7XNsJMzl4mObARUBUib+j2tf0U//JAAtTnYqvqCw=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.11/go.mod h1:aEUS4WrNk/+FxkBZZa7tVgp4pGH+kFGW40Y8rCPqt5g=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.19 h1:X1Tow7suZk9UCJHE1Iw9GMZJJl0dAnKXXP1NaSDHwmw=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.19/go.mod h1:/rARO8psX+4sfjUQXp5LLifjUt8DuATZ31WptNJTyQA=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.19 h1:JnQeStZvPHFHeyky/7LbMlyQjUa+jIBj36OlWm0pzIk=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.19/go.mod h1:HGyasyHvYdFQeJhvDHfH7HXkHh57htcJGKDZ+7z+I24=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.96.4 h1:4ExZyubQ6LQQVuF2Qp9OsfEvsTdAWh5Gfwf6PgIdLdk=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.96.4/go.mod h1:NF3JcMGOiARAss1ld3WGORCw71+4ExDD2cbbdKS5PpA=
+github.com/aws/aws-sdk-go-v2/service/signin v1.0.7 h1:Y2cAXlClHsXkkOvWZFXATr34b0hxxloeQu/pAZz2row=
+github.com/aws/aws-sdk-go-v2/service/signin v1.0.7/go.mod h1:idzZ7gmDeqeNrSPkdbtMp9qWMgcBwykA7P7Rzh5DXVU=
+github.com/aws/aws-sdk-go-v2/service/sso v1.30.12 h1:iSsvB9EtQ09YrsmIc44Heqlx5ByGErqhPK1ZQLppias=
+github.com/aws/aws-sdk-go-v2/service/sso v1.30.12/go.mod h1:fEWYKTRGoZNl8tZ77i61/ccwOMJdGxwOhWCkp6TXAr0=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.16 h1:EnUdUqRP1CNzt2DkV67tJx6XDN4xlfBFm+bzeNOQVb0=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.16/go.mod h1:Jic/xv0Rq/pFNCh3WwpH4BEqdbSAl+IyHro8LbibHD8=
+github.com/aws/aws-sdk-go-v2/service/sts v1.41.8 h1:XQTQTF75vnug2TXS8m7CVJfC2nniYPZnO1D4Np761Oo=
+github.com/aws/aws-sdk-go-v2/service/sts v1.41.8/go.mod h1:Xgx+PR1NUOjNmQY+tRMnouRp83JRM8pRMw/vCaVhPkI=
+github.com/aws/smithy-go v1.24.2 h1:FzA3bu/nt/vDvmnkg+R8Xl46gmzEDam6mZ1hzmwXFng=
+github.com/aws/smithy-go v1.24.2/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
+github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
+github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
+github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
+github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/cpuguy83/go-md2man/v2 v2.0.7 h1:zbFlGlXEAKlwXpmvle3d8Oe3YnkKIK4xSRTd3sHPnBo=
+github.com/cpuguy83/go-md2man/v2 v2.0.7/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
+github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
+github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/expr-lang/expr v1.17.8 h1:W1loDTT+0PQf5YteHSTpju2qfUfNoBt4yw9+wOEU9VM=
+github.com/expr-lang/expr v1.17.8/go.mod h1:8/vRC7+7HBzESEqt5kKpYXxrxkr31SaO8r40VO/1IT4=
 github.com/frankban/quicktest v1.13.0 h1:yNZif1OkDfNoDfb9zZa9aXIpejNR4F23Wely0c+Qdqk=
 github.com/frankban/quicktest v1.13.0/go.mod h1:qLE0fzW0VuyUAJgPU19zByoIr0HtCHN/r/VLSOOIySU=
-github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ=
-github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY=
-github.com/go-openapi/jsonreference v0.21.0 h1:Rs+Y7hSXT83Jacb7kFyjn4ijOuVGSvOdF2+tg1TRrwQ=
-github.com/go-openapi/jsonreference v0.21.0/go.mod h1:LmZmgsrTkVg9LG4EaHeY8cBDslNPMo06cago5JNLkm4=
-github.com/go-openapi/spec v0.21.0 h1:LTVzPc3p/RzRnkQqLRndbAzjY0d0BCL72A6j3CdL9ZY=
-github.com/go-openapi/spec v0.21.0/go.mod h1:78u6VdPw81XU44qEWGhtr982gJ5BWg2c0I5XwVMotYk=
-github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE=
-github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ=
-github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg=
-github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
-github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
-github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
-github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
-github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
-github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
-github.com/google/gops v0.3.28 h1:2Xr57tqKAmQYRAfG12E+yLcoa2Y42UJo2lOrUFL9ark=
-github.com/google/gops v0.3.28/go.mod h1:6f6+Nl8LcHrzJwi8+p0ii+vmBFSlB4f8cOOkTJ7sk4c=
-github.com/influxdata/line-protocol-corpus v0.0.0-20210519164801-ca6fa5da0184/go.mod h1:03nmhxzZ7Xk2pdG+lmMd7mHDfeVOYFyhOgwO61qWU98=
+github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k=
+github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
+github.com/go-openapi/jsonpointer v0.22.5 h1:8on/0Yp4uTb9f4XvTrM2+1CPrV05QPZXu+rvu2o9jcA=
+github.com/go-openapi/jsonpointer v0.22.5/go.mod h1:gyUR3sCvGSWchA2sUBJGluYMbe1zazrYWIkWPjjMUY0=
+github.com/go-openapi/jsonreference v0.21.5 h1:6uCGVXU/aNF13AQNggxfysJ+5ZcU4nEAe+pJyVWRdiE=
+github.com/go-openapi/jsonreference v0.21.5/go.mod h1:u25Bw85sX4E2jzFodh1FOKMTZLcfifd1Q+iKKOUxExw=
+github.com/go-openapi/spec v0.22.4 h1:4pxGjipMKu0FzFiu/DPwN3CTBRlVM2yLf/YTWorYfDQ=
+github.com/go-openapi/spec v0.22.4/go.mod h1:WQ6Ai0VPWMZgMT4XySjlRIE6GP1bGQOtEThn3gcWLtQ=
+github.com/go-openapi/swag v0.19.15 h1:D2NRCBzS9/pEY3gP9Nl8aDqGUcPFrwG2p+CNFrLyrCM=
+github.com/go-openapi/swag/conv v0.25.5 h1:wAXBYEXJjoKwE5+vc9YHhpQOFj2JYBMF2DUi+tGu97g=
+github.com/go-openapi/swag/conv v0.25.5/go.mod h1:CuJ1eWvh1c4ORKx7unQnFGyvBbNlRKbnRyAvDvzWA4k=
+github.com/go-openapi/swag/jsonname v0.25.5 h1:8p150i44rv/Drip4vWI3kGi9+4W9TdI3US3uUYSFhSo=
+github.com/go-openapi/swag/jsonname v0.25.5/go.mod h1:jNqqikyiAK56uS7n8sLkdaNY/uq6+D2m2LANat09pKU=
+github.com/go-openapi/swag/jsonutils v0.25.5 h1:XUZF8awQr75MXeC+/iaw5usY/iM7nXPDwdG3Jbl9vYo=
+github.com/go-openapi/swag/jsonutils v0.25.5/go.mod h1:48FXUaz8YsDAA9s5AnaUvAmry1UcLcNVWUjY42XkrN4=
+github.com/go-openapi/swag/jsonutils/fixtures_test v0.25.5 h1:SX6sE4FrGb4sEnnxbFL/25yZBb5Hcg1inLeErd86Y1U=
+github.com/go-openapi/swag/jsonutils/fixtures_test v0.25.5/go.mod h1:/2KvOTrKWjVA5Xli3DZWdMCZDzz3uV/T7bXwrKWPquo=
+github.com/go-openapi/swag/loading v0.25.5 h1:odQ/umlIZ1ZVRteI6ckSrvP6e2w9UTF5qgNdemJHjuU=
+github.com/go-openapi/swag/loading v0.25.5/go.mod h1:I8A8RaaQ4DApxhPSWLNYWh9NvmX2YKMoB9nwvv6oW6g=
+github.com/go-openapi/swag/stringutils v0.25.5 h1:NVkoDOA8YBgtAR/zvCx5rhJKtZF3IzXcDdwOsYzrB6M=
+github.com/go-openapi/swag/stringutils v0.25.5/go.mod h1:PKK8EZdu4QJq8iezt17HM8RXnLAzY7gW0O1KKarrZII=
+github.com/go-openapi/swag/typeutils v0.25.5 h1:EFJ+PCga2HfHGdo8s8VJXEVbeXRCYwzzr9u4rJk7L7E=
+github.com/go-openapi/swag/typeutils v0.25.5/go.mod h1:itmFmScAYE1bSD8C4rS0W+0InZUBrB2xSPbWt6DLGuc=
+github.com/go-openapi/swag/yamlutils v0.25.5 h1:kASCIS+oIeoc55j28T4o8KwlV2S4ZLPT6G0iq2SSbVQ=
+github.com/go-openapi/swag/yamlutils v0.25.5/go.mod h1:Gek1/SjjfbYvM+Iq4QGwa/2lEXde9n2j4a3wI3pNuOQ=
+github.com/go-openapi/testify/enable/yaml/v2 v2.4.0 h1:7SgOMTvJkM8yWrQlU8Jm18VeDPuAvB/xWrdxFJkoFag=
+github.com/go-openapi/testify/enable/yaml/v2 v2.4.0/go.mod h1:14iV8jyyQlinc9StD7w1xVPW3CO3q1Gj04Jy//Kw4VM=
+github.com/go-openapi/testify/v2 v2.4.0 h1:8nsPrHVCWkQ4p8h1EsRVymA2XABB4OT40gcvAu+voFM=
+github.com/go-openapi/testify/v2 v2.4.0/go.mod h1:HCPmvFFnheKK2BuwSA0TbbdxJ3I16pjwMkYkP4Ywn54=
+github.com/golang-jwt/jwt/v4 v4.5.2 h1:YtQM7lnr8iZ+j5q71MGKkNw9Mn7AjHM68uc9g5fXeUI=
+github.com/golang-jwt/jwt/v4 v4.5.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
+github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
+github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
+github.com/google/gops v0.3.29 h1:n98J2qSOK1NJvRjdLDcjgDryjpIBGhbaqph1mXKL0rY=
+github.com/google/gops v0.3.29/go.mod h1:8N3jZftuPazvUwtYY/ncG4iPrjp15ysNKLfq+QQPiwc=
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY=
+github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ=
+github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=
+github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg=
+github.com/influxdata/influxdb-client-go/v2 v2.14.0 h1:AjbBfJuq+QoaXNcrova8smSjwJdUHnwvfjMF71M1iI4=
+github.com/influxdata/influxdb-client-go/v2 v2.14.0/go.mod h1:Ahpm3QXKMJslpXl3IftVLVezreAUtBOTZssDrjZEFHI=
+github.com/influxdata/line-protocol v0.0.0-20210922203350-b1ad95c89adf h1:7JTmneyiNEwVBOHSjoMxiWAqB992atOeepeFYegn5RU=
+github.com/influxdata/line-protocol v0.0.0-20210922203350-b1ad95c89adf/go.mod h1:xaLFMmpvUxqXtVkUJfg9QmT88cDaCJ3ZKgdZ78oO8Qo=
 github.com/influxdata/line-protocol-corpus v0.0.0-20210922080147-aa28ccfb8937 h1:MHJNQ+p99hFATQm6ORoLmpUCF7ovjwEFshs/NHzAbig=
 github.com/influxdata/line-protocol-corpus v0.0.0-20210922080147-aa28ccfb8937/go.mod h1:BKR9c0uHSmRgM/se9JhFHtTT7JTO67X23MtKMHtZcpo=
-github.com/influxdata/line-protocol/v2 v2.0.0-20210312151457-c52fdecb625a/go.mod h1:6+9Xt5Sq1rWx+glMgxhcg2c0DUaehK+5TDcPZ76GypY=
-github.com/influxdata/line-protocol/v2 v2.1.0/go.mod h1:QKw43hdUBg3GTk2iC3iyCxksNj7PX9aUSeYOYE/ceHY=
-github.com/influxdata/line-protocol/v2 v2.2.1 h1:EAPkqJ9Km4uAxtMRgUubJyqAr6zgWM0dznKMLRauQRE=
-github.com/influxdata/line-protocol/v2 v2.2.1/go.mod h1:DmB3Cnh+3oxmG6LOBIxce4oaL4CPj3OmMPgvauXh+tM=
-github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
-github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
-github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA=
-github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
+github.com/klauspost/compress v1.18.4 h1:RPhnKRAQ4Fh8zU2FY/6ZFDwTVTxgJ/EMydqSTzE9a2c=
+github.com/klauspost/compress v1.18.4/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4=
 github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
 github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
@@ -47,88 +124,82 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
-github.com/linkedin/goavro/v2 v2.13.1 h1:4qZ5M0QzQFDRqccsroJlgOJznqAS/TpdvXg55h429+I=
-github.com/linkedin/goavro/v2 v2.13.1/go.mod h1:KXx+erlq+RPlGSPmLF7xGo6SAbh8sCQ53x064+ioxhk=
-github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
-github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
-github.com/nats-io/nats.go v1.36.0 h1:suEUPuWzTSse/XhESwqLxXGuj8vGRuPRoG7MoRN/qyU=
-github.com/nats-io/nats.go v1.36.0/go.mod h1:Ubdu4Nh9exXdSz0RVWRFBbRfrbSxOYd26oF0wkWclB8=
-github.com/nats-io/nkeys v0.4.7 h1:RwNJbbIdYCoClSDNY7QVKZlyb/wfT6ugvFCiKy6vDvI=
-github.com/nats-io/nkeys v0.4.7/go.mod h1:kqXRgRDPlGy7nGaEDMuYzmiJCIAAWDK0IMBtDmGD0nc=
+github.com/mattn/go-sqlite3 v1.14.34 h1:3NtcvcUnFBPsuRcno8pUtupspG/GM+9nZ88zgJcp6Zk=
+github.com/mattn/go-sqlite3 v1.14.34/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
+github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
+github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
+github.com/nats-io/nats.go v1.49.0 h1:yh/WvY59gXqYpgl33ZI+XoVPKyut/IcEaqtsiuTJpoE=
+github.com/nats-io/nats.go v1.49.0/go.mod h1:fDCn3mN5cY8HooHwE2ukiLb4p4G4ImmzvXyJt+tGwdw=
+github.com/nats-io/nkeys v0.4.15 h1:JACV5jRVO9V856KOapQ7x+EY8Jo3qw1vJt/9Jpwzkk4=
+github.com/nats-io/nkeys v0.4.15/go.mod h1:CpMchTXC9fxA5zrMo4KpySxNjiDVvr8ANOSZdiNfUrs=
 github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
 github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
-github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
-github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
-github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M=
-github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA=
+github.com/oapi-codegen/runtime v1.2.0 h1:RvKc1CVS1QeKSNzO97FBQbSMZyQ8s6rZd+LpmzwHMP4=
+github.com/oapi-codegen/runtime v1.2.0/go.mod h1:Y7ZhmmlE8ikZOmuHRRndiIm7nf3xcVv+YMweKgG1DT0=
+github.com/parquet-go/bitpack v1.0.0 h1:AUqzlKzPPXf2bCdjfj4sTeacrUwsT7NlcYDMUQxPcQA=
+github.com/parquet-go/bitpack v1.0.0/go.mod h1:XnVk9TH+O40eOOmvpAVZ7K2ocQFrQwysLMnc6M/8lgs=
+github.com/parquet-go/jsonlite v1.4.0 h1:RTG7prqfO0HD5egejU8MUDBN8oToMj55cgSV1I0zNW4=
+github.com/parquet-go/jsonlite v1.4.0/go.mod h1:nDjpkpL4EOtqs6NQugUsi0Rleq9sW/OtC1NnZEnxzF0=
+github.com/parquet-go/parquet-go v0.28.0 h1:ECyksyv8T2pOrlLsN7aWJIoQakyk/HtxQ2lchgS4els=
+github.com/parquet-go/parquet-go v0.28.0/go.mod h1:navtkAYr2LGoJVp141oXPlO/sxLvaOe3la2JEoD8+rg=
+github.com/pierrec/lz4/v4 v4.1.26 h1:GrpZw1gZttORinvzBdXPUXATeqlJjqUG/D87TKMnhjY=
+github.com/pierrec/lz4/v4 v4.1.26/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4=
+github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
+github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o=
+github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg=
+github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk=
+github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE=
+github.com/prometheus/common v0.67.5 h1:pIgK94WWlQt1WLwAC5j2ynLaBRDiinoAb86HZHTUGI4=
+github.com/prometheus/common v0.67.5/go.mod h1:SjE/0MzDEEAyrdr5Gqc6G+sXI67maCxzaT3A2+HqjUw=
+github.com/prometheus/procfs v0.20.0 h1:AA7aCvjxwAquZAlonN7888f2u4IN8WVeFgBi4k82M4Q=
+github.com/prometheus/procfs v0.20.0/go.mod h1:o9EMBZGRyvDrSPH1RqdxhojkuXstoe4UlK79eF5TGGo=
+github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
+github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
 github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
-github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
-github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.7.5/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
-github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
-github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
-github.com/swaggo/files v1.0.1 h1:J1bVJ4XHZNq0I46UU90611i9/YzdrF7x92oX1ig5IdE=
-github.com/swaggo/files v1.0.1/go.mod h1:0qXmMNH6sXNf+73t65aKeB+ApmgxdnkQzVTAj2uaMUg=
-github.com/swaggo/http-swagger v1.3.4 h1:q7t/XLx0n15H1Q9/tk3Y9L4n210XzJF5WtnDX64a5ww=
-github.com/swaggo/http-swagger v1.3.4/go.mod h1:9dAh0unqMBAlbp1uE2Uc2mQTxNMU/ha4UbucIg1MFkQ=
-github.com/swaggo/swag v1.16.3 h1:PnCYjPCah8FK4I26l2F/KQ4yz3sILcVUN3cTlBFA9Pg=
-github.com/swaggo/swag v1.16.3/go.mod h1:DImHIuOFXKpMFAQjcC7FG4m3Dg4+QuUgUzJmKjI/gRk=
-github.com/urfave/cli/v2 v2.27.1 h1:8xSQ6szndafKVRmfyeUMxkNUJQMjL1F2zmsZ+qHpfho=
-github.com/urfave/cli/v2 v2.27.1/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ=
-github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913 h1:+qGGcbkzsfDQNPPe9UDgpxAWQrhbbBXOYJFQDq/dtJw=
-github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913/go.mod h1:4aEEwZQutDLsQv2Deui4iYQ6DWTxR14g6m8Wv88+Xqk=
-github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
-golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
-golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI=
-golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM=
-golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
-golang.org/x/mod v0.18.0 h1:5+9lSbEzPSdWkH32vYPBwEpX8KwDbM52Ud9xBUvNlb0=
-golang.org/x/mod v0.18.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
-golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
-golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
-golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
-golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ=
-golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE=
-golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M=
-golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
-golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws=
-golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
-golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
-golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
-golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
-golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
-golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4=
-golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
-golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
-golang.org/x/tools v0.22.0 h1:gqSGLZqv+AI9lIQzniJ0nZDRG5GBPsSi+DRNHWNz6yA=
-golang.org/x/tools v0.22.0/go.mod h1:aCwcsjqvq7Yqt6TNyX7QMU2enbQ/Gt0bo6krSeEri+c=
-golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 h1:lZUw3E0/J3roVtGQ+SCrUrg3ON6NgVqpn3+iol9aGu4=
+github.com/santhosh-tekuri/jsonschema/v5 v5.3.1/go.mod h1:uToXkOrWAZ6/Oc07xWQrPOhJotwFIyu2bBVN41fcDUY=
+github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
+github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
+github.com/swaggo/files/v2 v2.0.2 h1:Bq4tgS/yxLB/3nwOMcul5oLEUKa877Ykgz3CJMVbQKU=
+github.com/swaggo/files/v2 v2.0.2/go.mod h1:TVqetIzZsO9OhHX1Am9sRf9LdrFZqoK49N37KON/jr0=
+github.com/swaggo/http-swagger/v2 v2.0.2 h1:FKCdLsl+sFCx60KFsyM0rDarwiUSZ8DqbfSyIKC9OBg=
+github.com/swaggo/http-swagger/v2 v2.0.2/go.mod h1:r7/GBkAWIfK6E/OLnE8fXnviHiDeAHmgIyooa4xm3AQ=
+github.com/swaggo/swag v1.16.6 h1:qBNcx53ZaX+M5dxVyTrgQ0PJ/ACK+NzhwcbieTt+9yI=
+github.com/swaggo/swag v1.16.6/go.mod h1:ngP2etMK5a0P3QBizic5MEwpRmluJZPHjXcMoj4Xesg=
+github.com/twpayne/go-geom v1.6.1 h1:iLE+Opv0Ihm/ABIcvQFGIiFBXd76oBIar9drAwHFhR4=
+github.com/twpayne/go-geom v1.6.1/go.mod h1:Kr+Nly6BswFsKM5sd31YaoWS5PeDDH2NftJTK7Gd028=
+github.com/urfave/cli/v2 v2.27.7 h1:bH59vdhbjLv3LAvIu6gd0usJHgoTTPhCFib8qqOwXYU=
+github.com/urfave/cli/v2 v2.27.7/go.mod h1:CyNAG/xg+iAOg0N4MPGZqVmv2rCoP267496AOXUZjA4=
+github.com/xrash/smetrics v0.0.0-20250705151800-55b8f293f342 h1:FnBeRrxr7OU4VvAzt5X7s6266i6cSVkkFPS0TuXWbIg=
+github.com/xrash/smetrics v0.0.0-20250705151800-55b8f293f342/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM=
+github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU=
+github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E=
+go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0=
+go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8=
+go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
+go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
+golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts=
+golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos=
+golang.org/x/mod v0.33.0 h1:tHFzIWbBifEmbwtGz65eaWyGiGZatSrT9prnU8DbVL8=
+golang.org/x/mod v0.33.0/go.mod h1:swjeQEj+6r7fODbD2cqrnje9PnziFuw4bmLbBZFrQ5w=
+golang.org/x/net v0.51.0 h1:94R/GTO7mt3/4wIKpcR5gkGmRLOuE/2hNGeWq/GBIFo=
+golang.org/x/net v0.51.0/go.mod h1:aamm+2QF5ogm02fjy5Bb7CQ0WMt1/WVM7FtyaTLlA9Y=
+golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4=
+golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
+golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k=
+golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
+golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk=
+golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA=
+golang.org/x/tools v0.42.0 h1:uNgphsn75Tdz5Ji2q36v/nsFSfR/9BRFvqhGBaJGd5k=
+golang.org/x/tools v0.42.0/go.mod h1:Ma6lCIwGZvHK6XtgbswSoWroEkhugApmsXyrUmBhfr0=
+google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=
+google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
-gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E=
-sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY=
+sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs=
+sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4=

internal/api/api.go

@@ -1,448 +1,25 @@
 // Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved.
+// All rights reserved. This file is part of cc-metric-store.
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
-package api
-
-import (
-	"bufio"
-	"encoding/json"
-	"errors"
-	"fmt"
-	"io"
-	"log"
-	"math"
-	"net/http"
-	"strconv"
-	"strings"
-
-	"github.com/ClusterCockpit/cc-metric-store/internal/memorystore"
-	"github.com/ClusterCockpit/cc-metric-store/internal/util"
-	"github.com/influxdata/line-protocol/v2/lineprotocol"
-)
 
+// Package api provides the REST API for cc-metric-store.
+//
 // @title       cc-metric-store REST API
 // @version     1.0.0
 // @description API for cc-metric-store
-
+//
 // @contact.name  ClusterCockpit Project
 // @contact.url   https://clustercockpit.org
 // @contact.email support@clustercockpit.org
-
+//
 // @license.name MIT License
 // @license.url  https://opensource.org/licenses/MIT
-
+//
 // @host     localhost:8082
 // @basePath /api/
-
+//
 // @securityDefinitions.apikey ApiKeyAuth
 // @in                         header
 // @name                       X-Auth-Token
-
-// ErrorResponse model
-type ErrorResponse struct {
-	// Statustext of Errorcode
-	Status string `json:"status"`
-	Error  string `json:"error"` // Error Message
-}
-
-type ApiMetricData struct {
-	Error      *string         `json:"error,omitempty"`
-	Data       util.FloatArray `json:"data,omitempty"`
-	From       int64           `json:"from"`
-	To         int64           `json:"to"`
-	Resolution int64           `json:"resolution"`
-	Avg        util.Float      `json:"avg"`
-	Min        util.Float      `json:"min"`
-	Max        util.Float      `json:"max"`
-}
-
-func handleError(err error, statusCode int, rw http.ResponseWriter) {
-	// log.Warnf("REST ERROR : %s", err.Error())
-	rw.Header().Add("Content-Type", "application/json")
-	rw.WriteHeader(statusCode)
-	json.NewEncoder(rw).Encode(ErrorResponse{
-		Status: http.StatusText(statusCode),
-		Error:  err.Error(),
-	})
-}
-
-// TODO: Optimize this, just like the stats endpoint!
-func (data *ApiMetricData) AddStats() {
-	n := 0
-	sum, min, max := 0.0, math.MaxFloat64, -math.MaxFloat64
-	for _, x := range data.Data {
-		if x.IsNaN() {
-			continue
-		}
-
-		n += 1
-		sum += float64(x)
-		min = math.Min(min, float64(x))
-		max = math.Max(max, float64(x))
-	}
-
-	if n > 0 {
-		avg := sum / float64(n)
-		data.Avg = util.Float(avg)
-		data.Min = util.Float(min)
-		data.Max = util.Float(max)
-	} else {
-		data.Avg, data.Min, data.Max = util.NaN, util.NaN, util.NaN
-	}
-}
-
-func (data *ApiMetricData) ScaleBy(f util.Float) {
-	if f == 0 || f == 1 {
-		return
-	}
-
-	data.Avg *= f
-	data.Min *= f
-	data.Max *= f
-	for i := 0; i < len(data.Data); i++ {
-		data.Data[i] *= f
-	}
-}
-
-func (data *ApiMetricData) PadDataWithNull(ms *memorystore.MemoryStore, from, to int64, metric string) {
-	minfo, ok := ms.Metrics[metric]
-	if !ok {
-		return
-	}
-
-	if (data.From / minfo.Frequency) > (from / minfo.Frequency) {
-		padfront := int((data.From / minfo.Frequency) - (from / minfo.Frequency))
-		ndata := make([]util.Float, 0, padfront+len(data.Data))
-		for i := 0; i < padfront; i++ {
-			ndata = append(ndata, util.NaN)
-		}
-		for j := 0; j < len(data.Data); j++ {
-			ndata = append(ndata, data.Data[j])
-		}
-		data.Data = ndata
-	}
-}
-
-// handleFree godoc
-// @summary
-// @tags free
-// @description This endpoint allows the users to free the Buffers from the
-// metric store. This endpoint offers the users to remove then systematically
-// and also allows then to prune the data under node, if they do not want to
-// remove the whole node.
-// @produce     json
-// @param       to        query    string        false  "up to timestamp"
-// @success     200            {string} string  "ok"
-// @failure     400            {object} api.ErrorResponse       "Bad Request"
-// @failure     401            {object} api.ErrorResponse       "Unauthorized"
-// @failure     403            {object} api.ErrorResponse       "Forbidden"
-// @failure     500            {object} api.ErrorResponse       "Internal Server Error"
-// @security    ApiKeyAuth
-// @router      /free/ [post]
-func handleFree(rw http.ResponseWriter, r *http.Request) {
-	rawTo := r.URL.Query().Get("to")
-	if rawTo == "" {
-		handleError(errors.New("'to' is a required query parameter"), http.StatusBadRequest, rw)
-		return
-	}
-
-	to, err := strconv.ParseInt(rawTo, 10, 64)
-	if err != nil {
-		handleError(err, http.StatusInternalServerError, rw)
-		return
-	}
-
-	// // TODO: lastCheckpoint might be modified by different go-routines.
-	// // Load it using the sync/atomic package?
-	// freeUpTo := lastCheckpoint.Unix()
-	// if to < freeUpTo {
-	// 	freeUpTo = to
-	// }
-
-	bodyDec := json.NewDecoder(r.Body)
-	var selectors [][]string
-	err = bodyDec.Decode(&selectors)
-	if err != nil {
-		http.Error(rw, err.Error(), http.StatusBadRequest)
-		return
-	}
-
-	ms := memorystore.GetMemoryStore()
-	n := 0
-	for _, sel := range selectors {
-		bn, err := ms.Free(sel, to)
-		if err != nil {
-			handleError(err, http.StatusInternalServerError, rw)
-			return
-		}
-
-		n += bn
-	}
-
-	rw.WriteHeader(http.StatusOK)
-	fmt.Fprintf(rw, "buffers freed: %d\n", n)
-}
-
-// handleWrite godoc
-// @summary Receive metrics in InfluxDB line-protocol
-// @tags write
-// @description Write data to the in-memory store in the InfluxDB line-protocol using [this format](https://github.com/ClusterCockpit/cc-specifications/blob/master/metrics/lineprotocol_alternative.md)
-
-// @accept      plain
-// @produce     json
-// @param       cluster        query string false "If the lines in the body do not have a cluster tag, use this value instead."
-// @success     200            {string} string  "ok"
-// @failure     400            {object} api.ErrorResponse       "Bad Request"
-// @failure     401            {object} api.ErrorResponse       "Unauthorized"
-// @failure     403            {object} api.ErrorResponse       "Forbidden"
-// @failure     500            {object} api.ErrorResponse       "Internal Server Error"
-// @security    ApiKeyAuth
-// @router      /write/ [post]
-func handleWrite(rw http.ResponseWriter, r *http.Request) {
-	bytes, err := io.ReadAll(r.Body)
-	rw.Header().Add("Content-Type", "application/json")
-	if err != nil {
-		handleError(err, http.StatusInternalServerError, rw)
-		return
-	}
-
-	ms := memorystore.GetMemoryStore()
-	dec := lineprotocol.NewDecoderWithBytes(bytes)
-	if err := decodeLine(dec, ms, r.URL.Query().Get("cluster")); err != nil {
-		log.Printf("/api/write error: %s", err.Error())
-		handleError(err, http.StatusBadRequest, rw)
-		return
-	}
-	rw.WriteHeader(http.StatusOK)
-}
-
-type ApiQueryRequest struct {
-	Cluster     string     `json:"cluster"`
-	Queries     []ApiQuery `json:"queries"`
-	ForAllNodes []string   `json:"for-all-nodes"`
-	From        int64      `json:"from"`
-	To          int64      `json:"to"`
-	WithStats   bool       `json:"with-stats"`
-	WithData    bool       `json:"with-data"`
-	WithPadding bool       `json:"with-padding"`
-}
-
-type ApiQueryResponse struct {
-	Queries []ApiQuery        `json:"queries,omitempty"`
-	Results [][]ApiMetricData `json:"results"`
-}
-
-type ApiQuery struct {
-	Type        *string    `json:"type,omitempty"`
-	SubType     *string    `json:"subtype,omitempty"`
-	Metric      string     `json:"metric"`
-	Hostname    string     `json:"host"`
-	Resolution  int64      `json:"resolution"`
-	TypeIds     []string   `json:"type-ids,omitempty"`
-	SubTypeIds  []string   `json:"subtype-ids,omitempty"`
-	ScaleFactor util.Float `json:"scale-by,omitempty"`
-	Aggregate   bool       `json:"aggreg"`
-}
-
-// handleQuery godoc
-// @summary    Query metrics
-// @tags query
-// @description This endpoint allows the users to retrieve data from the
-// in-memory database. The CCMS will return data in JSON format for the
-// specified interval requested by the user
-// @accept      json
-// @produce     json
-// @param       request body     api.ApiQueryRequest  true "API query payload object"
-// @success     200            {object} api.ApiQueryResponse  "API query response object"
-// @failure     400            {object} api.ErrorResponse       "Bad Request"
-// @failure     401   		   {object} api.ErrorResponse       "Unauthorized"
-// @failure     403            {object} api.ErrorResponse       "Forbidden"
-// @failure     500            {object} api.ErrorResponse       "Internal Server Error"
-// @security    ApiKeyAuth
-// @router      /query/ [get]
-func handleQuery(rw http.ResponseWriter, r *http.Request) {
-	var err error
-	ver := r.URL.Query().Get("version")
-	if ver == "" {
-		ver = "v2"
-	}
-	req := ApiQueryRequest{WithStats: true, WithData: true, WithPadding: true}
-	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
-		handleError(err, http.StatusBadRequest, rw)
-		return
-	}
-
-	ms := memorystore.GetMemoryStore()
-
-	response := ApiQueryResponse{
-		Results: make([][]ApiMetricData, 0, len(req.Queries)),
-	}
-	if req.ForAllNodes != nil {
-		nodes := ms.ListChildren([]string{req.Cluster})
-		for _, node := range nodes {
-			for _, metric := range req.ForAllNodes {
-				q := ApiQuery{
-					Metric:   metric,
-					Hostname: node,
-				}
-				req.Queries = append(req.Queries, q)
-				response.Queries = append(response.Queries, q)
-			}
-		}
-	}
-
-	for _, query := range req.Queries {
-		sels := make([]util.Selector, 0, 1)
-		if query.Aggregate || query.Type == nil {
-			sel := util.Selector{{String: req.Cluster}, {String: query.Hostname}}
-			if query.Type != nil {
-				if len(query.TypeIds) == 1 {
-					sel = append(sel, util.SelectorElement{String: *query.Type + query.TypeIds[0]})
-				} else {
-					ids := make([]string, len(query.TypeIds))
-					for i, id := range query.TypeIds {
-						ids[i] = *query.Type + id
-					}
-					sel = append(sel, util.SelectorElement{Group: ids})
-				}
-
-				if query.SubType != nil {
-					if len(query.SubTypeIds) == 1 {
-						sel = append(sel, util.SelectorElement{String: *query.SubType + query.SubTypeIds[0]})
-					} else {
-						ids := make([]string, len(query.SubTypeIds))
-						for i, id := range query.SubTypeIds {
-							ids[i] = *query.SubType + id
-						}
-						sel = append(sel, util.SelectorElement{Group: ids})
-					}
-				}
-			}
-			sels = append(sels, sel)
-		} else {
-			for _, typeId := range query.TypeIds {
-				if query.SubType != nil {
-					for _, subTypeId := range query.SubTypeIds {
-						sels = append(sels, util.Selector{
-							{String: req.Cluster},
-							{String: query.Hostname},
-							{String: *query.Type + typeId},
-							{String: *query.SubType + subTypeId},
-						})
-					}
-				} else {
-					sels = append(sels, util.Selector{
-						{String: req.Cluster},
-						{String: query.Hostname},
-						{String: *query.Type + typeId},
-					})
-				}
-			}
-		}
-
-		// log.Printf("query: %#v\n", query)
-		// log.Printf("sels: %#v\n", sels)
-
-		res := make([]ApiMetricData, 0, len(sels))
-		for _, sel := range sels {
-			data := ApiMetricData{}
-			if ver == "v1" {
-				data.Data, data.From, data.To, data.Resolution, err = ms.Read(sel, query.Metric, req.From, req.To, 0)
-			} else {
-				data.Data, data.From, data.To, data.Resolution, err = ms.Read(sel, query.Metric, req.From, req.To, query.Resolution)
-			}
-			if err != nil {
-				msg := err.Error()
-				data.Error = &msg
-				res = append(res, data)
-				continue
-			}
-
-			if req.WithStats {
-				data.AddStats()
-			}
-			if query.ScaleFactor != 0 {
-				data.ScaleBy(query.ScaleFactor)
-			}
-			if req.WithPadding {
-				data.PadDataWithNull(ms, req.From, req.To, query.Metric)
-			}
-			if !req.WithData {
-				data.Data = nil
-			}
-			res = append(res, data)
-		}
-		response.Results = append(response.Results, res)
-	}
-
-	rw.Header().Set("Content-Type", "application/json")
-	bw := bufio.NewWriter(rw)
-	defer bw.Flush()
-	if err := json.NewEncoder(bw).Encode(response); err != nil {
-		log.Print(err)
-		return
-	}
-}
-
-// handleDebug godoc
-// @summary Debug endpoint
-// @tags debug
-// @description This endpoint allows the users to print the content of
-// nodes/clusters/metrics to review the state of the data.
-// @produce     json
-// @param       selector        query    string            false "Selector"
-// @success     200            {string} string  "Debug dump"
-// @failure     400            {object} api.ErrorResponse       "Bad Request"
-// @failure     401            {object} api.ErrorResponse       "Unauthorized"
-// @failure     403            {object} api.ErrorResponse       "Forbidden"
-// @failure     500            {object} api.ErrorResponse       "Internal Server Error"
-// @security    ApiKeyAuth
-// @router      /debug/ [post]
-func handleDebug(rw http.ResponseWriter, r *http.Request) {
-	raw := r.URL.Query().Get("selector")
-	rw.Header().Add("Content-Type", "application/json")
-	selector := []string{}
-	if len(raw) != 0 {
-		selector = strings.Split(raw, ":")
-	}
-
-	ms := memorystore.GetMemoryStore()
-	if err := ms.DebugDump(bufio.NewWriter(rw), selector); err != nil {
-		handleError(err, http.StatusBadRequest, rw)
-		return
-	}
-}
-
-// handleHealthCheck godoc
-// @summary HealthCheck endpoint
-// @tags healthcheck
-// @description This endpoint allows the users to check if a node is healthy
-// @produce     json
-// @param       selector        query    string            false "Selector"
-// @success     200            {string} string  "Debug dump"
-// @failure     400            {object} api.ErrorResponse       "Bad Request"
-// @failure     401            {object} api.ErrorResponse       "Unauthorized"
-// @failure     403            {object} api.ErrorResponse       "Forbidden"
-// @failure     500            {object} api.ErrorResponse       "Internal Server Error"
-// @security    ApiKeyAuth
-// @router      /healthcheck/ [get]
-func handleHealthCheck(rw http.ResponseWriter, r *http.Request) {
-	rawCluster := r.URL.Query().Get("cluster")
-	rawNode := r.URL.Query().Get("node")
-
-	if rawCluster == "" || rawNode == "" {
-		handleError(errors.New("'cluster' and 'node' are required query parameter"), http.StatusBadRequest, rw)
-		return
-	}
-
-	rw.Header().Add("Content-Type", "application/json")
-
-	selector := []string{rawCluster, rawNode}
-
-	ms := memorystore.GetMemoryStore()
-	if err := ms.HealthCheck(bufio.NewWriter(rw), selector); err != nil {
-		handleError(err, http.StatusBadRequest, rw)
-		return
-	}
-}
+package api

internal/api/authentication.go

@@ -1,3 +1,8 @@
+// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
+// All rights reserved. This file is part of cc-metric-store.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
 package api
 
 import (
@@ -34,7 +39,7 @@ func authHandler(next http.Handler, publicKey ed25519.PublicKey) http.Handler {
 		// In case expiration and so on are specified, the Parse function
 		// already returns an error for expired tokens.
 		var err error
-		token, err = jwt.Parse(rawtoken, func(t *jwt.Token) (interface{}, error) {
+		token, err = jwt.Parse(rawtoken, func(t *jwt.Token) (any, error) {
 			if t.Method != jwt.SigningMethodEdDSA {
 				return nil, errors.New("only Ed25519/EdDSA supported")
 			}

internal/api/docs.go

@@ -25,11 +25,6 @@ const docTemplate = `{
     "paths": {
         "/debug/": {
             "post": {
-                "security": [
-                    {
-                        "ApiKeyAuth": []
-                    }
-                ],
                 "description": "This endpoint allows the users to print the content of",
                 "produces": [
                     "application/json"
@@ -77,16 +72,16 @@ const docTemplate = `{
                             "$ref": "#/definitions/api.ErrorResponse"
                         }
                     }
-                }
-            }
                 },
-        "/free/": {
-            "post": {
                 "security": [
                     {
                         "ApiKeyAuth": []
                     }
-                ],
+                ]
+            }
+        },
+        "/free/": {
+            "post": {
                 "description": "This endpoint allows the users to free the Buffers from the",
                 "produces": [
                     "application/json"
@@ -133,16 +128,16 @@ const docTemplate = `{
                             "$ref": "#/definitions/api.ErrorResponse"
                         }
                     }
-                }
-            }
                 },
-        "/healthcheck/": {
-            "get": {
                 "security": [
                     {
                         "ApiKeyAuth": []
                     }
-                ],
+                ]
+            }
+        },
+        "/healthcheck/": {
+            "get": {
                 "description": "This endpoint allows the users to check if a node is healthy",
                 "produces": [
                     "application/json"
@@ -190,16 +185,16 @@ const docTemplate = `{
                             "$ref": "#/definitions/api.ErrorResponse"
                         }
                     }
-                }
-            }
                 },
-        "/query/": {
-            "get": {
                 "security": [
                     {
                         "ApiKeyAuth": []
                     }
-                ],
+                ]
+            }
+        },
+        "/query/": {
+            "get": {
                 "description": "This endpoint allows the users to retrieve data from the",
                 "consumes": [
                     "application/json"
@@ -218,7 +213,7 @@ const docTemplate = `{
                         "in": "body",
                         "required": true,
                         "schema": {
-                            "$ref": "#/definitions/api.ApiQueryRequest"
+                            "$ref": "#/definitions/api.APIQueryRequest"
                         }
                     }
                 ],
@@ -226,7 +221,7 @@ const docTemplate = `{
                     "200": {
                         "description": "API query response object",
                         "schema": {
-                            "$ref": "#/definitions/api.ApiQueryResponse"
+                            "$ref": "#/definitions/api.APIQueryResponse"
                         }
                     },
                     "400": {
@@ -253,16 +248,16 @@ const docTemplate = `{
                             "$ref": "#/definitions/api.ErrorResponse"
                         }
                     }
-                }
-            }
                 },
-        "/write/": {
-            "post": {
                 "security": [
                     {
                         "ApiKeyAuth": []
                     }
-                ],
+                ]
+            }
+        },
+        "/write/": {
+            "post": {
                 "consumes": [
                     "text/plain"
                 ],
@@ -308,12 +303,17 @@ const docTemplate = `{
                             "$ref": "#/definitions/api.ErrorResponse"
                         }
                     }
+                },
+                "security": [
+                    {
+                        "ApiKeyAuth": []
                     }
+                ]
             }
         }
     },
     "definitions": {
-        "api.ApiMetricData": {
+        "api.APIMetricData": {
             "type": "object",
             "properties": {
                 "avg": {
@@ -345,7 +345,7 @@ const docTemplate = `{
                 }
             }
         },
-        "api.ApiQuery": {
+        "api.APIQuery": {
             "type": "object",
             "properties": {
                 "aggreg": {
@@ -383,7 +383,7 @@ const docTemplate = `{
                 }
             }
         },
-        "api.ApiQueryRequest": {
+        "api.APIQueryRequest": {
             "type": "object",
             "properties": {
                 "cluster": {
@@ -401,7 +401,7 @@ const docTemplate = `{
                 "queries": {
                     "type": "array",
                     "items": {
-                        "$ref": "#/definitions/api.ApiQuery"
+                        "$ref": "#/definitions/api.APIQuery"
                     }
                 },
                 "to": {
@@ -418,13 +418,13 @@ const docTemplate = `{
                 }
             }
         },
-        "api.ApiQueryResponse": {
+        "api.APIQueryResponse": {
             "type": "object",
             "properties": {
                 "queries": {
                     "type": "array",
                     "items": {
-                        "$ref": "#/definitions/api.ApiQuery"
+                        "$ref": "#/definitions/api.APIQuery"
                     }
                 },
                 "results": {
@@ -432,7 +432,7 @@ const docTemplate = `{
                     "items": {
                         "type": "array",
                         "items": {
-                            "$ref": "#/definitions/api.ApiMetricData"
+                            "$ref": "#/definitions/api.APIMetricData"
                         }
                     }
                 }

internal/api/lineprotocol.go

@@ -1,350 +0,0 @@
-package api
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"log"
-	"net"
-	"sync"
-	"time"
-
-	"github.com/ClusterCockpit/cc-metric-store/internal/avro"
-	"github.com/ClusterCockpit/cc-metric-store/internal/config"
-	"github.com/ClusterCockpit/cc-metric-store/internal/memorystore"
-	"github.com/ClusterCockpit/cc-metric-store/internal/util"
-	"github.com/influxdata/line-protocol/v2/lineprotocol"
-	"github.com/nats-io/nats.go"
-)
-
-// Each connection is handled in it's own goroutine. This is a blocking function.
-func ReceiveRaw(ctx context.Context,
-	listener net.Listener,
-	handleLine func(*lineprotocol.Decoder, string) error,
-) error {
-	var wg sync.WaitGroup
-
-	wg.Add(1)
-	go func() {
-		defer wg.Done()
-		<-ctx.Done()
-		if err := listener.Close(); err != nil {
-			log.Printf("listener.Close(): %s", err.Error())
-		}
-	}()
-
-	for {
-		conn, err := listener.Accept()
-		if err != nil {
-			if errors.Is(err, net.ErrClosed) {
-				break
-			}
-
-			log.Printf("listener.Accept(): %s", err.Error())
-		}
-
-		wg.Add(2)
-		go func() {
-			defer wg.Done()
-			defer conn.Close()
-
-			dec := lineprotocol.NewDecoder(conn)
-			connctx, cancel := context.WithCancel(context.Background())
-			defer cancel()
-			go func() {
-				defer wg.Done()
-				select {
-				case <-connctx.Done():
-					conn.Close()
-				case <-ctx.Done():
-					conn.Close()
-				}
-			}()
-
-			if err := handleLine(dec, "default"); err != nil {
-				if errors.Is(err, net.ErrClosed) {
-					return
-				}
-
-				log.Printf("%s: %s", conn.RemoteAddr().String(), err.Error())
-				errmsg := make([]byte, 128)
-				errmsg = append(errmsg, `error: `...)
-				errmsg = append(errmsg, err.Error()...)
-				errmsg = append(errmsg, '\n')
-				conn.Write(errmsg)
-			}
-		}()
-	}
-
-	wg.Wait()
-	return nil
-}
-
-// Connect to a nats server and subscribe to "updates". This is a blocking
-// function. handleLine will be called for each line recieved via nats.
-// Send `true` through the done channel for gracefull termination.
-func ReceiveNats(conf *config.NatsConfig,
-	ms *memorystore.MemoryStore,
-	workers int,
-	ctx context.Context,
-) error {
-	var opts []nats.Option
-	if conf.Username != "" && conf.Password != "" {
-		opts = append(opts, nats.UserInfo(conf.Username, conf.Password))
-	}
-
-	if conf.Credsfilepath != "" {
-		opts = append(opts, nats.UserCredentials(conf.Credsfilepath))
-	}
-
-	nc, err := nats.Connect(conf.Address, opts...)
-	if err != nil {
-		return err
-	}
-	defer nc.Close()
-
-	var wg sync.WaitGroup
-	var subs []*nats.Subscription
-
-	msgs := make(chan *nats.Msg, workers*2)
-
-	for _, sc := range conf.Subscriptions {
-		clusterTag := sc.ClusterTag
-		var sub *nats.Subscription
-		if workers > 1 {
-			wg.Add(workers)
-
-			for i := 0; i < workers; i++ {
-				go func() {
-					for m := range msgs {
-						dec := lineprotocol.NewDecoderWithBytes(m.Data)
-						if err := decodeLine(dec, ms, clusterTag); err != nil {
-							log.Printf("error: %s\n", err.Error())
-						}
-					}
-
-					wg.Done()
-				}()
-			}
-
-			sub, err = nc.Subscribe(sc.SubscribeTo, func(m *nats.Msg) {
-				msgs <- m
-			})
-		} else {
-			sub, err = nc.Subscribe(sc.SubscribeTo, func(m *nats.Msg) {
-				dec := lineprotocol.NewDecoderWithBytes(m.Data)
-				if err := decodeLine(dec, ms, clusterTag); err != nil {
-					log.Printf("error: %s\n", err.Error())
-				}
-			})
-		}
-
-		if err != nil {
-			return err
-		}
-		log.Printf("NATS subscription to '%s' on '%s' established\n", sc.SubscribeTo, conf.Address)
-		subs = append(subs, sub)
-	}
-
-	<-ctx.Done()
-	for _, sub := range subs {
-		err = sub.Unsubscribe()
-		if err != nil {
-			log.Printf("NATS unsubscribe failed: %s", err.Error())
-		}
-	}
-	close(msgs)
-	wg.Wait()
-
-	nc.Close()
-	log.Println("NATS connection closed")
-	return nil
-}
-
-// Place `prefix` in front of `buf` but if possible,
-// do that inplace in `buf`.
-func reorder(buf, prefix []byte) []byte {
-	n := len(prefix)
-	m := len(buf)
-	if cap(buf) < m+n {
-		return append(prefix[:n:n], buf...)
-	} else {
-		buf = buf[:n+m]
-		for i := m - 1; i >= 0; i-- {
-			buf[i+n] = buf[i]
-		}
-		for i := 0; i < n; i++ {
-			buf[i] = prefix[i]
-		}
-		return buf
-	}
-}
-
-// Decode lines using dec and make write calls to the MemoryStore.
-// If a line is missing its cluster tag, use clusterDefault as default.
-func decodeLine(dec *lineprotocol.Decoder,
-	ms *memorystore.MemoryStore,
-	clusterDefault string,
-) error {
-	// Reduce allocations in loop:
-	t := time.Now()
-	metric, metricBuf := memorystore.Metric{}, make([]byte, 0, 16)
-	selector := make([]string, 0, 4)
-	typeBuf, subTypeBuf := make([]byte, 0, 16), make([]byte, 0)
-
-	// Optimize for the case where all lines in a "batch" are about the same
-	// cluster and host. By using `WriteToLevel` (level = host), we do not need
-	// to take the root- and cluster-level lock as often.
-	var lvl *memorystore.Level = nil
-	prevCluster, prevHost := "", ""
-
-	var ok bool
-	for dec.Next() {
-		rawmeasurement, err := dec.Measurement()
-		if err != nil {
-			return err
-		}
-
-		// Needs to be copied because another call to dec.* would
-		// invalidate the returned slice.
-		metricBuf = append(metricBuf[:0], rawmeasurement...)
-
-		// The go compiler optimizes map[string(byteslice)] lookups:
-		metric.MetricConfig, ok = ms.Metrics[string(rawmeasurement)]
-		if !ok {
-			continue
-		}
-
-		typeBuf, subTypeBuf := typeBuf[:0], subTypeBuf[:0]
-		cluster, host := clusterDefault, ""
-		for {
-			key, val, err := dec.NextTag()
-			if err != nil {
-				return err
-			}
-			if key == nil {
-				break
-			}
-
-			// The go compiler optimizes string([]byte{...}) == "...":
-			switch string(key) {
-			case "cluster":
-				if string(val) == prevCluster {
-					cluster = prevCluster
-				} else {
-					cluster = string(val)
-					lvl = nil
-				}
-			case "hostname", "host":
-				if string(val) == prevHost {
-					host = prevHost
-				} else {
-					host = string(val)
-					lvl = nil
-				}
-			case "type":
-				if string(val) == "node" {
-					break
-				}
-
-				// We cannot be sure that the "type" tag comes before the "type-id" tag:
-				if len(typeBuf) == 0 {
-					typeBuf = append(typeBuf, val...)
-				} else {
-					typeBuf = reorder(typeBuf, val)
-				}
-			case "type-id":
-				typeBuf = append(typeBuf, val...)
-			case "stype":
-				// We cannot be sure that the "stype" tag comes before the "stype-id" tag:
-				if len(subTypeBuf) == 0 {
-					subTypeBuf = append(subTypeBuf, val...)
-				} else {
-					subTypeBuf = reorder(subTypeBuf, val)
-					// subTypeBuf = reorder(typeBuf, val)
-				}
-			case "stype-id":
-				subTypeBuf = append(subTypeBuf, val...)
-			default:
-				// Ignore unkown tags (cc-metric-collector might send us a unit for example that we do not need)
-				// return fmt.Errorf("unkown tag: '%s' (value: '%s')", string(key), string(val))
-			}
-		}
-
-		// If the cluster or host changed, the lvl was set to nil
-		if lvl == nil {
-			selector = selector[:2]
-			selector[0], selector[1] = cluster, host
-			lvl = ms.GetLevel(selector)
-			prevCluster, prevHost = cluster, host
-		}
-
-		// subtypes:
-		selector = selector[:0]
-		if len(typeBuf) > 0 {
-			selector = append(selector, string(typeBuf)) // <- Allocation :(
-			if len(subTypeBuf) > 0 {
-				selector = append(selector, string(subTypeBuf))
-			}
-		}
-
-		for {
-			key, val, err := dec.NextField()
-			if err != nil {
-				return err
-			}
-
-			if key == nil {
-				break
-			}
-
-			if string(key) != "value" {
-				return fmt.Errorf("host %s: unknown field: '%s' (value: %#v)", host, string(key), val)
-			}
-
-			if val.Kind() == lineprotocol.Float {
-				metric.Value = util.Float(val.FloatV())
-			} else if val.Kind() == lineprotocol.Int {
-				metric.Value = util.Float(val.IntV())
-			} else if val.Kind() == lineprotocol.Uint {
-				metric.Value = util.Float(val.UintV())
-			} else {
-				return fmt.Errorf("host %s: unsupported value type in message: %s", host, val.Kind().String())
-			}
-		}
-
-		if t, err = dec.Time(lineprotocol.Second, t); err != nil {
-			t = time.Now()
-			if t, err = dec.Time(lineprotocol.Millisecond, t); err != nil {
-				t = time.Now()
-				if t, err = dec.Time(lineprotocol.Microsecond, t); err != nil {
-					t = time.Now()
-					if t, err = dec.Time(lineprotocol.Nanosecond, t); err != nil {
-						return fmt.Errorf("host %s: timestamp : %#v with error : %#v", host, t, err.Error())
-					}
-				}
-			}
-		}
-
-		if err != nil {
-			return fmt.Errorf("host %s: timestamp : %#v with error : %#v", host, t, err.Error())
-		}
-
-		time := t.Unix()
-
-		if config.Keys.Checkpoints.FileFormat != "json" {
-			avro.LineProtocolMessages <- &avro.AvroStruct{
-				MetricName: string(metricBuf),
-				Cluster:    cluster,
-				Node:       host,
-				Selector:   append([]string{}, selector...),
-				Value:      metric.Value,
-				Timestamp:  time}
-		}
-
-		if err := ms.WriteToLevel(lvl, selector, time, []memorystore.Metric{metric}); err != nil {
-			return err
-		}
-	}
-	return nil
-}

internal/api/metricstore.go

@@ -0,0 +1,469 @@
+// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
+// All rights reserved. This file is part of cc-metric-store.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package api
+
+import (
+	"bufio"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"log"
+	"math"
+	"net/http"
+	"strconv"
+	"strings"
+
+	"github.com/ClusterCockpit/cc-backend/pkg/metricstore"
+	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
+	"github.com/ClusterCockpit/cc-lib/v2/schema"
+	"github.com/ClusterCockpit/cc-lib/v2/util"
+	"github.com/ClusterCockpit/cc-line-protocol/v2/lineprotocol"
+)
+
+// ErrorResponse model
+type ErrorResponse struct {
+	// Statustext of Errorcode
+	Status string `json:"status"`
+	Error  string `json:"error"` // Error Message
+}
+
+// DefaultAPIResponse model
+type DefaultAPIResponse struct {
+	Message string `json:"msg"`
+}
+
+// handleError writes a standardized JSON error response with the given status code.
+// It logs the error at WARN level and ensures proper Content-Type headers are set.
+func handleError(err error, statusCode int, rw http.ResponseWriter) {
+	cclog.Warnf("REST ERROR : %s", err.Error())
+	rw.Header().Add("Content-Type", "application/json")
+	rw.WriteHeader(statusCode)
+	if err := json.NewEncoder(rw).Encode(ErrorResponse{
+		Status: http.StatusText(statusCode),
+		Error:  err.Error(),
+	}); err != nil {
+		cclog.Errorf("Failed to encode error response: %v", err)
+	}
+}
+
+type APIMetricData struct {
+	Error      *string           `json:"error,omitempty"`
+	Data       schema.FloatArray `json:"data,omitempty"       swaggertype:"array,number"`
+	From       int64             `json:"from"`
+	To         int64             `json:"to"`
+	Resolution int64             `json:"resolution"`
+	Avg        schema.Float      `json:"avg"                  swaggertype:"number"`
+	Min        schema.Float      `json:"min"                  swaggertype:"number"`
+	Max        schema.Float      `json:"max"                  swaggertype:"number"`
+}
+
+// TODO: Optimize this, just like the stats endpoint!
+func (data *APIMetricData) AddStats() {
+	n := 0
+	sum, min, max := 0.0, math.MaxFloat64, -math.MaxFloat64
+	for _, x := range data.Data {
+		if x.IsNaN() {
+			continue
+		}
+
+		n += 1
+		sum += float64(x)
+		min = math.Min(min, float64(x))
+		max = math.Max(max, float64(x))
+	}
+
+	if n > 0 {
+		avg := sum / float64(n)
+		data.Avg = schema.Float(avg)
+		data.Min = schema.Float(min)
+		data.Max = schema.Float(max)
+	} else {
+		data.Avg, data.Min, data.Max = schema.NaN, schema.NaN, schema.NaN
+	}
+}
+
+func (data *APIMetricData) ScaleBy(f schema.Float) {
+	if f == 0 || f == 1 {
+		return
+	}
+
+	data.Avg *= f
+	data.Min *= f
+	data.Max *= f
+	for i := 0; i < len(data.Data); i++ {
+		data.Data[i] *= f
+	}
+}
+
+func (data *APIMetricData) PadDataWithNull(ms *metricstore.MemoryStore, from, to int64, metric string) {
+	minfo, ok := ms.Metrics[metric]
+	if !ok {
+		return
+	}
+
+	if (data.From / minfo.Frequency) > (from / minfo.Frequency) {
+		padfront := int((data.From / minfo.Frequency) - (from / minfo.Frequency))
+		ndata := make([]schema.Float, 0, padfront+len(data.Data))
+		for range padfront {
+			ndata = append(ndata, schema.NaN)
+		}
+		for j := 0; j < len(data.Data); j++ {
+			ndata = append(ndata, data.Data[j])
+		}
+		data.Data = ndata
+	}
+}
+
+type APIQueryRequest struct {
+	Cluster     string     `json:"cluster"`
+	Queries     []APIQuery `json:"queries"`
+	ForAllNodes []string   `json:"for-all-nodes"`
+	From        int64      `json:"from"`
+	To          int64      `json:"to"`
+	WithStats   bool       `json:"with-stats"`
+	WithData    bool       `json:"with-data"`
+	WithPadding bool       `json:"with-padding"`
+}
+
+type APIQueryResponse struct {
+	Queries []APIQuery        `json:"queries,omitempty"`
+	Results [][]APIMetricData `json:"results"`
+}
+
+type APIQuery struct {
+	Type        *string      `json:"type,omitempty"`
+	SubType     *string      `json:"subtype,omitempty"`
+	Metric      string       `json:"metric"`
+	Hostname    string       `json:"host"`
+	Resolution  int64        `json:"resolution"`
+	TypeIds     []string     `json:"type-ids,omitempty"`
+	SubTypeIds  []string     `json:"subtype-ids,omitempty"`
+	ScaleFactor schema.Float `json:"scale-by,omitempty" swaggertype:"number"`
+	Aggregate   bool         `json:"aggreg"`
+}
+
+// handleQuery godoc
+// @summary    Query metrics
+// @tags query
+// @description This endpoint allows the users to retrieve data from the
+// in-memory database. The CCMS will return data in JSON format for the
+// specified interval requested by the user
+// @accept      json
+// @produce     json
+// @param       request body     APIQueryRequest  true "API query payload object"
+// @success     200            {object} APIQueryResponse  "API query response object"
+// @failure     400            {object} ErrorResponse       "Bad Request"
+// @failure     401   		   {object} ErrorResponse       "Unauthorized"
+// @failure     403            {object} ErrorResponse       "Forbidden"
+// @failure     500            {object} ErrorResponse       "Internal Server Error"
+// @security    ApiKeyAuth
+// @router      /query/ [get]
+func handleQuery(rw http.ResponseWriter, r *http.Request) {
+	var err error
+	ver := r.URL.Query().Get("version")
+	if ver == "" {
+		ver = "v2"
+	}
+	req := APIQueryRequest{WithStats: true, WithData: true, WithPadding: true}
+	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
+		handleError(err, http.StatusBadRequest, rw)
+		return
+	}
+
+	ms := metricstore.GetMemoryStore()
+
+	response := APIQueryResponse{
+		Results: make([][]APIMetricData, 0, len(req.Queries)),
+	}
+	if req.ForAllNodes != nil {
+		nodes := ms.ListChildren([]string{req.Cluster})
+		for _, node := range nodes {
+			for _, metric := range req.ForAllNodes {
+				q := APIQuery{
+					Metric:   metric,
+					Hostname: node,
+				}
+				req.Queries = append(req.Queries, q)
+				response.Queries = append(response.Queries, q)
+			}
+		}
+	}
+
+	for _, query := range req.Queries {
+		sels := make([]util.Selector, 0, 1)
+		if query.Aggregate || query.Type == nil {
+			sel := util.Selector{{String: req.Cluster}, {String: query.Hostname}}
+			if query.Type != nil {
+				if len(query.TypeIds) == 1 {
+					sel = append(sel, util.SelectorElement{String: *query.Type + query.TypeIds[0]})
+				} else {
+					ids := make([]string, len(query.TypeIds))
+					for i, id := range query.TypeIds {
+						ids[i] = *query.Type + id
+					}
+					sel = append(sel, util.SelectorElement{Group: ids})
+				}
+
+				if query.SubType != nil {
+					if len(query.SubTypeIds) == 1 {
+						sel = append(sel, util.SelectorElement{String: *query.SubType + query.SubTypeIds[0]})
+					} else {
+						ids := make([]string, len(query.SubTypeIds))
+						for i, id := range query.SubTypeIds {
+							ids[i] = *query.SubType + id
+						}
+						sel = append(sel, util.SelectorElement{Group: ids})
+					}
+				}
+			}
+			sels = append(sels, sel)
+		} else {
+			for _, typeID := range query.TypeIds {
+				if query.SubType != nil {
+					for _, subTypeID := range query.SubTypeIds {
+						sels = append(sels, util.Selector{
+							{String: req.Cluster},
+							{String: query.Hostname},
+							{String: *query.Type + typeID},
+							{String: *query.SubType + subTypeID},
+						})
+					}
+				} else {
+					sels = append(sels, util.Selector{
+						{String: req.Cluster},
+						{String: query.Hostname},
+						{String: *query.Type + typeID},
+					})
+				}
+			}
+		}
+
+		// log.Printf("query: %#v\n", query)
+		// log.Printf("sels: %#v\n", sels)
+
+		res := make([]APIMetricData, 0, len(sels))
+		for _, sel := range sels {
+			data := APIMetricData{}
+
+			data.Data, data.From, data.To, data.Resolution, err = ms.Read(sel, query.Metric, req.From, req.To, query.Resolution)
+			if err != nil {
+				// Skip Error If Just Missing Host or Metric, Continue
+				// Empty Return For Metric Handled Gracefully By Frontend
+				if err != metricstore.ErrNoHostOrMetric {
+					msg := err.Error()
+					data.Error = &msg
+					res = append(res, data)
+				} else {
+					cclog.Warnf("failed to fetch '%s' from host '%s' (cluster: %s): %s", query.Metric, query.Hostname, req.Cluster, err.Error())
+				}
+				continue
+			}
+
+			if req.WithStats {
+				data.AddStats()
+			}
+			if query.ScaleFactor != 0 {
+				data.ScaleBy(query.ScaleFactor)
+			}
+			if req.WithPadding {
+				data.PadDataWithNull(ms, req.From, req.To, query.Metric)
+			}
+			if !req.WithData {
+				data.Data = nil
+			}
+			res = append(res, data)
+		}
+		response.Results = append(response.Results, res)
+	}
+
+	rw.Header().Set("Content-Type", "application/json")
+	bw := bufio.NewWriter(rw)
+	defer bw.Flush()
+	if err := json.NewEncoder(bw).Encode(response); err != nil {
+		log.Print(err)
+		return
+	}
+}
+
+// handleFree godoc
+// @summary
+// @tags free
+// @description This endpoint allows the users to free the Buffers from the
+// metric store. This endpoint offers the users to remove then systematically
+// and also allows then to prune the data under node, if they do not want to
+// remove the whole node.
+// @produce     json
+// @param       to        query    string        false  "up to timestamp"
+// @success     200            {string} string  "ok"
+// @failure     400            {object} ErrorResponse       "Bad Request"
+// @failure     401            {object} ErrorResponse       "Unauthorized"
+// @failure     403            {object} ErrorResponse       "Forbidden"
+// @failure     500            {object} ErrorResponse       "Internal Server Error"
+// @security    ApiKeyAuth
+// @router      /free/ [post]
+func freeMetrics(rw http.ResponseWriter, r *http.Request) {
+	rawTo := r.URL.Query().Get("to")
+	if rawTo == "" {
+		handleError(errors.New("'to' is a required query parameter"), http.StatusBadRequest, rw)
+		return
+	}
+
+	to, err := strconv.ParseInt(rawTo, 10, 64)
+	if err != nil {
+		handleError(err, http.StatusInternalServerError, rw)
+		return
+	}
+
+	bodyDec := json.NewDecoder(r.Body)
+	var selectors [][]string
+	err = bodyDec.Decode(&selectors)
+	if err != nil {
+		http.Error(rw, err.Error(), http.StatusBadRequest)
+		return
+	}
+
+	ms := metricstore.GetMemoryStore()
+	n := 0
+	for _, sel := range selectors {
+		bn, err := ms.Free(sel, to)
+		if err != nil {
+			handleError(err, http.StatusInternalServerError, rw)
+			return
+		}
+
+		n += bn
+	}
+
+	rw.WriteHeader(http.StatusOK)
+	fmt.Fprintf(rw, "buffers freed: %d\n", n)
+}
+
+// handleWrite godoc
+// @summary Receive metrics in InfluxDB line-protocol
+// @tags write
+// @description Write data to the in-memory store in the InfluxDB line-protocol using [this format](https://github.com/ClusterCockpit/cc-specifications/blob/master/metrics/lineprotocol_alternative.md)
+
+// @accept      plain
+// @produce     json
+// @param       cluster        query string false "If the lines in the body do not have a cluster tag, use this value instead."
+// @success     200            {string} string  "ok"
+// @failure     400            {object} ErrorResponse       "Bad Request"
+// @failure     401            {object} ErrorResponse       "Unauthorized"
+// @failure     403            {object} ErrorResponse       "Forbidden"
+// @failure     500            {object} ErrorResponse       "Internal Server Error"
+// @security    ApiKeyAuth
+// @router      /write/ [post]
+func writeMetrics(rw http.ResponseWriter, r *http.Request) {
+	rw.Header().Add("Content-Type", "application/json")
+
+	// Extract the "cluster" query parameter without allocating a url.Values map.
+	cluster := queryParam(r.URL.RawQuery, "cluster")
+
+	// Stream directly from the request body instead of copying it into a
+	// temporary buffer via io.ReadAll. The line-protocol decoder supports
+	// io.Reader natively, so this avoids the largest heap allocation.
+	ms := metricstore.GetMemoryStore()
+	dec := lineprotocol.NewDecoder(r.Body)
+	if err := metricstore.DecodeLine(dec, ms, cluster); err != nil {
+		cclog.Errorf("/api/write error: %s", err.Error())
+		handleError(err, http.StatusBadRequest, rw)
+		return
+	}
+	rw.WriteHeader(http.StatusOK)
+}
+
+// handleDebug godoc
+// @summary Debug endpoint
+// @tags debug
+// @description This endpoint allows the users to print the content of
+// nodes/clusters/metrics to review the state of the data.
+// @produce     json
+// @param       selector        query    string            false "Selector"
+// @success     200            {string} string  "Debug dump"
+// @failure     400            {object} ErrorResponse       "Bad Request"
+// @failure     401            {object} ErrorResponse       "Unauthorized"
+// @failure     403            {object} ErrorResponse       "Forbidden"
+// @failure     500            {object} ErrorResponse       "Internal Server Error"
+// @security    ApiKeyAuth
+// @router      /debug/ [post]
+func debugMetrics(rw http.ResponseWriter, r *http.Request) {
+	raw := r.URL.Query().Get("selector")
+	rw.Header().Add("Content-Type", "application/json")
+	selector := []string{}
+	if len(raw) != 0 {
+		selector = strings.Split(raw, ":")
+	}
+
+	ms := metricstore.GetMemoryStore()
+	if err := ms.DebugDump(bufio.NewWriter(rw), selector); err != nil {
+		handleError(err, http.StatusBadRequest, rw)
+		return
+	}
+}
+
+// handleHealthCheck godoc
+// @summary HealthCheck endpoint
+// @tags healthcheck
+// @description This endpoint allows the users to check if a node is healthy
+// @produce     json
+// @param       selector        query    string            false "Selector"
+// @success     200            {string} string  "Debug dump"
+// @failure     400            {object} ErrorResponse       "Bad Request"
+// @failure     401            {object} ErrorResponse       "Unauthorized"
+// @failure     403            {object} ErrorResponse       "Forbidden"
+// @failure     500            {object} ErrorResponse       "Internal Server Error"
+// @security    ApiKeyAuth
+// @router      /healthcheck/ [get]
+func metricsHealth(rw http.ResponseWriter, r *http.Request) {
+	req := metricstore.HealthCheckReq{}
+
+	dec := json.NewDecoder(r.Body)
+	dec.DisallowUnknownFields()
+
+	if err := dec.Decode(&req); err != nil {
+		handleError(fmt.Errorf("parsing request body failed: %w", err),
+			http.StatusBadRequest, rw)
+		return
+	}
+
+	rw.Header().Add("Content-Type", "application/json")
+
+	ms := metricstore.GetMemoryStore()
+	results, err := ms.HealthCheck(req.Cluster, req.Nodes, req.MetricNames)
+	if err != nil {
+		handleError(err, http.StatusBadRequest, rw)
+		return
+	}
+
+	rw.Header().Set("Content-Type", "application/json")
+	bw := bufio.NewWriter(rw)
+	defer bw.Flush()
+	if err := json.NewEncoder(bw).Encode(results); err != nil {
+		log.Print(err)
+		return
+	}
+}
+
+// queryParam extracts a single query parameter value from a raw query string
+// without allocating a url.Values map.
+func queryParam(rawQuery, key string) string {
+	for rawQuery != "" {
+		var kv string
+		if before, after, ok := strings.Cut(rawQuery, "&"); ok {
+			kv, rawQuery = before, after
+		} else {
+			kv, rawQuery = rawQuery, ""
+		}
+		if before, after, ok := strings.Cut(kv, "="); ok {
+			if before == key {
+				return after
+			}
+		} else if kv == key {
+			return ""
+		}
+	}
+	return ""
+}

internal/api/nodeprovider.go

@@ -0,0 +1,54 @@
+// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
+// All rights reserved. This file is part of cc-metric-store.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package api
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"time"
+)
+
+// BackendNodeProvider implements metricstore.NodeProvider by querying
+// the cc-backend /api/jobs/used_nodes endpoint.
+type BackendNodeProvider struct {
+	backendURL string
+	client     *http.Client
+}
+
+// NewBackendNodeProvider creates a new BackendNodeProvider that queries
+// the given cc-backend URL for used nodes information.
+func NewBackendNodeProvider(backendURL string) *BackendNodeProvider {
+	return &BackendNodeProvider{
+		backendURL: backendURL,
+		client: &http.Client{
+			Timeout: 10 * time.Second,
+		},
+	}
+}
+
+// GetUsedNodes returns a map of cluster names to sorted lists of unique hostnames
+// that are currently in use by jobs that started before the given timestamp.
+func (p *BackendNodeProvider) GetUsedNodes(ts int64) (map[string][]string, error) {
+	url := fmt.Sprintf("%s/api/jobs/used_nodes?ts=%d", p.backendURL, ts)
+
+	resp, err := p.client.Get(url)
+	if err != nil {
+		return nil, fmt.Errorf("querying used nodes from backend: %w", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("backend returned status %d", resp.StatusCode)
+	}
+
+	var result map[string][]string
+	if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
+		return nil, fmt.Errorf("decoding used nodes response: %w", err)
+	}
+
+	return result, nil
+}

internal/api/server.go

@@ -1,7 +1,8 @@
 // Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved.
+// All rights reserved. This file is part of cc-metric-store.
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
+
 package api
 
 import (
@@ -21,29 +22,29 @@ func MountRoutes(r *http.ServeMux) {
 		}
 		publicKey := ed25519.PublicKey(buf)
 		// Compatibility
-		r.Handle("POST /api/free", authHandler(http.HandlerFunc(handleFree), publicKey))
-		r.Handle("POST /api/write", authHandler(http.HandlerFunc(handleWrite), publicKey))
+		r.Handle("POST /api/free", authHandler(http.HandlerFunc(freeMetrics), publicKey))
+		r.Handle("POST /api/write", authHandler(http.HandlerFunc(writeMetrics), publicKey))
 		r.Handle("GET /api/query", authHandler(http.HandlerFunc(handleQuery), publicKey))
-		r.Handle("GET /api/debug", authHandler(http.HandlerFunc(handleDebug), publicKey))
-		r.Handle("GET /api/healthcheck", authHandler(http.HandlerFunc(handleHealthCheck), publicKey))
+		r.Handle("GET /api/debug", authHandler(http.HandlerFunc(debugMetrics), publicKey))
+		r.Handle("GET /api/healthcheck", authHandler(http.HandlerFunc(metricsHealth), publicKey))
 		// Refactor
-		r.Handle("POST /api/free/", authHandler(http.HandlerFunc(handleFree), publicKey))
-		r.Handle("POST /api/write/", authHandler(http.HandlerFunc(handleWrite), publicKey))
+		r.Handle("POST /api/free/", authHandler(http.HandlerFunc(freeMetrics), publicKey))
+		r.Handle("POST /api/write/", authHandler(http.HandlerFunc(writeMetrics), publicKey))
 		r.Handle("GET /api/query/", authHandler(http.HandlerFunc(handleQuery), publicKey))
-		r.Handle("GET /api/debug/", authHandler(http.HandlerFunc(handleDebug), publicKey))
-		r.Handle("GET /api/healthcheck/", authHandler(http.HandlerFunc(handleHealthCheck), publicKey))
+		r.Handle("GET /api/debug/", authHandler(http.HandlerFunc(debugMetrics), publicKey))
+		r.Handle("GET /api/healthcheck/", authHandler(http.HandlerFunc(metricsHealth), publicKey))
 	} else {
 		// Compatibility
-		r.HandleFunc("POST /api/free", handleFree)
-		r.HandleFunc("POST /api/write", handleWrite)
+		r.HandleFunc("POST /api/free", freeMetrics)
+		r.HandleFunc("POST /api/write", writeMetrics)
 		r.HandleFunc("GET /api/query", handleQuery)
-		r.HandleFunc("GET /api/debug", handleDebug)
-		r.HandleFunc("GET /api/healthcheck", handleHealthCheck)
+		r.HandleFunc("GET /api/debug", debugMetrics)
+		r.HandleFunc("GET /api/healthcheck", metricsHealth)
 		// Refactor
-		r.HandleFunc("POST /api/free/", handleFree)
-		r.HandleFunc("POST /api/write/", handleWrite)
+		r.HandleFunc("POST /api/free/", freeMetrics)
+		r.HandleFunc("POST /api/write/", writeMetrics)
 		r.HandleFunc("GET /api/query/", handleQuery)
-		r.HandleFunc("GET /api/debug/", handleDebug)
-		r.HandleFunc("GET /api/healthcheck/", handleHealthCheck)
+		r.HandleFunc("GET /api/debug/", debugMetrics)
+		r.HandleFunc("GET /api/healthcheck/", metricsHealth)
 	}
 }

internal/avro/avroCheckpoint.go

@@ -1,474 +0,0 @@
-package avro
-
-import (
-	"bufio"
-	"encoding/json"
-	"errors"
-	"fmt"
-	"log"
-	"os"
-	"path"
-	"sort"
-	"strconv"
-	"strings"
-	"sync"
-	"sync/atomic"
-	"time"
-
-	"github.com/ClusterCockpit/cc-metric-store/internal/config"
-	"github.com/ClusterCockpit/cc-metric-store/internal/util"
-	"github.com/linkedin/goavro/v2"
-)
-
-var NumWorkers int = 4
-
-var ErrNoNewData error = errors.New("no data in the pool")
-
-func (as *AvroStore) ToCheckpoint(dir string, dumpAll bool) (int, error) {
-	levels := make([]*AvroLevel, 0)
-	selectors := make([][]string, 0)
-	as.root.lock.RLock()
-	// Cluster
-	for sel1, l1 := range as.root.children {
-		l1.lock.RLock()
-		// Node
-		for sel2, l2 := range l1.children {
-			l2.lock.RLock()
-			// Frequency
-			for sel3, l3 := range l2.children {
-				levels = append(levels, l3)
-				selectors = append(selectors, []string{sel1, sel2, sel3})
-			}
-			l2.lock.RUnlock()
-		}
-		l1.lock.RUnlock()
-	}
-	as.root.lock.RUnlock()
-
-	type workItem struct {
-		level    *AvroLevel
-		dir      string
-		selector []string
-	}
-
-	n, errs := int32(0), int32(0)
-
-	var wg sync.WaitGroup
-	wg.Add(NumWorkers)
-	work := make(chan workItem, NumWorkers*2)
-	for range NumWorkers {
-		go func() {
-			defer wg.Done()
-
-			for workItem := range work {
-				var from int64 = getTimestamp(workItem.dir)
-
-				if err := workItem.level.toCheckpoint(workItem.dir, from, dumpAll); err != nil {
-					if err == ErrNoNewData {
-						continue
-					}
-
-					log.Printf("error while checkpointing %#v: %s", workItem.selector, err.Error())
-					atomic.AddInt32(&errs, 1)
-				} else {
-					atomic.AddInt32(&n, 1)
-				}
-			}
-		}()
-	}
-
-	for i := range len(levels) {
-		dir := path.Join(dir, path.Join(selectors[i]...))
-		work <- workItem{
-			level:    levels[i],
-			dir:      dir,
-			selector: selectors[i],
-		}
-	}
-
-	close(work)
-	wg.Wait()
-
-	if errs > 0 {
-		return int(n), fmt.Errorf("%d errors happend while creating avro checkpoints (%d successes)", errs, n)
-	}
-	return int(n), nil
-}
-
-// getTimestamp returns the timestamp from the directory name
-func getTimestamp(dir string) int64 {
-	// Extract the resolution and timestamp from the directory name
-	// The existing avro file will be in epoch timestamp format
-	// iterate over all the files in the directory and find the maximum timestamp
-	// and return it
-
-	resolution := path.Base(dir)
-	dir = path.Dir(dir)
-
-	files, err := os.ReadDir(dir)
-	if err != nil {
-		return 0
-	}
-	var maxTs int64 = 0
-
-	if len(files) == 0 {
-		return 0
-	}
-
-	for _, file := range files {
-		if file.IsDir() {
-			continue
-		}
-		name := file.Name()
-
-		if len(name) < 5 || !strings.HasSuffix(name, ".avro") || !strings.HasPrefix(name, resolution+"_") {
-			continue
-		}
-
-		ts, err := strconv.ParseInt(name[strings.Index(name, "_")+1:len(name)-5], 10, 64)
-		if err != nil {
-			fmt.Printf("error while parsing timestamp: %s\n", err.Error())
-			continue
-		}
-
-		if ts > maxTs {
-			maxTs = ts
-		}
-	}
-
-	interval, _ := time.ParseDuration(config.Keys.Checkpoints.Interval)
-	updateTime := time.Unix(maxTs, 0).Add(interval).Add(time.Duration(CheckpointBufferMinutes-1) * time.Minute).Unix()
-
-	if updateTime < time.Now().Unix() {
-		return 0
-	}
-
-	return maxTs
-}
-
-func (l *AvroLevel) toCheckpoint(dir string, from int64, dumpAll bool) error {
-	l.lock.Lock()
-	defer l.lock.Unlock()
-
-	// fmt.Printf("Checkpointing directory: %s\n", dir)
-	// filepath contains the resolution
-	int_res, _ := strconv.Atoi(path.Base(dir))
-
-	// find smallest overall timestamp in l.data map and delete it from l.data
-	var minTs int64 = int64(1<<63 - 1)
-	for ts, dat := range l.data {
-		if ts < minTs && len(dat) != 0 {
-			minTs = ts
-		}
-	}
-
-	if from == 0 && minTs != int64(1<<63-1) {
-		from = minTs
-	}
-
-	if from == 0 {
-		return ErrNoNewData
-	}
-
-	var schema string
-	var codec *goavro.Codec
-	record_list := make([]map[string]interface{}, 0)
-
-	var f *os.File
-
-	filePath := dir + fmt.Sprintf("_%d.avro", from)
-
-	var err error
-
-	fp_, err_ := os.Stat(filePath)
-	if errors.Is(err_, os.ErrNotExist) {
-		err = os.MkdirAll(path.Dir(dir), 0o755)
-		if err != nil {
-			return fmt.Errorf("failed to create directory: %v", err)
-		}
-	} else if fp_.Size() != 0 {
-		f, err = os.Open(filePath)
-		if err != nil {
-			return fmt.Errorf("failed to open existing avro file: %v", err)
-		}
-
-		br := bufio.NewReader(f)
-
-		reader, err := goavro.NewOCFReader(br)
-		if err != nil {
-			return fmt.Errorf("failed to create OCF reader: %v", err)
-		}
-		codec = reader.Codec()
-		schema = codec.Schema()
-
-		f.Close()
-	}
-
-	time_ref := time.Now().Add(time.Duration(-CheckpointBufferMinutes+1) * time.Minute).Unix()
-
-	if dumpAll {
-		time_ref = time.Now().Unix()
-	}
-
-	// Empty values
-	if len(l.data) == 0 {
-		// we checkpoint avro files every 60 seconds
-		repeat := 60 / int_res
-
-		for range repeat {
-			record_list = append(record_list, make(map[string]interface{}))
-		}
-	}
-
-	readFlag := true
-
-	for ts := range l.data {
-		flag := false
-		if ts < time_ref {
-			data := l.data[ts]
-
-			schema_gen, err := generateSchema(data)
-
-			if err != nil {
-				return err
-			}
-
-			flag, schema, err = compareSchema(schema, schema_gen)
-
-			if err != nil {
-				return fmt.Errorf("failed to compare read and generated schema: %v", err)
-			}
-			if flag && readFlag && !errors.Is(err_, os.ErrNotExist) {
-
-				f.Close()
-
-				f, err = os.Open(filePath)
-				if err != nil {
-					return fmt.Errorf("failed to open Avro file: %v", err)
-				}
-
-				br := bufio.NewReader(f)
-
-				ocfReader, err := goavro.NewOCFReader(br)
-				if err != nil {
-					return fmt.Errorf("failed to create OCF reader while changing schema: %v", err)
-				}
-
-				for ocfReader.Scan() {
-					record, err := ocfReader.Read()
-					if err != nil {
-						return fmt.Errorf("failed to read record: %v", err)
-					}
-
-					record_list = append(record_list, record.(map[string]interface{}))
-				}
-
-				f.Close()
-
-				err = os.Remove(filePath)
-				if err != nil {
-					return fmt.Errorf("failed to delete file: %v", err)
-				}
-
-				readFlag = false
-			}
-			codec, err = goavro.NewCodec(schema)
-			if err != nil {
-				return fmt.Errorf("failed to create codec after merged schema: %v", err)
-			}
-
-			record_list = append(record_list, generateRecord(data))
-			delete(l.data, ts)
-		}
-	}
-
-	if len(record_list) == 0 {
-		return ErrNoNewData
-	}
-
-	f, err = os.OpenFile(filePath, os.O_CREATE|os.O_APPEND|os.O_RDWR, 0o644)
-	if err != nil {
-		return fmt.Errorf("failed to append new avro file: %v", err)
-	}
-
-	// fmt.Printf("Codec : %#v\n", codec)
-
-	writer, err := goavro.NewOCFWriter(goavro.OCFConfig{
-		W:               f,
-		Codec:           codec,
-		CompressionName: goavro.CompressionDeflateLabel,
-	})
-	if err != nil {
-		return fmt.Errorf("failed to create OCF writer: %v", err)
-	}
-
-	// Append the new record
-	if err := writer.Append(record_list); err != nil {
-		return fmt.Errorf("failed to append record: %v", err)
-	}
-
-	f.Close()
-
-	return nil
-}
-
-func compareSchema(schemaRead, schemaGen string) (bool, string, error) {
-	var genSchema, readSchema AvroSchema
-
-	if schemaRead == "" {
-		return false, schemaGen, nil
-	}
-
-	// Unmarshal the schema strings into AvroSchema structs
-	if err := json.Unmarshal([]byte(schemaGen), &genSchema); err != nil {
-		return false, "", fmt.Errorf("failed to parse generated schema: %v", err)
-	}
-	if err := json.Unmarshal([]byte(schemaRead), &readSchema); err != nil {
-		return false, "", fmt.Errorf("failed to parse read schema: %v", err)
-	}
-
-	sort.Slice(genSchema.Fields, func(i, j int) bool {
-		return genSchema.Fields[i].Name < genSchema.Fields[j].Name
-	})
-
-	sort.Slice(readSchema.Fields, func(i, j int) bool {
-		return readSchema.Fields[i].Name < readSchema.Fields[j].Name
-	})
-
-	// Check if schemas are identical
-	schemasEqual := true
-	if len(genSchema.Fields) <= len(readSchema.Fields) {
-
-		for i := range genSchema.Fields {
-			if genSchema.Fields[i].Name != readSchema.Fields[i].Name {
-				schemasEqual = false
-				break
-			}
-		}
-
-		// If schemas are identical, return the read schema
-		if schemasEqual {
-			return false, schemaRead, nil
-		}
-	}
-
-	// Create a map to hold unique fields from both schemas
-	fieldMap := make(map[string]AvroField)
-
-	// Add fields from the read schema
-	for _, field := range readSchema.Fields {
-		fieldMap[field.Name] = field
-	}
-
-	// Add or update fields from the generated schema
-	for _, field := range genSchema.Fields {
-		fieldMap[field.Name] = field
-	}
-
-	// Create a union schema by collecting fields from the map
-	var mergedFields []AvroField
-	for _, field := range fieldMap {
-		mergedFields = append(mergedFields, field)
-	}
-
-	// Sort fields by name for consistency
-	sort.Slice(mergedFields, func(i, j int) bool {
-		return mergedFields[i].Name < mergedFields[j].Name
-	})
-
-	// Create the merged schema
-	mergedSchema := AvroSchema{
-		Type:   "record",
-		Name:   genSchema.Name,
-		Fields: mergedFields,
-	}
-
-	// Check if schemas are identical
-	schemasEqual = len(mergedSchema.Fields) == len(readSchema.Fields)
-	if schemasEqual {
-		for i := range mergedSchema.Fields {
-			if mergedSchema.Fields[i].Name != readSchema.Fields[i].Name {
-				schemasEqual = false
-				break
-			}
-		}
-
-		if schemasEqual {
-			return false, schemaRead, nil
-		}
-	}
-
-	// Marshal the merged schema back to JSON
-	mergedSchemaJson, err := json.Marshal(mergedSchema)
-	if err != nil {
-		return false, "", fmt.Errorf("failed to marshal merged schema: %v", err)
-	}
-
-	return true, string(mergedSchemaJson), nil
-}
-
-func generateSchema(data map[string]util.Float) (string, error) {
-
-	// Define the Avro schema structure
-	schema := map[string]interface{}{
-		"type":   "record",
-		"name":   "DataRecord",
-		"fields": []map[string]interface{}{},
-	}
-
-	fieldTracker := make(map[string]struct{})
-
-	for key := range data {
-		if _, exists := fieldTracker[key]; !exists {
-			key = correctKey(key)
-
-			field := map[string]interface{}{
-				"name":    key,
-				"type":    "double",
-				"default": -1.0,
-			}
-			schema["fields"] = append(schema["fields"].([]map[string]interface{}), field)
-			fieldTracker[key] = struct{}{}
-		}
-	}
-
-	schemaString, err := json.Marshal(schema)
-	if err != nil {
-		return "", fmt.Errorf("failed to marshal schema: %v", err)
-	}
-
-	return string(schemaString), nil
-}
-
-func generateRecord(data map[string]util.Float) map[string]interface{} {
-
-	record := make(map[string]interface{})
-
-	// Iterate through each map in data
-	for key, value := range data {
-		key = correctKey(key)
-
-		// Set the value in the record
-		record[key] = value.Double()
-	}
-
-	return record
-}
-
-func correctKey(key string) string {
-	// Replace any invalid characters in the key
-	// For example, replace spaces with underscores
-	key = strings.ReplaceAll(key, ":", "___")
-	key = strings.ReplaceAll(key, ".", "__")
-
-	return key
-}
-
-func ReplaceKey(key string) string {
-	// Replace any invalid characters in the key
-	// For example, replace spaces with underscores
-	key = strings.ReplaceAll(key, "___", ":")
-	key = strings.ReplaceAll(key, "__", ".")
-
-	return key
-}

internal/avro/avroHelper.go

@@ -1,80 +0,0 @@
-package avro
-
-import (
-	"context"
-	"fmt"
-	"strconv"
-	"sync"
-
-	"github.com/ClusterCockpit/cc-metric-store/internal/config"
-)
-
-func DataStaging(wg *sync.WaitGroup, ctx context.Context) {
-
-	// AvroPool is a pool of Avro writers.
-	go func() {
-		if config.Keys.Checkpoints.FileFormat == "json" {
-			wg.Done() // Mark this goroutine as done
-			return    // Exit the goroutine
-		}
-
-		defer wg.Done()
-
-		var avroLevel *AvroLevel
-		oldSelector := make([]string, 0)
-
-		for {
-			select {
-			case <-ctx.Done():
-				return
-			case val := <-LineProtocolMessages:
-				//Fetch the frequency of the metric from the global configuration
-				freq, err := config.Keys.GetMetricFrequency(val.MetricName)
-				if err != nil {
-					fmt.Printf("Error fetching metric frequency: %s\n", err)
-					continue
-				}
-
-				metricName := ""
-
-				for _, selector_name := range val.Selector {
-					metricName += selector_name + Delimiter
-				}
-
-				metricName += val.MetricName
-
-				// Create a new selector for the Avro level
-				// The selector is a slice of strings that represents the path to the
-				// Avro level. It is created by appending the cluster, node, and metric
-				// name to the selector.
-				var selector []string
-				selector = append(selector, val.Cluster, val.Node, strconv.FormatInt(freq, 10))
-
-				if !testEq(oldSelector, selector) {
-					// Get the Avro level for the metric
-					avroLevel = avroStore.root.findAvroLevelOrCreate(selector)
-
-					// If the Avro level is nil, create a new one
-					if avroLevel == nil {
-						fmt.Printf("Error creating or finding the level with cluster : %s, node : %s, metric : %s\n", val.Cluster, val.Node, val.MetricName)
-					}
-					oldSelector = append([]string{}, selector...)
-				}
-
-				avroLevel.addMetric(metricName, val.Value, val.Timestamp, int(freq))
-			}
-		}
-	}()
-}
-
-func testEq(a, b []string) bool {
-	if len(a) != len(b) {
-		return false
-	}
-	for i := range a {
-		if a[i] != b[i] {
-			return false
-		}
-	}
-	return true
-}

internal/avro/avroStruct.go

@@ -1,161 +0,0 @@
-package avro
-
-import (
-	"sync"
-
-	"github.com/ClusterCockpit/cc-metric-store/internal/util"
-)
-
-var LineProtocolMessages = make(chan *AvroStruct)
-var Delimiter = "ZZZZZ"
-
-// CheckpointBufferMinutes should always be in minutes.
-// Its controls the amount of data to hold for given amount of time.
-var CheckpointBufferMinutes = 3
-
-type AvroStruct struct {
-	MetricName string
-	Cluster    string
-	Node       string
-	Selector   []string
-	Value      util.Float
-	Timestamp  int64
-}
-
-type AvroStore struct {
-	root AvroLevel
-}
-
-var avroStore AvroStore
-
-type AvroLevel struct {
-	children map[string]*AvroLevel
-	data     map[int64]map[string]util.Float
-	lock     sync.RWMutex
-}
-
-type AvroField struct {
-	Name    string      `json:"name"`
-	Type    interface{} `json:"type"`
-	Default interface{} `json:"default,omitempty"`
-}
-
-type AvroSchema struct {
-	Type   string      `json:"type"`
-	Name   string      `json:"name"`
-	Fields []AvroField `json:"fields"`
-}
-
-func (l *AvroLevel) findAvroLevelOrCreate(selector []string) *AvroLevel {
-	if len(selector) == 0 {
-		return l
-	}
-
-	// Allow concurrent reads:
-	l.lock.RLock()
-	var child *AvroLevel
-	var ok bool
-	if l.children == nil {
-		// Children map needs to be created...
-		l.lock.RUnlock()
-	} else {
-		child, ok := l.children[selector[0]]
-		l.lock.RUnlock()
-		if ok {
-			return child.findAvroLevelOrCreate(selector[1:])
-		}
-	}
-
-	// The level does not exist, take write lock for unqiue access:
-	l.lock.Lock()
-	// While this thread waited for the write lock, another thread
-	// could have created the child node.
-	if l.children != nil {
-		child, ok = l.children[selector[0]]
-		if ok {
-			l.lock.Unlock()
-			return child.findAvroLevelOrCreate(selector[1:])
-		}
-	}
-
-	child = &AvroLevel{
-		data:     make(map[int64]map[string]util.Float, 0),
-		children: nil,
-	}
-
-	if l.children != nil {
-		l.children[selector[0]] = child
-	} else {
-		l.children = map[string]*AvroLevel{selector[0]: child}
-	}
-	l.lock.Unlock()
-	return child.findAvroLevelOrCreate(selector[1:])
-}
-
-func (l *AvroLevel) addMetric(metricName string, value util.Float, timestamp int64, Freq int) {
-	l.lock.Lock()
-	defer l.lock.Unlock()
-
-	KeyCounter := int(CheckpointBufferMinutes * 60 / Freq)
-
-	// Create keys in advance for the given amount of time
-	if len(l.data) != KeyCounter {
-		if len(l.data) == 0 {
-			for i := range KeyCounter {
-				l.data[timestamp+int64(i*Freq)] = make(map[string]util.Float, 0)
-			}
-		} else {
-			//Get the last timestamp
-			var lastTs int64
-			for ts := range l.data {
-				if ts > lastTs {
-					lastTs = ts
-				}
-			}
-			// Create keys for the next KeyCounter timestamps
-			l.data[lastTs+int64(Freq)] = make(map[string]util.Float, 0)
-		}
-	}
-
-	closestTs := int64(0)
-	minDiff := int64(Freq) + 1 // Start with diff just outside the valid range
-	found := false
-
-	// Iterate over timestamps and choose the one which is within range.
-	// Since its epoch time, we check if the difference is less than 60 seconds.
-	for ts, dat := range l.data {
-		// Check if timestamp is within range
-		diff := timestamp - ts
-		if diff < -int64(Freq) || diff > int64(Freq) {
-			continue
-		}
-
-		// Metric already present at this timestamp — skip
-		if _, ok := dat[metricName]; ok {
-			continue
-		}
-
-		// Check if this is the closest timestamp so far
-		if Abs(diff) < minDiff {
-			minDiff = Abs(diff)
-			closestTs = ts
-			found = true
-		}
-	}
-
-	if found {
-		l.data[closestTs][metricName] = value
-	}
-}
-
-func GetAvroStore() *AvroStore {
-	return &avroStore
-}
-
-// Abs returns the absolute value of x.
-func Abs(x int64) int64 {
-	if x < 0 {
-		return -x
-	}
-	return x
-}

internal/config/config.go

@@ -1,123 +1,81 @@
+// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
+// All rights reserved. This file is part of cc-metric-store.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
 package config
 
 import (
+	"bytes"
 	"encoding/json"
 	"fmt"
-	"log"
-	"os"
+
+	"github.com/ClusterCockpit/cc-backend/pkg/metricstore"
+	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
 )
 
-// For aggregation over multiple values at different cpus/sockets/..., not time!
-type AggregationStrategy int
-
-const (
-	NoAggregation AggregationStrategy = iota
-	SumAggregation
-	AvgAggregation
-)
-
-func (as *AggregationStrategy) UnmarshalJSON(data []byte) error {
-	var str string
-	if err := json.Unmarshal(data, &str); err != nil {
-		return err
-	}
-
-	switch str {
-	case "":
-		*as = NoAggregation
-	case "sum":
-		*as = SumAggregation
-	case "avg":
-		*as = AvgAggregation
-	default:
-		return fmt.Errorf("invalid aggregation strategy: %#v", str)
-	}
-	return nil
-}
-
-type MetricConfig struct {
-	// Interval in seconds at which measurements will arive.
-	Frequency int64 `json:"frequency"`
-
-	// Can be 'sum', 'avg' or null. Describes how to aggregate metrics from the same timestep over the hierarchy.
-	Aggregation AggregationStrategy `json:"aggregation"`
-
-	// Private, used internally...
-	Offset int
-}
-
-type HttpConfig struct {
-	// Address to bind to, for example "0.0.0.0:8081"
-	Address string `json:"address"`
-
-	// If not the empty string, use https with this as the certificate file
-	CertFile string `json:"https-cert-file"`
-
-	// If not the empty string, use https with this as the key file
-	KeyFile string `json:"https-key-file"`
-}
-
-type NatsConfig struct {
-	// Address of the nats server
-	Address string `json:"address"`
-
-	// Username/Password, optional
-	Username string `json:"username"`
-	Password string `json:"password"`
-
-	//Creds file path
-	Credsfilepath string `json:"creds-file-path"`
-
-	Subscriptions []struct {
-		// Channel name
-		SubscribeTo string `json:"subscribe-to"`
-
-		// Allow lines without a cluster tag, use this as default, optional
-		ClusterTag string `json:"cluster-tag"`
-	} `json:"subscriptions"`
-}
+var metrics map[string]metricstore.MetricConfig
 
 type Config struct {
-	Metrics     map[string]MetricConfig `json:"metrics"`
-	HttpConfig  *HttpConfig             `json:"http-api"`
-	Checkpoints struct {
-		FileFormat string `json:"file-format"`
-		Interval   string `json:"interval"`
-		RootDir    string `json:"directory"`
-		Restore    string `json:"restore"`
-	} `json:"checkpoints"`
+	Address    string `json:"addr"`
+	CertFile   string `json:"https-cert-file"`
+	KeyFile    string `json:"https-key-file"`
+	User       string `json:"user"`
+	Group      string `json:"group"`
+	BackendURL string `json:"backend-url"`
 	Debug      struct {
 		DumpToFile string `json:"dump-to-file"`
 		EnableGops bool   `json:"gops"`
 	} `json:"debug"`
-	RetentionInMemory string `json:"retention-in-memory"`
 	JwtPublicKey string `json:"jwt-public-key"`
-	Archive           struct {
-		Interval      string `json:"interval"`
-		RootDir       string `json:"directory"`
-		DeleteInstead bool   `json:"delete-instead"`
-	} `json:"archive"`
-	Nats []*NatsConfig `json:"nats"`
 }
 
 var Keys Config
 
-func Init(file string) {
-	configFile, err := os.Open(file)
-	if err != nil {
-		log.Fatal(err)
-	}
-	defer configFile.Close()
-	dec := json.NewDecoder(configFile)
+type metricConfigJSON struct {
+	Frequency   int64  `json:"frequency"`
+	Aggregation string `json:"aggregation"`
+}
+
+func InitMetrics(metricConfig json.RawMessage) {
+	Validate(metricConfigSchema, metricConfig)
+
+	var tempMetrics map[string]metricConfigJSON
+	dec := json.NewDecoder(bytes.NewReader(metricConfig))
 	dec.DisallowUnknownFields()
-	if err := dec.Decode(&Keys); err != nil {
-		log.Fatal(err)
+	if err := dec.Decode(&tempMetrics); err != nil {
+		cclog.Abortf("Config Init: Could not decode config file '%s'.\nError: %s\n", metricConfig, err.Error())
+	}
+
+	metrics = make(map[string]metricstore.MetricConfig)
+	for name, cfg := range tempMetrics {
+		agg, err := metricstore.AssignAggregationStrategy(cfg.Aggregation)
+		if err != nil {
+			cclog.Warnf("Could not parse aggregation strategy for metric '%s': %s", name, err.Error())
+		}
+		metrics[name] = metricstore.MetricConfig{
+			Frequency:   cfg.Frequency,
+			Aggregation: agg,
+		}
 	}
 }
 
-func (c *Config) GetMetricFrequency(metricName string) (int64, error) {
-	if metric, ok := c.Metrics[metricName]; ok {
+func Init(mainConfig json.RawMessage) {
+	Validate(configSchema, mainConfig)
+	dec := json.NewDecoder(bytes.NewReader(mainConfig))
+	dec.DisallowUnknownFields()
+	if err := dec.Decode(&Keys); err != nil {
+		cclog.Abortf("Config Init: Could not decode config file '%s'.\nError: %s\n", mainConfig, err.Error())
+	}
+}
+
+func GetMetricFrequency(metricName string) (int64, error) {
+	if metric, ok := metrics[metricName]; ok {
 		return metric.Frequency, nil
 	}
 	return 0, fmt.Errorf("metric %s not found", metricName)
 }
+
+func GetMetrics() map[string]metricstore.MetricConfig {
+	return metrics
+}

internal/config/metricSchema.go

@@ -0,0 +1,26 @@
+// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
+// All rights reserved. This file is part of cc-metric-store.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package config
+
+var metricConfigSchema = `
+{
+  "type": "object",
+  "description": "Map of metric names to their configuration.",
+  "additionalProperties": {
+    "type": "object",
+    "properties": {
+      "frequency": {
+        "description": "Sampling frequency in seconds.",
+        "type": "integer"
+      },
+      "aggregation": {
+        "description": "Aggregation strategy: 'sum', 'avg', or 'null'.",
+        "type": ["string", "null"]
+      }
+    },
+    "required": ["frequency", "aggregation"]
+  }
+}`

internal/config/schema.go

@@ -0,0 +1,55 @@
+// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
+// All rights reserved. This file is part of cc-metric-store.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package config
+
+var configSchema = `
+{
+  "type": "object",
+  "properties": {
+    "addr": {
+      "description": "Address where the http (or https) server will listen on (for example: 'localhost:8080').",
+      "type": "string"
+    },
+    "https-cert-file": {
+      "description": "Filepath to SSL certificate. If also https-key-file is set, use HTTPS.",
+      "type": "string"
+    },
+    "https-key-file": {
+      "description": "Filepath to SSL key file. If also https-cert-file is set, use HTTPS.",
+      "type": "string"
+    },
+    "user": {
+      "description": "Drop root permissions once the port was taken. Only applicable if using privileged port.",
+      "type": "string"
+    },
+    "group": {
+      "description": "Drop root permissions once the port was taken. Only applicable if using privileged port.",
+      "type": "string"
+    },
+    "backend-url": {
+      "description": "URL of cc-backend for querying job information (e.g., 'https://localhost:8080').",
+      "type": "string"
+    },
+    "debug": {
+      "description": "Debug options.",
+      "type": "object",
+      "properties": {
+        "dump-to-file": {
+          "description": "Path to file for dumping internal state.",
+          "type": "string"
+        },
+        "gops": {
+          "description": "Enable gops agent for debugging.",
+          "type": "boolean"
+        }
+      }
+    },
+    "jwt-public-key": {
+      "description": "Ed25519 public key for JWT verification.",
+      "type": "string"
+    }
+  }
+}`

internal/config/validate.go

@@ -0,0 +1,29 @@
+// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
+// All rights reserved. This file is part of cc-metric-store.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package config
+
+import (
+	"encoding/json"
+
+	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
+	"github.com/santhosh-tekuri/jsonschema/v5"
+)
+
+func Validate(schema string, instance json.RawMessage) {
+	sch, err := jsonschema.CompileString("schema.json", schema)
+	if err != nil {
+		cclog.Fatalf("%#v", err)
+	}
+
+	var v any
+	if err := json.Unmarshal([]byte(instance), &v); err != nil {
+		cclog.Fatal(err)
+	}
+
+	if err = sch.Validate(v); err != nil {
+		cclog.Fatalf("%#v", err)
+	}
+}

internal/memorystore/archive.go

@@ -1,185 +0,0 @@
-package memorystore
-
-import (
-	"archive/zip"
-	"bufio"
-	"context"
-	"errors"
-	"fmt"
-	"io"
-	"log"
-	"os"
-	"path/filepath"
-	"sync"
-	"sync/atomic"
-	"time"
-
-	"github.com/ClusterCockpit/cc-metric-store/internal/config"
-)
-
-func Archiving(wg *sync.WaitGroup, ctx context.Context) {
-	go func() {
-		defer wg.Done()
-		d, err := time.ParseDuration(config.Keys.Archive.Interval)
-		if err != nil {
-			log.Fatal(err)
-		}
-		if d <= 0 {
-			return
-		}
-
-		ticks := func() <-chan time.Time {
-			if d <= 0 {
-				return nil
-			}
-			return time.NewTicker(d).C
-		}()
-		for {
-			select {
-			case <-ctx.Done():
-				return
-			case <-ticks:
-				t := time.Now().Add(-d)
-				log.Printf("start archiving checkpoints (older than %s)...\n", t.Format(time.RFC3339))
-				n, err := ArchiveCheckpoints(config.Keys.Checkpoints.RootDir, config.Keys.Archive.RootDir, t.Unix(), config.Keys.Archive.DeleteInstead)
-				if err != nil {
-					log.Printf("archiving failed: %s\n", err.Error())
-				} else {
-					log.Printf("done: %d files zipped and moved to archive\n", n)
-				}
-			}
-		}
-	}()
-}
-
-var ErrNoNewData error = errors.New("all data already archived")
-
-// ZIP all checkpoint files older than `from` together and write them to the `archiveDir`,
-// deleting them from the `checkpointsDir`.
-func ArchiveCheckpoints(checkpointsDir, archiveDir string, from int64, deleteInstead bool) (int, error) {
-	entries1, err := os.ReadDir(checkpointsDir)
-	if err != nil {
-		return 0, err
-	}
-
-	type workItem struct {
-		cdir, adir    string
-		cluster, host string
-	}
-
-	var wg sync.WaitGroup
-	n, errs := int32(0), int32(0)
-	work := make(chan workItem, NumWorkers)
-
-	wg.Add(NumWorkers)
-	for worker := 0; worker < NumWorkers; worker++ {
-		go func() {
-			defer wg.Done()
-			for workItem := range work {
-				m, err := archiveCheckpoints(workItem.cdir, workItem.adir, from, deleteInstead)
-				if err != nil {
-					log.Printf("error while archiving %s/%s: %s", workItem.cluster, workItem.host, err.Error())
-					atomic.AddInt32(&errs, 1)
-				}
-				atomic.AddInt32(&n, int32(m))
-			}
-		}()
-	}
-
-	for _, de1 := range entries1 {
-		entries2, e := os.ReadDir(filepath.Join(checkpointsDir, de1.Name()))
-		if e != nil {
-			err = e
-		}
-
-		for _, de2 := range entries2 {
-			cdir := filepath.Join(checkpointsDir, de1.Name(), de2.Name())
-			adir := filepath.Join(archiveDir, de1.Name(), de2.Name())
-			work <- workItem{
-				adir: adir, cdir: cdir,
-				cluster: de1.Name(), host: de2.Name(),
-			}
-		}
-	}
-
-	close(work)
-	wg.Wait()
-
-	if err != nil {
-		return int(n), err
-	}
-
-	if errs > 0 {
-		return int(n), fmt.Errorf("%d errors happend while archiving (%d successes)", errs, n)
-	}
-	return int(n), nil
-}
-
-// Helper function for `ArchiveCheckpoints`.
-func archiveCheckpoints(dir string, archiveDir string, from int64, deleteInstead bool) (int, error) {
-	entries, err := os.ReadDir(dir)
-	if err != nil {
-		return 0, err
-	}
-
-	extension := config.Keys.Checkpoints.FileFormat
-	files, err := findFiles(entries, from, extension, false)
-	if err != nil {
-		return 0, err
-	}
-
-	if deleteInstead {
-		n := 0
-		for _, checkpoint := range files {
-			filename := filepath.Join(dir, checkpoint)
-			if err = os.Remove(filename); err != nil {
-				return n, err
-			}
-			n += 1
-		}
-		return n, nil
-	}
-
-	filename := filepath.Join(archiveDir, fmt.Sprintf("%d.zip", from))
-	f, err := os.OpenFile(filename, os.O_CREATE|os.O_WRONLY, 0o644)
-	if err != nil && os.IsNotExist(err) {
-		err = os.MkdirAll(archiveDir, 0o755)
-		if err == nil {
-			f, err = os.OpenFile(filename, os.O_CREATE|os.O_WRONLY, 0o644)
-		}
-	}
-	if err != nil {
-		return 0, err
-	}
-	defer f.Close()
-	bw := bufio.NewWriter(f)
-	defer bw.Flush()
-	zw := zip.NewWriter(bw)
-	defer zw.Close()
-
-	n := 0
-	for _, checkpoint := range files {
-		filename := filepath.Join(dir, checkpoint)
-		r, err := os.Open(filename)
-		if err != nil {
-			return n, err
-		}
-		defer r.Close()
-
-		w, err := zw.Create(checkpoint)
-		if err != nil {
-			return n, err
-		}
-
-		if _, err = io.Copy(w, r); err != nil {
-			return n, err
-		}
-
-		if err = os.Remove(filename); err != nil {
-			return n, err
-		}
-		n += 1
-	}
-
-	return n, nil
-}

internal/memorystore/buffer.go

@@ -1,233 +0,0 @@
-package memorystore
-
-import (
-	"errors"
-	"sync"
-
-	"github.com/ClusterCockpit/cc-metric-store/internal/util"
-)
-
-// Default buffer capacity.
-// `buffer.data` will only ever grow up to it's capacity and a new link
-// in the buffer chain will be created if needed so that no copying
-// of data or reallocation needs to happen on writes.
-const (
-	BUFFER_CAP int = 512
-)
-
-// So that we can reuse allocations
-var bufferPool sync.Pool = sync.Pool{
-	New: func() interface{} {
-		return &buffer{
-			data: make([]util.Float, 0, BUFFER_CAP),
-		}
-	},
-}
-
-var (
-	ErrNoData           error = errors.New("no data for this metric/level")
-	ErrDataDoesNotAlign error = errors.New("data from lower granularities does not align")
-)
-
-// Each metric on each level has it's own buffer.
-// This is where the actual values go.
-// If `cap(data)` is reached, a new buffer is created and
-// becomes the new head of a buffer list.
-type buffer struct {
-	prev      *buffer
-	next      *buffer
-	data      []util.Float
-	frequency int64
-	start     int64
-	archived  bool
-	closed    bool
-}
-
-func newBuffer(ts, freq int64) *buffer {
-	b := bufferPool.Get().(*buffer)
-	b.frequency = freq
-	b.start = ts - (freq / 2)
-	b.prev = nil
-	b.next = nil
-	b.archived = false
-	b.closed = false
-	b.data = b.data[:0]
-	return b
-}
-
-// If a new buffer was created, the new head is returnd.
-// Otherwise, the existing buffer is returnd.
-// Normaly, only "newer" data should be written, but if the value would
-// end up in the same buffer anyways it is allowed.
-func (b *buffer) write(ts int64, value util.Float) (*buffer, error) {
-	if ts < b.start {
-		return nil, errors.New("cannot write value to buffer from past")
-	}
-
-	// idx := int((ts - b.start + (b.frequency / 3)) / b.frequency)
-	idx := int((ts - b.start) / b.frequency)
-	if idx >= cap(b.data) {
-		newbuf := newBuffer(ts, b.frequency)
-		newbuf.prev = b
-		b.next = newbuf
-		b.close()
-		b = newbuf
-		idx = 0
-	}
-
-	// Overwriting value or writing value from past
-	if idx < len(b.data) {
-		b.data[idx] = value
-		return b, nil
-	}
-
-	// Fill up unwritten slots with NaN
-	for i := len(b.data); i < idx; i++ {
-		b.data = append(b.data, util.NaN)
-	}
-
-	b.data = append(b.data, value)
-	return b, nil
-}
-
-func (b *buffer) end() int64 {
-	return b.firstWrite() + int64(len(b.data))*b.frequency
-}
-
-func (b *buffer) firstWrite() int64 {
-	return b.start + (b.frequency / 2)
-}
-
-func (b *buffer) close() {}
-
-/*
-func (b *buffer) close() {
-	if b.closed {
-		return
-	}
-
-	b.closed = true
-	n, sum, min, max := 0, 0., math.MaxFloat64, -math.MaxFloat64
-	for _, x := range b.data {
-		if x.IsNaN() {
-			continue
-		}
-
-		n += 1
-		f := float64(x)
-		sum += f
-		min = math.Min(min, f)
-		max = math.Max(max, f)
-	}
-
-	b.statisticts.samples = n
-	if n > 0 {
-		b.statisticts.avg = Float(sum / float64(n))
-		b.statisticts.min = Float(min)
-		b.statisticts.max = Float(max)
-	} else {
-		b.statisticts.avg = NaN
-		b.statisticts.min = NaN
-		b.statisticts.max = NaN
-	}
-}
-*/
-
-// func interpolate(idx int, data []Float) Float {
-// 	if idx == 0 || idx+1 == len(data) {
-// 		return NaN
-// 	}
-// 	return (data[idx-1] + data[idx+1]) / 2.0
-// }
-
-// Return all known values from `from` to `to`. Gaps of information are represented as NaN.
-// Simple linear interpolation is done between the two neighboring cells if possible.
-// If values at the start or end are missing, instead of NaN values, the second and thrid
-// return values contain the actual `from`/`to`.
-// This function goes back the buffer chain if `from` is older than the currents buffer start.
-// The loaded values are added to `data` and `data` is returned, possibly with a shorter length.
-// If `data` is not long enough to hold all values, this function will panic!
-func (b *buffer) read(from, to int64, data []util.Float) ([]util.Float, int64, int64, error) {
-	if from < b.firstWrite() {
-		if b.prev != nil {
-			return b.prev.read(from, to, data)
-		}
-		from = b.firstWrite()
-	}
-
-	i := 0
-	t := from
-	for ; t < to; t += b.frequency {
-		idx := int((t - b.start) / b.frequency)
-		if idx >= cap(b.data) {
-			if b.next == nil {
-				break
-			}
-			b = b.next
-			idx = 0
-		}
-
-		if idx >= len(b.data) {
-			if b.next == nil || to <= b.next.start {
-				break
-			}
-			data[i] += util.NaN
-		} else if t < b.start {
-			data[i] += util.NaN
-			// } else if b.data[idx].IsNaN() {
-			// 	data[i] += interpolate(idx, b.data)
-		} else {
-			data[i] += b.data[idx]
-		}
-		i++
-	}
-
-	return data[:i], from, t, nil
-}
-
-// Returns true if this buffer needs to be freed.
-func (b *buffer) free(t int64) (delme bool, n int) {
-	if b.prev != nil {
-		delme, m := b.prev.free(t)
-		n += m
-		if delme {
-			b.prev.next = nil
-			if cap(b.prev.data) == BUFFER_CAP {
-				bufferPool.Put(b.prev)
-			}
-			b.prev = nil
-		}
-	}
-
-	end := b.end()
-	if end < t {
-		return true, n + 1
-	}
-
-	return false, n
-}
-
-// Call `callback` on every buffer that contains data in the range from `from` to `to`.
-func (b *buffer) iterFromTo(from, to int64, callback func(b *buffer) error) error {
-	if b == nil {
-		return nil
-	}
-
-	if err := b.prev.iterFromTo(from, to, callback); err != nil {
-		return err
-	}
-
-	if from <= b.end() && b.start <= to {
-		return callback(b)
-	}
-
-	return nil
-}
-
-func (b *buffer) count() int64 {
-	res := int64(len(b.data))
-	if b.prev != nil {
-		res += b.prev.count()
-	}
-	return res
-}

internal/memorystore/checkpoint.go

@@ -1,767 +0,0 @@
-package memorystore
-
-import (
-	"bufio"
-	"context"
-	"encoding/json"
-	"errors"
-	"fmt"
-	"io/fs"
-	"log"
-	"os"
-	"path"
-	"path/filepath"
-	"runtime"
-	"sort"
-	"strconv"
-	"strings"
-	"sync"
-	"sync/atomic"
-	"time"
-
-	"github.com/ClusterCockpit/cc-metric-store/internal/avro"
-	"github.com/ClusterCockpit/cc-metric-store/internal/config"
-	"github.com/ClusterCockpit/cc-metric-store/internal/util"
-	"github.com/linkedin/goavro/v2"
-)
-
-// Whenever changed, update MarshalJSON as well!
-type CheckpointMetrics struct {
-	Data      []util.Float `json:"data"`
-	Frequency int64        `json:"frequency"`
-	Start     int64        `json:"start"`
-}
-
-type CheckpointFile struct {
-	Metrics  map[string]*CheckpointMetrics `json:"metrics"`
-	Children map[string]*CheckpointFile    `json:"children"`
-	From     int64                         `json:"from"`
-	To       int64                         `json:"to"`
-}
-
-var lastCheckpoint time.Time
-
-func Checkpointing(wg *sync.WaitGroup, ctx context.Context) {
-	lastCheckpoint = time.Now()
-
-	if config.Keys.Checkpoints.FileFormat == "json" {
-		ms := GetMemoryStore()
-
-		go func() {
-			defer wg.Done()
-			d, err := time.ParseDuration(config.Keys.Checkpoints.Interval)
-			if err != nil {
-				log.Fatal(err)
-			}
-			if d <= 0 {
-				return
-			}
-
-			ticks := func() <-chan time.Time {
-				if d <= 0 {
-					return nil
-				}
-				return time.NewTicker(d).C
-			}()
-			for {
-				select {
-				case <-ctx.Done():
-					return
-				case <-ticks:
-					log.Printf("start checkpointing (starting at %s)...\n", lastCheckpoint.Format(time.RFC3339))
-					now := time.Now()
-					n, err := ms.ToCheckpoint(config.Keys.Checkpoints.RootDir,
-						lastCheckpoint.Unix(), now.Unix())
-					if err != nil {
-						log.Printf("checkpointing failed: %s\n", err.Error())
-					} else {
-						log.Printf("done: %d checkpoint files created\n", n)
-						lastCheckpoint = now
-					}
-				}
-			}
-		}()
-	} else {
-		go func() {
-			defer wg.Done()
-			d, _ := time.ParseDuration("1m")
-
-			select {
-			case <-ctx.Done():
-				return
-			case <-time.After(time.Duration(avro.CheckpointBufferMinutes) * time.Minute):
-				// This is the first tick untill we collect the data for given minutes.
-				avro.GetAvroStore().ToCheckpoint(config.Keys.Checkpoints.RootDir, false)
-				// log.Printf("Checkpointing %d avro files", count)
-
-			}
-
-			ticks := func() <-chan time.Time {
-				if d <= 0 {
-					return nil
-				}
-				return time.NewTicker(d).C
-			}()
-
-			for {
-				select {
-				case <-ctx.Done():
-					return
-				case <-ticks:
-					// Regular ticks of 1 minute to write data.
-					avro.GetAvroStore().ToCheckpoint(config.Keys.Checkpoints.RootDir, false)
-					// log.Printf("Checkpointing %d avro files", count)
-				}
-			}
-		}()
-	}
-}
-
-// As `Float` implements a custom MarshalJSON() function,
-// serializing an array of such types has more overhead
-// than one would assume (because of extra allocations, interfaces and so on).
-func (cm *CheckpointMetrics) MarshalJSON() ([]byte, error) {
-	buf := make([]byte, 0, 128+len(cm.Data)*8)
-	buf = append(buf, `{"frequency":`...)
-	buf = strconv.AppendInt(buf, cm.Frequency, 10)
-	buf = append(buf, `,"start":`...)
-	buf = strconv.AppendInt(buf, cm.Start, 10)
-	buf = append(buf, `,"data":[`...)
-	for i, x := range cm.Data {
-		if i != 0 {
-			buf = append(buf, ',')
-		}
-		if x.IsNaN() {
-			buf = append(buf, `null`...)
-		} else {
-			buf = strconv.AppendFloat(buf, float64(x), 'f', 1, 32)
-		}
-	}
-	buf = append(buf, `]}`...)
-	return buf, nil
-}
-
-// Metrics stored at the lowest 2 levels are not stored away (root and cluster)!
-// On a per-host basis a new JSON file is created. I have no idea if this will scale.
-// The good thing: Only a host at a time is locked, so this function can run
-// in parallel to writes/reads.
-func (m *MemoryStore) ToCheckpoint(dir string, from, to int64) (int, error) {
-	levels := make([]*Level, 0)
-	selectors := make([][]string, 0)
-	m.root.lock.RLock()
-	for sel1, l1 := range m.root.children {
-		l1.lock.RLock()
-		for sel2, l2 := range l1.children {
-			levels = append(levels, l2)
-			selectors = append(selectors, []string{sel1, sel2})
-		}
-		l1.lock.RUnlock()
-	}
-	m.root.lock.RUnlock()
-
-	type workItem struct {
-		level    *Level
-		dir      string
-		selector []string
-	}
-
-	n, errs := int32(0), int32(0)
-
-	var wg sync.WaitGroup
-	wg.Add(NumWorkers)
-	work := make(chan workItem, NumWorkers*2)
-	for worker := 0; worker < NumWorkers; worker++ {
-		go func() {
-			defer wg.Done()
-
-			for workItem := range work {
-				if err := workItem.level.toCheckpoint(workItem.dir, from, to, m); err != nil {
-					if err == ErrNoNewData {
-						continue
-					}
-
-					log.Printf("error while checkpointing %#v: %s", workItem.selector, err.Error())
-					atomic.AddInt32(&errs, 1)
-				} else {
-					atomic.AddInt32(&n, 1)
-				}
-			}
-		}()
-	}
-
-	for i := 0; i < len(levels); i++ {
-		dir := path.Join(dir, path.Join(selectors[i]...))
-		work <- workItem{
-			level:    levels[i],
-			dir:      dir,
-			selector: selectors[i],
-		}
-	}
-
-	close(work)
-	wg.Wait()
-
-	if errs > 0 {
-		return int(n), fmt.Errorf("%d errors happend while creating checkpoints (%d successes)", errs, n)
-	}
-	return int(n), nil
-}
-
-func (l *Level) toCheckpointFile(from, to int64, m *MemoryStore) (*CheckpointFile, error) {
-	l.lock.RLock()
-	defer l.lock.RUnlock()
-
-	retval := &CheckpointFile{
-		From:     from,
-		To:       to,
-		Metrics:  make(map[string]*CheckpointMetrics),
-		Children: make(map[string]*CheckpointFile),
-	}
-
-	for metric, minfo := range m.Metrics {
-		b := l.metrics[minfo.Offset]
-		if b == nil {
-			continue
-		}
-
-		allArchived := true
-		b.iterFromTo(from, to, func(b *buffer) error {
-			if !b.archived {
-				allArchived = false
-			}
-			return nil
-		})
-
-		if allArchived {
-			continue
-		}
-
-		data := make([]util.Float, (to-from)/b.frequency+1)
-		data, start, end, err := b.read(from, to, data)
-		if err != nil {
-			return nil, err
-		}
-
-		for i := int((end - start) / b.frequency); i < len(data); i++ {
-			data[i] = util.NaN
-		}
-
-		retval.Metrics[metric] = &CheckpointMetrics{
-			Frequency: b.frequency,
-			Start:     start,
-			Data:      data,
-		}
-	}
-
-	for name, child := range l.children {
-		val, err := child.toCheckpointFile(from, to, m)
-		if err != nil {
-			return nil, err
-		}
-
-		if val != nil {
-			retval.Children[name] = val
-		}
-	}
-
-	if len(retval.Children) == 0 && len(retval.Metrics) == 0 {
-		return nil, nil
-	}
-
-	return retval, nil
-}
-
-func (l *Level) toCheckpoint(dir string, from, to int64, m *MemoryStore) error {
-	cf, err := l.toCheckpointFile(from, to, m)
-	if err != nil {
-		return err
-	}
-
-	if cf == nil {
-		return ErrNoNewData
-	}
-
-	filepath := path.Join(dir, fmt.Sprintf("%d.json", from))
-	f, err := os.OpenFile(filepath, os.O_CREATE|os.O_WRONLY, 0o644)
-	if err != nil && os.IsNotExist(err) {
-		err = os.MkdirAll(dir, 0o755)
-		if err == nil {
-			f, err = os.OpenFile(filepath, os.O_CREATE|os.O_WRONLY, 0o644)
-		}
-	}
-	if err != nil {
-		return err
-	}
-	defer f.Close()
-
-	bw := bufio.NewWriter(f)
-	if err = json.NewEncoder(bw).Encode(cf); err != nil {
-		return err
-	}
-
-	return bw.Flush()
-}
-
-func (m *MemoryStore) FromCheckpoint(dir string, from int64, extension string) (int, error) {
-	var wg sync.WaitGroup
-	work := make(chan [2]string, NumWorkers)
-	n, errs := int32(0), int32(0)
-
-	wg.Add(NumWorkers)
-	for worker := 0; worker < NumWorkers; worker++ {
-		go func() {
-			defer wg.Done()
-			for host := range work {
-				lvl := m.root.findLevelOrCreate(host[:], len(m.Metrics))
-				nn, err := lvl.fromCheckpoint(m, filepath.Join(dir, host[0], host[1]), from, extension)
-				if err != nil {
-					log.Fatalf("error while loading checkpoints: %s", err.Error())
-					atomic.AddInt32(&errs, 1)
-				}
-				atomic.AddInt32(&n, int32(nn))
-			}
-		}()
-	}
-
-	i := 0
-	clustersDir, err := os.ReadDir(dir)
-	for _, clusterDir := range clustersDir {
-		if !clusterDir.IsDir() {
-			err = errors.New("expected only directories at first level of checkpoints/ directory")
-			goto done
-		}
-
-		hostsDir, e := os.ReadDir(filepath.Join(dir, clusterDir.Name()))
-		if e != nil {
-			err = e
-			goto done
-		}
-
-		for _, hostDir := range hostsDir {
-			if !hostDir.IsDir() {
-				err = errors.New("expected only directories at second level of checkpoints/ directory")
-				goto done
-			}
-
-			i++
-			if i%NumWorkers == 0 && i > 100 {
-				// Forcing garbage collection runs here regulary during the loading of checkpoints
-				// will decrease the total heap size after loading everything back to memory is done.
-				// While loading data, the heap will grow fast, so the GC target size will double
-				// almost always. By forcing GCs here, we can keep it growing more slowly so that
-				// at the end, less memory is wasted.
-				runtime.GC()
-			}
-
-			work <- [2]string{clusterDir.Name(), hostDir.Name()}
-		}
-	}
-done:
-	close(work)
-	wg.Wait()
-
-	if err != nil {
-		return int(n), err
-	}
-
-	if errs > 0 {
-		return int(n), fmt.Errorf("%d errors happend while creating checkpoints (%d successes)", errs, n)
-	}
-	return int(n), nil
-}
-
-// Metrics stored at the lowest 2 levels are not loaded (root and cluster)!
-// This function can only be called once and before the very first write or read.
-// Different host's data is loaded to memory in parallel.
-func (m *MemoryStore) FromCheckpointFiles(dir string, from int64) (int, error) {
-
-	if _, err := os.Stat(dir); os.IsNotExist(err) {
-		// The directory does not exist, so create it using os.MkdirAll()
-		err := os.MkdirAll(dir, 0755) // 0755 sets the permissions for the directory
-		if err != nil {
-			log.Fatalf("Error creating directory: %#v\n", err)
-		}
-		fmt.Printf("%#v Directory created successfully.\n", dir)
-	}
-
-	// Config read (replace with your actual config read)
-	fileFormat := config.Keys.Checkpoints.FileFormat
-	if fileFormat == "" {
-		fileFormat = "avro"
-	}
-
-	// Map to easily get the fallback format
-	oppositeFormat := map[string]string{
-		"json": "avro",
-		"avro": "json",
-	}
-
-	// First, attempt to load the specified format
-	if found, err := checkFilesWithExtension(dir, fileFormat); err != nil {
-		return 0, fmt.Errorf("error checking files with extension: %v", err)
-	} else if found {
-		log.Printf("Loading %s files because fileformat is %s\n", fileFormat, fileFormat)
-		return m.FromCheckpoint(dir, from, fileFormat)
-	}
-
-	// If not found, attempt the opposite format
-	altFormat := oppositeFormat[fileFormat]
-	if found, err := checkFilesWithExtension(dir, altFormat); err != nil {
-		return 0, fmt.Errorf("error checking files with extension: %v", err)
-	} else if found {
-		log.Printf("Loading %s files but fileformat is %s\n", altFormat, fileFormat)
-		return m.FromCheckpoint(dir, from, altFormat)
-	}
-
-	log.Println("No valid checkpoint files found in the directory.")
-	return 0, nil
-}
-
-func checkFilesWithExtension(dir string, extension string) (bool, error) {
-	found := false
-
-	err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
-		if err != nil {
-			return fmt.Errorf("error accessing path %s: %v", path, err)
-		}
-		if !info.IsDir() && filepath.Ext(info.Name()) == "."+extension {
-			found = true
-			return nil
-		}
-		return nil
-	})
-
-	if err != nil {
-		return false, fmt.Errorf("error walking through directories: %s", err)
-	}
-
-	return found, nil
-}
-
-func (l *Level) loadAvroFile(m *MemoryStore, f *os.File, from int64) error {
-	br := bufio.NewReader(f)
-
-	fileName := f.Name()[strings.LastIndex(f.Name(), "/")+1:]
-	resolution, err := strconv.ParseInt(fileName[0:strings.Index(fileName, "_")], 10, 64)
-	if err != nil {
-		return fmt.Errorf("error while reading avro file (resolution parsing) : %s", err)
-	}
-
-	from_timestamp, err := strconv.ParseInt(fileName[strings.Index(fileName, "_")+1:len(fileName)-5], 10, 64)
-
-	// Same logic according to lineprotocol
-	from_timestamp -= (resolution / 2)
-
-	if err != nil {
-		return fmt.Errorf("error converting timestamp from the avro file : %s", err)
-	}
-
-	// fmt.Printf("File : %s with resolution : %d\n", fileName, resolution)
-
-	var recordCounter int64 = 0
-
-	// Create a new OCF reader from the buffered reader
-	ocfReader, err := goavro.NewOCFReader(br)
-	if err != nil {
-		panic(err)
-	}
-
-	metricsData := make(map[string]util.FloatArray)
-
-	for ocfReader.Scan() {
-		datum, err := ocfReader.Read()
-		if err != nil {
-			return fmt.Errorf("error while reading avro file : %s", err)
-		}
-
-		record, ok := datum.(map[string]interface{})
-		if !ok {
-			panic("failed to assert datum as map[string]interface{}")
-		}
-
-		for key, value := range record {
-			metricsData[key] = append(metricsData[key], util.ConvertToFloat(value.(float64)))
-		}
-
-		recordCounter += 1
-	}
-
-	to := (from_timestamp + (recordCounter / (60 / resolution) * 60))
-	if to < from {
-		return nil
-	}
-
-	for key, floatArray := range metricsData {
-		metricName := avro.ReplaceKey(key)
-
-		if strings.Contains(metricName, avro.Delimiter) {
-			subString := strings.Split(metricName, avro.Delimiter)
-
-			lvl := l
-
-			for i := 0; i < len(subString)-1; i++ {
-
-				sel := subString[i]
-
-				if lvl.children == nil {
-					lvl.children = make(map[string]*Level)
-				}
-
-				child, ok := lvl.children[sel]
-				if !ok {
-					child = &Level{
-						metrics:  make([]*buffer, len(m.Metrics)),
-						children: nil,
-					}
-					lvl.children[sel] = child
-				}
-				lvl = child
-			}
-
-			leafMetricName := subString[len(subString)-1]
-			err = lvl.createBuffer(m, leafMetricName, floatArray, from_timestamp, resolution)
-			if err != nil {
-				return fmt.Errorf("error while creating buffers from avroReader : %s", err)
-			}
-		} else {
-			err = l.createBuffer(m, metricName, floatArray, from_timestamp, resolution)
-			if err != nil {
-				return fmt.Errorf("error while creating buffers from avroReader : %s", err)
-			}
-		}
-
-	}
-
-	return nil
-}
-
-func (l *Level) createBuffer(m *MemoryStore, metricName string, floatArray util.FloatArray, from int64, resolution int64) error {
-	n := len(floatArray)
-	b := &buffer{
-		frequency: resolution,
-		start:     from,
-		data:      floatArray[0:n:n],
-		prev:      nil,
-		next:      nil,
-		archived:  true,
-	}
-	b.close()
-
-	minfo, ok := m.Metrics[metricName]
-	if !ok {
-		return nil
-		// return errors.New("Unkown metric: " + name)
-	}
-
-	prev := l.metrics[minfo.Offset]
-	if prev == nil {
-		l.metrics[minfo.Offset] = b
-	} else {
-		if prev.start > b.start {
-			return errors.New("wooops")
-		}
-
-		b.prev = prev
-		prev.next = b
-
-		missingCount := ((int(b.start) - int(prev.start)) - len(prev.data)*int(b.frequency))
-		if missingCount > 0 {
-			missingCount /= int(b.frequency)
-
-			for range missingCount {
-				prev.data = append(prev.data, util.NaN)
-			}
-
-			prev.data = prev.data[0:len(prev.data):len(prev.data)]
-		}
-	}
-	l.metrics[minfo.Offset] = b
-
-	return nil
-}
-
-func (l *Level) loadJsonFile(m *MemoryStore, f *os.File, from int64) error {
-	br := bufio.NewReader(f)
-	cf := &CheckpointFile{}
-	if err := json.NewDecoder(br).Decode(cf); err != nil {
-		return err
-	}
-
-	if cf.To != 0 && cf.To < from {
-		return nil
-	}
-
-	if err := l.loadFile(cf, m); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func (l *Level) loadFile(cf *CheckpointFile, m *MemoryStore) error {
-	for name, metric := range cf.Metrics {
-		n := len(metric.Data)
-		b := &buffer{
-			frequency: metric.Frequency,
-			start:     metric.Start,
-			data:      metric.Data[0:n:n], // Space is wasted here :(
-			prev:      nil,
-			next:      nil,
-			archived:  true,
-		}
-		b.close()
-
-		minfo, ok := m.Metrics[name]
-		if !ok {
-			continue
-			// return errors.New("Unkown metric: " + name)
-		}
-
-		prev := l.metrics[minfo.Offset]
-		if prev == nil {
-			l.metrics[minfo.Offset] = b
-		} else {
-			if prev.start > b.start {
-				return errors.New("wooops")
-			}
-
-			b.prev = prev
-			prev.next = b
-		}
-		l.metrics[minfo.Offset] = b
-	}
-
-	if len(cf.Children) > 0 && l.children == nil {
-		l.children = make(map[string]*Level)
-	}
-
-	for sel, childCf := range cf.Children {
-		child, ok := l.children[sel]
-		if !ok {
-			child = &Level{
-				metrics:  make([]*buffer, len(m.Metrics)),
-				children: nil,
-			}
-			l.children[sel] = child
-		}
-
-		if err := child.loadFile(childCf, m); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func (l *Level) fromCheckpoint(m *MemoryStore, dir string, from int64, extension string) (int, error) {
-	direntries, err := os.ReadDir(dir)
-	if err != nil {
-		if os.IsNotExist(err) {
-			return 0, nil
-		}
-
-		return 0, err
-	}
-
-	allFiles := make([]fs.DirEntry, 0)
-	filesLoaded := 0
-	for _, e := range direntries {
-		if e.IsDir() {
-			child := &Level{
-				metrics:  make([]*buffer, len(m.Metrics)),
-				children: make(map[string]*Level),
-			}
-
-			files, err := child.fromCheckpoint(m, path.Join(dir, e.Name()), from, extension)
-			filesLoaded += files
-			if err != nil {
-				return filesLoaded, err
-			}
-
-			l.children[e.Name()] = child
-		} else if strings.HasSuffix(e.Name(), "."+extension) {
-			allFiles = append(allFiles, e)
-		} else {
-			continue
-		}
-	}
-
-	files, err := findFiles(allFiles, from, extension, true)
-	if err != nil {
-		return filesLoaded, err
-	}
-
-	loaders := map[string]func(*MemoryStore, *os.File, int64) error{
-		"json": l.loadJsonFile,
-		"avro": l.loadAvroFile,
-	}
-
-	loader := loaders[extension]
-
-	for _, filename := range files {
-		f, err := os.Open(path.Join(dir, filename))
-		if err != nil {
-			return filesLoaded, err
-		}
-		defer f.Close()
-
-		if err = loader(m, f, from); err != nil {
-			return filesLoaded, err
-		}
-
-		filesLoaded += 1
-	}
-
-	return filesLoaded, nil
-}
-
-// This will probably get very slow over time!
-// A solution could be some sort of an index file in which all other files
-// and the timespan they contain is listed.
-func findFiles(direntries []fs.DirEntry, t int64, extension string, findMoreRecentFiles bool) ([]string, error) {
-	nums := map[string]int64{}
-	for _, e := range direntries {
-		if !strings.HasSuffix(e.Name(), "."+extension) {
-			continue
-		}
-
-		ts, err := strconv.ParseInt(e.Name()[strings.Index(e.Name(), "_")+1:len(e.Name())-5], 10, 64)
-		if err != nil {
-			return nil, err
-		}
-		nums[e.Name()] = ts
-	}
-
-	sort.Slice(direntries, func(i, j int) bool {
-		a, b := direntries[i], direntries[j]
-		return nums[a.Name()] < nums[b.Name()]
-	})
-
-	filenames := make([]string, 0)
-	for i := 0; i < len(direntries); i++ {
-		e := direntries[i]
-		ts1 := nums[e.Name()]
-
-		if findMoreRecentFiles && t <= ts1 {
-			filenames = append(filenames, e.Name())
-		}
-		if i == len(direntries)-1 {
-			continue
-		}
-
-		enext := direntries[i+1]
-		ts2 := nums[enext.Name()]
-
-		if findMoreRecentFiles {
-			if ts1 < t && t < ts2 {
-				filenames = append(filenames, e.Name())
-			}
-		} else {
-			if ts2 < t {
-				filenames = append(filenames, e.Name())
-			}
-		}
-	}
-
-	return filenames, nil
-}

internal/memorystore/debug.go

@@ -1,107 +0,0 @@
-package memorystore
-
-import (
-	"bufio"
-	"fmt"
-	"strconv"
-)
-
-func (b *buffer) debugDump(buf []byte) []byte {
-	if b.prev != nil {
-		buf = b.prev.debugDump(buf)
-	}
-
-	start, len, end := b.start, len(b.data), b.start+b.frequency*int64(len(b.data))
-	buf = append(buf, `{"start":`...)
-	buf = strconv.AppendInt(buf, start, 10)
-	buf = append(buf, `,"len":`...)
-	buf = strconv.AppendInt(buf, int64(len), 10)
-	buf = append(buf, `,"end":`...)
-	buf = strconv.AppendInt(buf, end, 10)
-	if b.archived {
-		buf = append(buf, `,"saved":true`...)
-	}
-	if b.next != nil {
-		buf = append(buf, `},`...)
-	} else {
-		buf = append(buf, `}`...)
-	}
-	return buf
-}
-
-func (l *Level) debugDump(m *MemoryStore, w *bufio.Writer, lvlname string, buf []byte, depth int) ([]byte, error) {
-	l.lock.RLock()
-	defer l.lock.RUnlock()
-	for i := 0; i < depth; i++ {
-		buf = append(buf, '\t')
-	}
-	buf = append(buf, '"')
-	buf = append(buf, lvlname...)
-	buf = append(buf, "\":{\n"...)
-	depth += 1
-	objitems := 0
-	for name, mc := range m.Metrics {
-		if b := l.metrics[mc.Offset]; b != nil {
-			for i := 0; i < depth; i++ {
-				buf = append(buf, '\t')
-			}
-
-			buf = append(buf, '"')
-			buf = append(buf, name...)
-			buf = append(buf, `":[`...)
-			buf = b.debugDump(buf)
-			buf = append(buf, "],\n"...)
-			objitems++
-		}
-	}
-
-	for name, lvl := range l.children {
-		_, err := w.Write(buf)
-		if err != nil {
-			return nil, err
-		}
-
-		buf = buf[0:0]
-		buf, err = lvl.debugDump(m, w, name, buf, depth)
-		if err != nil {
-			return nil, err
-		}
-
-		buf = append(buf, ',', '\n')
-		objitems++
-	}
-
-	// remove final `,`:
-	if objitems > 0 {
-		buf = append(buf[0:len(buf)-1], '\n')
-	}
-
-	depth -= 1
-	for i := 0; i < depth; i++ {
-		buf = append(buf, '\t')
-	}
-	buf = append(buf, '}')
-	return buf, nil
-}
-
-func (m *MemoryStore) DebugDump(w *bufio.Writer, selector []string) error {
-	lvl := m.root.findLevel(selector)
-	if lvl == nil {
-		return fmt.Errorf("not found: %#v", selector)
-	}
-
-	buf := make([]byte, 0, 2048)
-	buf = append(buf, "{"...)
-
-	buf, err := lvl.debugDump(m, w, "data", buf, 0)
-	if err != nil {
-		return err
-	}
-
-	buf = append(buf, "}\n"...)
-	if _, err = w.Write(buf); err != nil {
-		return err
-	}
-
-	return w.Flush()
-}

internal/memorystore/healthcheck.go

@@ -1,88 +0,0 @@
-package memorystore
-
-import (
-	"bufio"
-	"fmt"
-	"time"
-)
-
-// This is a threshold that allows a node to be healthy with certain number of data points missing.
-// Suppose a node does not receive last 5 data points, then healthCheck endpoint will still say a
-// node is healthy. Anything more than 5 missing points in metrics of the node will deem the node unhealthy.
-const MaxMissingDataPoints int64 = 5
-
-// This is a threshold which allows upto certain number of metrics in a node to be unhealthly.
-// Works with MaxMissingDataPoints. Say 5 metrics (including submetrics) do not receive the last
-// MaxMissingDataPoints data points, then the node will be deemed healthy. Any more metrics that does
-// not receive data for MaxMissingDataPoints data points will deem the node unhealthy.
-const MaxUnhealthyMetrics int64 = 5
-
-func (b *buffer) healthCheck() int64 {
-
-	// Check if the buffer is empty
-	if b.data == nil {
-		return 1
-	}
-
-	buffer_end := b.start + b.frequency*int64(len(b.data))
-	t := time.Now().Unix()
-
-	// Check if the buffer is too old
-	if t-buffer_end > MaxMissingDataPoints*b.frequency {
-		return 1
-	}
-
-	return 0
-}
-
-func (l *Level) healthCheck(m *MemoryStore, count int64) (int64, error) {
-	l.lock.RLock()
-	defer l.lock.RUnlock()
-
-	for _, mc := range m.Metrics {
-		if b := l.metrics[mc.Offset]; b != nil {
-			count += b.healthCheck()
-		}
-	}
-
-	for _, lvl := range l.children {
-		c, err := lvl.healthCheck(m, 0)
-		if err != nil {
-			return 0, err
-		}
-		count += c
-	}
-
-	return count, nil
-}
-
-func (m *MemoryStore) HealthCheck(w *bufio.Writer, selector []string) error {
-	lvl := m.root.findLevel(selector)
-	if lvl == nil {
-		return fmt.Errorf("not found: %#v", selector)
-	}
-
-	buf := make([]byte, 0, 25)
-	// buf = append(buf, "{"...)
-
-	var count int64 = 0
-
-	unhealthyMetricsCount, err := lvl.healthCheck(m, count)
-	if err != nil {
-		return err
-	}
-
-	if unhealthyMetricsCount < MaxUnhealthyMetrics {
-		buf = append(buf, "Healthy"...)
-	} else {
-		buf = append(buf, "Unhealthy"...)
-	}
-
-	// buf = append(buf, "}\n"...)
-
-	if _, err = w.Write(buf); err != nil {
-		return err
-	}
-
-	return w.Flush()
-}

internal/memorystore/level.go

@@ -1,187 +0,0 @@
-package memorystore
-
-import (
-	"sync"
-	"unsafe"
-
-	"github.com/ClusterCockpit/cc-metric-store/internal/util"
-)
-
-// Could also be called "node" as this forms a node in a tree structure.
-// Called Level because "node" might be confusing here.
-// Can be both a leaf or a inner node. In this tree structue, inner nodes can
-// also hold data (in `metrics`).
-type Level struct {
-	children map[string]*Level
-	metrics  []*buffer
-	lock     sync.RWMutex
-}
-
-// Find the correct level for the given selector, creating it if
-// it does not exist. Example selector in the context of the
-// ClusterCockpit could be: []string{ "emmy", "host123", "cpu0" }.
-// This function would probably benefit a lot from `level.children` beeing a `sync.Map`?
-func (l *Level) findLevelOrCreate(selector []string, nMetrics int) *Level {
-	if len(selector) == 0 {
-		return l
-	}
-
-	// Allow concurrent reads:
-	l.lock.RLock()
-	var child *Level
-	var ok bool
-	if l.children == nil {
-		// Children map needs to be created...
-		l.lock.RUnlock()
-	} else {
-		child, ok := l.children[selector[0]]
-		l.lock.RUnlock()
-		if ok {
-			return child.findLevelOrCreate(selector[1:], nMetrics)
-		}
-	}
-
-	// The level does not exist, take write lock for unqiue access:
-	l.lock.Lock()
-	// While this thread waited for the write lock, another thread
-	// could have created the child node.
-	if l.children != nil {
-		child, ok = l.children[selector[0]]
-		if ok {
-			l.lock.Unlock()
-			return child.findLevelOrCreate(selector[1:], nMetrics)
-		}
-	}
-
-	child = &Level{
-		metrics:  make([]*buffer, nMetrics),
-		children: nil,
-	}
-
-	if l.children != nil {
-		l.children[selector[0]] = child
-	} else {
-		l.children = map[string]*Level{selector[0]: child}
-	}
-	l.lock.Unlock()
-	return child.findLevelOrCreate(selector[1:], nMetrics)
-}
-
-func (l *Level) free(t int64) (int, error) {
-	l.lock.Lock()
-	defer l.lock.Unlock()
-
-	n := 0
-	for i, b := range l.metrics {
-		if b != nil {
-			delme, m := b.free(t)
-			n += m
-			if delme {
-				if cap(b.data) == BUFFER_CAP {
-					bufferPool.Put(b)
-				}
-				l.metrics[i] = nil
-			}
-		}
-	}
-
-	for _, l := range l.children {
-		m, err := l.free(t)
-		n += m
-		if err != nil {
-			return n, err
-		}
-	}
-
-	return n, nil
-}
-
-func (l *Level) sizeInBytes() int64 {
-	l.lock.RLock()
-	defer l.lock.RUnlock()
-	size := int64(0)
-
-	for _, b := range l.metrics {
-		if b != nil {
-			size += b.count() * int64(unsafe.Sizeof(util.Float(0)))
-		}
-	}
-
-	for _, child := range l.children {
-		size += child.sizeInBytes()
-	}
-
-	return size
-}
-
-func (l *Level) findLevel(selector []string) *Level {
-	if len(selector) == 0 {
-		return l
-	}
-
-	l.lock.RLock()
-	defer l.lock.RUnlock()
-
-	lvl := l.children[selector[0]]
-	if lvl == nil {
-		return nil
-	}
-
-	return lvl.findLevel(selector[1:])
-}
-
-func (l *Level) findBuffers(selector util.Selector, offset int, f func(b *buffer) error) error {
-	l.lock.RLock()
-	defer l.lock.RUnlock()
-
-	if len(selector) == 0 {
-		b := l.metrics[offset]
-		if b != nil {
-			return f(b)
-		}
-
-		for _, lvl := range l.children {
-			err := lvl.findBuffers(nil, offset, f)
-			if err != nil {
-				return err
-			}
-		}
-		return nil
-	}
-
-	sel := selector[0]
-	if len(sel.String) != 0 && l.children != nil {
-		lvl, ok := l.children[sel.String]
-		if ok {
-			err := lvl.findBuffers(selector[1:], offset, f)
-			if err != nil {
-				return err
-			}
-		}
-		return nil
-	}
-
-	if sel.Group != nil && l.children != nil {
-		for _, key := range sel.Group {
-			lvl, ok := l.children[key]
-			if ok {
-				err := lvl.findBuffers(selector[1:], offset, f)
-				if err != nil {
-					return err
-				}
-			}
-		}
-		return nil
-	}
-
-	if sel.Any && l.children != nil {
-		for _, lvl := range l.children {
-			if err := lvl.findBuffers(selector[1:], offset, f); err != nil {
-				return err
-			}
-		}
-		return nil
-	}
-
-	return nil
-}

internal/memorystore/memorystore.go

@@ -1,373 +0,0 @@
-package memorystore
-
-import (
-	"context"
-	"errors"
-	"log"
-	"runtime"
-	"sync"
-	"time"
-
-	"github.com/ClusterCockpit/cc-metric-store/internal/avro"
-	"github.com/ClusterCockpit/cc-metric-store/internal/config"
-	"github.com/ClusterCockpit/cc-metric-store/internal/util"
-	"github.com/ClusterCockpit/cc-metric-store/pkg/resampler"
-)
-
-var (
-	singleton  sync.Once
-	msInstance *MemoryStore
-)
-
-var NumWorkers int = 4
-
-func init() {
-	maxWorkers := 10
-	NumWorkers = runtime.NumCPU()/2 + 1
-	if NumWorkers > maxWorkers {
-		NumWorkers = maxWorkers
-	}
-}
-
-type Metric struct {
-	Name         string
-	Value        util.Float
-	MetricConfig config.MetricConfig
-}
-
-type MemoryStore struct {
-	Metrics map[string]config.MetricConfig
-	root    Level
-}
-
-// Create a new, initialized instance of a MemoryStore.
-// Will panic if values in the metric configurations are invalid.
-func Init(metrics map[string]config.MetricConfig) {
-	singleton.Do(func() {
-		offset := 0
-		for key, cfg := range metrics {
-			if cfg.Frequency == 0 {
-				panic("invalid frequency")
-			}
-
-			metrics[key] = config.MetricConfig{
-				Frequency:   cfg.Frequency,
-				Aggregation: cfg.Aggregation,
-				Offset:      offset,
-			}
-			offset += 1
-		}
-
-		msInstance = &MemoryStore{
-			root: Level{
-				metrics:  make([]*buffer, len(metrics)),
-				children: make(map[string]*Level),
-			},
-			Metrics: metrics,
-		}
-	})
-}
-
-func GetMemoryStore() *MemoryStore {
-	if msInstance == nil {
-		log.Fatalf("MemoryStore not initialized!")
-	}
-
-	return msInstance
-}
-
-func Shutdown() {
-	log.Printf("Writing to '%s'...\n", config.Keys.Checkpoints.RootDir)
-	var files int
-	var err error
-
-	ms := GetMemoryStore()
-
-	if config.Keys.Checkpoints.FileFormat == "json" {
-		files, err = ms.ToCheckpoint(config.Keys.Checkpoints.RootDir, lastCheckpoint.Unix(), time.Now().Unix())
-	} else {
-		files, err = avro.GetAvroStore().ToCheckpoint(config.Keys.Checkpoints.RootDir, true)
-		close(avro.LineProtocolMessages)
-	}
-
-	if err != nil {
-		log.Printf("Writing checkpoint failed: %s\n", err.Error())
-	}
-	log.Printf("Done! (%d files written)\n", files)
-
-	// ms.PrintHeirarchy()
-}
-
-// func (m *MemoryStore) PrintHeirarchy() {
-// 	m.root.lock.Lock()
-// 	defer m.root.lock.Unlock()
-
-// 	fmt.Printf("Root : \n")
-
-// 	for lvl1, sel1 := range m.root.children {
-// 		fmt.Printf("\t%s\n", lvl1)
-// 		for lvl2, sel2 := range sel1.children {
-// 			fmt.Printf("\t\t%s\n", lvl2)
-// 			if lvl1 == "fritz" && lvl2 == "f0201" {
-
-// 				for name, met := range m.Metrics {
-// 					mt := sel2.metrics[met.Offset]
-
-// 					fmt.Printf("\t\t\t\t%s\n", name)
-// 					fmt.Printf("\t\t\t\t")
-
-// 					for mt != nil {
-// 						// if name == "cpu_load" {
-// 						fmt.Printf("%d(%d) -> %#v", mt.start, len(mt.data), mt.data)
-// 						// }
-// 						mt = mt.prev
-// 					}
-// 					fmt.Printf("\n")
-
-// 				}
-// 			}
-// 			for lvl3, sel3 := range sel2.children {
-// 				if lvl1 == "fritz" && lvl2 == "f0201" && lvl3 == "hwthread70" {
-
-// 					fmt.Printf("\t\t\t\t\t%s\n", lvl3)
-
-// 					for name, met := range m.Metrics {
-// 						mt := sel3.metrics[met.Offset]
-
-// 						fmt.Printf("\t\t\t\t\t\t%s\n", name)
-
-// 						fmt.Printf("\t\t\t\t\t\t")
-
-// 						for mt != nil {
-// 							// if name == "clock" {
-// 							fmt.Printf("%d(%d) -> %#v", mt.start, len(mt.data), mt.data)
-
-// 							mt = mt.prev
-// 						}
-// 						fmt.Printf("\n")
-
-// 					}
-
-// 					// for i, _ := range sel3.metrics {
-// 					// 	fmt.Printf("\t\t\t\t\t%s\n", getName(configmetrics, i))
-// 					// }
-// 				}
-// 			}
-// 		}
-// 	}
-
-// }
-
-func getName(m *MemoryStore, i int) string {
-	for key, val := range m.Metrics {
-		if val.Offset == i {
-			return key
-		}
-	}
-	return ""
-}
-
-func Retention(wg *sync.WaitGroup, ctx context.Context) {
-	ms := GetMemoryStore()
-
-	go func() {
-		defer wg.Done()
-		d, err := time.ParseDuration(config.Keys.RetentionInMemory)
-		if err != nil {
-			log.Fatal(err)
-		}
-		if d <= 0 {
-			return
-		}
-
-		ticks := func() <-chan time.Time {
-			d := d / 2
-			if d <= 0 {
-				return nil
-			}
-			return time.NewTicker(d).C
-		}()
-		for {
-			select {
-			case <-ctx.Done():
-				return
-			case <-ticks:
-				t := time.Now().Add(-d)
-				log.Printf("start freeing buffers (older than %s)...\n", t.Format(time.RFC3339))
-				freed, err := ms.Free(nil, t.Unix())
-				if err != nil {
-					log.Printf("freeing up buffers failed: %s\n", err.Error())
-				} else {
-					log.Printf("done: %d buffers freed\n", freed)
-				}
-			}
-		}
-	}()
-}
-
-// Write all values in `metrics` to the level specified by `selector` for time `ts`.
-// Look at `findLevelOrCreate` for how selectors work.
-func (m *MemoryStore) Write(selector []string, ts int64, metrics []Metric) error {
-	var ok bool
-	for i, metric := range metrics {
-		if metric.MetricConfig.Frequency == 0 {
-			metric.MetricConfig, ok = m.Metrics[metric.Name]
-			if !ok {
-				metric.MetricConfig.Frequency = 0
-			}
-			metrics[i] = metric
-		}
-	}
-
-	return m.WriteToLevel(&m.root, selector, ts, metrics)
-}
-
-func (m *MemoryStore) GetLevel(selector []string) *Level {
-	return m.root.findLevelOrCreate(selector, len(m.Metrics))
-}
-
-// Assumes that `minfo` in `metrics` is filled in!
-func (m *MemoryStore) WriteToLevel(l *Level, selector []string, ts int64, metrics []Metric) error {
-	l = l.findLevelOrCreate(selector, len(m.Metrics))
-	l.lock.Lock()
-	defer l.lock.Unlock()
-
-	for _, metric := range metrics {
-		if metric.MetricConfig.Frequency == 0 {
-			continue
-		}
-
-		b := l.metrics[metric.MetricConfig.Offset]
-		if b == nil {
-			// First write to this metric and level
-			b = newBuffer(ts, metric.MetricConfig.Frequency)
-			l.metrics[metric.MetricConfig.Offset] = b
-		}
-
-		nb, err := b.write(ts, metric.Value)
-		if err != nil {
-			return err
-		}
-
-		// Last write created a new buffer...
-		if b != nb {
-			l.metrics[metric.MetricConfig.Offset] = nb
-		}
-	}
-	return nil
-}
-
-// Returns all values for metric `metric` from `from` to `to` for the selected level(s).
-// If the level does not hold the metric itself, the data will be aggregated recursively from the children.
-// The second and third return value are the actual from/to for the data. Those can be different from
-// the range asked for if no data was available.
-func (m *MemoryStore) Read(selector util.Selector, metric string, from, to, resolution int64) ([]util.Float, int64, int64, int64, error) {
-	if from > to {
-		return nil, 0, 0, 0, errors.New("invalid time range")
-	}
-
-	minfo, ok := m.Metrics[metric]
-	if !ok {
-		return nil, 0, 0, 0, errors.New("unkown metric: " + metric)
-	}
-
-	n, data := 0, make([]util.Float, (to-from)/minfo.Frequency+1)
-
-	err := m.root.findBuffers(selector, minfo.Offset, func(b *buffer) error {
-		cdata, cfrom, cto, err := b.read(from, to, data)
-		if err != nil {
-			return err
-		}
-
-		if n == 0 {
-			from, to = cfrom, cto
-		} else if from != cfrom || to != cto || len(data) != len(cdata) {
-			missingfront, missingback := int((from-cfrom)/minfo.Frequency), int((to-cto)/minfo.Frequency)
-			if missingfront != 0 {
-				return ErrDataDoesNotAlign
-			}
-
-			newlen := len(cdata) - missingback
-			if newlen < 1 {
-				return ErrDataDoesNotAlign
-			}
-			cdata = cdata[0:newlen]
-			if len(cdata) != len(data) {
-				return ErrDataDoesNotAlign
-			}
-
-			from, to = cfrom, cto
-		}
-
-		data = cdata
-		n += 1
-		return nil
-	})
-
-	if err != nil {
-		return nil, 0, 0, 0, err
-	} else if n == 0 {
-		return nil, 0, 0, 0, errors.New("metric or host not found")
-	} else if n > 1 {
-		if minfo.Aggregation == config.AvgAggregation {
-			normalize := 1. / util.Float(n)
-			for i := 0; i < len(data); i++ {
-				data[i] *= normalize
-			}
-		} else if minfo.Aggregation != config.SumAggregation {
-			return nil, 0, 0, 0, errors.New("invalid aggregation")
-		}
-	}
-
-	data, resolution, err = resampler.LargestTriangleThreeBucket(data, minfo.Frequency, resolution)
-
-	if err != nil {
-		return nil, 0, 0, 0, err
-	}
-
-	return data, from, to, resolution, nil
-}
-
-// Release all buffers for the selected level and all its children that contain only
-// values older than `t`.
-func (m *MemoryStore) Free(selector []string, t int64) (int, error) {
-	return m.GetLevel(selector).free(t)
-}
-
-func (m *MemoryStore) FreeAll() error {
-	for k := range m.root.children {
-		delete(m.root.children, k)
-	}
-
-	return nil
-}
-
-func (m *MemoryStore) SizeInBytes() int64 {
-	return m.root.sizeInBytes()
-}
-
-// Given a selector, return a list of all children of the level selected.
-func (m *MemoryStore) ListChildren(selector []string) []string {
-	lvl := &m.root
-	for lvl != nil && len(selector) != 0 {
-		lvl.lock.RLock()
-		next := lvl.children[selector[0]]
-		lvl.lock.RUnlock()
-		lvl = next
-		selector = selector[1:]
-	}
-
-	if lvl == nil {
-		return nil
-	}
-
-	lvl.lock.RLock()
-	defer lvl.lock.RUnlock()
-
-	children := make([]string, 0, len(lvl.children))
-	for child := range lvl.children {
-		children = append(children, child)
-	}
-
-	return children
-}

internal/memorystore/stats.go

@@ -1,120 +0,0 @@
-package memorystore
-
-import (
-	"errors"
-	"math"
-
-	"github.com/ClusterCockpit/cc-metric-store/internal/config"
-	"github.com/ClusterCockpit/cc-metric-store/internal/util"
-)
-
-type Stats struct {
-	Samples int
-	Avg     util.Float
-	Min     util.Float
-	Max     util.Float
-}
-
-func (b *buffer) stats(from, to int64) (Stats, int64, int64, error) {
-	if from < b.start {
-		if b.prev != nil {
-			return b.prev.stats(from, to)
-		}
-		from = b.start
-	}
-
-	// TODO: Check if b.closed and if so and the full buffer is queried,
-	// use b.statistics instead of iterating over the buffer.
-
-	samples := 0
-	sum, min, max := 0.0, math.MaxFloat32, -math.MaxFloat32
-
-	var t int64
-	for t = from; t < to; t += b.frequency {
-		idx := int((t - b.start) / b.frequency)
-		if idx >= cap(b.data) {
-			b = b.next
-			if b == nil {
-				break
-			}
-			idx = 0
-		}
-
-		if t < b.start || idx >= len(b.data) {
-			continue
-		}
-
-		xf := float64(b.data[idx])
-		if math.IsNaN(xf) {
-			continue
-		}
-
-		samples += 1
-		sum += xf
-		min = math.Min(min, xf)
-		max = math.Max(max, xf)
-	}
-
-	return Stats{
-		Samples: samples,
-		Avg:     util.Float(sum) / util.Float(samples),
-		Min:     util.Float(min),
-		Max:     util.Float(max),
-	}, from, t, nil
-}
-
-// Returns statistics for the requested metric on the selected node/level.
-// Data is aggregated to the selected level the same way as in `MemoryStore.Read`.
-// If `Stats.Samples` is zero, the statistics should not be considered as valid.
-func (m *MemoryStore) Stats(selector util.Selector, metric string, from, to int64) (*Stats, int64, int64, error) {
-	if from > to {
-		return nil, 0, 0, errors.New("invalid time range")
-	}
-
-	minfo, ok := m.Metrics[metric]
-	if !ok {
-		return nil, 0, 0, errors.New("unkown metric: " + metric)
-	}
-
-	n, samples := 0, 0
-	avg, min, max := util.Float(0), math.MaxFloat32, -math.MaxFloat32
-	err := m.root.findBuffers(selector, minfo.Offset, func(b *buffer) error {
-		stats, cfrom, cto, err := b.stats(from, to)
-		if err != nil {
-			return err
-		}
-
-		if n == 0 {
-			from, to = cfrom, cto
-		} else if from != cfrom || to != cto {
-			return ErrDataDoesNotAlign
-		}
-
-		samples += stats.Samples
-		avg += stats.Avg
-		min = math.Min(min, float64(stats.Min))
-		max = math.Max(max, float64(stats.Max))
-		n += 1
-		return nil
-	})
-	if err != nil {
-		return nil, 0, 0, err
-	}
-
-	if n == 0 {
-		return nil, 0, 0, ErrNoData
-	}
-
-	if minfo.Aggregation == config.AvgAggregation {
-		avg /= util.Float(n)
-	} else if n > 1 && minfo.Aggregation != config.SumAggregation {
-		return nil, 0, 0, errors.New("invalid aggregation")
-	}
-
-	return &Stats{
-		Samples: samples,
-		Avg:     avg,
-		Min:     util.Float(min),
-		Max:     util.Float(max),
-	}, from, to, nil
-}

internal/runtimeEnv/setup.go

@@ -1,140 +0,0 @@
-// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved.
-// Use of this source code is governed by a MIT-style
-// license that can be found in the LICENSE file.
-package runtimeEnv
-
-import (
-	"bufio"
-	"errors"
-	"fmt"
-	"os"
-	"os/exec"
-	"os/user"
-	"strconv"
-	"strings"
-	"syscall"
-)
-
-// Very simple and limited .env file reader.
-// All variable definitions found are directly
-// added to the processes environment.
-func LoadEnv(file string) error {
-	f, err := os.Open(file)
-	if err != nil {
-		// log.Error("Error while opening .env file")
-		return err
-	}
-
-	defer f.Close()
-	s := bufio.NewScanner(bufio.NewReader(f))
-	for s.Scan() {
-		line := s.Text()
-		if strings.HasPrefix(line, "#") || len(line) == 0 {
-			continue
-		}
-
-		if strings.Contains(line, "#") {
-			return errors.New("'#' are only supported at the start of a line")
-		}
-
-		line = strings.TrimPrefix(line, "export ")
-		parts := strings.SplitN(line, "=", 2)
-		if len(parts) != 2 {
-			return fmt.Errorf("RUNTIME/SETUP > unsupported line: %#v", line)
-		}
-
-		key := strings.TrimSpace(parts[0])
-		val := strings.TrimSpace(parts[1])
-		if strings.HasPrefix(val, "\"") {
-			if !strings.HasSuffix(val, "\"") {
-				return fmt.Errorf("RUNTIME/SETUP > unsupported line: %#v", line)
-			}
-
-			runes := []rune(val[1 : len(val)-1])
-			sb := strings.Builder{}
-			for i := 0; i < len(runes); i++ {
-				if runes[i] == '\\' {
-					i++
-					switch runes[i] {
-					case 'n':
-						sb.WriteRune('\n')
-					case 'r':
-						sb.WriteRune('\r')
-					case 't':
-						sb.WriteRune('\t')
-					case '"':
-						sb.WriteRune('"')
-					default:
-						return fmt.Errorf("RUNTIME/SETUP > unsupported escape sequence in quoted string: backslash %#v", runes[i])
-					}
-					continue
-				}
-				sb.WriteRune(runes[i])
-			}
-
-			val = sb.String()
-		}
-
-		os.Setenv(key, val)
-	}
-
-	return s.Err()
-}
-
-// Changes the processes user and group to that
-// specified in the config.json. The go runtime
-// takes care of all threads (and not only the calling one)
-// executing the underlying systemcall.
-func DropPrivileges(username string, group string) error {
-	if group != "" {
-		g, err := user.LookupGroup(group)
-		if err != nil {
-			// log.Warn("Error while looking up group")
-			return err
-		}
-
-		gid, _ := strconv.Atoi(g.Gid)
-		if err := syscall.Setgid(gid); err != nil {
-			// log.Warn("Error while setting gid")
-			return err
-		}
-	}
-
-	if username != "" {
-		u, err := user.Lookup(username)
-		if err != nil {
-			// log.Warn("Error while looking up user")
-			return err
-		}
-
-		uid, _ := strconv.Atoi(u.Uid)
-		if err := syscall.Setuid(uid); err != nil {
-			// log.Warn("Error while setting uid")
-			return err
-		}
-	}
-
-	return nil
-}
-
-// If started via systemd, inform systemd that we are running:
-// https://www.freedesktop.org/software/systemd/man/sd_notify.html
-func SystemdNotifiy(ready bool, status string) {
-	if os.Getenv("NOTIFY_SOCKET") == "" {
-		// Not started using systemd
-		return
-	}
-
-	args := []string{fmt.Sprintf("--pid=%d", os.Getpid())}
-	if ready {
-		args = append(args, "--ready")
-	}
-
-	if status != "" {
-		args = append(args, fmt.Sprintf("--status=%s", status))
-	}
-
-	cmd := exec.Command("systemd-notify", args...)
-	cmd.Run() // errors ignored on purpose, there is not much to do anyways.
-}

internal/util/float.go

@@ -1,76 +0,0 @@
-package util
-
-import (
-	"math"
-	"strconv"
-)
-
-// Go's JSON encoder for floats does not support NaN (https://github.com/golang/go/issues/3480).
-// This program uses NaN as a signal for missing data.
-// For the HTTP JSON API to be able to handle NaN values,
-// we have to use our own type which implements encoding/json.Marshaler itself.
-type Float float64
-
-var (
-	NaN         Float  = Float(math.NaN())
-	nullAsBytes []byte = []byte("null")
-)
-
-func (f Float) IsNaN() bool {
-	return math.IsNaN(float64(f))
-}
-
-func (f Float) MarshalJSON() ([]byte, error) {
-	if math.IsNaN(float64(f)) {
-		return nullAsBytes, nil
-	}
-
-	return strconv.AppendFloat(make([]byte, 0, 10), float64(f), 'f', 3, 64), nil
-}
-
-func (f Float) Double() float64 {
-	return float64(f)
-}
-
-func (f *Float) UnmarshalJSON(input []byte) error {
-	if string(input) == "null" {
-		*f = NaN
-		return nil
-	}
-
-	val, err := strconv.ParseFloat(string(input), 64)
-	if err != nil {
-		return err
-	}
-	*f = Float(val)
-	return nil
-}
-
-// Same as `[]Float`, but can be marshaled to JSON with less allocations.
-type FloatArray []Float
-
-func ConvertToFloat(input float64) Float {
-	if input == -1.0 {
-		return NaN
-	} else {
-		return Float(input)
-	}
-}
-
-func (fa FloatArray) MarshalJSON() ([]byte, error) {
-	buf := make([]byte, 0, 2+len(fa)*8)
-	buf = append(buf, '[')
-	for i := 0; i < len(fa); i++ {
-		if i != 0 {
-			buf = append(buf, ',')
-		}
-
-		if fa[i].IsNaN() {
-			buf = append(buf, `null`...)
-		} else {
-			buf = strconv.AppendFloat(buf, float64(fa[i]), 'f', 3, 64)
-		}
-	}
-	buf = append(buf, ']')
-	return buf, nil
-}

internal/util/selector.go

@@ -1,51 +0,0 @@
-package util
-
-import (
-	"encoding/json"
-	"errors"
-)
-
-type SelectorElement struct {
-	String string
-	Group  []string
-	Any    bool
-}
-
-func (se *SelectorElement) UnmarshalJSON(input []byte) error {
-	if input[0] == '"' {
-		if err := json.Unmarshal(input, &se.String); err != nil {
-			return err
-		}
-
-		if se.String == "*" {
-			se.Any = true
-			se.String = ""
-		}
-
-		return nil
-	}
-
-	if input[0] == '[' {
-		return json.Unmarshal(input, &se.Group)
-	}
-
-	return errors.New("the Go SelectorElement type can only be a string or an array of strings")
-}
-
-func (se *SelectorElement) MarshalJSON() ([]byte, error) {
-	if se.Any {
-		return []byte("\"*\""), nil
-	}
-
-	if se.String != "" {
-		return json.Marshal(se.String)
-	}
-
-	if se.Group != nil {
-		return json.Marshal(se.Group)
-	}
-
-	return nil, errors.New("a Go Selector must be a non-empty string or a non-empty slice of strings")
-}
-
-type Selector []SelectorElement

pkg/resampler/resampler.go

@@ -1,122 +0,0 @@
-package resampler
-
-import (
-	"errors"
-	"fmt"
-	"math"
-
-	"github.com/ClusterCockpit/cc-metric-store/internal/util"
-)
-
-func SimpleResampler(data []util.Float, old_frequency int64, new_frequency int64) ([]util.Float, int64, error) {
-	if old_frequency == 0 || new_frequency == 0 || new_frequency <= old_frequency {
-		return data, old_frequency, nil
-	}
-
-	if new_frequency%old_frequency != 0 {
-		return nil, 0, errors.New("new sampling frequency should be multiple of the old frequency")
-	}
-
-	var step int = int(new_frequency / old_frequency)
-	var new_data_length = len(data) / step
-
-	if new_data_length == 0 || len(data) < 100 || new_data_length >= len(data) {
-		return data, old_frequency, nil
-	}
-
-	new_data := make([]util.Float, new_data_length)
-
-	for i := 0; i < new_data_length; i++ {
-		new_data[i] = data[i*step]
-	}
-
-	return new_data, new_frequency, nil
-}
-
-// Inspired by one of the algorithms from https://skemman.is/bitstream/1946/15343/3/SS_MSthesis.pdf
-// Adapted from https://github.com/haoel/downsampling/blob/master/core/lttb.go
-func LargestTriangleThreeBucket(data []util.Float, old_frequency int64, new_frequency int64) ([]util.Float, int64, error) {
-
-	if old_frequency == 0 || new_frequency == 0 || new_frequency <= old_frequency {
-		return data, old_frequency, nil
-	}
-
-	if new_frequency%old_frequency != 0 {
-		return nil, 0, fmt.Errorf("new sampling frequency : %d should be multiple of the old frequency : %d", new_frequency, old_frequency)
-	}
-
-	var step int = int(new_frequency / old_frequency)
-	var new_data_length = len(data) / step
-
-	if new_data_length == 0 || len(data) < 100 || new_data_length >= len(data) {
-		return data, old_frequency, nil
-	}
-
-	new_data := make([]util.Float, 0, new_data_length)
-
-	// Bucket size. Leave room for start and end data points
-	bucketSize := float64(len(data)-2) / float64(new_data_length-2)
-
-	new_data = append(new_data, data[0]) // Always add the first point
-
-	// We have 3 pointers represent for
-	// > bucketLow - the current bucket's beginning location
-	// > bucketMiddle - the current bucket's ending location,
-	//                  also the beginning location of next bucket
-	// > bucketHight - the next bucket's ending location.
-	bucketLow := 1
-	bucketMiddle := int(math.Floor(bucketSize)) + 1
-
-	var prevMaxAreaPoint int
-
-	for i := 0; i < new_data_length-2; i++ {
-
-		bucketHigh := int(math.Floor(float64(i+2)*bucketSize)) + 1
-		if bucketHigh >= len(data)-1 {
-			bucketHigh = len(data) - 2
-		}
-
-		// Calculate point average for next bucket (containing c)
-		avgPointX, avgPointY := calculateAverageDataPoint(data[bucketMiddle:bucketHigh+1], int64(bucketMiddle))
-
-		// Get the range for current bucket
-		currBucketStart := bucketLow
-		currBucketEnd := bucketMiddle
-
-		// Point a
-		pointX := prevMaxAreaPoint
-		pointY := data[prevMaxAreaPoint]
-
-		maxArea := -1.0
-
-		var maxAreaPoint int
-		flag_ := 0
-		for ; currBucketStart < currBucketEnd; currBucketStart++ {
-
-			area := calculateTriangleArea(util.Float(pointX), pointY, avgPointX, avgPointY, util.Float(currBucketStart), data[currBucketStart])
-			if area > maxArea {
-				maxArea = area
-				maxAreaPoint = currBucketStart
-			}
-			if math.IsNaN(float64(avgPointY)) {
-				flag_ = 1
-			}
-		}
-
-		if flag_ == 1 {
-			new_data = append(new_data, util.NaN) // Pick this point from the bucket
-
-		} else {
-			new_data = append(new_data, data[maxAreaPoint]) // Pick this point from the bucket
-		}
-		prevMaxAreaPoint = maxAreaPoint // This MaxArea point is the next's prevMAxAreaPoint
-
-		//move to the next window
-		bucketLow = bucketMiddle
-		bucketMiddle = bucketHigh
-	}
-
-	new_data = append(new_data, data[len(data)-1]) // Always add last
-
-	return new_data, new_frequency, nil
-}

pkg/resampler/util.go

@@ -1,35 +0,0 @@
-package resampler
-
-import (
-	"math"
-
-	"github.com/ClusterCockpit/cc-metric-store/internal/util"
-)
-
-func calculateTriangleArea(paX, paY, pbX, pbY, pcX, pcY util.Float) float64 {
-	area := ((paX-pcX)*(pbY-paY) - (paX-pbX)*(pcY-paY)) * 0.5
-	return math.Abs(float64(area))
-}
-
-func calculateAverageDataPoint(points []util.Float, xStart int64) (avgX util.Float, avgY util.Float) {
-	flag := 0
-	for _, point := range points {
-		avgX += util.Float(xStart)
-		avgY += point
-		xStart++
-		if math.IsNaN(float64(point)) {
-			flag = 1
-		}
-	}
-
-	l := util.Float(len(points))
-
-	avgX /= l
-	avgY /= l
-
-	if flag == 1 {
-		return avgX, util.NaN
-	} else {
-		return avgX, avgY
-	}
-}

tools.go

@@ -1,8 +0,0 @@
-//go:build tools
-// +build tools
-
-package tools
-
-import (
-	_ "github.com/swaggo/swag/cmd/swag"
-)

tools/tools.go

@@ -0,0 +1,5 @@
+//go:build tools
+
+package tools
+
+import _ "github.com/swaggo/swag/cmd/swag"