Remove WithUnit query option

.claude/settings.json

@@ -1,84 +0,0 @@
-{
-  "hooks": {
-    "PostToolUse": [
-      {
-        "matcher": "Task",
-        "hooks": [
-          {
-            "type": "command",
-            "command": "entire hooks claude-code post-task"
-          }
-        ]
-      },
-      {
-        "matcher": "TodoWrite",
-        "hooks": [
-          {
-            "type": "command",
-            "command": "entire hooks claude-code post-todo"
-          }
-        ]
-      }
-    ],
-    "PreToolUse": [
-      {
-        "matcher": "Task",
-        "hooks": [
-          {
-            "type": "command",
-            "command": "entire hooks claude-code pre-task"
-          }
-        ]
-      }
-    ],
-    "SessionEnd": [
-      {
-        "matcher": "",
-        "hooks": [
-          {
-            "type": "command",
-            "command": "entire hooks claude-code session-end"
-          }
-        ]
-      }
-    ],
-    "SessionStart": [
-      {
-        "matcher": "",
-        "hooks": [
-          {
-            "type": "command",
-            "command": "entire hooks claude-code session-start"
-          }
-        ]
-      }
-    ],
-    "Stop": [
-      {
-        "matcher": "",
-        "hooks": [
-          {
-            "type": "command",
-            "command": "entire hooks claude-code stop"
-          }
-        ]
-      }
-    ],
-    "UserPromptSubmit": [
-      {
-        "matcher": "",
-        "hooks": [
-          {
-            "type": "command",
-            "command": "entire hooks claude-code user-prompt-submit"
-          }
-        ]
-      }
-    ]
-  },
-  "permissions": {
-    "deny": [
-      "Read(./.entire/metadata/**)"
-    ]
-  }
-}

.entire/.gitignore

@@ -1,4 +0,0 @@
-tmp/
-settings.local.json
-metadata/
-logs/

.entire/settings.json

@@ -1,4 +0,0 @@
-{
-  "enabled": true,
-  "telemetry": true
-}

.github/dependabot.yml

@@ -1,11 +0,0 @@
-# To get started with Dependabot version updates, you'll need to specify which
-# package ecosystems to update and where the package manifests are located.
-# Please see the documentation for all configuration options:
-# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file
-
-version: 2
-updates:
-  - package-ecosystem: "gomod"
-    directory: "/"
-    schedule:
-      interval: "weekly"

.github/workflows/Release.yml

@@ -0,0 +1,314 @@
+# See: https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions
+
+# Workflow name
+name: Release
+
+# Run on tag push
+on:
+ push:
+  tags:
+  - '**'
+
+jobs:
+
+  #
+  # Build on AlmaLinux 8.5 using golang-1.18.2
+  #
+  AlmaLinux-RPM-build:
+    runs-on: ubuntu-latest
+    # See: https://hub.docker.com/_/almalinux
+    container: almalinux:8.5
+    # The job outputs link to the outputs of the 'rpmrename' step
+    # Only job outputs can be used in child jobs
+    outputs:
+      rpm : ${{steps.rpmrename.outputs.RPM}}
+      srpm : ${{steps.rpmrename.outputs.SRPM}}
+    steps:
+
+    # Use dnf to install development packages
+    - name: Install development packages
+      run: |
+          dnf --assumeyes group install "Development Tools" "RPM Development Tools"
+          dnf --assumeyes install wget openssl-devel diffutils delve which
+
+    # Checkout git repository and submodules
+    # fetch-depth must be 0 to use git describe
+    # See: https://github.com/marketplace/actions/checkout
+    - name: Checkout
+      uses: actions/checkout@v2
+      with:
+        submodules: recursive
+        fetch-depth: 0
+
+    # Use dnf to install build dependencies
+    - name: Install build dependencies
+      run: |
+          wget -q http://mirror.centos.org/centos/8-stream/AppStream/x86_64/os/Packages/golang-1.18.2-1.module_el8.7.0+1173+5d37c0fd.x86_64.rpm \
+                  http://mirror.centos.org/centos/8-stream/AppStream/x86_64/os/Packages/golang-bin-1.18.2-1.module_el8.7.0+1173+5d37c0fd.x86_64.rpm \
+                  http://mirror.centos.org/centos/8-stream/AppStream/x86_64/os/Packages/golang-src-1.18.2-1.module_el8.7.0+1173+5d37c0fd.noarch.rpm \
+                  http://mirror.centos.org/centos/8-stream/AppStream/x86_64/os/Packages/go-toolset-1.18.2-1.module_el8.7.0+1173+5d37c0fd.x86_64.rpm
+          rpm -i go*.rpm
+
+    - name: RPM build MetricStore
+      id: rpmbuild
+      run: make RPM
+
+    # AlmaLinux 8.5 is a derivate of RedHat Enterprise Linux 8 (UBI8),
+    # so the created RPM both contain the substring 'el8' in the RPM file names
+    # This step replaces the substring 'el8' to 'alma85'. It uses the move operation
+    # because it is unclear whether the default AlmaLinux 8.5 container contains the 
+    # 'rename' command. This way we also get the new names for output.
+    - name: Rename RPMs (s/el8/alma85/)
+      id: rpmrename
+      run: |
+        OLD_RPM="${{steps.rpmbuild.outputs.RPM}}"
+        OLD_SRPM="${{steps.rpmbuild.outputs.SRPM}}"
+        NEW_RPM="${OLD_RPM/el8/alma85}"
+        NEW_SRPM=${OLD_SRPM/el8/alma85}
+        mv "${OLD_RPM}" "${NEW_RPM}"
+        mv "${OLD_SRPM}" "${NEW_SRPM}"
+        echo "::set-output name=SRPM::${NEW_SRPM}"
+        echo "::set-output name=RPM::${NEW_RPM}"
+
+    # See: https://github.com/actions/upload-artifact
+    - name: Save RPM as artifact
+      uses: actions/upload-artifact@v2
+      with:
+        name: cc-metric-store RPM for AlmaLinux 8.5
+        path: ${{ steps.rpmrename.outputs.RPM }}
+    - name: Save SRPM as artifact
+      uses: actions/upload-artifact@v2
+      with:
+        name: cc-metric-store SRPM for AlmaLinux 8.5
+        path: ${{ steps.rpmrename.outputs.SRPM }}
+
+  #
+  # Build on UBI 8 using golang-1.18.2
+  #
+  UBI-8-RPM-build:
+    runs-on: ubuntu-latest
+    # See: https://catalog.redhat.com/software/containers/ubi8/ubi/5c359854d70cc534b3a3784e?container-tabs=gti
+    container: registry.access.redhat.com/ubi8/ubi:8.5-226.1645809065
+    # The job outputs link to the outputs of the 'rpmbuild' step
+    outputs:
+      rpm : ${{steps.rpmbuild.outputs.RPM}}
+      srpm : ${{steps.rpmbuild.outputs.SRPM}}
+    steps:
+
+    # Use dnf to install development packages
+    - name: Install development packages
+      run: dnf --assumeyes --disableplugin=subscription-manager install rpm-build go-srpm-macros rpm-build-libs rpm-libs gcc make python38 git wget openssl-devel diffutils delve which
+
+    # Checkout git repository and submodules
+    # fetch-depth must be 0 to use git describe
+    # See: https://github.com/marketplace/actions/checkout
+    - name: Checkout
+      uses: actions/checkout@v2
+      with:
+        submodules: recursive
+        fetch-depth: 0
+
+    # Use dnf to install build dependencies
+    - name: Install build dependencies
+      run: |
+          wget -q http://mirror.centos.org/centos/8-stream/AppStream/x86_64/os/Packages/golang-1.18.2-1.module_el8.7.0+1173+5d37c0fd.x86_64.rpm \
+                  http://mirror.centos.org/centos/8-stream/AppStream/x86_64/os/Packages/golang-bin-1.18.2-1.module_el8.7.0+1173+5d37c0fd.x86_64.rpm \
+                  http://mirror.centos.org/centos/8-stream/AppStream/x86_64/os/Packages/golang-src-1.18.2-1.module_el8.7.0+1173+5d37c0fd.noarch.rpm \
+                  http://mirror.centos.org/centos/8-stream/AppStream/x86_64/os/Packages/go-toolset-1.18.2-1.module_el8.7.0+1173+5d37c0fd.x86_64.rpm
+          rpm -i go*.rpm
+
+    - name: RPM build MetricStore
+      id: rpmbuild
+      run: make RPM
+
+    # See: https://github.com/actions/upload-artifact
+    - name: Save RPM as artifact
+      uses: actions/upload-artifact@v2
+      with:
+        name: cc-metric-store RPM for UBI 8
+        path: ${{ steps.rpmbuild.outputs.RPM }}
+    - name: Save SRPM as artifact
+      uses: actions/upload-artifact@v2
+      with:
+        name: cc-metric-store SRPM for UBI 8
+        path: ${{ steps.rpmbuild.outputs.SRPM }}
+
+  #
+  # Build on Ubuntu 20.04 using official go 1.19.1 package
+  #
+  Ubuntu-focal-build:
+    runs-on: ubuntu-latest
+    container: ubuntu:20.04
+    # The job outputs link to the outputs of the 'debrename' step
+    # Only job outputs can be used in child jobs
+    outputs:
+      deb : ${{steps.debrename.outputs.DEB}}
+    steps:
+    # Use apt to install development packages
+    - name: Install development packages
+      run: |
+          apt update && apt --assume-yes upgrade
+          apt --assume-yes install build-essential sed git wget bash
+    # Checkout git repository and submodules
+    # fetch-depth must be 0 to use git describe
+    # See: https://github.com/marketplace/actions/checkout
+    - name: Checkout
+      uses: actions/checkout@v2
+      with:
+        submodules: recursive
+        fetch-depth: 0
+    # Use official golang package
+    - name: Install Golang
+      run: |
+          wget -q https://go.dev/dl/go1.19.1.linux-amd64.tar.gz
+          tar -C /usr/local -xzf go1.19.1.linux-amd64.tar.gz
+          export PATH=/usr/local/go/bin:/usr/local/go/pkg/tool/linux_amd64:$PATH
+          go version
+    - name: DEB build MetricStore
+      id: dpkg-build
+      run: |
+          export PATH=/usr/local/go/bin:/usr/local/go/pkg/tool/linux_amd64:$PATH
+          make DEB
+    - name: Rename DEB (add '_ubuntu20.04')
+      id: debrename
+      run: |
+        OLD_DEB_NAME=$(echo "${{steps.dpkg-build.outputs.DEB}}" | rev | cut -d '.' -f 2- | rev)
+        NEW_DEB_FILE="${OLD_DEB_NAME}_ubuntu20.04.deb"
+        mv "${{steps.dpkg-build.outputs.DEB}}" "${NEW_DEB_FILE}"
+        echo "::set-output name=DEB::${NEW_DEB_FILE}"
+    # See: https://github.com/actions/upload-artifact
+    - name: Save DEB as artifact
+      uses: actions/upload-artifact@v2
+      with:
+        name: cc-metric-store DEB for Ubuntu 20.04
+        path: ${{ steps.debrename.outputs.DEB }}
+
+  #
+  # Build on Ubuntu 20.04 using official go 1.19.1 package
+  #
+  Ubuntu-jammy-build:
+    runs-on: ubuntu-latest
+    container: ubuntu:22.04
+    # The job outputs link to the outputs of the 'debrename' step
+    # Only job outputs can be used in child jobs
+    outputs:
+      deb : ${{steps.debrename.outputs.DEB}}
+    steps:
+    # Use apt to install development packages
+    - name: Install development packages
+      run: |
+          apt update && apt --assume-yes upgrade
+          apt --assume-yes install build-essential sed git wget bash
+    # Checkout git repository and submodules
+    # fetch-depth must be 0 to use git describe
+    # See: https://github.com/marketplace/actions/checkout
+    - name: Checkout
+      uses: actions/checkout@v2
+      with:
+        submodules: recursive
+        fetch-depth: 0
+    # Use official golang package
+    - name: Install Golang
+      run: |
+          wget -q https://go.dev/dl/go1.19.1.linux-amd64.tar.gz
+          tar -C /usr/local -xzf go1.19.1.linux-amd64.tar.gz
+          export PATH=/usr/local/go/bin:/usr/local/go/pkg/tool/linux_amd64:$PATH
+          go version
+    - name: DEB build MetricStore
+      id: dpkg-build
+      run: |
+          export PATH=/usr/local/go/bin:/usr/local/go/pkg/tool/linux_amd64:$PATH
+          make DEB
+    - name: Rename DEB (add '_ubuntu22.04')
+      id: debrename
+      run: |
+        OLD_DEB_NAME=$(echo "${{steps.dpkg-build.outputs.DEB}}" | rev | cut -d '.' -f 2- | rev)
+        NEW_DEB_FILE="${OLD_DEB_NAME}_ubuntu22.04.deb"
+        mv "${{steps.dpkg-build.outputs.DEB}}" "${NEW_DEB_FILE}"
+        echo "::set-output name=DEB::${NEW_DEB_FILE}"
+    # See: https://github.com/actions/upload-artifact
+    - name: Save DEB as artifact
+      uses: actions/upload-artifact@v2
+      with:
+        name: cc-metric-store DEB for Ubuntu 22.04
+        path: ${{ steps.debrename.outputs.DEB }}
+
+  #
+  # Create release with fresh RPMs
+  #
+  Release:
+    runs-on: ubuntu-latest
+    # We need the RPMs, so add dependency
+    needs: [AlmaLinux-RPM-build, UBI-8-RPM-build, Ubuntu-focal-build, Ubuntu-jammy-build]
+
+    steps:
+    # See: https://github.com/actions/download-artifact
+    - name: Download AlmaLinux 8.5 RPM
+      uses: actions/download-artifact@v2
+      with:
+        name: cc-metric-store RPM for AlmaLinux 8.5
+    - name: Download AlmaLinux 8.5 SRPM
+      uses: actions/download-artifact@v2
+      with:
+        name: cc-metric-store SRPM for AlmaLinux 8.5
+    
+    - name: Download UBI 8 RPM
+      uses: actions/download-artifact@v2
+      with:
+        name: cc-metric-store RPM for UBI 8
+    - name: Download UBI 8 SRPM
+      uses: actions/download-artifact@v2
+      with:
+        name: cc-metric-store SRPM for UBI 8
+
+    - name: Download Ubuntu 20.04 DEB
+      uses: actions/download-artifact@v2
+      with:
+        name: cc-metric-store DEB for Ubuntu 20.04
+
+    - name: Download Ubuntu 22.04 DEB
+      uses: actions/download-artifact@v2
+      with:
+        name: cc-metric-store DEB for Ubuntu 22.04
+
+    # The download actions do not publish the name of the downloaded file,
+    # so we re-use the job outputs of the parent jobs. The files are all
+    # downloaded to the current folder.
+    # The gh-release action afterwards does not accept file lists but all
+    # files have to be listed at 'files'. The step creates one output per
+    # RPM package (2 per distro)
+    - name: Set RPM variables
+      id: files
+      run: |
+        ALMA_85_RPM=$(basename "${{ needs.AlmaLinux-RPM-build.outputs.rpm}}")
+        ALMA_85_SRPM=$(basename "${{ needs.AlmaLinux-RPM-build.outputs.srpm}}")
+        UBI_8_RPM=$(basename "${{ needs.UBI-8-RPM-build.outputs.rpm}}")
+        UBI_8_SRPM=$(basename "${{ needs.UBI-8-RPM-build.outputs.srpm}}")
+        U_2004_DEB=$(basename "${{ needs.Ubuntu-focal-build.outputs.deb}}")
+        U_2204_DEB=$(basename "${{ needs.Ubuntu-jammy-build.outputs.deb}}")
+        echo "ALMA_85_RPM::${ALMA_85_RPM}"
+        echo "ALMA_85_SRPM::${ALMA_85_SRPM}"
+        echo "UBI_8_RPM::${UBI_8_RPM}"
+        echo "UBI_8_SRPM::${UBI_8_SRPM}"
+        echo "U_2004_DEB::${U_2004_DEB}"
+        echo "U_2204_DEB::${U_2204_DEB}"
+        echo "::set-output name=ALMA_85_RPM::${ALMA_85_RPM}"
+        echo "::set-output name=ALMA_85_SRPM::${ALMA_85_SRPM}"
+        echo "::set-output name=UBI_8_RPM::${UBI_8_RPM}"
+        echo "::set-output name=UBI_8_SRPM::${UBI_8_SRPM}"
+        echo "::set-output name=U_2004_DEB::${U_2004_DEB}"
+        echo "::set-output name=U_2204_DEB::${U_2204_DEB}"
+
+    # See: https://github.com/softprops/action-gh-release
+    - name: Release
+      uses: softprops/action-gh-release@v1
+      if: startsWith(github.ref, 'refs/tags/')
+      with:
+        name: cc-metric-store-${{github.ref_name}}
+        files: |
+         ${{ steps.files.outputs.ALMA_85_RPM }}
+         ${{ steps.files.outputs.ALMA_85_SRPM }}
+         ${{ steps.files.outputs.UBI_8_RPM }}
+         ${{ steps.files.outputs.UBI_8_SRPM }}
+         ${{ steps.files.outputs.U_2004_DEB }}
+         ${{ steps.files.outputs.U_2204_DEB }}

.github/workflows/test.yml

@@ -5,14 +5,14 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Install Go
-        uses: actions/setup-go@v4
+        uses: actions/setup-go@v2
         with:
-          go-version: 1.22.x
+          go-version: 1.16.x
       - name: Checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v2
       - name: Build, Vet & Test
         run: |
           go build ./...
           go vet ./...
-          go test  ./...
+          go test -v ./...
 

.gitignore

@@ -4,6 +4,7 @@
 *.dll
 *.so
 *.dylib
+/cc-metric-store
 
 # Test binary, built with `go test -c`
 *.test
@@ -15,16 +16,5 @@
 # vendor/
 
 # Project specific ignores
-/cc-metric-store
 /var
-
+/configs
-aditya.creds
-test.creds
-/config.json
-
-migrateTimestamps.pl
-sample.txt
-sample_alex.txt
-sample_fritz.txt
-
-test_ccms_write_api.sh

.goreleaser.yaml

@@ -1,64 +0,0 @@
-version: 2
-before:
-  hooks:
-    - go mod tidy
-builds:
-  - env:
-      - CGO_ENABLED=0
-    goos:
-      - linux
-    goarch:
-      - amd64
-    goamd64:
-      - v3
-    id: "cc-metric-store"
-    binary: cc-metric-store
-    main: ./cmd/cc-metric-store
-    ldflags:
-      - -s -w -X main.version={{.Version}}
-      - -X main.commit={{.Commit}} -X main.date={{.Date}}
-    tags:
-      - static_build
-archives:
-  - formats: tar.gz
-    # this name template makes the OS and Arch compatible with the results of uname.
-    name_template: >-
-      {{ .ProjectName }}_
-      {{- title .Os }}_
-      {{- if eq .Arch "amd64" }}x86_64
-      {{- else }}{{ .Arch }}{{ end }}
-      {{- if .Arm }}v{{ .Arm }}{{ end }}
-checksum:
-  name_template: "checksums.txt"
-snapshot:
-  version_template: "{{ incpatch .Version }}-next"
-changelog:
-  sort: asc
-  filters:
-    include:
-      - "^feat:"
-      - "^fix:"
-      - "^sec:"
-      - "^docs:"
-  groups:
-    - title: "Dependency updates"
-      regexp: '^.*?(feat|fix)\(deps\)!?:.+$'
-      order: 300
-    - title: "New Features"
-      regexp: '^.*?feat(\([[:word:]]+\))??!?:.+$'
-      order: 100
-    - title: "Security updates"
-      regexp: '^.*?sec(\([[:word:]]+\))??!?:.+$'
-      order: 150
-    - title: "Bug fixes"
-      regexp: '^.*?fix(\([[:word:]]+\))??!?:.+$'
-      order: 200
-    - title: "Documentation updates"
-      regexp: ^.*?doc(\([[:word:]]+\))??!?:.+$
-      order: 400
-release:
-  draft: false
-  footer: |
-    Please check out the [Release Notes](https://github.com/ClusterCockpit/cc-metric-store/blob/master/ReleaseNotes.md) for further details on breaking changes.
-
-# vim: set ts=2 sw=2 tw=0 fo=cnqoj

CLAUDE.md

@@ -1,77 +0,0 @@
-# CLAUDE.md
-
-This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
-
-## Project Overview
-
-cc-metric-store is an in-memory time-series database for HPC cluster metrics, part of the ClusterCockpit monitoring suite. Data is indexed by a hierarchical tree (cluster → host → socket/cpu/gpu) and accessed via selectors. The core storage engine lives in `cc-backend/pkg/metricstore`; this repo provides the HTTP API wrapper.
-
-## Build Commands
-
-```bash
-make                 # Build binary, copy config template, create checkpoint dirs
-make clean           # Clean build cache and binary
-make distclean       # Also remove ./var and config.json
-make swagger         # Regenerate Swagger from source comments
-make test            # Run go build, go vet, go test
-```
-
-## Testing
-
-```bash
-go test -v ./...              # Run tests
-go test -bench=. -race -v ./...  # With benchmarks and race detector
-```
-
-Integration test scripts in `/endpoint-test-scripts/` for manual API testing.
-
-## Running
-
-```bash
-./cc-metric-store                           # Uses ./config.json
-./cc-metric-store -config /path/to/config.json
-./cc-metric-store -dev                      # Enable Swagger UI at /swagger/
-./cc-metric-store -loglevel debug           # debug|info|warn|err|crit
-```
-
-## Architecture
-
-**Entry point:** `cmd/cc-metric-store/main.go`
-- `run()` → parse flags, init logging/config, connect NATS
-- `runServer()` → init metricstore from cc-backend, start HTTP server
-
-**Key packages:**
-- `internal/api/` - REST endpoints (query, write, free, debug, healthcheck) and JWT auth (Ed25519)
-- `internal/config/` - Config loading and JSON schema validation
-- External: `cc-backend/pkg/metricstore` - actual time-series storage engine
-
-**API endpoints** (all support optional JWT auth):
-- `GET /api/query/` - Query metrics with selectors
-- `POST /api/write/` - Write metrics (InfluxDB line protocol)
-- `POST /api/free/` - Free buffers up to timestamp
-- `GET /api/debug/` - Dump internal state
-- `GET /api/healthcheck/` - Node health status
-
-## Selectors
-
-Data is accessed via hierarchical selectors:
-```
-["cluster1", "host1", "cpu0"]           # Specific CPU
-["cluster1", "host1", ["cpu4", "cpu5"]] # Multiple CPUs
-["cluster1", "host1"]                   # Entire node (all CPUs implied)
-```
-
-## Configuration
-
-Config file structure (see `configs/config.json`):
-- `main` - Server address, TLS certs, JWT public key, user/group for privilege drop
-- `metrics` - Per-metric frequency and aggregation strategy (sum/avg/null)
-- `metric-store` - Checkpoints, memory cap, retention, cleanup mode, NATS subscriptions
-- `nats` - Optional NATS connection for receiving metrics
-
-## Test JWT
-
-For testing with JWT auth enabled:
-```
-eyJ0eXAiOiJKV1QiLCJhbGciOiJFZERTQSJ9.eyJ1c2VyIjoiYWRtaW4iLCJyb2xlcyI6WyJST0xFX0FETUlOIiwiUk9MRV9BTkFMWVNUIiwiUk9MRV9VU0VSIl19.d-3_3FZTsadPjDEdsWrrQ7nS0edMAR4zjl-eK7rJU3HziNBfI9PDHDIpJVHTNN5E5SlLGLFXctWyKAkwhXL-Dw
-```

Makefile

@@ -1,41 +1,113 @@
-TARGET = ./cc-metric-store
-VAR = ./var/checkpoints/
-VERSION = 1.5.0
-GIT_HASH := $(shell git rev-parse --short HEAD || echo 'development')
-CURRENT_TIME = $(shell date +"%Y-%m-%d:T%H:%M:%S")
-LD_FLAGS = '-s -X main.date=${CURRENT_TIME} -X main.version=${VERSION} -X main.commit=${GIT_HASH}'
 
-.PHONY: clean distclean test swagger $(TARGET)
+APP = cc-metric-store
+GOSRC_APP        := cc-metric-store.go
+GOSRC_FILES      := api.go \
+					memstore.go \
+					archive.go \
+					debug.go \
+					float.go \
+					lineprotocol.go \
+					selector.go \
+					stats.go
 
-.NOTPARALLEL:
 
-$(TARGET): config.json $(VAR)
-	$(info ===>  BUILD cc-metric-store)
-	@go build -ldflags=${LD_FLAGS} ./cmd/cc-metric-store
 
-config.json:
+BINDIR ?= bin
-	@cp ./configs/config.json config.json
 
-$(VAR):
-	@mkdir -p $(VAR)
 
-swagger:
+.PHONY: all
-	$(info ===>  GENERATE swagger)
+all: $(APP)
-	@go run github.com/swaggo/swag/cmd/swag init -d ./internal/api -g api.go -o ./api
-	@mv ./api/docs.go ./internal/api/docs.go
 
+$(APP): $(GOSRC) $(GOSRC_APP)
+	go get
+	go build -o $(APP) $(GOSRC_APP) $(GOSRC_FILES)
+
+install: $(APP)
+	@WORKSPACE=$(PREFIX)
+	@if [ -z "$${WORKSPACE}" ]; then exit 1; fi
+	@mkdir --parents --verbose $${WORKSPACE}/usr/$(BINDIR)
+	@install -Dpm 755 $(APP) $${WORKSPACE}/usr/$(BINDIR)/$(APP)
+	@install -Dpm 600 config.json $${WORKSPACE}/etc/$(APP)/$(APP).json
+
+.PHONY: clean
+.ONESHELL:
 clean:
-	$(info ===>  CLEAN)
+	rm -f $(APP)
-	@go clean
-	@rm -f $(TARGET)
 
-distclean: clean
+.PHONY: fmt
-	@rm -rf ./var
+fmt:
-	@rm -f config.json
+	go fmt $(GOSRC_APP)
 
-test:
+# Examine Go source code and reports suspicious constructs
-	$(info ===>  TESTING)
+.PHONY: vet
-	@go clean -testcache
+vet:
-	@go build ./...
+	go vet ./...
-	@go vet ./...
+
-	@go test ./...
+# Run linter for the Go programming language.
+# Using static analysis, it finds bugs and performance issues, offers simplifications, and enforces style rules
+.PHONY: staticcheck
+staticcheck:
+	go install honnef.co/go/tools/cmd/staticcheck@latest
+	$$(go env GOPATH)/bin/staticcheck ./...
+
+.ONESHELL:
+.PHONY: RPM
+RPM: scripts/cc-metric-store.spec
+	@WORKSPACE="$${PWD}"
+	@SPECFILE="$${WORKSPACE}/scripts/cc-metric-store.spec"
+	# Setup RPM build tree
+	@eval $$(rpm --eval "ARCH='%{_arch}' RPMDIR='%{_rpmdir}' SOURCEDIR='%{_sourcedir}' SPECDIR='%{_specdir}' SRPMDIR='%{_srcrpmdir}' BUILDDIR='%{_builddir}'")
+	@mkdir --parents --verbose "$${RPMDIR}" "$${SOURCEDIR}" "$${SPECDIR}" "$${SRPMDIR}" "$${BUILDDIR}"
+	# Create source tarball
+	@COMMITISH="HEAD"
+	@VERS=$$(git describe --tags $${COMMITISH})
+	@VERS=$${VERS#v}
+	@VERS=$$(echo $$VERS | sed -e s+'-'+'_'+g)
+	@if [ "$${VERS}" = "" ]; then VERS="0.0.1"; fi
+	@eval $$(rpmspec --query --queryformat "NAME='%{name}' VERSION='%{version}' RELEASE='%{release}' NVR='%{NVR}' NVRA='%{NVRA}'" --define="VERS $${VERS}" "$${SPECFILE}")
+	@PREFIX="$${NAME}-$${VERSION}"
+	@FORMAT="tar.gz"
+	@SRCFILE="$${SOURCEDIR}/$${PREFIX}.$${FORMAT}"
+	@git archive --verbose --format "$${FORMAT}" --prefix="$${PREFIX}/" --output="$${SRCFILE}" $${COMMITISH}
+	# Build RPM and SRPM
+	@rpmbuild -ba --define="VERS $${VERS}" --rmsource --clean "$${SPECFILE}"
+	# Report RPMs and SRPMs when in GitHub Workflow
+	@if [[ "$${GITHUB_ACTIONS}" == true ]]; then
+	@     RPMFILE="$${RPMDIR}/$${ARCH}/$${NVRA}.rpm"
+	@     SRPMFILE="$${SRPMDIR}/$${NVR}.src.rpm"
+	@     echo "RPM: $${RPMFILE}"
+	@     echo "SRPM: $${SRPMFILE}"
+	@     echo "::set-output name=SRPM::$${SRPMFILE}"
+	@     echo "::set-output name=RPM::$${RPMFILE}"
+	@fi
+
+.ONESHELL:
+.PHONY: DEB
+DEB: scripts/cc-metric-store.deb.control $(APP)
+	@BASEDIR=$${PWD}
+	@WORKSPACE=$${PWD}/.dpkgbuild
+	@DEBIANDIR=$${WORKSPACE}/debian
+	@DEBIANBINDIR=$${WORKSPACE}/DEBIAN
+	@mkdir --parents --verbose $$WORKSPACE $$DEBIANBINDIR
+	#@mkdir --parents --verbose $$DEBIANDIR
+	@CONTROLFILE="$${BASEDIR}/scripts/cc-metric-store.deb.control"
+	@COMMITISH="HEAD"
+	@VERS=$$(git describe --tags --abbrev=0 $${COMMITISH})
+	@VERS=$${VERS#v}
+	@VERS=$$(echo $$VERS | sed -e s+'-'+'_'+g)
+	@if [ "$${VERS}" = "" ]; then VERS="0.0.1"; fi
+	@ARCH=$$(uname -m)
+	@ARCH=$$(echo $$ARCH | sed -e s+'_'+'-'+g)
+	@if [ "$${ARCH}" = "x86-64" ]; then ARCH=amd64; fi
+	@PREFIX="$${NAME}-$${VERSION}_$${ARCH}"
+	@SIZE_BYTES=$$(du -bcs --exclude=.dpkgbuild "$$WORKSPACE"/ | awk '{print $$1}' | head -1 | sed -e 's/^0\+//')
+	@SIZE="$$(awk -v size="$$SIZE_BYTES" 'BEGIN {print (size/1024)+1}' | awk '{print int($$0)}')"
+	#@sed -e s+"{VERSION}"+"$$VERS"+g -e s+"{INSTALLED_SIZE}"+"$$SIZE"+g -e s+"{ARCH}"+"$$ARCH"+g $$CONTROLFILE > $${DEBIANDIR}/control
+	@sed -e s+"{VERSION}"+"$$VERS"+g -e s+"{INSTALLED_SIZE}"+"$$SIZE"+g -e s+"{ARCH}"+"$$ARCH"+g $$CONTROLFILE > $${DEBIANBINDIR}/control
+	@make PREFIX=$${WORKSPACE} install
+	@DEB_FILE="cc-metric-store_$${VERS}_$${ARCH}.deb"
+	@dpkg-deb -b $${WORKSPACE} "$$DEB_FILE"
+	@rm -r "$${WORKSPACE}"
+	@if [ "$${GITHUB_ACTIONS}" = "true" ]; then
+	@     echo "::set-output name=DEB::$${DEB_FILE}"
+	@fi

README.md

@@ -2,68 +2,18 @@
 
 [![Build & Test](https://github.com/ClusterCockpit/cc-metric-store/actions/workflows/test.yml/badge.svg)](https://github.com/ClusterCockpit/cc-metric-store/actions/workflows/test.yml)
 
-The cc-metric-store provides a simple in-memory time series database for storing
+The cc-metric-store provides a simple in-memory time series database for storing metrics of cluster nodes at preconfigured intervals. It is meant to be used as part of the [ClusterCockpit suite](https://github.com/ClusterCockpit). As all data is kept in-memory (but written to disk as compressed JSON for long term storage), accessing it is very fast. It also provides aggregations over time *and* nodes/sockets/cpus.
-metrics of cluster nodes at preconfigured intervals. It is meant to be used as
-part of the [ClusterCockpit suite](https://github.com/ClusterCockpit). As all
-data is kept in-memory, accessing it is very fast. It also provides topology aware
-aggregations over time _and_ nodes/sockets/cpus.
 
-The storage engine is provided by the
+There are major limitations: Data only gets written to disk at periodic checkpoints, not as soon as it is received.
-[cc-backend](https://github.com/ClusterCockpit/cc-backend) package
-(`cc-backend/pkg/metricstore`). This repository provides the HTTP API wrapper.
 
-The [NATS.io](https://nats.io/) based writing endpoint and the HTTP write
+Go look at the `TODO.md` file and the [GitHub Issues](https://github.com/ClusterCockpit/cc-metric-store/issues) for a progress overview. Things work, but are not properly tested.
-endpoint both consume messages in [this format of the InfluxDB line
+The [NATS.io](https://nats.io/) based writing endpoint consumes messages in [this format of the InfluxDB line protocol](https://github.com/ClusterCockpit/cc-specifications/blob/master/metrics/lineprotocol_alternative.md).
-protocol](https://github.com/ClusterCockpit/cc-specifications/blob/master/metrics/lineprotocol_alternative.md).
 
-## Building
+### REST API Endpoints
 
-`cc-metric-store` can be built using the provided `Makefile`.
+The REST API is documented in [openapi.yaml](./openapi.yaml) in the OpenAPI 3.0 format.
-It supports the following targets:
 
-- `make`: Build the application, copy an example configuration file and generate
+### Run tests
-  checkpoint folders if required.
-- `make clean`: Clean the golang build cache and application binary
-- `make distclean`: In addition to the clean target also remove the `./var`
-  folder and `config.json`
-- `make swagger`: Regenerate the Swagger files from the source comments.
-- `make test`: Run tests and basic checks (`go build`, `go vet`, `go test`).
-
-## Running
-
-```sh
-./cc-metric-store                              # Uses ./config.json
-./cc-metric-store -config /path/to/config.json
-./cc-metric-store -dev                         # Enable Swagger UI at /swagger/
-./cc-metric-store -loglevel debug              # debug|info|warn (default)|err|crit
-./cc-metric-store -logdate                     # Add date and time to log messages
-./cc-metric-store -version                     # Show version information and exit
-./cc-metric-store -gops                        # Enable gops agent for debugging
-```
-
-## REST API Endpoints
-
-The REST API is documented in [swagger.json](./api/swagger.json). You can
-explore and try the REST API using the integrated [SwaggerUI web
-interface](http://localhost:8082/swagger/) (requires the `-dev` flag).
-
-For more information on the `cc-metric-store` REST API have a look at the
-ClusterCockpit documentation [website](https://clustercockpit.org/docs/reference/cc-metric-store/ccms-rest-api/).
-
-All endpoints support both trailing-slash and non-trailing-slash variants:
-
-| Method | Path                | Description                            |
-| ------ | ------------------- | -------------------------------------- |
-| `GET`  | `/api/query/`       | Query metrics with selectors           |
-| `POST` | `/api/write/`       | Write metrics (InfluxDB line protocol) |
-| `POST` | `/api/free/`        | Free buffers up to a timestamp         |
-| `GET`  | `/api/debug/`       | Dump internal state                    |
-| `GET`  | `/api/healthcheck/` | Check node health status               |
-
-If `jwt-public-key` is set in `config.json`, all endpoints require JWT
-authentication using an Ed25519 key (`Authorization: Bearer <token>`).
-
-## Run tests
 
 Some benchmarks concurrently access the `MemoryStore`, so enabling the
 [Race Detector](https://golang.org/doc/articles/race_detector) might be useful.
@@ -78,16 +28,18 @@ go test -v ./...
 go test -bench=. -race -v ./...
 ```
 
-## What are these selectors mentioned in the code?
+### What are these selectors mentioned in the code?
 
-The cc-metric-store works as a time-series database and uses the InfluxDB line
+Tags in InfluxDB are used to build indexes over the stored data. InfluxDB-Tags have no
-protocol as input format. Unlike InfluxDB, the data is indexed by one single
+relation to each other, they do not depend on each other and have no hierarchy.
-strictly hierarchical tree structure. A selector is built out of the tags in the
+Different tags build up different indexes (I am no expert at all, but this is how i think they work).
-InfluxDB line protocol, and can be used to select a node (not in the sense of a
+
-compute node, can also be a socket, cpu, ...) in that tree. The implementation
+This project also works as a time-series database and uses the InfluxDB line protocol.
-calls those nodes `level` to avoid confusion. It is impossible to access data
+Unlike InfluxDB, the data is indexed by one single strictly hierarchical tree structure.
-only by knowing the _socket_ or _cpu_ tag — all higher up levels have to be
+A selector is build out of the tags in the InfluxDB line protocol, and can be used to select
-specified as well.
+a node (not in the sense of a compute node, can also be a socket, cpu, ...) in that tree.
+The implementation calls those nodes `level` to avoid confusion.
+It is impossible to access data only by knowing the *socket* or *cpu* tag, all higher up levels have to be specified as well.
 
 This is what the hierarchy currently looks like:
 
@@ -101,165 +53,49 @@ This is what the hierarchy currently looks like:
     - cpu3
     - cpu4
     - ...
-    - gpu1
-    - gpu2
   - host2
   - ...
 - cluster2
 - ...
 
 Example selectors:
-
 1. `["cluster1", "host1", "cpu0"]`: Select only the cpu0 of host1 in cluster1
 2. `["cluster1", "host1", ["cpu4", "cpu5", "cpu6", "cpu7"]]`: Select only CPUs 4-7 of host1 in cluster1
-3. `["cluster1", "host1"]`: Select the complete node. If querying for a CPU-specific metric such as flops, all CPUs are implied
+3. `["cluster1", "host1"]`: Select the complete node. If querying for a CPU-specific metric such as floats, all CPUs are implied
 
-## Config file
+### Config file
 
-The config file is a JSON document with four top-level sections.
+All durations are specified as string that will be parsed [like this](https://pkg.go.dev/time#ParseDuration) (Allowed suffixes: `s`, `m`, `h`, ...).
 
-### `main`
+- `metrics`: Map of metric-name to objects with the following properties
+    - `frequency`: Timestep/Interval/Resolution of this metric
+    - `aggregation`: Can be `"sum"`, `"avg"` or `null`
+        - `null` means aggregation across nodes is forbidden for this metric
+        - `"sum"` means that values from the child levels are summed up for the parent level
+        - `"avg"` means that values from the child levels are averaged for the parent level
+    - `scope`: Unused at the moment, should be something like `"node"`, `"socket"` or `"hwthread"`
+- `nats`:
+    - `address`: Url of NATS.io server, example: "nats://localhost:4222"
+    - `username` and `password`: Optional, if provided use those for the connection
+    - `subscriptions`:
+        - `subscribe-to`: Where to expect the measurements to be published
+        - `cluster-tag`: Default value for the cluster tag
+- `http-api`:
+    - `address`: Address to bind to, for example `0.0.0.0:8080`
+    - `https-cert-file` and `https-key-file`: Optional, if provided enable HTTPS using those files as certificate/key
+- `jwt-public-key`: Base64 encoded string, use this to verify requests to the HTTP API
+- `retention-on-memory`: Keep all values in memory for at least that amount of time
+- `checkpoints`:
+    - `interval`: Do checkpoints every X seconds/minutes/hours
+    - `directory`: Path to a directory
+    - `restore`: After a restart, load the last X seconds/minutes/hours of data back into memory
+- `archive`:
+    - `interval`: Move and compress all checkpoints not needed anymore every X seconds/minutes/hours
+    - `directory`: Path to a directory
 
-```json
+### Test the complete setup (excluding ClusterCockpit itself)
-"main": {
-  "addr": "0.0.0.0:8082",
-  "https-cert-file": "",
-  "https-key-file": "",
-  "jwt-public-key": "<base64-encoded Ed25519 public key>",
-  "user": "",
-  "group": "",
-  "backend-url": ""
-}
-```
 
-- `addr`: Address and port to listen on (default: `0.0.0.0:8082`)
+There are two ways for sending data to the cc-metric-store, both of which are supported by the [cc-metric-collector](https://github.com/ClusterCockpit/cc-metric-collector). This example uses Nats, the alternative is to use HTTP.
-- `https-cert-file` / `https-key-file`: Paths to TLS certificate/key for HTTPS
-- `jwt-public-key`: Base64-encoded Ed25519 public key for JWT authentication. If empty, no auth is required.
-- `user` / `group`: Drop privileges to this user/group after startup
-- `backend-url`: Optional URL of a cc-backend instance used as node provider
-
-### `metrics`
-
-Per-metric configuration. Each key is the metric name:
-
-```json
-"metrics": {
-  "cpu_load": { "frequency": 60, "aggregation": null },
-  "flops_any": { "frequency": 60, "aggregation": "sum" },
-  "cpu_user":  { "frequency": 60, "aggregation": "avg" }
-}
-```
-
-- `frequency`: Sampling interval in seconds
-- `aggregation`: How to aggregate sub-level data: `"sum"`, `"avg"`, or `null` (no aggregation)
-
-### `metric-store`
-
-```json
-"metric-store": {
-  "checkpoints": {
-    "file-format": "wal",
-    "directory": "./var/checkpoints"
-  },
-  "memory-cap": 100,
-  "retention-in-memory": "24h",
-  "num-workers": 0,
-  "cleanup": {
-    "mode": "archive",
-    "directory": "./var/archive"
-  },
-  "nats-subscriptions": [
-    { "subscribe-to": "hpc-nats", "cluster-tag": "fritz" }
-  ]
-}
-```
-
-- `checkpoints.file-format`: Checkpoint format: `"json"` (default, human-readable) or `"wal"` (binary WAL, crash-safe). See [Checkpoint formats](#checkpoint-formats) below.
-- `checkpoints.directory`: Root directory for checkpoint files (organized as `<dir>/<cluster>/<host>/`)
-- `memory-cap`: Approximate memory cap in MB for metric buffers
-- `retention-in-memory`: How long to keep data in memory (e.g. `"48h"`)
-- `num-workers`: Number of parallel workers for checkpoint/archive I/O (0 = auto, capped at 10)
-- `cleanup.mode`: What to do with data older than `retention-in-memory`: `"archive"` (write Parquet) or `"delete"`
-- `cleanup.directory`: Root directory for Parquet archive files (required when `mode` is `"archive"`)
-- `nats-subscriptions`: List of NATS subjects to subscribe to, with associated cluster tag
-
-### Checkpoint formats
-
-The `checkpoints.file-format` field controls how in-memory data is persisted to disk.
-
-**`"json"` (default)** — human-readable JSON snapshots written periodically. Each
-snapshot is stored as `<dir>/<cluster>/<host>/<timestamp>.json` and contains the
-full metric hierarchy. Easy to inspect and recover manually, but larger on disk
-and slower to write.
-
-**`"wal"`** — binary Write-Ahead Log format designed for crash safety. Two file
-types are used per host:
-
-- `current.wal` — append-only binary log. Every incoming data point is appended
-  immediately (magic `0xCC1DA7A1`, 4-byte CRC32 per record). Truncated trailing
-  records from unclean shutdowns are silently skipped on restart.
-- `<timestamp>.bin` — binary snapshot written at each checkpoint interval
-  (magic `0xCC5B0001`). Contains the complete hierarchical metric state
-  column-by-column. Written atomically via a `.tmp` rename.
-
-On startup the most recent `.bin` snapshot is loaded, then any remaining WAL
-entries are replayed on top. The WAL is rotated (old file deleted, new one
-started) after each successful snapshot.
-
-The `"wal"` option is the default and will be the only supported option in the
-future. The `"json"` checkpoint format is still provided to migrate from
-previous cc-metric-store version.
-
-### Parquet archive
-
-When `cleanup.mode` is `"archive"`, data that ages out of the in-memory
-retention window is written to [Apache Parquet](https://parquet.apache.org/)
-files before being freed. Files are organized as:
-
-```
-<cleanup.directory>/
-  <cluster>/
-    <timestamp>.parquet
-```
-
-One Parquet file is produced per cluster per cleanup run, consolidating all
-hosts. Rows use a long (tidy) schema:
-
-| Column      | Type    | Description                                                             |
-| ----------- | ------- | ----------------------------------------------------------------------- |
-| `cluster`   | string  | Cluster name                                                            |
-| `hostname`  | string  | Host name                                                               |
-| `metric`    | string  | Metric name                                                             |
-| `scope`     | string  | Hardware scope (`node`, `socket`, `core`, `hwthread`, `accelerator`, …) |
-| `scope_id`  | string  | Numeric ID within the scope (e.g. `"0"`)                                |
-| `timestamp` | int64   | Unix timestamp (seconds)                                                |
-| `frequency` | int64   | Sampling interval in seconds                                            |
-| `value`     | float32 | Metric value                                                            |
-
-Files are compressed with Zstandard and sorted by `(cluster, hostname, metric,
-timestamp)` for efficient columnar reads. The `cpu` prefix in the tree is
-treated as an alias for `hwthread` scope.
-
-### `nats`
-
-```json
-"nats": {
-  "address": "nats://0.0.0.0:4222",
-  "username": "root",
-  "password": "root"
-}
-```
-
-NATS connection is optional. If not configured, only the HTTP write endpoint is available.
-
-For more information see the ClusterCockpit documentation [website](https://clustercockpit.org/docs/reference/cc-metric-store/ccms-configuration/).
-
-## Test the complete setup (excluding cc-backend itself)
-
-There are two ways for sending data to the cc-metric-store, both of which are
-supported by the
-[cc-metric-collector](https://github.com/ClusterCockpit/cc-metric-collector).
-This example uses NATS; the alternative is to use HTTP.
 
 ```sh
 # Only needed once, downloads the docker image
@@ -269,9 +105,7 @@ docker pull nats:latest
 docker run -p 4222:4222 -ti nats:latest
 ```
 
-Second, build and start the
+Second, build and start the [cc-metric-collector](https://github.com/ClusterCockpit/cc-metric-collector) using the following as Sink-Config:
-[cc-metric-collector](https://github.com/ClusterCockpit/cc-metric-collector)
-using the following as Sink-Config:
 
 ```json
 {
@@ -282,43 +116,28 @@ using the following as Sink-Config:
 }
 ```
 
-Third, build and start the metric store. For this example here, the
+Third, build and start the metric store. For this example here, the `config.json` file
-`config.json` file already in the repository should work just fine.
+already in the repository should work just fine.
 
 ```sh
 # Assuming you have a clone of this repo in ./cc-metric-store:
 cd cc-metric-store
-make
+go get
+go build
 ./cc-metric-store
 ```
 
-And finally, use the API to fetch some data. The API is protected by JWT based
+And finally, use the API to fetch some data. The API is protected by JWT based authentication if `jwt-public-key` is set in `config.json`. You can use this JWT for testing: `eyJ0eXAiOiJKV1QiLCJhbGciOiJFZERTQSJ9.eyJ1c2VyIjoiYWRtaW4iLCJyb2xlcyI6WyJST0xFX0FETUlOIiwiUk9MRV9BTkFMWVNUIiwiUk9MRV9VU0VSIl19.d-3_3FZTsadPjDEdsWrrQ7nS0edMAR4zjl-eK7rJU3HziNBfI9PDHDIpJVHTNN5E5SlLGLFXctWyKAkwhXL-Dw`
-authentication if `jwt-public-key` is set in `config.json`. You can use this JWT
-for testing:
-`eyJ0eXAiOiJKV1QiLCJhbGciOiJFZERTQSJ9.eyJ1c2VyIjoiYWRtaW4iLCJyb2xlcyI6WyJST0xFX0FETUlOIiwiUk9MRV9BTkFMWVNUIiwiUk9MRV9VU0VSIl19.d-3_3FZTsadPjDEdsWrrQ7nS0edMAR4zjl-eK7rJU3HziNBfI9PDHDIpJVHTNN5E5SlLGLFXctWyKAkwhXL-Dw`
 
 ```sh
 JWT="eyJ0eXAiOiJKV1QiLCJhbGciOiJFZERTQSJ9.eyJ1c2VyIjoiYWRtaW4iLCJyb2xlcyI6WyJST0xFX0FETUlOIiwiUk9MRV9BTkFMWVNUIiwiUk9MRV9VU0VSIl19.d-3_3FZTsadPjDEdsWrrQ7nS0edMAR4zjl-eK7rJU3HziNBfI9PDHDIpJVHTNN5E5SlLGLFXctWyKAkwhXL-Dw"
 
-# If the collector and store and nats-server have been running for at least 60 seconds on the same host:
+# If the collector and store and nats-server have been running for at least 60 seconds on the same host, you may run:
-curl -H "Authorization: Bearer $JWT" \
+curl -H "Authorization: Bearer $JWT" -D - "http://localhost:8080/api/query" -d "{ \"cluster\": \"testcluster\", \"from\": $(expr $(date +%s) - 60), \"to\": $(date +%s), \"queries\": [{
-     "http://localhost:8082/api/query/" \
+  \"metric\": \"load_one\",
-     -d '{
+  \"host\": \"$(hostname)\"
-       "cluster": "testcluster",
+}] }"
-       "from": '"$(expr $(date +%s) - 60)"',
+
-       "to": '"$(date +%s)"',
+# ...
-       "queries": [{ "metric": "cpu_load", "host": "'"$(hostname)"'" }]
-     }'
 ```
 
-For debugging, the debug endpoint dumps the current content to stdout:
-
-```sh
-JWT="eyJ0eXAiOiJKV1QiLCJhbGciOiJFZERTQSJ9.eyJ1c2VyIjoiYWRtaW4iLCJyb2xlcyI6WyJST0xFX0FETUlOIiwiUk9MRV9BTkFMWVNUIiwiUk9MRV9VU0VSIl19.d-3_3FZTsadPjDEdsWrrQ7nS0edMAR4zjl-eK7rJU3HziNBfI9PDHDIpJVHTNN5E5SlLGLFXctWyKAkwhXL-Dw"
-
-# Dump everything
-curl -H "Authorization: Bearer $JWT" "http://localhost:8082/api/debug/"
-
-# Dump a specific selector (colon-separated path)
-curl -H "Authorization: Bearer $JWT" "http://localhost:8082/api/debug/?selector=testcluster:host1"
-```

ReleaseNotes.md

@@ -1,35 +0,0 @@
-# `cc-metric-store` version 1.5.0
-
-This is a major release of `cc-metric-store`, the metric timeseries cache
-implementation of ClusterCockpit. Since the storage engine is now part of
-`cc-backend` we will follow the version number of `cc-backend`.
-For release specific notes visit the [ClusterCockpit Documentation](https://clusterockpit.org/docs/release/).
-
-## Breaking changes
-
-- The internal `memorystore`, `avro`, `resampler`, and `util` packages have been
-  removed. The storage engine is now provided by the
-  [`cc-backend`](https://github.com/ClusterCockpit/cc-backend) package
-  (`cc-backend/pkg/metricstore`). This repository is now the HTTP API wrapper
-  only.
-- The configuration schema has changed. Refer to `configs/config.json` for the
-  updated structure.
-
-## Notable changes
-
-- **Storage engine extracted to `cc-backend` library**: The entire in-memory
-  time-series storage engine was moved to `cc-backend/pkg/metricstore`. This
-  reduces duplication in the ClusterCockpit suite and enables shared maintenance
-  of the storage layer.
-- **HealthCheck API endpoint**: New `GET /api/healthcheck/` endpoint reports the
-  health status of cluster nodes.
-- **Dynamic memory management**: Memory limits can now be adjusted at runtime via
-  a callback from the `cc-backend` library.
-- **Configuration schema validation**: The config and metric config JSON schemas
-  have been updated and are now validated against the structs they describe.
-- **Startup refactored**: Application startup has been split into `cli.go` and
-  `server.go` for clearer separation of concerns.
-- **`go fix` applied**: Codebase updated to current Go idioms.
-- **Dependency upgrades**: `nats.go` bumped from 1.36.0 to 1.47.0;
-  `cc-lib` updated to v2.8.0; `cc-backend` updated to v1.5.0; various other
-  module upgrades.

TODO.md

@@ -0,0 +1,14 @@
+# TODO
+
+- Improve checkpoints/archives
+    - Store information in each buffer if already archived
+    - Do not create new checkpoint if all buffers already archived
+- Missing Testcases:
+    - General tests
+    - Check for corner cases that should fail gracefully
+    - Write a more realistic `ToArchive`/`FromArchive` tests
+- Optimization: Once a buffer is full, calculate min, max and avg
+    - Calculate averages buffer-wise, average weighted by length of buffer
+    - Only the head-buffer needs to be fully traversed
+- Optimization: If aggregating over hwthreads/cores/sockets cache those results and reuse some of that for new queres aggregating only over the newer data
+- ...

api.go

@@ -0,0 +1,424 @@
+package main
+
+import (
+	"bufio"
+	"context"
+	"crypto/ed25519"
+	"encoding/base64"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"log"
+	"math"
+	"net/http"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/golang-jwt/jwt/v4"
+	"github.com/gorilla/mux"
+	"github.com/influxdata/line-protocol/v2/lineprotocol"
+)
+
+type ApiMetricData struct {
+	Error *string    `json:"error,omitempty"`
+	From  int64      `json:"from"`
+	To    int64      `json:"to"`
+	Data  FloatArray `json:"data,omitempty"`
+	Avg   Float      `json:"avg"`
+	Min   Float      `json:"min"`
+	Max   Float      `json:"max"`
+	Unit  string     `json:"unit,omitempty"`
+}
+
+// TODO: Optimize this, just like the stats endpoint!
+func (data *ApiMetricData) AddStats() {
+	n := 0
+	sum, min, max := 0.0, math.MaxFloat64, -math.MaxFloat64
+	for _, x := range data.Data {
+		if x.IsNaN() {
+			continue
+		}
+
+		n += 1
+		sum += float64(x)
+		min = math.Min(min, float64(x))
+		max = math.Max(max, float64(x))
+	}
+
+	if n > 0 {
+		avg := sum / float64(n)
+		data.Avg = Float(avg)
+		data.Min = Float(min)
+		data.Max = Float(max)
+	} else {
+		data.Avg, data.Min, data.Max = NaN, NaN, NaN
+	}
+}
+
+func (data *ApiMetricData) ScaleBy(f Float) {
+	if f == 0 || f == 1 {
+		return
+	}
+
+	data.Avg *= f
+	data.Min *= f
+	data.Max *= f
+	for i := 0; i < len(data.Data); i++ {
+		data.Data[i] *= f
+	}
+}
+
+func (data *ApiMetricData) PadDataWithNull(from, to int64, metric string) {
+	minfo, ok := memoryStore.metrics[metric]
+	if !ok {
+		return
+	}
+
+	if (data.From / minfo.Frequency) > (from / minfo.Frequency) {
+		padfront := int((data.From / minfo.Frequency) - (from / minfo.Frequency))
+		ndata := make([]Float, 0, padfront+len(data.Data))
+		for i := 0; i < padfront; i++ {
+			ndata = append(ndata, NaN)
+		}
+		for j := 0; j < len(data.Data); j++ {
+			ndata = append(ndata, data.Data[j])
+		}
+		data.Data = ndata
+	}
+}
+
+func handleFree(rw http.ResponseWriter, r *http.Request) {
+	rawTo := r.URL.Query().Get("to")
+	if rawTo == "" {
+		http.Error(rw, "'to' is a required query parameter", http.StatusBadRequest)
+		return
+	}
+
+	to, err := strconv.ParseInt(rawTo, 10, 64)
+	if err != nil {
+		http.Error(rw, err.Error(), http.StatusBadRequest)
+		return
+	}
+
+	// TODO: lastCheckpoint might be modified by different go-routines.
+	// Load it using the sync/atomic package?
+	freeUpTo := lastCheckpoint.Unix()
+	if to < freeUpTo {
+		freeUpTo = to
+	}
+
+	if r.Method != http.MethodPost {
+		http.Error(rw, "Method Not Allowed", http.StatusMethodNotAllowed)
+		return
+	}
+
+	bodyDec := json.NewDecoder(r.Body)
+	var selectors [][]string
+	err = bodyDec.Decode(&selectors)
+	if err != nil {
+		http.Error(rw, err.Error(), http.StatusBadRequest)
+		return
+	}
+
+	n := 0
+	for _, sel := range selectors {
+		bn, err := memoryStore.Free(sel, freeUpTo)
+		if err != nil {
+			http.Error(rw, err.Error(), http.StatusInternalServerError)
+			return
+		}
+
+		n += bn
+	}
+
+	rw.WriteHeader(http.StatusOK)
+	rw.Write([]byte(fmt.Sprintf("buffers freed: %d\n", n)))
+}
+
+func handleWrite(rw http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodPost {
+		http.Error(rw, "Method Not Allowed", http.StatusMethodNotAllowed)
+		return
+	}
+
+	bytes, err := io.ReadAll(r.Body)
+	if err != nil {
+		log.Printf("error while reading request body: %s", err.Error())
+		http.Error(rw, err.Error(), http.StatusInternalServerError)
+		return
+	}
+
+	if debugDump != io.Discard {
+		now := time.Now()
+		msg := make([]byte, 0, 512)
+		msg = append(msg, "\n--- local unix time: "...)
+		msg = strconv.AppendInt(msg, now.Unix(), 10)
+		msg = append(msg, " ---\n"...)
+
+		debugDumpLock.Lock()
+		defer debugDumpLock.Unlock()
+		if _, err := debugDump.Write(msg); err != nil {
+			log.Printf("error while writing to debug dump: %s", err.Error())
+		}
+		if _, err := debugDump.Write(bytes); err != nil {
+			log.Printf("error while writing to debug dump: %s", err.Error())
+		}
+		return
+	}
+
+	dec := lineprotocol.NewDecoderWithBytes(bytes)
+	if err := decodeLine(dec, r.URL.Query().Get("cluster")); err != nil {
+		log.Printf("/api/write error: %s", err.Error())
+		http.Error(rw, err.Error(), http.StatusBadRequest)
+		return
+	}
+	rw.WriteHeader(http.StatusOK)
+}
+
+type ApiQueryRequest struct {
+	Cluster     string     `json:"cluster"`
+	From        int64      `json:"from"`
+	To          int64      `json:"to"`
+	WithStats   bool       `json:"with-stats"`
+	WithUnit    bool       `json:"with-unit,omitempty"`
+	WithData    bool       `json:"with-data"`
+	WithPadding bool       `json:"with-padding"`
+	Queries     []ApiQuery `json:"queries"`
+	ForAllNodes []string   `json:"for-all-nodes"`
+}
+
+type ApiQueryResponse struct {
+	Queries []ApiQuery        `json:"queries,omitempty"`
+	Results [][]ApiMetricData `json:"results"`
+}
+
+type ApiQuery struct {
+	Metric      string   `json:"metric"`
+	Hostname    string   `json:"host"`
+	Aggregate   bool     `json:"aggreg"`
+	ScaleFactor Float    `json:"scale-by,omitempty"`
+	Type        *string  `json:"type,omitempty"`
+	TypeIds     []string `json:"type-ids,omitempty"`
+	SubType     *string  `json:"subtype,omitempty"`
+	SubTypeIds  []string `json:"subtype-ids,omitempty"`
+}
+
+func handleQuery(rw http.ResponseWriter, r *http.Request) {
+	var err error
+	var req ApiQueryRequest = ApiQueryRequest{WithStats: true, WithData: true, WithPadding: true}
+	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
+		http.Error(rw, err.Error(), http.StatusBadRequest)
+		return
+	}
+
+	response := ApiQueryResponse{
+		Results: make([][]ApiMetricData, 0, len(req.Queries)),
+	}
+	if req.ForAllNodes != nil {
+		nodes := memoryStore.ListChildren([]string{req.Cluster})
+		for _, node := range nodes {
+			for _, metric := range req.ForAllNodes {
+				q := ApiQuery{
+					Metric:   metric,
+					Hostname: node,
+				}
+				req.Queries = append(req.Queries, q)
+				response.Queries = append(response.Queries, q)
+			}
+		}
+	}
+
+	for _, query := range req.Queries {
+		sels := make([]Selector, 0, 1)
+		if query.Aggregate || query.Type == nil {
+			sel := Selector{{String: req.Cluster}, {String: query.Hostname}}
+			if query.Type != nil {
+				if len(query.TypeIds) == 1 {
+					sel = append(sel, SelectorElement{String: *query.Type + query.TypeIds[0]})
+				} else {
+					ids := make([]string, len(query.TypeIds))
+					for i, id := range query.TypeIds {
+						ids[i] = *query.Type + id
+					}
+					sel = append(sel, SelectorElement{Group: ids})
+				}
+
+				if query.SubType != nil {
+					if len(query.SubTypeIds) == 1 {
+						sel = append(sel, SelectorElement{String: *query.SubType + query.SubTypeIds[0]})
+					} else {
+						ids := make([]string, len(query.SubTypeIds))
+						for i, id := range query.SubTypeIds {
+							ids[i] = *query.SubType + id
+						}
+						sel = append(sel, SelectorElement{Group: ids})
+					}
+				}
+			}
+			sels = append(sels, sel)
+		} else {
+			for _, typeId := range query.TypeIds {
+				if query.SubType != nil {
+					for _, subTypeId := range query.SubTypeIds {
+						sels = append(sels, Selector{
+							{String: req.Cluster}, {String: query.Hostname},
+							{String: *query.Type + typeId},
+							{String: *query.SubType + subTypeId}})
+					}
+				} else {
+					sels = append(sels, Selector{
+						{String: req.Cluster},
+						{String: query.Hostname},
+						{String: *query.Type + typeId}})
+				}
+			}
+		}
+
+		// log.Printf("query: %#v\n", query)
+		// log.Printf("sels: %#v\n", sels)
+
+		res := make([]ApiMetricData, 0, len(sels))
+		for _, sel := range sels {
+			data := ApiMetricData{}
+			data.Data, data.From, data.To, data.Unit, err = memoryStore.Read(sel, query.Metric, req.From, req.To)
+			// log.Printf("data: %#v, %#v, %#v, %#v", data.Data, data.From, data.To, err)
+			if err != nil {
+				msg := err.Error()
+				data.Error = &msg
+				res = append(res, data)
+				continue
+			}
+
+			if req.WithStats {
+				data.AddStats()
+			}
+			if query.ScaleFactor != 0 {
+				data.ScaleBy(query.ScaleFactor)
+			}
+			if req.WithPadding {
+				data.PadDataWithNull(req.From, req.To, query.Metric)
+			}
+			if !req.WithData {
+				data.Data = nil
+			}
+			res = append(res, data)
+		}
+		response.Results = append(response.Results, res)
+	}
+
+	rw.Header().Set("Content-Type", "application/json")
+	bw := bufio.NewWriter(rw)
+	defer bw.Flush()
+	if err := json.NewEncoder(bw).Encode(response); err != nil {
+		log.Print(err)
+		return
+	}
+}
+
+func authentication(next http.Handler, publicKey ed25519.PublicKey) http.Handler {
+	cacheLock := sync.RWMutex{}
+	cache := map[string]*jwt.Token{}
+
+	return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
+		authheader := r.Header.Get("Authorization")
+		if authheader == "" || !strings.HasPrefix(authheader, "Bearer ") {
+			http.Error(rw, "Use JWT Authentication", http.StatusUnauthorized)
+			return
+		}
+
+		rawtoken := authheader[len("Bearer "):]
+		cacheLock.RLock()
+		token, ok := cache[rawtoken]
+		cacheLock.RUnlock()
+		if ok && token.Claims.Valid() == nil {
+			next.ServeHTTP(rw, r)
+			return
+		}
+
+		// The actual token is ignored for now.
+		// In case expiration and so on are specified, the Parse function
+		// already returns an error for expired tokens.
+		var err error
+		token, err = jwt.Parse(rawtoken, func(t *jwt.Token) (interface{}, error) {
+			if t.Method != jwt.SigningMethodEdDSA {
+				return nil, errors.New("only Ed25519/EdDSA supported")
+			}
+
+			return publicKey, nil
+		})
+
+		if err != nil {
+			http.Error(rw, err.Error(), http.StatusUnauthorized)
+			return
+		}
+
+		cacheLock.Lock()
+		cache[rawtoken] = token
+		cacheLock.Unlock()
+
+		// Let request through...
+		next.ServeHTTP(rw, r)
+	})
+}
+
+func StartApiServer(ctx context.Context, httpConfig *HttpConfig) error {
+	r := mux.NewRouter()
+
+	r.HandleFunc("/api/free", handleFree)
+	r.HandleFunc("/api/write", handleWrite)
+	r.HandleFunc("/api/query", handleQuery)
+	r.HandleFunc("/api/debug", func(rw http.ResponseWriter, r *http.Request) {
+		raw := r.URL.Query().Get("selector")
+		selector := []string{}
+		if len(raw) != 0 {
+			selector = strings.Split(raw, ":")
+		}
+
+		if err := memoryStore.DebugDump(bufio.NewWriter(rw), selector); err != nil {
+			rw.WriteHeader(http.StatusBadRequest)
+			rw.Write([]byte(err.Error()))
+		}
+	})
+
+	server := &http.Server{
+		Handler:      r,
+		Addr:         httpConfig.Address,
+		WriteTimeout: 30 * time.Second,
+		ReadTimeout:  30 * time.Second,
+	}
+
+	if len(conf.JwtPublicKey) > 0 {
+		buf, err := base64.StdEncoding.DecodeString(conf.JwtPublicKey)
+		if err != nil {
+			return err
+		}
+		publicKey := ed25519.PublicKey(buf)
+		server.Handler = authentication(server.Handler, publicKey)
+	}
+
+	go func() {
+		if httpConfig.CertFile != "" && httpConfig.KeyFile != "" {
+			log.Printf("API https endpoint listening on '%s'\n", httpConfig.Address)
+			err := server.ListenAndServeTLS(httpConfig.CertFile, httpConfig.KeyFile)
+			if err != nil && err != http.ErrServerClosed {
+				log.Println(err)
+			}
+		} else {
+			log.Printf("API http endpoint listening on '%s'\n", httpConfig.Address)
+			err := server.ListenAndServe()
+			if err != nil && err != http.ErrServerClosed {
+				log.Println(err)
+			}
+		}
+	}()
+
+	for {
+		<-ctx.Done()
+		err := server.Shutdown(context.Background())
+		log.Println("API server shut down")
+		return err
+	}
+}

api/swagger.json

@@ -1,456 +0,0 @@
-{
-    "swagger": "2.0",
-    "info": {
-        "description": "API for cc-metric-store",
-        "title": "cc-metric-store REST API",
-        "contact": {
-            "name": "ClusterCockpit Project",
-            "url": "https://clustercockpit.org",
-            "email": "support@clustercockpit.org"
-        },
-        "license": {
-            "name": "MIT License",
-            "url": "https://opensource.org/licenses/MIT"
-        },
-        "version": "1.0.0"
-    },
-    "host": "localhost:8082",
-    "basePath": "/api/",
-    "paths": {
-        "/debug/": {
-            "post": {
-                "description": "This endpoint allows the users to print the content of",
-                "produces": [
-                    "application/json"
-                ],
-                "tags": [
-                    "debug"
-                ],
-                "summary": "Debug endpoint",
-                "parameters": [
-                    {
-                        "type": "string",
-                        "description": "Selector",
-                        "name": "selector",
-                        "in": "query"
-                    }
-                ],
-                "responses": {
-                    "200": {
-                        "description": "Debug dump",
-                        "schema": {
-                            "type": "string"
-                        }
-                    },
-                    "400": {
-                        "description": "Bad Request",
-                        "schema": {
-                            "$ref": "#/definitions/api.ErrorResponse"
-                        }
-                    },
-                    "401": {
-                        "description": "Unauthorized",
-                        "schema": {
-                            "$ref": "#/definitions/api.ErrorResponse"
-                        }
-                    },
-                    "403": {
-                        "description": "Forbidden",
-                        "schema": {
-                            "$ref": "#/definitions/api.ErrorResponse"
-                        }
-                    },
-                    "500": {
-                        "description": "Internal Server Error",
-                        "schema": {
-                            "$ref": "#/definitions/api.ErrorResponse"
-                        }
-                    }
-                },
-                "security": [
-                    {
-                        "ApiKeyAuth": []
-                    }
-                ]
-            }
-        },
-        "/free/": {
-            "post": {
-                "description": "This endpoint allows the users to free the Buffers from the",
-                "produces": [
-                    "application/json"
-                ],
-                "tags": [
-                    "free"
-                ],
-                "parameters": [
-                    {
-                        "type": "string",
-                        "description": "up to timestamp",
-                        "name": "to",
-                        "in": "query"
-                    }
-                ],
-                "responses": {
-                    "200": {
-                        "description": "ok",
-                        "schema": {
-                            "type": "string"
-                        }
-                    },
-                    "400": {
-                        "description": "Bad Request",
-                        "schema": {
-                            "$ref": "#/definitions/api.ErrorResponse"
-                        }
-                    },
-                    "401": {
-                        "description": "Unauthorized",
-                        "schema": {
-                            "$ref": "#/definitions/api.ErrorResponse"
-                        }
-                    },
-                    "403": {
-                        "description": "Forbidden",
-                        "schema": {
-                            "$ref": "#/definitions/api.ErrorResponse"
-                        }
-                    },
-                    "500": {
-                        "description": "Internal Server Error",
-                        "schema": {
-                            "$ref": "#/definitions/api.ErrorResponse"
-                        }
-                    }
-                },
-                "security": [
-                    {
-                        "ApiKeyAuth": []
-                    }
-                ]
-            }
-        },
-        "/healthcheck/": {
-            "get": {
-                "description": "This endpoint allows the users to check if a node is healthy",
-                "produces": [
-                    "application/json"
-                ],
-                "tags": [
-                    "healthcheck"
-                ],
-                "summary": "HealthCheck endpoint",
-                "parameters": [
-                    {
-                        "type": "string",
-                        "description": "Selector",
-                        "name": "selector",
-                        "in": "query"
-                    }
-                ],
-                "responses": {
-                    "200": {
-                        "description": "Debug dump",
-                        "schema": {
-                            "type": "string"
-                        }
-                    },
-                    "400": {
-                        "description": "Bad Request",
-                        "schema": {
-                            "$ref": "#/definitions/api.ErrorResponse"
-                        }
-                    },
-                    "401": {
-                        "description": "Unauthorized",
-                        "schema": {
-                            "$ref": "#/definitions/api.ErrorResponse"
-                        }
-                    },
-                    "403": {
-                        "description": "Forbidden",
-                        "schema": {
-                            "$ref": "#/definitions/api.ErrorResponse"
-                        }
-                    },
-                    "500": {
-                        "description": "Internal Server Error",
-                        "schema": {
-                            "$ref": "#/definitions/api.ErrorResponse"
-                        }
-                    }
-                },
-                "security": [
-                    {
-                        "ApiKeyAuth": []
-                    }
-                ]
-            }
-        },
-        "/query/": {
-            "get": {
-                "description": "This endpoint allows the users to retrieve data from the",
-                "consumes": [
-                    "application/json"
-                ],
-                "produces": [
-                    "application/json"
-                ],
-                "tags": [
-                    "query"
-                ],
-                "summary": "Query metrics",
-                "parameters": [
-                    {
-                        "description": "API query payload object",
-                        "name": "request",
-                        "in": "body",
-                        "required": true,
-                        "schema": {
-                            "$ref": "#/definitions/api.APIQueryRequest"
-                        }
-                    }
-                ],
-                "responses": {
-                    "200": {
-                        "description": "API query response object",
-                        "schema": {
-                            "$ref": "#/definitions/api.APIQueryResponse"
-                        }
-                    },
-                    "400": {
-                        "description": "Bad Request",
-                        "schema": {
-                            "$ref": "#/definitions/api.ErrorResponse"
-                        }
-                    },
-                    "401": {
-                        "description": "Unauthorized",
-                        "schema": {
-                            "$ref": "#/definitions/api.ErrorResponse"
-                        }
-                    },
-                    "403": {
-                        "description": "Forbidden",
-                        "schema": {
-                            "$ref": "#/definitions/api.ErrorResponse"
-                        }
-                    },
-                    "500": {
-                        "description": "Internal Server Error",
-                        "schema": {
-                            "$ref": "#/definitions/api.ErrorResponse"
-                        }
-                    }
-                },
-                "security": [
-                    {
-                        "ApiKeyAuth": []
-                    }
-                ]
-            }
-        },
-        "/write/": {
-            "post": {
-                "consumes": [
-                    "text/plain"
-                ],
-                "produces": [
-                    "application/json"
-                ],
-                "parameters": [
-                    {
-                        "type": "string",
-                        "description": "If the lines in the body do not have a cluster tag, use this value instead.",
-                        "name": "cluster",
-                        "in": "query"
-                    }
-                ],
-                "responses": {
-                    "200": {
-                        "description": "ok",
-                        "schema": {
-                            "type": "string"
-                        }
-                    },
-                    "400": {
-                        "description": "Bad Request",
-                        "schema": {
-                            "$ref": "#/definitions/api.ErrorResponse"
-                        }
-                    },
-                    "401": {
-                        "description": "Unauthorized",
-                        "schema": {
-                            "$ref": "#/definitions/api.ErrorResponse"
-                        }
-                    },
-                    "403": {
-                        "description": "Forbidden",
-                        "schema": {
-                            "$ref": "#/definitions/api.ErrorResponse"
-                        }
-                    },
-                    "500": {
-                        "description": "Internal Server Error",
-                        "schema": {
-                            "$ref": "#/definitions/api.ErrorResponse"
-                        }
-                    }
-                },
-                "security": [
-                    {
-                        "ApiKeyAuth": []
-                    }
-                ]
-            }
-        }
-    },
-    "definitions": {
-        "api.APIMetricData": {
-            "type": "object",
-            "properties": {
-                "avg": {
-                    "type": "number"
-                },
-                "data": {
-                    "type": "array",
-                    "items": {
-                        "type": "number"
-                    }
-                },
-                "error": {
-                    "type": "string"
-                },
-                "from": {
-                    "type": "integer"
-                },
-                "max": {
-                    "type": "number"
-                },
-                "min": {
-                    "type": "number"
-                },
-                "resolution": {
-                    "type": "integer"
-                },
-                "to": {
-                    "type": "integer"
-                }
-            }
-        },
-        "api.APIQuery": {
-            "type": "object",
-            "properties": {
-                "aggreg": {
-                    "type": "boolean"
-                },
-                "host": {
-                    "type": "string"
-                },
-                "metric": {
-                    "type": "string"
-                },
-                "resolution": {
-                    "type": "integer"
-                },
-                "scale-by": {
-                    "type": "number"
-                },
-                "subtype": {
-                    "type": "string"
-                },
-                "subtype-ids": {
-                    "type": "array",
-                    "items": {
-                        "type": "string"
-                    }
-                },
-                "type": {
-                    "type": "string"
-                },
-                "type-ids": {
-                    "type": "array",
-                    "items": {
-                        "type": "string"
-                    }
-                }
-            }
-        },
-        "api.APIQueryRequest": {
-            "type": "object",
-            "properties": {
-                "cluster": {
-                    "type": "string"
-                },
-                "for-all-nodes": {
-                    "type": "array",
-                    "items": {
-                        "type": "string"
-                    }
-                },
-                "from": {
-                    "type": "integer"
-                },
-                "queries": {
-                    "type": "array",
-                    "items": {
-                        "$ref": "#/definitions/api.APIQuery"
-                    }
-                },
-                "to": {
-                    "type": "integer"
-                },
-                "with-data": {
-                    "type": "boolean"
-                },
-                "with-padding": {
-                    "type": "boolean"
-                },
-                "with-stats": {
-                    "type": "boolean"
-                }
-            }
-        },
-        "api.APIQueryResponse": {
-            "type": "object",
-            "properties": {
-                "queries": {
-                    "type": "array",
-                    "items": {
-                        "$ref": "#/definitions/api.APIQuery"
-                    }
-                },
-                "results": {
-                    "type": "array",
-                    "items": {
-                        "type": "array",
-                        "items": {
-                            "$ref": "#/definitions/api.APIMetricData"
-                        }
-                    }
-                }
-            }
-        },
-        "api.ErrorResponse": {
-            "type": "object",
-            "properties": {
-                "error": {
-                    "description": "Error Message",
-                    "type": "string"
-                },
-                "status": {
-                    "description": "Statustext of Errorcode",
-                    "type": "string"
-                }
-            }
-        }
-    },
-    "securityDefinitions": {
-        "ApiKeyAuth": {
-            "type": "apiKey",
-            "name": "X-Auth-Token",
-            "in": "header"
-        }
-    }
-}

api/swagger.yaml

@@ -1,294 +0,0 @@
-basePath: /api/
-definitions:
-  api.APIMetricData:
-    properties:
-      avg:
-        type: number
-      data:
-        items:
-          type: number
-        type: array
-      error:
-        type: string
-      from:
-        type: integer
-      max:
-        type: number
-      min:
-        type: number
-      resolution:
-        type: integer
-      to:
-        type: integer
-    type: object
-  api.APIQuery:
-    properties:
-      aggreg:
-        type: boolean
-      host:
-        type: string
-      metric:
-        type: string
-      resolution:
-        type: integer
-      scale-by:
-        type: number
-      subtype:
-        type: string
-      subtype-ids:
-        items:
-          type: string
-        type: array
-      type:
-        type: string
-      type-ids:
-        items:
-          type: string
-        type: array
-    type: object
-  api.APIQueryRequest:
-    properties:
-      cluster:
-        type: string
-      for-all-nodes:
-        items:
-          type: string
-        type: array
-      from:
-        type: integer
-      queries:
-        items:
-          $ref: '#/definitions/api.APIQuery'
-        type: array
-      to:
-        type: integer
-      with-data:
-        type: boolean
-      with-padding:
-        type: boolean
-      with-stats:
-        type: boolean
-    type: object
-  api.APIQueryResponse:
-    properties:
-      queries:
-        items:
-          $ref: '#/definitions/api.APIQuery'
-        type: array
-      results:
-        items:
-          items:
-            $ref: '#/definitions/api.APIMetricData'
-          type: array
-        type: array
-    type: object
-  api.ErrorResponse:
-    properties:
-      error:
-        description: Error Message
-        type: string
-      status:
-        description: Statustext of Errorcode
-        type: string
-    type: object
-host: localhost:8082
-info:
-  contact:
-    email: support@clustercockpit.org
-    name: ClusterCockpit Project
-    url: https://clustercockpit.org
-  description: API for cc-metric-store
-  license:
-    name: MIT License
-    url: https://opensource.org/licenses/MIT
-  title: cc-metric-store REST API
-  version: 1.0.0
-paths:
-  /debug/:
-    post:
-      description: This endpoint allows the users to print the content of
-      parameters:
-      - description: Selector
-        in: query
-        name: selector
-        type: string
-      produces:
-      - application/json
-      responses:
-        "200":
-          description: Debug dump
-          schema:
-            type: string
-        "400":
-          description: Bad Request
-          schema:
-            $ref: '#/definitions/api.ErrorResponse'
-        "401":
-          description: Unauthorized
-          schema:
-            $ref: '#/definitions/api.ErrorResponse'
-        "403":
-          description: Forbidden
-          schema:
-            $ref: '#/definitions/api.ErrorResponse'
-        "500":
-          description: Internal Server Error
-          schema:
-            $ref: '#/definitions/api.ErrorResponse'
-      security:
-      - ApiKeyAuth: []
-      summary: Debug endpoint
-      tags:
-      - debug
-  /free/:
-    post:
-      description: This endpoint allows the users to free the Buffers from the
-      parameters:
-      - description: up to timestamp
-        in: query
-        name: to
-        type: string
-      produces:
-      - application/json
-      responses:
-        "200":
-          description: ok
-          schema:
-            type: string
-        "400":
-          description: Bad Request
-          schema:
-            $ref: '#/definitions/api.ErrorResponse'
-        "401":
-          description: Unauthorized
-          schema:
-            $ref: '#/definitions/api.ErrorResponse'
-        "403":
-          description: Forbidden
-          schema:
-            $ref: '#/definitions/api.ErrorResponse'
-        "500":
-          description: Internal Server Error
-          schema:
-            $ref: '#/definitions/api.ErrorResponse'
-      security:
-      - ApiKeyAuth: []
-      tags:
-      - free
-  /healthcheck/:
-    get:
-      description: This endpoint allows the users to check if a node is healthy
-      parameters:
-      - description: Selector
-        in: query
-        name: selector
-        type: string
-      produces:
-      - application/json
-      responses:
-        "200":
-          description: Debug dump
-          schema:
-            type: string
-        "400":
-          description: Bad Request
-          schema:
-            $ref: '#/definitions/api.ErrorResponse'
-        "401":
-          description: Unauthorized
-          schema:
-            $ref: '#/definitions/api.ErrorResponse'
-        "403":
-          description: Forbidden
-          schema:
-            $ref: '#/definitions/api.ErrorResponse'
-        "500":
-          description: Internal Server Error
-          schema:
-            $ref: '#/definitions/api.ErrorResponse'
-      security:
-      - ApiKeyAuth: []
-      summary: HealthCheck endpoint
-      tags:
-      - healthcheck
-  /query/:
-    get:
-      consumes:
-      - application/json
-      description: This endpoint allows the users to retrieve data from the
-      parameters:
-      - description: API query payload object
-        in: body
-        name: request
-        required: true
-        schema:
-          $ref: '#/definitions/api.APIQueryRequest'
-      produces:
-      - application/json
-      responses:
-        "200":
-          description: API query response object
-          schema:
-            $ref: '#/definitions/api.APIQueryResponse'
-        "400":
-          description: Bad Request
-          schema:
-            $ref: '#/definitions/api.ErrorResponse'
-        "401":
-          description: Unauthorized
-          schema:
-            $ref: '#/definitions/api.ErrorResponse'
-        "403":
-          description: Forbidden
-          schema:
-            $ref: '#/definitions/api.ErrorResponse'
-        "500":
-          description: Internal Server Error
-          schema:
-            $ref: '#/definitions/api.ErrorResponse'
-      security:
-      - ApiKeyAuth: []
-      summary: Query metrics
-      tags:
-      - query
-  /write/:
-    post:
-      consumes:
-      - text/plain
-      parameters:
-      - description: If the lines in the body do not have a cluster tag, use this
-          value instead.
-        in: query
-        name: cluster
-        type: string
-      produces:
-      - application/json
-      responses:
-        "200":
-          description: ok
-          schema:
-            type: string
-        "400":
-          description: Bad Request
-          schema:
-            $ref: '#/definitions/api.ErrorResponse'
-        "401":
-          description: Unauthorized
-          schema:
-            $ref: '#/definitions/api.ErrorResponse'
-        "403":
-          description: Forbidden
-          schema:
-            $ref: '#/definitions/api.ErrorResponse'
-        "500":
-          description: Internal Server Error
-          schema:
-            $ref: '#/definitions/api.ErrorResponse'
-      security:
-      - ApiKeyAuth: []
-securityDefinitions:
-  ApiKeyAuth:
-    in: header
-    name: X-Auth-Token
-    type: apiKey
-swagger: "2.0"

archive.go

@@ -0,0 +1,601 @@
+package main
+
+import (
+	"archive/zip"
+	"bufio"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"io/fs"
+	"log"
+	"os"
+	"path"
+	"path/filepath"
+	"runtime"
+	"sort"
+	"strconv"
+	"strings"
+	"sync"
+	"sync/atomic"
+)
+
+// Whenever changed, update MarshalJSON as well!
+type CheckpointMetrics struct {
+	Frequency int64   `json:"frequency"`
+	Start     int64   `json:"start"`
+	Unit      string  `json:"unit,omitempty"`
+	Data      []Float `json:"data"`
+}
+
+// As `Float` implements a custom MarshalJSON() function,
+// serializing an array of such types has more overhead
+// than one would assume (because of extra allocations, interfaces and so on).
+func (cm *CheckpointMetrics) MarshalJSON() ([]byte, error) {
+	buf := make([]byte, 0, 128+len(cm.Data)*8)
+	buf = append(buf, `{"frequency":`...)
+	buf = strconv.AppendInt(buf, cm.Frequency, 10)
+	buf = append(buf, `,"start":`...)
+	buf = strconv.AppendInt(buf, cm.Start, 10)
+	buf = append(buf, fmt.Sprintf(`,"unit":"%s"`, cm.Unit)...)
+	buf = append(buf, `,"data":[`...)
+	for i, x := range cm.Data {
+		if i != 0 {
+			buf = append(buf, ',')
+		}
+		if x.IsNaN() {
+			buf = append(buf, `null`...)
+		} else {
+			buf = strconv.AppendFloat(buf, float64(x), 'f', 1, 32)
+		}
+	}
+	buf = append(buf, `]}`...)
+	return buf, nil
+}
+
+type CheckpointFile struct {
+	From     int64                         `json:"from"`
+	To       int64                         `json:"to"`
+	Metrics  map[string]*CheckpointMetrics `json:"metrics"`
+	Children map[string]*CheckpointFile    `json:"children"`
+}
+
+var ErrNoNewData error = errors.New("all data already archived")
+
+var NumWorkers int = 4
+
+func init() {
+	maxWorkers := 10
+	NumWorkers = runtime.NumCPU()/2 + 1
+	if NumWorkers > maxWorkers {
+		NumWorkers = maxWorkers
+	}
+}
+
+// Metrics stored at the lowest 2 levels are not stored away (root and cluster)!
+// On a per-host basis a new JSON file is created. I have no idea if this will scale.
+// The good thing: Only a host at a time is locked, so this function can run
+// in parallel to writes/reads.
+func (m *MemoryStore) ToCheckpoint(dir string, from, to int64) (int, error) {
+	levels := make([]*level, 0)
+	selectors := make([][]string, 0)
+	m.root.lock.RLock()
+	for sel1, l1 := range m.root.children {
+		l1.lock.RLock()
+		for sel2, l2 := range l1.children {
+			levels = append(levels, l2)
+			selectors = append(selectors, []string{sel1, sel2})
+		}
+		l1.lock.RUnlock()
+	}
+	m.root.lock.RUnlock()
+
+	type workItem struct {
+		level    *level
+		dir      string
+		selector []string
+	}
+
+	n, errs := int32(0), int32(0)
+
+	var wg sync.WaitGroup
+	wg.Add(NumWorkers)
+	work := make(chan workItem, NumWorkers*2)
+	for worker := 0; worker < NumWorkers; worker++ {
+		go func() {
+			defer wg.Done()
+
+			for workItem := range work {
+				if err := workItem.level.toCheckpoint(workItem.dir, from, to, m); err != nil {
+					if err == ErrNoNewData {
+						continue
+					}
+
+					log.Printf("error while checkpointing %#v: %s", workItem.selector, err.Error())
+					atomic.AddInt32(&errs, 1)
+				} else {
+					atomic.AddInt32(&n, 1)
+				}
+			}
+		}()
+	}
+
+	for i := 0; i < len(levels); i++ {
+		dir := path.Join(dir, path.Join(selectors[i]...))
+		work <- workItem{
+			level:    levels[i],
+			dir:      dir,
+			selector: selectors[i],
+		}
+	}
+
+	close(work)
+	wg.Wait()
+
+	if errs > 0 {
+		return int(n), fmt.Errorf("%d errors happend while creating checkpoints (%d successes)", errs, n)
+	}
+	return int(n), nil
+}
+
+func (l *level) toCheckpointFile(from, to int64, m *MemoryStore) (*CheckpointFile, error) {
+	l.lock.RLock()
+	defer l.lock.RUnlock()
+
+	retval := &CheckpointFile{
+		From:     from,
+		To:       to,
+		Metrics:  make(map[string]*CheckpointMetrics),
+		Children: make(map[string]*CheckpointFile),
+	}
+
+	for metric, minfo := range m.metrics {
+		b := l.metrics[minfo.offset]
+		if b == nil {
+			continue
+		}
+
+		allArchived := true
+		b.iterFromTo(from, to, func(b *buffer) error {
+			if !b.archived {
+				allArchived = false
+			}
+			return nil
+		})
+
+		if allArchived {
+			continue
+		}
+
+		data := make([]Float, (to-from)/b.frequency+1)
+		data, start, end, err := b.read(from, to, data)
+		if err != nil {
+			return nil, err
+		}
+
+		for i := int((end - start) / b.frequency); i < len(data); i++ {
+			data[i] = NaN
+		}
+
+		retval.Metrics[metric] = &CheckpointMetrics{
+			Frequency: b.frequency,
+			Start:     start,
+			Data:      data,
+			Unit:      b.unit,
+		}
+	}
+
+	for name, child := range l.children {
+		val, err := child.toCheckpointFile(from, to, m)
+		if err != nil {
+			return nil, err
+		}
+
+		if val != nil {
+			retval.Children[name] = val
+		}
+	}
+
+	if len(retval.Children) == 0 && len(retval.Metrics) == 0 {
+		return nil, nil
+	}
+
+	return retval, nil
+}
+
+func (l *level) toCheckpoint(dir string, from, to int64, m *MemoryStore) error {
+	cf, err := l.toCheckpointFile(from, to, m)
+	if err != nil {
+		return err
+	}
+
+	if cf == nil {
+		return ErrNoNewData
+	}
+
+	filepath := path.Join(dir, fmt.Sprintf("%d.json", from))
+	f, err := os.OpenFile(filepath, os.O_CREATE|os.O_WRONLY, 0644)
+	if err != nil && os.IsNotExist(err) {
+		err = os.MkdirAll(dir, 0755)
+		if err == nil {
+			f, err = os.OpenFile(filepath, os.O_CREATE|os.O_WRONLY, 0644)
+		}
+	}
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	bw := bufio.NewWriter(f)
+	if err = json.NewEncoder(bw).Encode(cf); err != nil {
+		return err
+	}
+
+	return bw.Flush()
+}
+
+// Metrics stored at the lowest 2 levels are not loaded (root and cluster)!
+// This function can only be called once and before the very first write or read.
+// Different host's data is loaded to memory in parallel.
+func (m *MemoryStore) FromCheckpoint(dir string, from int64) (int, error) {
+	var wg sync.WaitGroup
+	work := make(chan [2]string, NumWorkers)
+	n, errs := int32(0), int32(0)
+
+	wg.Add(NumWorkers)
+	for worker := 0; worker < NumWorkers; worker++ {
+		go func() {
+			defer wg.Done()
+			for host := range work {
+				lvl := m.root.findLevelOrCreate(host[:], len(m.metrics))
+				nn, err := lvl.fromCheckpoint(filepath.Join(dir, host[0], host[1]), from, m)
+				if err != nil {
+					log.Fatalf("error while loading checkpoints: %s", err.Error())
+					atomic.AddInt32(&errs, 1)
+				}
+				atomic.AddInt32(&n, int32(nn))
+			}
+		}()
+	}
+
+	i := 0
+	clustersDir, err := os.ReadDir(dir)
+	for _, clusterDir := range clustersDir {
+		if !clusterDir.IsDir() {
+			err = errors.New("expected only directories at first level of checkpoints/ directory")
+			goto done
+		}
+
+		hostsDir, e := os.ReadDir(filepath.Join(dir, clusterDir.Name()))
+		if e != nil {
+			err = e
+			goto done
+		}
+
+		for _, hostDir := range hostsDir {
+			if !hostDir.IsDir() {
+				err = errors.New("expected only directories at second level of checkpoints/ directory")
+				goto done
+			}
+
+			i++
+			if i%NumWorkers == 0 && i > 100 {
+				// Forcing garbage collection runs here regulary during the loading of checkpoints
+				// will decrease the total heap size after loading everything back to memory is done.
+				// While loading data, the heap will grow fast, so the GC target size will double
+				// almost always. By forcing GCs here, we can keep it growing more slowly so that
+				// at the end, less memory is wasted.
+				runtime.GC()
+			}
+
+			work <- [2]string{clusterDir.Name(), hostDir.Name()}
+		}
+	}
+done:
+	close(work)
+	wg.Wait()
+
+	if err != nil {
+		return int(n), err
+	}
+
+	if errs > 0 {
+		return int(n), fmt.Errorf("%d errors happend while creating checkpoints (%d successes)", errs, n)
+	}
+	return int(n), nil
+}
+
+func (l *level) loadFile(cf *CheckpointFile, m *MemoryStore) error {
+	for name, metric := range cf.Metrics {
+		n := len(metric.Data)
+		b := &buffer{
+			frequency: metric.Frequency,
+			start:     metric.Start,
+			data:      metric.Data[0:n:n], // Space is wasted here :(
+			unit:      metric.Unit,
+			prev:      nil,
+			next:      nil,
+			archived:  true,
+		}
+		b.close()
+
+		minfo, ok := m.metrics[name]
+		if !ok {
+			continue
+			// return errors.New("Unkown metric: " + name)
+		}
+
+		prev := l.metrics[minfo.offset]
+		if prev == nil {
+			l.metrics[minfo.offset] = b
+		} else {
+			if prev.start > b.start {
+				return errors.New("wooops")
+			}
+
+			b.prev = prev
+			prev.next = b
+		}
+		l.metrics[minfo.offset] = b
+	}
+
+	if len(cf.Children) > 0 && l.children == nil {
+		l.children = make(map[string]*level)
+	}
+
+	for sel, childCf := range cf.Children {
+		child, ok := l.children[sel]
+		if !ok {
+			child = &level{
+				metrics:  make([]*buffer, len(m.metrics)),
+				children: nil,
+			}
+			l.children[sel] = child
+		}
+
+		if err := child.loadFile(childCf, m); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (l *level) fromCheckpoint(dir string, from int64, m *MemoryStore) (int, error) {
+	direntries, err := os.ReadDir(dir)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return 0, nil
+		}
+
+		return 0, err
+	}
+
+	jsonFiles := make([]fs.DirEntry, 0)
+	filesLoaded := 0
+	for _, e := range direntries {
+		if e.IsDir() {
+			child := &level{
+				metrics:  make([]*buffer, len(m.metrics)),
+				children: make(map[string]*level),
+			}
+
+			files, err := child.fromCheckpoint(path.Join(dir, e.Name()), from, m)
+			filesLoaded += files
+			if err != nil {
+				return filesLoaded, err
+			}
+
+			l.children[e.Name()] = child
+		} else if strings.HasSuffix(e.Name(), ".json") {
+			jsonFiles = append(jsonFiles, e)
+		} else {
+			return filesLoaded, errors.New("unexpected file: " + dir + "/" + e.Name())
+		}
+	}
+
+	files, err := findFiles(jsonFiles, from, true)
+	if err != nil {
+		return filesLoaded, err
+	}
+
+	for _, filename := range files {
+		f, err := os.Open(path.Join(dir, filename))
+		if err != nil {
+			return filesLoaded, err
+		}
+		defer f.Close()
+
+		br := bufio.NewReader(f)
+		cf := &CheckpointFile{}
+		if err = json.NewDecoder(br).Decode(cf); err != nil {
+			return filesLoaded, err
+		}
+
+		if cf.To != 0 && cf.To < from {
+			continue
+		}
+
+		if err = l.loadFile(cf, m); err != nil {
+			return filesLoaded, err
+		}
+
+		filesLoaded += 1
+	}
+
+	return filesLoaded, nil
+}
+
+// This will probably get very slow over time!
+// A solution could be some sort of an index file in which all other files
+// and the timespan they contain is listed.
+func findFiles(direntries []fs.DirEntry, t int64, findMoreRecentFiles bool) ([]string, error) {
+	nums := map[string]int64{}
+	for _, e := range direntries {
+		ts, err := strconv.ParseInt(strings.TrimSuffix(e.Name(), ".json"), 10, 64)
+		if err != nil {
+			return nil, err
+		}
+		nums[e.Name()] = ts
+	}
+
+	sort.Slice(direntries, func(i, j int) bool {
+		a, b := direntries[i], direntries[j]
+		return nums[a.Name()] < nums[b.Name()]
+	})
+
+	filenames := make([]string, 0)
+	for i := 0; i < len(direntries); i++ {
+		e := direntries[i]
+		ts1 := nums[e.Name()]
+
+		if findMoreRecentFiles && t <= ts1 || i == len(direntries)-1 {
+			filenames = append(filenames, e.Name())
+			continue
+		}
+
+		enext := direntries[i+1]
+		ts2 := nums[enext.Name()]
+
+		if findMoreRecentFiles {
+			if ts1 < t && t < ts2 {
+				filenames = append(filenames, e.Name())
+			}
+		} else {
+			if ts2 < t {
+				filenames = append(filenames, e.Name())
+			}
+		}
+	}
+
+	return filenames, nil
+}
+
+// ZIP all checkpoint files older than `from` together and write them to the `archiveDir`,
+// deleting them from the `checkpointsDir`.
+func ArchiveCheckpoints(checkpointsDir, archiveDir string, from int64, deleteInstead bool) (int, error) {
+	entries1, err := os.ReadDir(checkpointsDir)
+	if err != nil {
+		return 0, err
+	}
+
+	type workItem struct {
+		cdir, adir    string
+		cluster, host string
+	}
+
+	var wg sync.WaitGroup
+	n, errs := int32(0), int32(0)
+	work := make(chan workItem, NumWorkers)
+
+	wg.Add(NumWorkers)
+	for worker := 0; worker < NumWorkers; worker++ {
+		go func() {
+			defer wg.Done()
+			for workItem := range work {
+				m, err := archiveCheckpoints(workItem.cdir, workItem.adir, from, deleteInstead)
+				if err != nil {
+					log.Printf("error while archiving %s/%s: %s", workItem.cluster, workItem.host, err.Error())
+					atomic.AddInt32(&errs, 1)
+				}
+				atomic.AddInt32(&n, int32(m))
+			}
+		}()
+	}
+
+	for _, de1 := range entries1 {
+		entries2, e := os.ReadDir(filepath.Join(checkpointsDir, de1.Name()))
+		if e != nil {
+			err = e
+		}
+
+		for _, de2 := range entries2 {
+			cdir := filepath.Join(checkpointsDir, de1.Name(), de2.Name())
+			adir := filepath.Join(archiveDir, de1.Name(), de2.Name())
+			work <- workItem{
+				adir: adir, cdir: cdir,
+				cluster: de1.Name(), host: de2.Name(),
+			}
+		}
+	}
+
+	close(work)
+	wg.Wait()
+
+	if err != nil {
+		return int(n), err
+	}
+
+	if errs > 0 {
+		return int(n), fmt.Errorf("%d errors happend while archiving (%d successes)", errs, n)
+	}
+	return int(n), nil
+}
+
+// Helper function for `ArchiveCheckpoints`.
+func archiveCheckpoints(dir string, archiveDir string, from int64, deleteInstead bool) (int, error) {
+	entries, err := os.ReadDir(dir)
+	if err != nil {
+		return 0, err
+	}
+
+	files, err := findFiles(entries, from, false)
+	if err != nil {
+		return 0, err
+	}
+
+	if deleteInstead {
+		n := 0
+		for _, checkpoint := range files {
+			filename := filepath.Join(dir, checkpoint)
+			if err = os.Remove(filename); err != nil {
+				return n, err
+			}
+			n += 1
+		}
+		return n, nil
+	}
+
+	filename := filepath.Join(archiveDir, fmt.Sprintf("%d.zip", from))
+	f, err := os.OpenFile(filename, os.O_CREATE|os.O_WRONLY, 0644)
+	if err != nil && os.IsNotExist(err) {
+		err = os.MkdirAll(archiveDir, 0755)
+		if err == nil {
+			f, err = os.OpenFile(filename, os.O_CREATE|os.O_WRONLY, 0644)
+		}
+	}
+	if err != nil {
+		return 0, err
+	}
+	defer f.Close()
+	bw := bufio.NewWriter(f)
+	defer bw.Flush()
+	zw := zip.NewWriter(bw)
+	defer zw.Close()
+
+	n := 0
+	for _, checkpoint := range files {
+		filename := filepath.Join(dir, checkpoint)
+		r, err := os.Open(filename)
+		if err != nil {
+			return n, err
+		}
+		defer r.Close()
+
+		w, err := zw.Create(checkpoint)
+		if err != nil {
+			return n, err
+		}
+
+		if _, err = io.Copy(w, r); err != nil {
+			return n, err
+		}
+
+		if err = os.Remove(filename); err != nil {
+			return n, err
+		}
+		n += 1
+	}
+
+	return n, nil
+}

archive_test.go

@@ -0,0 +1,33 @@
+package main
+
+import (
+	"bufio"
+	"log"
+	"os"
+	"testing"
+	"time"
+)
+
+func TestFromCheckpoint(t *testing.T) {
+	m := NewMemoryStore(map[string]MetricConfig{
+		"cpi":       {Frequency: 5, Aggregation: AvgAggregation},
+		"flops_any": {Frequency: 5, Aggregation: SumAggregation},
+		"flops_dp":  {Frequency: 5, Aggregation: SumAggregation},
+		"flops_sp":  {Frequency: 5, Aggregation: SumAggregation},
+	})
+
+	startupTime := time.Now()
+	files, err := m.FromCheckpoint("./testdata/checkpoints", 1692628930)
+	loadedData := m.SizeInBytes() / 1024 / 1024 // In MB
+	if err != nil {
+		t.Fatal(err)
+	} else {
+		log.Printf("Checkpoints loaded (%d files, %d MB, that took %fs)\n", files, loadedData, time.Since(startupTime).Seconds())
+	}
+
+	m.DebugDump(bufio.NewWriter(os.Stdout), nil)
+
+	if files != 2 {
+		t.Errorf("expected: %d, got: %d\n", 2, files)
+	}
+}

cc-metric-store.go

@@ -0,0 +1,349 @@
+package main
+
+import (
+	"bufio"
+	"context"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"io"
+	"log"
+	"os"
+	"os/signal"
+	"runtime"
+	"runtime/debug"
+	"sync"
+	"syscall"
+	"time"
+
+	"github.com/google/gops/agent"
+)
+
+// For aggregation over multiple values at different cpus/sockets/..., not time!
+type AggregationStrategy int
+
+const (
+	NoAggregation AggregationStrategy = iota
+	SumAggregation
+	AvgAggregation
+)
+
+func (as *AggregationStrategy) UnmarshalJSON(data []byte) error {
+	var str string
+	if err := json.Unmarshal(data, &str); err != nil {
+		return err
+	}
+
+	switch str {
+	case "":
+		*as = NoAggregation
+	case "sum":
+		*as = SumAggregation
+	case "avg":
+		*as = AvgAggregation
+	default:
+		return fmt.Errorf("invalid aggregation strategy: %#v", str)
+	}
+	return nil
+}
+
+type MetricConfig struct {
+	// Interval in seconds at which measurements will arive.
+	Frequency int64 `json:"frequency"`
+
+	// Can be 'sum', 'avg' or null. Describes how to aggregate metrics from the same timestep over the hierarchy.
+	Aggregation AggregationStrategy `json:"aggregation"`
+
+	// Private, used internally...
+	offset int
+}
+
+type HttpConfig struct {
+	// Address to bind to, for example "0.0.0.0:8081"
+	Address string `json:"address"`
+
+	// If not the empty string, use https with this as the certificate file
+	CertFile string `json:"https-cert-file"`
+
+	// If not the empty string, use https with this as the key file
+	KeyFile string `json:"https-key-file"`
+}
+
+type NatsConfig struct {
+	// Address of the nats server
+	Address string `json:"address"`
+
+	// Username/Password, optional
+	Username string `json:"username"`
+	Password string `json:"password"`
+
+	Subscriptions []struct {
+		// Channel name
+		SubscribeTo string `json:"subscribe-to"`
+
+		// Allow lines without a cluster tag, use this as default, optional
+		ClusterTag string `json:"cluster-tag"`
+	} `json:"subscriptions"`
+}
+
+type Config struct {
+	Metrics           map[string]MetricConfig `json:"metrics"`
+	RetentionInMemory string                  `json:"retention-in-memory"`
+	Nats              []*NatsConfig           `json:"nats"`
+	JwtPublicKey      string                  `json:"jwt-public-key"`
+	HttpConfig        *HttpConfig             `json:"http-api"`
+	Checkpoints       struct {
+		Interval string `json:"interval"`
+		RootDir  string `json:"directory"`
+		Restore  string `json:"restore"`
+	} `json:"checkpoints"`
+	Archive struct {
+		Interval      string `json:"interval"`
+		RootDir       string `json:"directory"`
+		DeleteInstead bool   `json:"delete-instead"`
+	} `json:"archive"`
+	Debug struct {
+		EnableGops bool   `json:"gops"`
+		DumpToFile string `json:"dump-to-file"`
+	} `json:"debug"`
+}
+
+var conf Config
+var memoryStore *MemoryStore = nil
+var lastCheckpoint time.Time
+
+var debugDumpLock sync.Mutex
+var debugDump io.Writer = io.Discard
+
+func loadConfiguration(file string) Config {
+	var config Config
+	configFile, err := os.Open(file)
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer configFile.Close()
+	dec := json.NewDecoder(configFile)
+	dec.DisallowUnknownFields()
+	if err := dec.Decode(&config); err != nil {
+		log.Fatal(err)
+	}
+	return config
+}
+
+func intervals(wg *sync.WaitGroup, ctx context.Context) {
+	wg.Add(3)
+	// go func() {
+	// 	defer wg.Done()
+	// 	ticks := time.Tick(30 * time.Minute)
+	// 	for {
+	// 		select {
+	// 		case <-ctx.Done():
+	// 			return
+	// 		case <-ticks:
+	// 			runtime.GC()
+	// 		}
+	// 	}
+	// }()
+
+	go func() {
+		defer wg.Done()
+		d, err := time.ParseDuration(conf.RetentionInMemory)
+		if err != nil {
+			log.Fatal(err)
+		}
+		if d <= 0 {
+			return
+		}
+
+		ticks := time.Tick(d / 2)
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			case <-ticks:
+				t := time.Now().Add(-d)
+				log.Printf("start freeing buffers (older than %s)...\n", t.Format(time.RFC3339))
+				freed, err := memoryStore.Free(nil, t.Unix())
+				if err != nil {
+					log.Printf("freeing up buffers failed: %s\n", err.Error())
+				} else {
+					log.Printf("done: %d buffers freed\n", freed)
+				}
+			}
+		}
+	}()
+
+	lastCheckpoint = time.Now()
+	go func() {
+		defer wg.Done()
+		d, err := time.ParseDuration(conf.Checkpoints.Interval)
+		if err != nil {
+			log.Fatal(err)
+		}
+		if d <= 0 {
+			return
+		}
+
+		ticks := time.Tick(d)
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			case <-ticks:
+				log.Printf("start checkpointing (starting at %s)...\n", lastCheckpoint.Format(time.RFC3339))
+				now := time.Now()
+				n, err := memoryStore.ToCheckpoint(conf.Checkpoints.RootDir,
+					lastCheckpoint.Unix(), now.Unix())
+				if err != nil {
+					log.Printf("checkpointing failed: %s\n", err.Error())
+				} else {
+					log.Printf("done: %d checkpoint files created\n", n)
+					lastCheckpoint = now
+				}
+			}
+		}
+	}()
+
+	go func() {
+		defer wg.Done()
+		d, err := time.ParseDuration(conf.Archive.Interval)
+		if err != nil {
+			log.Fatal(err)
+		}
+		if d <= 0 {
+			return
+		}
+
+		ticks := time.Tick(d)
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			case <-ticks:
+				t := time.Now().Add(-d)
+				log.Printf("start archiving checkpoints (older than %s)...\n", t.Format(time.RFC3339))
+				n, err := ArchiveCheckpoints(conf.Checkpoints.RootDir, conf.Archive.RootDir, t.Unix(), conf.Archive.DeleteInstead)
+				if err != nil {
+					log.Printf("archiving failed: %s\n", err.Error())
+				} else {
+					log.Printf("done: %d files zipped and moved to archive\n", n)
+				}
+			}
+		}
+	}()
+}
+
+func main() {
+	var configFile string
+	var enableGopsAgent bool
+	flag.StringVar(&configFile, "config", "./config.json", "configuration file")
+	flag.BoolVar(&enableGopsAgent, "gops", false, "Listen via github.com/google/gops/agent")
+	flag.Parse()
+
+	startupTime := time.Now()
+	conf = loadConfiguration(configFile)
+	memoryStore = NewMemoryStore(conf.Metrics)
+
+	if enableGopsAgent || conf.Debug.EnableGops {
+		if err := agent.Listen(agent.Options{}); err != nil {
+			log.Fatal(err)
+		}
+	}
+
+	if conf.Debug.DumpToFile != "" {
+		f, err := os.Create(conf.Debug.DumpToFile)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		debugDump = f
+	}
+
+	d, err := time.ParseDuration(conf.Checkpoints.Restore)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	restoreFrom := startupTime.Add(-d)
+	log.Printf("Loading checkpoints newer than %s\n", restoreFrom.Format(time.RFC3339))
+	files, err := memoryStore.FromCheckpoint(conf.Checkpoints.RootDir, restoreFrom.Unix())
+	loadedData := memoryStore.SizeInBytes() / 1024 / 1024 // In MB
+	if err != nil {
+		log.Fatalf("Loading checkpoints failed: %s\n", err.Error())
+	} else {
+		log.Printf("Checkpoints loaded (%d files, %d MB, that took %fs)\n", files, loadedData, time.Since(startupTime).Seconds())
+	}
+
+	// Try to use less memory by forcing a GC run here and then
+	// lowering the target percentage. The default of 100 means
+	// that only once the ratio of new allocations execeds the
+	// previously active heap, a GC is triggered.
+	// Forcing a GC here will set the "previously active heap"
+	// to a minumum.
+	runtime.GC()
+	if loadedData > 1000 && os.Getenv("GOGC") == "" {
+		debug.SetGCPercent(10)
+	}
+
+	ctx, shutdown := context.WithCancel(context.Background())
+
+	var wg sync.WaitGroup
+	sigs := make(chan os.Signal, 1)
+	signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM, syscall.SIGUSR1)
+	go func() {
+		for {
+			sig := <-sigs
+			if sig == syscall.SIGUSR1 {
+				memoryStore.DebugDump(bufio.NewWriter(os.Stdout), nil)
+				continue
+			}
+
+			log.Println("Shuting down...")
+			shutdown()
+		}
+	}()
+
+	intervals(&wg, ctx)
+
+	wg.Add(1)
+
+	go func() {
+		err := StartApiServer(ctx, conf.HttpConfig)
+		if err != nil {
+			log.Fatal(err)
+		}
+		wg.Done()
+	}()
+
+	if conf.Nats != nil {
+		for _, natsConf := range conf.Nats {
+			// TODO: When multiple nats configs share a URL, do a single connect.
+			wg.Add(1)
+			nc := natsConf
+			go func() {
+				// err := ReceiveNats(conf.Nats, decodeLine, runtime.NumCPU()-1, ctx)
+				err := ReceiveNats(nc, decodeLine, 1, ctx)
+
+				if err != nil {
+					log.Fatal(err)
+				}
+				wg.Done()
+			}()
+		}
+	}
+
+	wg.Wait()
+
+	log.Printf("Writing to '%s'...\n", conf.Checkpoints.RootDir)
+	files, err = memoryStore.ToCheckpoint(conf.Checkpoints.RootDir, lastCheckpoint.Unix(), time.Now().Unix())
+	if err != nil {
+		log.Printf("Writing checkpoint failed: %s\n", err.Error())
+	}
+	log.Printf("Done! (%d files written)\n", files)
+
+	if closer, ok := debugDump.(io.Closer); ok {
+		if err := closer.Close(); err != nil {
+			log.Printf("error: %s", err.Error())
+		}
+	}
+}

cmd/cc-metric-store/main.go

@@ -1,173 +0,0 @@
-// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved. This file is part of cc-metric-store.
-// Use of this source code is governed by a MIT-style
-// license that can be found in the LICENSE file.
-
-package main
-
-import (
-	"context"
-	"flag"
-	"fmt"
-	"os"
-	"os/signal"
-	"runtime/debug"
-	"sync"
-	"syscall"
-
-	"github.com/ClusterCockpit/cc-backend/pkg/metricstore"
-	ccconf "github.com/ClusterCockpit/cc-lib/v2/ccConfig"
-	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
-	"github.com/ClusterCockpit/cc-lib/v2/nats"
-	"github.com/ClusterCockpit/cc-lib/v2/runtime"
-	"github.com/ClusterCockpit/cc-metric-store/internal/api"
-	"github.com/ClusterCockpit/cc-metric-store/internal/config"
-	"github.com/google/gops/agent"
-)
-
-const (
-	envGOGC = "GOGC"
-)
-
-var (
-	date    string
-	commit  string
-	version string
-)
-
-var (
-	flagGops, flagVersion, flagDev, flagLogDateTime bool
-	flagConfigFile, flagLogLevel                    string
-)
-
-func printVersion() {
-	fmt.Printf("Version:\t%s\n", version)
-	fmt.Printf("Git hash:\t%s\n", commit)
-	fmt.Printf("Build time:\t%s\n", date)
-}
-
-func runServer(ctx context.Context) error {
-	var wg sync.WaitGroup
-
-	mscfg := ccconf.GetPackageConfig("metrics")
-	if mscfg == nil {
-		return fmt.Errorf("missing metrics configuration")
-	}
-	config.InitMetrics(mscfg)
-
-	mscfg = ccconf.GetPackageConfig("metric-store")
-	if mscfg == nil {
-		return fmt.Errorf("missing metricstore configuration")
-	}
-
-	metricstore.Init(mscfg, config.GetMetrics(), &wg)
-
-	// Set GC percent if not configured
-	if os.Getenv(envGOGC) == "" {
-		debug.SetGCPercent(15)
-	}
-
-	if config.Keys.BackendURL != "" {
-		ms := metricstore.GetMemoryStore()
-		ms.SetNodeProvider(api.NewBackendNodeProvider(config.Keys.BackendURL))
-		cclog.Infof("Node provider configured with backend URL: %s", config.Keys.BackendURL)
-	}
-
-	// Initialize HTTP server
-	srv, err := NewServer(version, commit, date)
-	if err != nil {
-		return fmt.Errorf("creating server: %w", err)
-	}
-
-	// Channel to collect errors from server
-	errChan := make(chan error, 1)
-
-	// Start HTTP server
-	wg.Go(func() {
-		if err := srv.Start(ctx); err != nil {
-			errChan <- err
-		}
-	})
-
-	// Handle shutdown signals
-	wg.Add(1)
-	sigs := make(chan os.Signal, 1)
-	signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
-	go func() {
-		defer wg.Done()
-		select {
-		case <-sigs:
-			cclog.Info("Shutdown signal received")
-		case <-ctx.Done():
-		}
-
-		runtime.SystemdNotify(false, "Shutting down ...")
-		srv.Shutdown(ctx)
-	}()
-
-	runtime.SystemdNotify(true, "running")
-
-	// Wait for completion or errors
-	go func() {
-		wg.Wait()
-		close(errChan)
-	}()
-
-	// Wait for either server startup error or shutdown completion
-	if err := <-errChan; err != nil {
-		return err
-	}
-
-	cclog.Print("Graceful shutdown completed!")
-	return nil
-}
-
-func run() error {
-	flag.BoolVar(&flagGops, "gops", false, "Listen via github.com/google/gops/agent (for debugging)")
-	flag.BoolVar(&flagDev, "dev", false, "Enable development component: Swagger UI")
-	flag.BoolVar(&flagVersion, "version", false, "Show version information and exit")
-	flag.BoolVar(&flagLogDateTime, "logdate", false, "Set this flag to add date and time to log messages")
-	flag.StringVar(&flagConfigFile, "config", "./config.json", "Specify alternative path to `config.json`")
-	flag.StringVar(&flagLogLevel, "loglevel", "warn", "Sets the logging level: `[debug, info, warn (default), err, crit]`")
-	flag.Parse()
-
-	if flagVersion {
-		printVersion()
-		return nil
-	}
-
-	cclog.Init(flagLogLevel, flagLogDateTime)
-
-	if flagGops || config.Keys.Debug.EnableGops {
-		if err := agent.Listen(agent.Options{}); err != nil {
-			return fmt.Errorf("starting gops agent: %w", err)
-		}
-	}
-
-	ccconf.Init(flagConfigFile)
-
-	cfg := ccconf.GetPackageConfig("main")
-	if cfg == nil {
-		return fmt.Errorf("main configuration must be present")
-	}
-
-	config.Init(cfg)
-
-	natsConfig := ccconf.GetPackageConfig("nats")
-	if err := nats.Init(natsConfig); err != nil {
-		cclog.Warnf("initializing (optional) nats client: %s", err.Error())
-	}
-	nats.Connect()
-
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-
-	return runServer(ctx)
-}
-
-func main() {
-	if err := run(); err != nil {
-		cclog.Error(err.Error())
-		os.Exit(1)
-	}
-}

cmd/cc-metric-store/server.go

@@ -1,142 +0,0 @@
-// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved. This file is part of cc-metric-store.
-// Use of this source code is governed by a MIT-style
-// license that can be found in the LICENSE file.
-
-// Package main provides the entry point for the ClusterCockpit metric store server.
-// This file contains HTTP server setup, routing configuration.
-package main
-
-import (
-	"context"
-	"crypto/tls"
-	"fmt"
-	"net"
-	"net/http"
-	"time"
-
-	"github.com/ClusterCockpit/cc-backend/pkg/metricstore"
-	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
-	"github.com/ClusterCockpit/cc-lib/v2/nats"
-	"github.com/ClusterCockpit/cc-lib/v2/runtime"
-	"github.com/ClusterCockpit/cc-metric-store/internal/api"
-	"github.com/ClusterCockpit/cc-metric-store/internal/config"
-	httpSwagger "github.com/swaggo/http-swagger/v2"
-)
-
-// Server encapsulates the HTTP server state and dependencies
-type Server struct {
-	router *http.ServeMux
-	server *http.Server
-}
-
-// NewServer creates and initializes a new Server instance
-func NewServer(version, commit, buildDate string) (*Server, error) {
-	s := &Server{
-		router: http.NewServeMux(),
-	}
-
-	if err := s.init(); err != nil {
-		return nil, err
-	}
-
-	return s, nil
-}
-
-func (s *Server) init() error {
-	api.MountRoutes(s.router)
-
-	if flagDev {
-		cclog.Print("Enable Swagger UI!")
-		s.router.HandleFunc("GET /swagger/", httpSwagger.Handler(
-			httpSwagger.URL("http://"+config.Keys.Address+"/swagger/doc.json")))
-	}
-
-	return nil
-}
-
-// Server timeout defaults (in seconds)
-const (
-	defaultReadTimeout  = 30
-	defaultWriteTimeout = 30
-)
-
-func (s *Server) Start(ctx context.Context) error {
-	// Use configurable timeouts with defaults
-	readTimeout := time.Duration(defaultReadTimeout) * time.Second
-	writeTimeout := time.Duration(defaultWriteTimeout) * time.Second
-
-	s.server = &http.Server{
-		ReadTimeout:  readTimeout,
-		WriteTimeout: writeTimeout,
-		Handler:      s.router,
-		Addr:         config.Keys.Address,
-	}
-
-	// Start http or https server
-	listener, err := net.Listen("tcp", config.Keys.Address)
-	if err != nil {
-		return fmt.Errorf("starting listener on '%s': %w", config.Keys.Address, err)
-	}
-
-	if config.Keys.CertFile != "" && config.Keys.KeyFile != "" {
-		cert, err := tls.LoadX509KeyPair(
-			config.Keys.CertFile, config.Keys.KeyFile)
-		if err != nil {
-			return fmt.Errorf("loading X509 keypair (check 'https-cert-file' and 'https-key-file' in config.json): %w", err)
-		}
-		listener = tls.NewListener(listener, &tls.Config{
-			Certificates: []tls.Certificate{cert},
-			CipherSuites: []uint16{
-				tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
-				tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
-			},
-			MinVersion:               tls.VersionTLS12,
-			PreferServerCipherSuites: true,
-		})
-		cclog.Infof("HTTPS server listening at %s...", config.Keys.Address)
-	} else {
-		cclog.Infof("HTTP server listening at %s...", config.Keys.Address)
-	}
-
-	// Because this program will want to bind to a privileged port (like 80), the listener must
-	// be established first, then the user can be changed, and after that,
-	// the actual http server can be started.
-	if err := runtime.DropPrivileges(config.Keys.Group, config.Keys.User); err != nil {
-		return fmt.Errorf("dropping privileges: %w", err)
-	}
-
-	// Handle context cancellation for graceful shutdown
-	go func() {
-		<-ctx.Done()
-		shutdownCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
-		defer cancel()
-		if err := s.server.Shutdown(shutdownCtx); err != nil {
-			cclog.Errorf("Server shutdown error: %v", err)
-		}
-	}()
-
-	if err = s.server.Serve(listener); err != nil && err != http.ErrServerClosed {
-		return fmt.Errorf("server failed: %w", err)
-	}
-	return nil
-}
-
-func (s *Server) Shutdown(ctx context.Context) {
-	// Create a shutdown context with timeout
-	shutdownCtx, cancel := context.WithTimeout(ctx, 30*time.Second)
-	defer cancel()
-
-	nc := nats.GetClient()
-	if nc != nil {
-		nc.Close()
-	}
-
-	// First shut down the server gracefully (waiting for all ongoing requests)
-	if err := s.server.Shutdown(shutdownCtx); err != nil {
-		cclog.Errorf("Server shutdown error: %v", err)
-	}
-
-	// Archive all the metric store data
-	metricstore.Shutdown()
-}

config.json

@@ -0,0 +1,23 @@
+{
+    "metrics": {
+        "cpi":  { "frequency": 15, "aggregation": "avg" },
+        "flops_any":  { "frequency": 15, "aggregation": "sum" },
+        "flops_dp":   { "frequency": 15, "aggregation": "sum" },
+        "flops_sp":   { "frequency": 15, "aggregation": "sum" }
+    },
+    "checkpoints": {
+        "interval": "12h",
+        "directory": "./testdata/checkpoints",
+        "restore": "48h"
+    },
+    "archive": {
+        "interval": "168h",
+        "directory": "./testdata/archive"
+    },
+    "http-api": {
+        "address": "127.0.0.1:8081",
+        "https-cert-file": null,
+        "https-key-file": null
+    },
+    "retention-in-memory": "48h"
+}

configs/config-nats.json

@@ -1,196 +0,0 @@
-{
-    "metrics": {
-        "debug_metric": {
-            "frequency": 60,
-            "aggregation": "avg"
-        },
-        "clock": {
-            "frequency": 60,
-            "aggregation": "avg"
-        },
-        "cpu_idle": {
-            "frequency": 60,
-            "aggregation": "avg"
-        },
-        "cpu_iowait": {
-            "frequency": 60,
-            "aggregation": "avg"
-        },
-        "cpu_irq": {
-            "frequency": 60,
-            "aggregation": "avg"
-        },
-        "cpu_system": {
-            "frequency": 60,
-            "aggregation": "avg"
-        },
-        "cpu_user": {
-            "frequency": 60,
-            "aggregation": "avg"
-        },
-        "nv_mem_util": {
-            "frequency": 60,
-            "aggregation": "avg"
-        },
-        "nv_temp": {
-            "frequency": 60,
-            "aggregation": "avg"
-        },
-        "nv_sm_clock": {
-            "frequency": 60,
-            "aggregation": "avg"
-        },
-        "acc_utilization": {
-            "frequency": 60,
-            "aggregation": "avg"
-        },
-        "acc_mem_used": {
-            "frequency": 60,
-            "aggregation": "sum"
-        },
-        "acc_power": {
-            "frequency": 60,
-            "aggregation": "sum"
-        },
-        "flops_any": {
-            "frequency": 60,
-            "aggregation": "sum"
-        },
-        "flops_dp": {
-            "frequency": 60,
-            "aggregation": "sum"
-        },
-        "flops_sp": {
-            "frequency": 60,
-            "aggregation": "sum"
-        },
-        "ib_recv": {
-            "frequency": 60,
-            "aggregation": "sum"
-        },
-        "ib_xmit": {
-            "frequency": 60,
-            "aggregation": "sum"
-        },
-        "ib_recv_pkts": {
-            "frequency": 60,
-            "aggregation": "sum"
-        },
-        "ib_xmit_pkts": {
-            "frequency": 60,
-            "aggregation": "sum"
-        },
-        "cpu_power": {
-            "frequency": 60,
-            "aggregation": "sum"
-        },
-        "core_power": {
-            "frequency": 60,
-            "aggregation": "sum"
-        },
-        "mem_power": {
-            "frequency": 60,
-            "aggregation": "sum"
-        },
-        "ipc": {
-            "frequency": 60,
-            "aggregation": "avg"
-        },
-        "cpu_load": {
-            "frequency": 60,
-            "aggregation": null
-        },
-        "lustre_close": {
-            "frequency": 60,
-            "aggregation": null
-        },
-        "lustre_open": {
-            "frequency": 60,
-            "aggregation": null
-        },
-        "lustre_statfs": {
-            "frequency": 60,
-            "aggregation": null
-        },
-        "lustre_read_bytes": {
-            "frequency": 60,
-            "aggregation": null
-        },
-        "lustre_write_bytes": {
-            "frequency": 60,
-            "aggregation": null
-        },
-        "net_bw": {
-            "frequency": 60,
-            "aggregation": null
-        },
-        "file_bw": {
-            "frequency": 60,
-            "aggregation": null
-        },
-        "mem_bw": {
-            "frequency": 60,
-            "aggregation": "sum"
-        },
-        "mem_cached": {
-            "frequency": 60,
-            "aggregation": null
-        },
-        "mem_used": {
-            "frequency": 60,
-            "aggregation": null
-        },
-        "net_bytes_in": {
-            "frequency": 60,
-            "aggregation": null
-        },
-        "net_bytes_out": {
-            "frequency": 60,
-            "aggregation": null
-        },
-        "nfs4_read": {
-            "frequency": 60,
-            "aggregation": null
-        },
-        "nfs4_total": {
-            "frequency": 60,
-            "aggregation": null
-        },
-        "nfs4_write": {
-            "frequency": 60,
-            "aggregation": null
-        },
-        "vectorization_ratio": {
-            "frequency": 60,
-            "aggregation": "avg"
-        }
-    },
-    "checkpoints": {
-        "interval": "12h",
-        "directory": "./var/checkpoints",
-        "restore": "48h"
-    },
-    "archive": {
-        "interval": "50h",
-        "directory": "./var/archive"
-    },
-    "http-api": {
-        "address": "localhost:8082",
-        "https-cert-file": null,
-        "https-key-file": null
-    },
-    "retention-in-memory": "48h",
-    "nats": [
-        {
-            "address": "nats://localhost:4222",
-            "creds-file-path": "test.creds",
-            "subscriptions": [
-                {
-                    "subscribe-to": "ee-hpc-nats",
-                    "cluster-tag": "fritz2"
-                }
-            ]
-        }
-    ],
-    "jwt-public-key": "kzfYrYy+TzpanWZHJ5qSdMj5uKUWgq74BWhQG6copP0="
-}

configs/config.json

@@ -1,202 +0,0 @@
-{
-  "main": {
-    "addr": "0.0.0.0:8082",
-    "https-cert-file": "",
-    "https-key-file": "",
-    "jwt-public-key": "kzfYrYy+TzpanWZHJ5qSdMj5uKUWgq74BWhQG6copP0="
-  },
-  "nats": {
-    "address": "nats://0.0.0.0:4222",
-    "username": "root",
-    "password": "root"
-  },
-  "metric-store": {
-    "checkpoints": {
-      "interval": "12h",
-      "directory": "./var/checkpoints"
-    },
-    "memory-cap": 100,
-    "retention-in-memory": "48h",
-    "cleanup": {
-      "mode": "archive",
-      "interval": "48h",
-      "directory": "./var/archive"
-    },
-    "nats-subscriptions": [
-      {
-        "subscribe-to": "hpc-nats",
-        "cluster-tag": "fritz"
-      },
-      {
-        "subscribe-to": "hpc-nats",
-        "cluster-tag": "alex"
-      }
-    ]
-  },
-  "metrics": {
-    "debug_metric": {
-      "frequency": 60,
-      "aggregation": "avg"
-    },
-    "clock": {
-      "frequency": 60,
-      "aggregation": "avg"
-    },
-    "cpu_idle": {
-      "frequency": 60,
-      "aggregation": "avg"
-    },
-    "cpu_iowait": {
-      "frequency": 60,
-      "aggregation": "avg"
-    },
-    "cpu_irq": {
-      "frequency": 60,
-      "aggregation": "avg"
-    },
-    "cpu_system": {
-      "frequency": 60,
-      "aggregation": "avg"
-    },
-    "cpu_user": {
-      "frequency": 60,
-      "aggregation": "avg"
-    },
-    "nv_mem_util": {
-      "frequency": 60,
-      "aggregation": "avg"
-    },
-    "nv_temp": {
-      "frequency": 60,
-      "aggregation": "avg"
-    },
-    "nv_sm_clock": {
-      "frequency": 60,
-      "aggregation": "avg"
-    },
-    "acc_utilization": {
-      "frequency": 60,
-      "aggregation": "avg"
-    },
-    "acc_mem_used": {
-      "frequency": 60,
-      "aggregation": "sum"
-    },
-    "acc_power": {
-      "frequency": 60,
-      "aggregation": "sum"
-    },
-    "flops_any": {
-      "frequency": 60,
-      "aggregation": "sum"
-    },
-    "flops_dp": {
-      "frequency": 60,
-      "aggregation": "sum"
-    },
-    "flops_sp": {
-      "frequency": 60,
-      "aggregation": "sum"
-    },
-    "ib_recv": {
-      "frequency": 60,
-      "aggregation": "sum"
-    },
-    "ib_xmit": {
-      "frequency": 60,
-      "aggregation": "sum"
-    },
-    "ib_recv_pkts": {
-      "frequency": 60,
-      "aggregation": "sum"
-    },
-    "ib_xmit_pkts": {
-      "frequency": 60,
-      "aggregation": "sum"
-    },
-    "cpu_power": {
-      "frequency": 60,
-      "aggregation": "sum"
-    },
-    "core_power": {
-      "frequency": 60,
-      "aggregation": "sum"
-    },
-    "mem_power": {
-      "frequency": 60,
-      "aggregation": "sum"
-    },
-    "ipc": {
-      "frequency": 60,
-      "aggregation": "avg"
-    },
-    "cpu_load": {
-      "frequency": 60,
-      "aggregation": null
-    },
-    "lustre_close": {
-      "frequency": 60,
-      "aggregation": null
-    },
-    "lustre_open": {
-      "frequency": 60,
-      "aggregation": null
-    },
-    "lustre_statfs": {
-      "frequency": 60,
-      "aggregation": null
-    },
-    "lustre_read_bytes": {
-      "frequency": 60,
-      "aggregation": null
-    },
-    "lustre_write_bytes": {
-      "frequency": 60,
-      "aggregation": null
-    },
-    "net_bw": {
-      "frequency": 60,
-      "aggregation": null
-    },
-    "file_bw": {
-      "frequency": 60,
-      "aggregation": null
-    },
-    "mem_bw": {
-      "frequency": 60,
-      "aggregation": "sum"
-    },
-    "mem_cached": {
-      "frequency": 60,
-      "aggregation": null
-    },
-    "mem_used": {
-      "frequency": 60,
-      "aggregation": null
-    },
-    "net_bytes_in": {
-      "frequency": 60,
-      "aggregation": null
-    },
-    "net_bytes_out": {
-      "frequency": 60,
-      "aggregation": null
-    },
-    "nfs4_read": {
-      "frequency": 60,
-      "aggregation": null
-    },
-    "nfs4_total": {
-      "frequency": 60,
-      "aggregation": null
-    },
-    "nfs4_write": {
-      "frequency": 60,
-      "aggregation": null
-    },
-    "vectorization_ratio": {
-      "frequency": 60,
-      "aggregation": "avg"
-    }
-  }
-}

debug.go

@@ -0,0 +1,110 @@
+package main
+
+import (
+	"bufio"
+	"fmt"
+	"strconv"
+)
+
+func (b *buffer) debugDump(buf []byte) []byte {
+	if b.prev != nil {
+		buf = b.prev.debugDump(buf)
+	}
+
+	start, len, end := b.start, len(b.data), b.start+b.frequency*int64(len(b.data))
+	buf = append(buf, `{"start":`...)
+	buf = strconv.AppendInt(buf, start, 10)
+	buf = append(buf, `,"len":`...)
+	buf = strconv.AppendInt(buf, int64(len), 10)
+	buf = append(buf, `,"end":`...)
+	buf = strconv.AppendInt(buf, end, 10)
+	if b.archived {
+		buf = append(buf, `,"saved":true`...)
+	}
+	if b.unit != "" {
+		buf = append(buf, fmt.Sprintf(`,"unit":"%s"`, b.unit)...)
+	}
+	if b.next != nil {
+		buf = append(buf, `},`...)
+	} else {
+		buf = append(buf, `}`...)
+	}
+	return buf
+}
+
+func (l *level) debugDump(m *MemoryStore, w *bufio.Writer, lvlname string, buf []byte, depth int) ([]byte, error) {
+	l.lock.RLock()
+	defer l.lock.RUnlock()
+	for i := 0; i < depth; i++ {
+		buf = append(buf, '\t')
+	}
+	buf = append(buf, '"')
+	buf = append(buf, lvlname...)
+	buf = append(buf, "\":{\n"...)
+	depth += 1
+	objitems := 0
+	for name, mc := range m.metrics {
+		if b := l.metrics[mc.offset]; b != nil {
+			for i := 0; i < depth; i++ {
+				buf = append(buf, '\t')
+			}
+
+			buf = append(buf, '"')
+			buf = append(buf, name...)
+			buf = append(buf, `":[`...)
+			buf = b.debugDump(buf)
+			buf = append(buf, "],\n"...)
+			objitems++
+		}
+	}
+
+	for name, lvl := range l.children {
+		_, err := w.Write(buf)
+		if err != nil {
+			return nil, err
+		}
+
+		buf = buf[0:0]
+		buf, err = lvl.debugDump(m, w, name, buf, depth)
+		if err != nil {
+			return nil, err
+		}
+
+		buf = append(buf, ',', '\n')
+		objitems++
+	}
+
+	// remove final `,`:
+	if objitems > 0 {
+		buf = append(buf[0:len(buf)-1], '\n')
+	}
+
+	depth -= 1
+	for i := 0; i < depth; i++ {
+		buf = append(buf, '\t')
+	}
+	buf = append(buf, '}')
+	return buf, nil
+}
+
+func (m *MemoryStore) DebugDump(w *bufio.Writer, selector []string) error {
+	lvl := m.root.findLevel(selector)
+	if lvl == nil {
+		return fmt.Errorf("not found: %#v", selector)
+	}
+
+	buf := make([]byte, 0, 2048)
+	buf = append(buf, "{"...)
+
+	buf, err := lvl.debugDump(m, w, "data", buf, 0)
+	if err != nil {
+		return err
+	}
+
+	buf = append(buf, "}\n"...)
+	if _, err = w.Write(buf); err != nil {
+		return err
+	}
+
+	return w.Flush()
+}

endpoint-test-scripts/test_ccms_api.sh

@@ -1,6 +0,0 @@
-JWT="eyJ0eXAiOiJKV1QiLCJhbGciOiJFZERTQSJ9.eyJ1c2VyIjoiYWRtaW4iLCJyb2xlcyI6WyJST0xFX0FETUlOIiwiUk9MRV9BTkFMWVNUIiwiUk9MRV9VU0VSIl19.d-3_3FZTsadPjDEdsWrrQ7nS0edMAR4zjl-eK7rJU3HziNBfI9PDHDIpJVHTNN5E5SlLGLFXctWyKAkwhXL-Dw"
-
-# If the collector and store and nats-server have been running for at least 60 seconds on the same host, you may run:
-curl -X 'GET' 'http://localhost:8081/api/query/' -H "Authorization: Bearer $JWT" -d "{ \"cluster\": \"alex\", \"from\": 1720879275, \"to\": 1720964715, \"queries\": [{\"metric\": \"cpu_load\",\"host\": \"a0124\"}] }"
-
-# ...

endpoint-test-scripts/test_ccms_debug_api.sh

@@ -1,6 +0,0 @@
-JWT="eyJ0eXAiOiJKV1QiLCJhbGciOiJFZERTQSJ9.eyJ1c2VyIjoiYWRtaW4iLCJyb2xlcyI6WyJST0xFX0FETUlOIiwiUk9MRV9BTkFMWVNUIiwiUk9MRV9VU0VSIl19.d-3_3FZTsadPjDEdsWrrQ7nS0edMAR4zjl-eK7rJU3HziNBfI9PDHDIpJVHTNN5E5SlLGLFXctWyKAkwhXL-Dw"
-
-# If the collector and store and nats-server have been running for at least 60 seconds on the same host, you may run:
-curl -X 'GET' 'http://localhost:8082/api/debug/' -H "Authorization: Bearer $JWT"
-
-# ...

endpoint-test-scripts/test_ccms_free_api.sh

@@ -1,6 +0,0 @@
-JWT="eyJ0eXAiOiJKV1QiLCJhbGciOiJFZERTQSJ9.eyJ1c2VyIjoiYWRtaW4iLCJyb2xlcyI6WyJST0xFX0FETUlOIiwiUk9MRV9BTkFMWVNUIiwiUk9MRV9VU0VSIl19.d-3_3FZTsadPjDEdsWrrQ7nS0edMAR4zjl-eK7rJU3HziNBfI9PDHDIpJVHTNN5E5SlLGLFXctWyKAkwhXL-Dw"
-
-# If the collector and store and nats-server have been running for at least 60 seconds on the same host, you may run:
-curl -X 'POST' 'http://localhost:8082/api/free/?to=1724536800' -H "Authorization: Bearer $JWT" -d "[     [    \"alex\",     \"a0329\",         \"memoryDomain2\"     ],     [        \"alex\", \"a0903\"     ],[        \"fritz\", \"f0201\"     ]  ]"
-
-# ...

endpoint-test-scripts/test_ccms_health_check_api.sh

@@ -1,6 +0,0 @@
-JWT="eyJ0eXAiOiJKV1QiLCJhbGciOiJFZERTQSJ9.eyJ1c2VyIjoiYWRtaW4iLCJyb2xlcyI6WyJST0xFX0FETUlOIiwiUk9MRV9BTkFMWVNUIiwiUk9MRV9VU0VSIl19.d-3_3FZTsadPjDEdsWrrQ7nS0edMAR4zjl-eK7rJU3HziNBfI9PDHDIpJVHTNN5E5SlLGLFXctWyKAkwhXL-Dw"
-
-# If the collector and store and nats-server have been running for at least 60 seconds on the same host, you may run:
-curl -X 'GET' 'http://localhost:8082/api/healthcheck/?cluster=alex&node=a0903' -H "Authorization: Bearer $JWT"
-
-# ...

endpoint-test-scripts/test_ccms_write_api.sh

@@ -1,110 +0,0 @@
-JWT="eyJ0eXAiOiJKV1QiLCJhbGciOiJFZERTQSJ9.eyJ1c2VyIjoiYWRtaW4iLCJyb2xlcyI6WyJST0xFX0FETUlOIiwiUk9MRV9BTkFMWVNUIiwiUk9MRV9VU0VSIl19.d-3_3FZTsadPjDEdsWrrQ7nS0edMAR4zjl-eK7rJU3HziNBfI9PDHDIpJVHTNN5E5SlLGLFXctWyKAkwhXL-Dw"
-
-# curl -X 'POST' 'http://localhost:8082/api/write/?cluster=alex' -H "Authorization: Bearer $JWT" -d $'cpu_load,cluster=alex,hostname=a042,type=hwthread,type-id=0 value=35.0 1725827464642231296'
-
-rm sample_fritz.txt
-rm sample_alex.txt
-
-while [ true ]; do
-    echo "Alex Metrics for hwthread types and type-ids"
-    timestamp="$(date '+%s')"
-    echo "Timestamp : "+$timestamp
-    for metric in cpu_load cpu_user flops_any cpu_irq cpu_system ipc cpu_idle cpu_iowait core_power clock; do
-        for hostname in a0603 a0903 a0832 a0329 a0702 a0122 a1624 a0731 a0224 a0704 a0631 a0225 a0222 a0427 a0603 a0429 a0833 a0705 a0901 a0601 a0227 a0804 a0322 a0226 a0126 a0129 a0605 a0801 a0934 a1622 a0902 a0428 a0537 a1623 a1722 a0228 a0701 a0326 a0327 a0123 a0321 a1621 a0323 a0124 a0534 a0931 a0324 a0933 a0424 a0905 a0128 a0532 a0805 a0521 a0535 a0932 a0127 a0325 a0633 a0831 a0803 a0426 a0425 a0229 a1721 a0602 a0632 a0223 a0422 a0423 a0536 a0328 a0703 anvme7 a0125 a0221 a0604 a0802 a0522 a0531 a0533 a0904; do
-            for id in {0..127}; do
-                echo "$metric,cluster=alex,hostname=$hostname,type=hwthread,type-id=$id value=$((1 + RANDOM % 100)).0 $timestamp" >>sample_alex.txt
-            done
-        done
-    done
-
-    curl -X 'POST' 'http://localhost:8082/api/write/?cluster=alex' -H "Authorization: Bearer $JWT" --data-binary @sample_alex.txt
-
-    echo "Fritz Metrics for hwthread types and type-ids"
-    for metric in cpu_load cpu_user flops_any cpu_irq cpu_system ipc cpu_idle cpu_iowait core_power clock; do
-        for hostname in f0201 f0202 f0203 f0204 f0205 f0206 f0207 f0208 f0209 f0210 f0211 f0212 f0213 f0214 f0215 f0217 f0218 f0219 f0220 f0221 f0222 f0223 f0224 f0225 f0226 f0227 f0228 f0229 f0230 f0231 f0232 f0233 f0234 f0235 f0236 f0237 f0238 f0239 f0240 f0241 f0242 f0243 f0244 f0245 f0246 f0247 f0248 f0249 f0250 f0251 f0252 f0253 f0254 f0255 f0256 f0257 f0258 f0259 f0260 f0261 f0262 f0263 f0264 f0378; do
-            for id in {0..71}; do
-                echo "$metric,cluster=fritz,hostname=$hostname,type=hwthread,type-id=$id value=$((1 + RANDOM % 100)).0 $timestamp" >>sample_fritz.txt
-            done
-        done
-    done
-
-    curl -X 'POST' 'http://localhost:8082/api/write/?cluster=fritz' -H "Authorization: Bearer $JWT" --data-binary @sample_fritz.txt
-
-    rm sample_fritz.txt
-    rm sample_alex.txt
-
-    echo "Alex Metrics for accelerator types and type-ids"
-    for metric in cpu_load cpu_user flops_any cpu_irq cpu_system ipc cpu_idle cpu_iowait core_power clock; do
-        for hostname in a0603 a0903 a0832 a0329 a0702 a0122 a1624 a0731 a0224 a0704 a0631 a0225 a0222 a0427 a0603 a0429 a0833 a0705 a0901 a0601 a0227 a0804 a0322 a0226 a0126 a0129 a0605 a0801 a0934 a1622 a0902 a0428 a0537 a1623 a1722 a0228 a0701 a0326 a0327 a0123 a0321 a1621 a0323 a0124 a0534 a0931 a0324 a0933 a0424 a0905 a0128 a0532 a0805 a0521 a0535 a0932 a0127 a0325 a0633 a0831 a0803 a0426 a0425 a0229 a1721 a0602 a0632 a0223 a0422 a0423 a0536 a0328 a0703 anvme7 a0125 a0221 a0604 a0802 a0522 a0531 a0533 a0904; do
-            for id in 00000000:49:00.0 00000000:0E:00.0 00000000:D1:00.0 00000000:90:00.0 00000000:13:00.0 00000000:96:00.0 00000000:CC:00.0 00000000:4F:00.0; do
-                echo "$metric,cluster=alex,hostname=$hostname,type=accelerator,type-id=$id value=$((1 + RANDOM % 100)).0 $timestamp" >>sample_alex.txt
-            done
-        done
-    done
-
-    curl -X 'POST' 'http://localhost:8082/api/write/?cluster=alex' -H "Authorization: Bearer $JWT" --data-binary @sample_alex.txt
-
-    rm sample_alex.txt
-
-    echo "Alex Metrics for memoryDomain types and type-ids"
-    for metric in cpu_load cpu_user flops_any cpu_irq cpu_system ipc cpu_idle cpu_iowait core_power clock; do
-        for hostname in a0603 a0903 a0832 a0329 a0702 a0122 a1624 a0731 a0224 a0704 a0631 a0225 a0222 a0427 a0603 a0429 a0833 a0705 a0901 a0601 a0227 a0804 a0322 a0226 a0126 a0129 a0605 a0801 a0934 a1622 a0902 a0428 a0537 a1623 a1722 a0228 a0701 a0326 a0327 a0123 a0321 a1621 a0323 a0124 a0534 a0931 a0324 a0933 a0424 a0905 a0128 a0532 a0805 a0521 a0535 a0932 a0127 a0325 a0633 a0831 a0803 a0426 a0425 a0229 a1721 a0602 a0632 a0223 a0422 a0423 a0536 a0328 a0703 anvme7 a0125 a0221 a0604 a0802 a0522 a0531 a0533 a0904; do
-            for id in {0..7}; do
-                echo "$metric,cluster=alex,hostname=$hostname,type=memoryDomain,type-id=$id value=$((1 + RANDOM % 100)).0 $timestamp" >>sample_alex.txt
-            done
-        done
-    done
-
-    curl -X 'POST' 'http://localhost:8082/api/write/?cluster=alex' -H "Authorization: Bearer $JWT" --data-binary @sample_alex.txt
-
-    rm sample_alex.txt
-
-    echo "Alex Metrics for socket types and type-ids"
-    for metric in cpu_load cpu_user flops_any cpu_irq cpu_system ipc cpu_idle cpu_iowait core_power clock; do
-        for hostname in a0603 a0903 a0832 a0329 a0702 a0122 a1624 a0731 a0224 a0704 a0631 a0225 a0222 a0427 a0603 a0429 a0833 a0705 a0901 a0601 a0227 a0804 a0322 a0226 a0126 a0129 a0605 a0801 a0934 a1622 a0902 a0428 a0537 a1623 a1722 a0228 a0701 a0326 a0327 a0123 a0321 a1621 a0323 a0124 a0534 a0931 a0324 a0933 a0424 a0905 a0128 a0532 a0805 a0521 a0535 a0932 a0127 a0325 a0633 a0831 a0803 a0426 a0425 a0229 a1721 a0602 a0632 a0223 a0422 a0423 a0536 a0328 a0703 anvme7 a0125 a0221 a0604 a0802 a0522 a0531 a0533 a0904; do
-            for id in {0..1}; do
-                echo "$metric,cluster=alex,hostname=$hostname,type=socket,type-id=$id value=$((1 + RANDOM % 100)).0 $timestamp" >>sample_alex.txt
-            done
-        done
-    done
-
-    curl -X 'POST' 'http://localhost:8082/api/write/?cluster=alex' -H "Authorization: Bearer $JWT" --data-binary @sample_alex.txt
-
-    echo "Fritz Metrics for socket types and type-ids"
-    for metric in cpu_load cpu_user flops_any cpu_irq cpu_system ipc cpu_idle cpu_iowait core_power clock; do
-        for hostname in f0201 f0202 f0203 f0204 f0205 f0206 f0207 f0208 f0209 f0210 f0211 f0212 f0213 f0214 f0215 f0217 f0218 f0219 f0220 f0221 f0222 f0223 f0224 f0225 f0226 f0227 f0228 f0229 f0230 f0231 f0232 f0233 f0234 f0235 f0236 f0237 f0238 f0239 f0240 f0241 f0242 f0243 f0244 f0245 f0246 f0247 f0248 f0249 f0250 f0251 f0252 f0253 f0254 f0255 f0256 f0257 f0258 f0259 f0260 f0261 f0262 f0263 f0264 f0378; do
-            for id in {0..1}; do
-                echo "$metric,cluster=fritz,hostname=$hostname,type=socket,type-id=$id value=$((1 + RANDOM % 100)).0 $timestamp" >>sample_fritz.txt
-            done
-        done
-    done
-
-    curl -X 'POST' 'http://localhost:8082/api/write/?cluster=fritz' -H "Authorization: Bearer $JWT" --data-binary @sample_fritz.txt
-
-    rm sample_fritz.txt
-    rm sample_alex.txt
-
-    echo "Alex Metrics for nodes"
-    for metric in cpu_irq cpu_load mem_cached net_bytes_in cpu_user cpu_idle nfs4_read mem_used nfs4_write nfs4_total ib_xmit ib_xmit_pkts net_bytes_out cpu_iowait ib_recv cpu_system ib_recv_pkts; do
-        for hostname in a0603 a0903 a0832 a0329 a0702 a0122 a1624 a0731 a0224 a0704 a0631 a0225 a0222 a0427 a0603 a0429 a0833 a0705 a0901 a0601 a0227 a0804 a0322 a0226 a0126 a0129 a0605 a0801 a0934 a1622 a0902 a0428 a0537 a1623 a1722 a0228 a0701 a0326 a0327 a0123 a0321 a1621 a0323 a0124 a0534 a0931 a0324 a0933 a0424 a0905 a0128 a0532 a0805 a0521 a0535 a0932 a0127 a0325 a0633 a0831 a0803 a0426 a0425 a0229 a1721 a0602 a0632 a0223 a0422 a0423 a0536 a0328 a0703 anvme7 a0125 a0221 a0604 a0802 a0522 a0531 a0533 a0904; do
-            echo "$metric,cluster=alex,hostname=$hostname,type=node value=$((1 + RANDOM % 100)).0 $timestamp" >>sample_alex.txt
-        done
-    done
-
-    curl -X 'POST' 'http://localhost:8082/api/write/?cluster=alex' -H "Authorization: Bearer $JWT" --data-binary @sample_alex.txt
-
-    echo "Fritz Metrics for nodes"
-    for metric in cpu_irq cpu_load mem_cached net_bytes_in cpu_user cpu_idle nfs4_read mem_used nfs4_write nfs4_total ib_xmit ib_xmit_pkts net_bytes_out cpu_iowait ib_recv cpu_system ib_recv_pkts; do
-        for hostname in f0201 f0202 f0203 f0204 f0205 f0206 f0207 f0208 f0209 f0210 f0211 f0212 f0213 f0214 f0215 f0217 f0218 f0219 f0220 f0221 f0222 f0223 f0224 f0225 f0226 f0227 f0228 f0229 f0230 f0231 f0232 f0233 f0234 f0235 f0236 f0237 f0238 f0239 f0240 f0241 f0242 f0243 f0244 f0245 f0246 f0247 f0248 f0249 f0250 f0251 f0252 f0253 f0254 f0255 f0256 f0257 f0258 f0259 f0260 f0261 f0262 f0263 f0264 f0378; do
-            echo "$metric,cluster=fritz,hostname=$hostname,type=node value=$((1 + RANDOM % 100)).0 $timestamp" >>sample_fritz.txt
-        done
-    done
-
-    curl -X 'POST' 'http://localhost:8082/api/write/?cluster=fritz' -H "Authorization: Bearer $JWT" --data-binary @sample_fritz.txt
-
-    rm sample_fritz.txt
-    rm sample_alex.txt
-
-    sleep 1m
-done
-# curl -X 'POST' 'http://localhost:8081/api/write/?cluster=alex' -H "Authorization: Bearer $JWT" -d $'cpu_load,cluster=alex,hostname=a042,type=hwthread,type-id=0 value=35.0 1725827464642231296'

float.go

@@ -0,0 +1,63 @@
+package main
+
+import (
+	"math"
+	"strconv"
+)
+
+// Go's JSON encoder for floats does not support NaN (https://github.com/golang/go/issues/3480).
+// This program uses NaN as a signal for missing data.
+// For the HTTP JSON API to be able to handle NaN values,
+// we have to use our own type which implements encoding/json.Marshaler itself.
+type Float float64
+
+var NaN Float = Float(math.NaN())
+var nullAsBytes []byte = []byte("null")
+
+func (f Float) IsNaN() bool {
+	return math.IsNaN(float64(f))
+}
+
+func (f Float) MarshalJSON() ([]byte, error) {
+	if math.IsNaN(float64(f)) {
+		return nullAsBytes, nil
+	}
+
+	return strconv.AppendFloat(make([]byte, 0, 10), float64(f), 'f', 3, 64), nil
+}
+
+func (f *Float) UnmarshalJSON(input []byte) error {
+	if string(input) == "null" {
+		*f = NaN
+		return nil
+	}
+
+	val, err := strconv.ParseFloat(string(input), 64)
+	if err != nil {
+		return err
+	}
+	*f = Float(val)
+	return nil
+}
+
+// Same as `[]Float`, but can be marshaled to JSON with less allocations.
+type FloatArray []Float
+
+func (fa FloatArray) MarshalJSON() ([]byte, error) {
+	buf := make([]byte, 0, 2+len(fa)*8)
+	buf = append(buf, '[')
+	for i := 0; i < len(fa); i++ {
+		if i != 0 {
+			buf = append(buf, ',')
+		}
+
+		if fa[i].IsNaN() {
+			buf = append(buf, `null`...)
+		} else {
+			buf = strconv.AppendFloat(buf, float64(fa[i]), 'f', 3, 64)
+
+		}
+	}
+	buf = append(buf, ']')
+	return buf, nil
+}

go.mod

@@ -1,76 +1,13 @@
 module github.com/ClusterCockpit/cc-metric-store
 
-go 1.25.0
+go 1.16
 
 require (
-	github.com/ClusterCockpit/cc-backend v1.5.0
+	github.com/golang-jwt/jwt/v4 v4.0.0
-	github.com/ClusterCockpit/cc-lib/v2 v2.8.0
+	github.com/golang/protobuf v1.5.2 // indirect
-	github.com/ClusterCockpit/cc-line-protocol/v2 v2.4.0
+	github.com/google/gops v0.3.22
-	github.com/golang-jwt/jwt/v4 v4.5.2
+	github.com/gorilla/mux v1.8.0
-	github.com/google/gops v0.3.29
+	github.com/influxdata/line-protocol/v2 v2.2.0
-	github.com/santhosh-tekuri/jsonschema/v5 v5.3.1
+	github.com/nats-io/nats-server/v2 v2.2.6 // indirect
-	github.com/swaggo/http-swagger/v2 v2.0.2
+	github.com/nats-io/nats.go v1.11.0
-	github.com/swaggo/swag v1.16.6
-)
-
-require (
-	github.com/KyleBanks/depth v1.2.1 // indirect
-	github.com/andybalholm/brotli v1.2.0 // indirect
-	github.com/aws/aws-sdk-go-v2 v1.41.3 // indirect
-	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.6 // indirect
-	github.com/aws/aws-sdk-go-v2/config v1.32.11 // indirect
-	github.com/aws/aws-sdk-go-v2/credentials v1.19.11 // indirect
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.19 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.19 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.19 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.5 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.20 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.6 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.11 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.19 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.19 // indirect
-	github.com/aws/aws-sdk-go-v2/service/s3 v1.96.4 // indirect
-	github.com/aws/aws-sdk-go-v2/service/signin v1.0.7 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.30.12 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.16 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.41.8 // indirect
-	github.com/aws/smithy-go v1.24.2 // indirect
-	github.com/cpuguy83/go-md2man/v2 v2.0.7 // indirect
-	github.com/fsnotify/fsnotify v1.9.0 // indirect
-	github.com/go-openapi/jsonpointer v0.22.5 // indirect
-	github.com/go-openapi/jsonreference v0.21.5 // indirect
-	github.com/go-openapi/spec v0.22.4 // indirect
-	github.com/go-openapi/swag/conv v0.25.5 // indirect
-	github.com/go-openapi/swag/jsonname v0.25.5 // indirect
-	github.com/go-openapi/swag/jsonutils v0.25.5 // indirect
-	github.com/go-openapi/swag/loading v0.25.5 // indirect
-	github.com/go-openapi/swag/stringutils v0.25.5 // indirect
-	github.com/go-openapi/swag/typeutils v0.25.5 // indirect
-	github.com/go-openapi/swag/yamlutils v0.25.5 // indirect
-	github.com/google/uuid v1.6.0 // indirect
-	github.com/klauspost/compress v1.18.4 // indirect
-	github.com/mattn/go-sqlite3 v1.14.34 // indirect
-	github.com/nats-io/nats.go v1.49.0 // indirect
-	github.com/nats-io/nkeys v0.4.15 // indirect
-	github.com/nats-io/nuid v1.0.1 // indirect
-	github.com/parquet-go/bitpack v1.0.0 // indirect
-	github.com/parquet-go/jsonlite v1.4.0 // indirect
-	github.com/parquet-go/parquet-go v0.28.0 // indirect
-	github.com/pierrec/lz4/v4 v4.1.26 // indirect
-	github.com/russross/blackfriday/v2 v2.1.0 // indirect
-	github.com/swaggo/files/v2 v2.0.2 // indirect
-	github.com/twpayne/go-geom v1.6.1 // indirect
-	github.com/urfave/cli/v2 v2.27.7 // indirect
-	github.com/xrash/smetrics v0.0.0-20250705151800-55b8f293f342 // indirect
-	go.yaml.in/yaml/v2 v2.4.3 // indirect
-	go.yaml.in/yaml/v3 v3.0.4 // indirect
-	golang.org/x/crypto v0.48.0 // indirect
-	golang.org/x/mod v0.33.0 // indirect
-	golang.org/x/sync v0.19.0 // indirect
-	golang.org/x/sys v0.41.0 // indirect
-	golang.org/x/text v0.34.0 // indirect
-	golang.org/x/tools v0.42.0 // indirect
-	google.golang.org/protobuf v1.36.11 // indirect
-	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
-	sigs.k8s.io/yaml v1.6.0 // indirect
 )

go.sum

@@ -1,205 +1,110 @@
-github.com/ClusterCockpit/cc-backend v1.5.0 h1:KGAxcb6wB9h2ICX1im3ldHJvmkslzV980aweUtAXCKM=
+github.com/StackExchange/wmi v1.2.1 h1:VIkavFPXSjcnS+O8yTq7NI32k0R5Aj+v39y29VYDOSA=
-github.com/ClusterCockpit/cc-backend v1.5.0/go.mod h1:ASdOqR16C0b23KSvzh0BafydIvgA3RzZocQ/hQ1hhN8=
+github.com/StackExchange/wmi v1.2.1/go.mod h1:rcmrprowKIVzvc+NUiLncP2uuArMWLCbu9SBzvHz7e8=
-github.com/ClusterCockpit/cc-lib/v2 v2.8.0 h1:ROduRzRuusi+6kLB991AAu3Pp2AHOasQJFJc7JU/n/E=
+github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
-github.com/ClusterCockpit/cc-lib/v2 v2.8.0/go.mod h1:FwD8vnTIbBM3ngeLNKmCvp9FoSjQZm7xnuaVxEKR23o=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/ClusterCockpit/cc-line-protocol/v2 v2.4.0 h1:hIzxgTBWcmCIHtoDKDkSCsKCOCOwUC34sFsbD2wcW0Q=
+github.com/frankban/quicktest v1.11.0/go.mod h1:K+q6oSqb0W0Ininfk863uOk1lMy69l/P6txr3mVT54s=
-github.com/ClusterCockpit/cc-line-protocol/v2 v2.4.0/go.mod h1:y42qUu+YFmu5fdNuUAS4VbbIKxVjxCvbVqFdpdh8ahY=
+github.com/frankban/quicktest v1.11.2/go.mod h1:K+q6oSqb0W0Ininfk863uOk1lMy69l/P6txr3mVT54s=
-github.com/DATA-DOG/go-sqlmock v1.5.2 h1:OcvFkGmslmlZibjAjaHm3L//6LiuBgolP7OputlJIzU=
-github.com/DATA-DOG/go-sqlmock v1.5.2/go.mod h1:88MAG/4G7SMwSE3CeA0ZKzrT5CiOU3OJ+JlNzwDqpNU=
-github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc=
-github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE=
-github.com/NVIDIA/go-nvml v0.13.0-1 h1:OLX8Jq3dONuPOQPC7rndB6+iDmDakw0XTYgzMxObkEw=
-github.com/NVIDIA/go-nvml v0.13.0-1/go.mod h1:+KNA7c7gIBH7SKSJ1ntlwkfN80zdx8ovl4hrK3LmPt4=
-github.com/alecthomas/assert/v2 v2.10.0 h1:jjRCHsj6hBJhkmhznrCzoNpbA3zqy0fYiUcYZP/GkPY=
-github.com/alecthomas/assert/v2 v2.10.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k=
-github.com/alecthomas/repr v0.4.0 h1:GhI2A8MACjfegCPVq9f1FLvIBS+DrQ2KQBFZP1iFzXc=
-github.com/alecthomas/repr v0.4.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4=
-github.com/andybalholm/brotli v1.2.0 h1:ukwgCxwYrmACq68yiUqwIWnGY0cTPox/M94sVwToPjQ=
-github.com/andybalholm/brotli v1.2.0/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY=
-github.com/apapsch/go-jsonmerge/v2 v2.0.0 h1:axGnT1gRIfimI7gJifB699GoE/oq+F2MU7Dml6nw9rQ=
-github.com/apapsch/go-jsonmerge/v2 v2.0.0/go.mod h1:lvDnEdqiQrp0O42VQGgmlKpxL1AP2+08jFMw88y4klk=
-github.com/aws/aws-sdk-go-v2 v1.41.3 h1:4kQ/fa22KjDt13QCy1+bYADvdgcxpfH18f0zP542kZA=
-github.com/aws/aws-sdk-go-v2 v1.41.3/go.mod h1:mwsPRE8ceUUpiTgF7QmQIJ7lgsKUPQOUl3o72QBrE1o=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.6 h1:N4lRUXZpZ1KVEUn6hxtco/1d2lgYhNn1fHkkl8WhlyQ=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.6/go.mod h1:lyw7GFp3qENLh7kwzf7iMzAxDn+NzjXEAGjKS2UOKqI=
-github.com/aws/aws-sdk-go-v2/config v1.32.11 h1:ftxI5sgz8jZkckuUHXfC/wMUc8u3fG1vQS0plr2F2Zs=
-github.com/aws/aws-sdk-go-v2/config v1.32.11/go.mod h1:twF11+6ps9aNRKEDimksp923o44w/Thk9+8YIlzWMmo=
-github.com/aws/aws-sdk-go-v2/credentials v1.19.11 h1:NdV8cwCcAXrCWyxArt58BrvZJ9pZ9Fhf9w6Uh5W3Uyc=
-github.com/aws/aws-sdk-go-v2/credentials v1.19.11/go.mod h1:30yY2zqkMPdrvxBqzI9xQCM+WrlrZKSOpSJEsylVU+8=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.19 h1:INUvJxmhdEbVulJYHI061k4TVuS3jzzthNvjqvVvTKM=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.19/go.mod h1:FpZN2QISLdEBWkayloda+sZjVJL+e9Gl0k1SyTgcswU=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.19 h1:/sECfyq2JTifMI2JPyZ4bdRN77zJmr6SrS1eL3augIA=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.19/go.mod h1:dMf8A5oAqr9/oxOfLkC/c2LU/uMcALP0Rgn2BD5LWn0=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.19 h1:AWeJMk33GTBf6J20XJe6qZoRSJo0WfUhsMdUKhoODXE=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.19/go.mod h1:+GWrYoaAsV7/4pNHpwh1kiNLXkKaSoppxQq9lbH8Ejw=
-github.com/aws/aws-sdk-go-v2/internal/ini v1.8.5 h1:clHU5fm//kWS1C2HgtgWxfQbFbx4b6rx+5jzhgX9HrI=
-github.com/aws/aws-sdk-go-v2/internal/ini v1.8.5/go.mod h1:O3h0IK87yXci+kg6flUKzJnWeziQUKciKrLjcatSNcY=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.20 h1:qi3e/dmpdONhj1RyIZdi6DKKpDXS5Lb8ftr3p7cyHJc=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.20/go.mod h1:V1K+TeJVD5JOk3D9e5tsX2KUdL7BlB+FV6cBhdobN8c=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.6 h1:XAq62tBTJP/85lFD5oqOOe7YYgWxY9LvWq8plyDvDVg=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.6/go.mod h1:x0nZssQ3qZSnIcePWLvcoFisRXJzcTVvYpAAdYX8+GI=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.11 h1:BYf7XNsJMzl4mObARUBUib+j2tf0U//JAAtTnYqvqCw=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.11/go.mod h1:aEUS4WrNk/+FxkBZZa7tVgp4pGH+kFGW40Y8rCPqt5g=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.19 h1:X1Tow7suZk9UCJHE1Iw9GMZJJl0dAnKXXP1NaSDHwmw=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.19/go.mod h1:/rARO8psX+4sfjUQXp5LLifjUt8DuATZ31WptNJTyQA=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.19 h1:JnQeStZvPHFHeyky/7LbMlyQjUa+jIBj36OlWm0pzIk=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.19/go.mod h1:HGyasyHvYdFQeJhvDHfH7HXkHh57htcJGKDZ+7z+I24=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.96.4 h1:4ExZyubQ6LQQVuF2Qp9OsfEvsTdAWh5Gfwf6PgIdLdk=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.96.4/go.mod h1:NF3JcMGOiARAss1ld3WGORCw71+4ExDD2cbbdKS5PpA=
-github.com/aws/aws-sdk-go-v2/service/signin v1.0.7 h1:Y2cAXlClHsXkkOvWZFXATr34b0hxxloeQu/pAZz2row=
-github.com/aws/aws-sdk-go-v2/service/signin v1.0.7/go.mod h1:idzZ7gmDeqeNrSPkdbtMp9qWMgcBwykA7P7Rzh5DXVU=
-github.com/aws/aws-sdk-go-v2/service/sso v1.30.12 h1:iSsvB9EtQ09YrsmIc44Heqlx5ByGErqhPK1ZQLppias=
-github.com/aws/aws-sdk-go-v2/service/sso v1.30.12/go.mod h1:fEWYKTRGoZNl8tZ77i61/ccwOMJdGxwOhWCkp6TXAr0=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.16 h1:EnUdUqRP1CNzt2DkV67tJx6XDN4xlfBFm+bzeNOQVb0=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.16/go.mod h1:Jic/xv0Rq/pFNCh3WwpH4BEqdbSAl+IyHro8LbibHD8=
-github.com/aws/aws-sdk-go-v2/service/sts v1.41.8 h1:XQTQTF75vnug2TXS8m7CVJfC2nniYPZnO1D4Np761Oo=
-github.com/aws/aws-sdk-go-v2/service/sts v1.41.8/go.mod h1:Xgx+PR1NUOjNmQY+tRMnouRp83JRM8pRMw/vCaVhPkI=
-github.com/aws/smithy-go v1.24.2 h1:FzA3bu/nt/vDvmnkg+R8Xl46gmzEDam6mZ1hzmwXFng=
-github.com/aws/smithy-go v1.24.2/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
-github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
-github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
-github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
-github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
-github.com/cpuguy83/go-md2man/v2 v2.0.7 h1:zbFlGlXEAKlwXpmvle3d8Oe3YnkKIK4xSRTd3sHPnBo=
-github.com/cpuguy83/go-md2man/v2 v2.0.7/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
-github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
-github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/expr-lang/expr v1.17.8 h1:W1loDTT+0PQf5YteHSTpju2qfUfNoBt4yw9+wOEU9VM=
-github.com/expr-lang/expr v1.17.8/go.mod h1:8/vRC7+7HBzESEqt5kKpYXxrxkr31SaO8r40VO/1IT4=
-github.com/frankban/quicktest v1.13.0 h1:yNZif1OkDfNoDfb9zZa9aXIpejNR4F23Wely0c+Qdqk=
 github.com/frankban/quicktest v1.13.0/go.mod h1:qLE0fzW0VuyUAJgPU19zByoIr0HtCHN/r/VLSOOIySU=
-github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k=
+github.com/go-ole/go-ole v1.2.5/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
-github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
+github.com/go-ole/go-ole v1.2.6-0.20210915003542-8b1f7f90f6b1 h1:4dntyT+x6QTOSCIrgczbQ+ockAEha0cfxD5Wi0iCzjY=
-github.com/go-openapi/jsonpointer v0.22.5 h1:8on/0Yp4uTb9f4XvTrM2+1CPrV05QPZXu+rvu2o9jcA=
+github.com/go-ole/go-ole v1.2.6-0.20210915003542-8b1f7f90f6b1/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
-github.com/go-openapi/jsonpointer v0.22.5/go.mod h1:gyUR3sCvGSWchA2sUBJGluYMbe1zazrYWIkWPjjMUY0=
+github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY=
-github.com/go-openapi/jsonreference v0.21.5 h1:6uCGVXU/aNF13AQNggxfysJ+5ZcU4nEAe+pJyVWRdiE=
+github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I=
-github.com/go-openapi/jsonreference v0.21.5/go.mod h1:u25Bw85sX4E2jzFodh1FOKMTZLcfifd1Q+iKKOUxExw=
+github.com/golang-jwt/jwt/v4 v4.0.0 h1:RAqyYixv1p7uEnocuy8P1nru5wprCh/MH2BIlW5z5/o=
-github.com/go-openapi/spec v0.22.4 h1:4pxGjipMKu0FzFiu/DPwN3CTBRlVM2yLf/YTWorYfDQ=
+github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg=
-github.com/go-openapi/spec v0.22.4/go.mod h1:WQ6Ai0VPWMZgMT4XySjlRIE6GP1bGQOtEThn3gcWLtQ=
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
-github.com/go-openapi/swag v0.19.15 h1:D2NRCBzS9/pEY3gP9Nl8aDqGUcPFrwG2p+CNFrLyrCM=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
-github.com/go-openapi/swag/conv v0.25.5 h1:wAXBYEXJjoKwE5+vc9YHhpQOFj2JYBMF2DUi+tGu97g=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
-github.com/go-openapi/swag/conv v0.25.5/go.mod h1:CuJ1eWvh1c4ORKx7unQnFGyvBbNlRKbnRyAvDvzWA4k=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
-github.com/go-openapi/swag/jsonname v0.25.5 h1:8p150i44rv/Drip4vWI3kGi9+4W9TdI3US3uUYSFhSo=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
-github.com/go-openapi/swag/jsonname v0.25.5/go.mod h1:jNqqikyiAK56uS7n8sLkdaNY/uq6+D2m2LANat09pKU=
+github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
-github.com/go-openapi/swag/jsonutils v0.25.5 h1:XUZF8awQr75MXeC+/iaw5usY/iM7nXPDwdG3Jbl9vYo=
+github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
-github.com/go-openapi/swag/jsonutils v0.25.5/go.mod h1:48FXUaz8YsDAA9s5AnaUvAmry1UcLcNVWUjY42XkrN4=
+github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
-github.com/go-openapi/swag/jsonutils/fixtures_test v0.25.5 h1:SX6sE4FrGb4sEnnxbFL/25yZBb5Hcg1inLeErd86Y1U=
+github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
-github.com/go-openapi/swag/jsonutils/fixtures_test v0.25.5/go.mod h1:/2KvOTrKWjVA5Xli3DZWdMCZDzz3uV/T7bXwrKWPquo=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
-github.com/go-openapi/swag/loading v0.25.5 h1:odQ/umlIZ1ZVRteI6ckSrvP6e2w9UTF5qgNdemJHjuU=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
-github.com/go-openapi/swag/loading v0.25.5/go.mod h1:I8A8RaaQ4DApxhPSWLNYWh9NvmX2YKMoB9nwvv6oW6g=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/go-openapi/swag/stringutils v0.25.5 h1:NVkoDOA8YBgtAR/zvCx5rhJKtZF3IzXcDdwOsYzrB6M=
+github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/go-openapi/swag/stringutils v0.25.5/go.mod h1:PKK8EZdu4QJq8iezt17HM8RXnLAzY7gW0O1KKarrZII=
+github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/go-openapi/swag/typeutils v0.25.5 h1:EFJ+PCga2HfHGdo8s8VJXEVbeXRCYwzzr9u4rJk7L7E=
+github.com/google/gops v0.3.22 h1:lyvhDxfPLHAOR2xIYwjPhN387qHxyU21Sk9sz/GhmhQ=
-github.com/go-openapi/swag/typeutils v0.25.5/go.mod h1:itmFmScAYE1bSD8C4rS0W+0InZUBrB2xSPbWt6DLGuc=
+github.com/google/gops v0.3.22/go.mod h1:7diIdLsqpCihPSX3fQagksT/Ku/y4RL9LHTlKyEUDl8=
-github.com/go-openapi/swag/yamlutils v0.25.5 h1:kASCIS+oIeoc55j28T4o8KwlV2S4ZLPT6G0iq2SSbVQ=
+github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
-github.com/go-openapi/swag/yamlutils v0.25.5/go.mod h1:Gek1/SjjfbYvM+Iq4QGwa/2lEXde9n2j4a3wI3pNuOQ=
+github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
-github.com/go-openapi/testify/enable/yaml/v2 v2.4.0 h1:7SgOMTvJkM8yWrQlU8Jm18VeDPuAvB/xWrdxFJkoFag=
+github.com/influxdata/line-protocol-corpus v0.0.0-20210519164801-ca6fa5da0184/go.mod h1:03nmhxzZ7Xk2pdG+lmMd7mHDfeVOYFyhOgwO61qWU98=
-github.com/go-openapi/testify/enable/yaml/v2 v2.4.0/go.mod h1:14iV8jyyQlinc9StD7w1xVPW3CO3q1Gj04Jy//Kw4VM=
-github.com/go-openapi/testify/v2 v2.4.0 h1:8nsPrHVCWkQ4p8h1EsRVymA2XABB4OT40gcvAu+voFM=
-github.com/go-openapi/testify/v2 v2.4.0/go.mod h1:HCPmvFFnheKK2BuwSA0TbbdxJ3I16pjwMkYkP4Ywn54=
-github.com/golang-jwt/jwt/v4 v4.5.2 h1:YtQM7lnr8iZ+j5q71MGKkNw9Mn7AjHM68uc9g5fXeUI=
-github.com/golang-jwt/jwt/v4 v4.5.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
-github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
-github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
-github.com/google/gops v0.3.29 h1:n98J2qSOK1NJvRjdLDcjgDryjpIBGhbaqph1mXKL0rY=
-github.com/google/gops v0.3.29/go.mod h1:8N3jZftuPazvUwtYY/ncG4iPrjp15ysNKLfq+QQPiwc=
-github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
-github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY=
-github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ=
-github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=
-github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg=
-github.com/influxdata/influxdb-client-go/v2 v2.14.0 h1:AjbBfJuq+QoaXNcrova8smSjwJdUHnwvfjMF71M1iI4=
-github.com/influxdata/influxdb-client-go/v2 v2.14.0/go.mod h1:Ahpm3QXKMJslpXl3IftVLVezreAUtBOTZssDrjZEFHI=
-github.com/influxdata/line-protocol v0.0.0-20210922203350-b1ad95c89adf h1:7JTmneyiNEwVBOHSjoMxiWAqB992atOeepeFYegn5RU=
-github.com/influxdata/line-protocol v0.0.0-20210922203350-b1ad95c89adf/go.mod h1:xaLFMmpvUxqXtVkUJfg9QmT88cDaCJ3ZKgdZ78oO8Qo=
-github.com/influxdata/line-protocol-corpus v0.0.0-20210922080147-aa28ccfb8937 h1:MHJNQ+p99hFATQm6ORoLmpUCF7ovjwEFshs/NHzAbig=
 github.com/influxdata/line-protocol-corpus v0.0.0-20210922080147-aa28ccfb8937/go.mod h1:BKR9c0uHSmRgM/se9JhFHtTT7JTO67X23MtKMHtZcpo=
-github.com/klauspost/compress v1.18.4 h1:RPhnKRAQ4Fh8zU2FY/6ZFDwTVTxgJ/EMydqSTzE9a2c=
+github.com/influxdata/line-protocol/v2 v2.0.0-20210312151457-c52fdecb625a/go.mod h1:6+9Xt5Sq1rWx+glMgxhcg2c0DUaehK+5TDcPZ76GypY=
-github.com/klauspost/compress v1.18.4/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4=
+github.com/influxdata/line-protocol/v2 v2.1.0/go.mod h1:QKw43hdUBg3GTk2iC3iyCxksNj7PX9aUSeYOYE/ceHY=
+github.com/influxdata/line-protocol/v2 v2.2.0 h1:UPmAqE15Hw5zu9E10SYhoXVLWnEJkWnuCbaCiRsA3c0=
+github.com/influxdata/line-protocol/v2 v2.2.0/go.mod h1:DmB3Cnh+3oxmG6LOBIxce4oaL4CPj3OmMPgvauXh+tM=
+github.com/keybase/go-ps v0.0.0-20190827175125-91aafc93ba19 h1:WjT3fLi9n8YWh/Ih8Q1LHAPsTqGddPcHqscN+PJ3i68=
+github.com/keybase/go-ps v0.0.0-20190827175125-91aafc93ba19/go.mod h1:hY+WOq6m2FpbvyrI93sMaypsttvaIL5nhVR92dTMUcQ=
+github.com/klauspost/compress v1.11.12 h1:famVnQVu7QwryBN4jNseQdUKES71ZAOnB6UQQJPZvqk=
+github.com/klauspost/compress v1.11.12/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
 github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
-github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
-github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
-github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
-github.com/mattn/go-sqlite3 v1.14.34 h1:3NtcvcUnFBPsuRcno8pUtupspG/GM+9nZ88zgJcp6Zk=
+github.com/minio/highwayhash v1.0.1 h1:dZ6IIu8Z14VlC0VpfKofAhCy74wu/Qb5gcn52yWoz/0=
-github.com/mattn/go-sqlite3 v1.14.34/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
+github.com/minio/highwayhash v1.0.1/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY=
-github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
+github.com/nats-io/jwt v1.2.2 h1:w3GMTO969dFg+UOKTmmyuu7IGdusK+7Ytlt//OYH/uU=
-github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
+github.com/nats-io/jwt v1.2.2/go.mod h1:/xX356yQA6LuXI9xWW7mZNpxgF2mBmGecH+Fj34sP5Q=
-github.com/nats-io/nats.go v1.49.0 h1:yh/WvY59gXqYpgl33ZI+XoVPKyut/IcEaqtsiuTJpoE=
+github.com/nats-io/jwt/v2 v2.0.2 h1:ejVCLO8gu6/4bOKIHQpmB5UhhUJfAQw55yvLWpfmKjI=
-github.com/nats-io/nats.go v1.49.0/go.mod h1:fDCn3mN5cY8HooHwE2ukiLb4p4G4ImmzvXyJt+tGwdw=
+github.com/nats-io/jwt/v2 v2.0.2/go.mod h1:VRP+deawSXyhNjXmxPCHskrR6Mq50BqpEI5SEcNiGlY=
-github.com/nats-io/nkeys v0.4.15 h1:JACV5jRVO9V856KOapQ7x+EY8Jo3qw1vJt/9Jpwzkk4=
+github.com/nats-io/nats-server/v2 v2.2.6 h1:FPK9wWx9pagxcw14s8W9rlfzfyHm61uNLnJyybZbn48=
-github.com/nats-io/nkeys v0.4.15/go.mod h1:CpMchTXC9fxA5zrMo4KpySxNjiDVvr8ANOSZdiNfUrs=
+github.com/nats-io/nats-server/v2 v2.2.6/go.mod h1:sEnFaxqe09cDmfMgACxZbziXnhQFhwk+aKkZjBBRYrI=
+github.com/nats-io/nats.go v1.11.0 h1:L263PZkrmkRJRJT2YHU8GwWWvEvmr9/LUKuJTXsF32k=
+github.com/nats-io/nats.go v1.11.0/go.mod h1:BPko4oXsySz4aSWeFgOHLZs3G4Jq4ZAyE6/zMCxRT6w=
+github.com/nats-io/nkeys v0.2.0/go.mod h1:XdZpAbhgyyODYqjTawOnIOI7VlbKSarI9Gfy1tqEu/s=
+github.com/nats-io/nkeys v0.3.0 h1:cgM5tL53EvYRU+2YLXIK0G2mJtK12Ft9oeooSZMA2G8=
+github.com/nats-io/nkeys v0.3.0/go.mod h1:gvUNGjVcM2IPr5rCsRsC6Wb3Hr2CQAm08dsxtV6A5y4=
 github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
 github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
-github.com/oapi-codegen/runtime v1.2.0 h1:RvKc1CVS1QeKSNzO97FBQbSMZyQ8s6rZd+LpmzwHMP4=
+github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
-github.com/oapi-codegen/runtime v1.2.0/go.mod h1:Y7ZhmmlE8ikZOmuHRRndiIm7nf3xcVv+YMweKgG1DT0=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/parquet-go/bitpack v1.0.0 h1:AUqzlKzPPXf2bCdjfj4sTeacrUwsT7NlcYDMUQxPcQA=
+github.com/shirou/gopsutil/v3 v3.21.9 h1:Vn4MUz2uXhqLSiCbGFRc0DILbMVLAY92DSkT8bsYrHg=
-github.com/parquet-go/bitpack v1.0.0/go.mod h1:XnVk9TH+O40eOOmvpAVZ7K2ocQFrQwysLMnc6M/8lgs=
+github.com/shirou/gopsutil/v3 v3.21.9/go.mod h1:YWp/H8Qs5fVmf17v7JNZzA0mPJ+mS2e9JdiUF9LlKzQ=
-github.com/parquet-go/jsonlite v1.4.0 h1:RTG7prqfO0HD5egejU8MUDBN8oToMj55cgSV1I0zNW4=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/parquet-go/jsonlite v1.4.0/go.mod h1:nDjpkpL4EOtqs6NQugUsi0Rleq9sW/OtC1NnZEnxzF0=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/parquet-go/parquet-go v0.28.0 h1:ECyksyv8T2pOrlLsN7aWJIoQakyk/HtxQ2lchgS4els=
+github.com/tklauser/go-sysconf v0.3.9 h1:JeUVdAOWhhxVcU6Eqr/ATFHgXk/mmiItdKeJPev3vTo=
-github.com/parquet-go/parquet-go v0.28.0/go.mod h1:navtkAYr2LGoJVp141oXPlO/sxLvaOe3la2JEoD8+rg=
+github.com/tklauser/go-sysconf v0.3.9/go.mod h1:11DU/5sG7UexIrp/O6g35hrWzu0JxlwQ3LSFUzyeuhs=
-github.com/pierrec/lz4/v4 v4.1.26 h1:GrpZw1gZttORinvzBdXPUXATeqlJjqUG/D87TKMnhjY=
+github.com/tklauser/numcpus v0.3.0 h1:ILuRUQBtssgnxw0XXIjKUC56fgnOrFoQQ/4+DeU2biQ=
-github.com/pierrec/lz4/v4 v4.1.26/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4=
+github.com/tklauser/numcpus v0.3.0/go.mod h1:yFGUr7TUHQRAhyqBcEg0Ge34zDBAsIvJJcyE6boqnA8=
-github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
+github.com/xlab/treeprint v1.1.0 h1:G/1DjNkPpfZCFt9CSh6b5/nY4VimlbHF3Rh4obvtzDk=
-github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/xlab/treeprint v1.1.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0=
-github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg=
+golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk=
+golang.org/x/crypto v0.0.0-20210314154223-e6e6c4f2bb5b h1:wSOdpTq0/eI46Ez/LkDwIsAKA71YP2SRKBODiRWM0as=
-github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE=
+golang.org/x/crypto v0.0.0-20210314154223-e6e6c4f2bb5b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
-github.com/prometheus/common v0.67.5 h1:pIgK94WWlQt1WLwAC5j2ynLaBRDiinoAb86HZHTUGI4=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-github.com/prometheus/common v0.67.5/go.mod h1:SjE/0MzDEEAyrdr5Gqc6G+sXI67maCxzaT3A2+HqjUw=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
-github.com/prometheus/procfs v0.20.0 h1:AA7aCvjxwAquZAlonN7888f2u4IN8WVeFgBi4k82M4Q=
+golang.org/x/sys v0.0.0-20190130150945-aca44879d564/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-github.com/prometheus/procfs v0.20.0/go.mod h1:o9EMBZGRyvDrSPH1RqdxhojkuXstoe4UlK79eF5TGGo=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
+golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68 h1:nxC68pudNYkKU6jWhgrqdreuFiOQWj1Fs7T3VrH4Pjw=
-github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 h1:lZUw3E0/J3roVtGQ+SCrUrg3ON6NgVqpn3+iol9aGu4=
+golang.org/x/sys v0.0.0-20210816074244-15123e1e1f71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-github.com/santhosh-tekuri/jsonschema/v5 v5.3.1/go.mod h1:uToXkOrWAZ6/Oc07xWQrPOhJotwFIyu2bBVN41fcDUY=
+golang.org/x/sys v0.0.0-20210902050250-f475640dd07b h1:S7hKs0Flbq0bbc9xgYt4stIEG1zNDFqyrPwAX2Wj/sE=
-github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
+golang.org/x/sys v0.0.0-20210902050250-f475640dd07b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
-github.com/swaggo/files/v2 v2.0.2 h1:Bq4tgS/yxLB/3nwOMcul5oLEUKa877Ykgz3CJMVbQKU=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-github.com/swaggo/files/v2 v2.0.2/go.mod h1:TVqetIzZsO9OhHX1Am9sRf9LdrFZqoK49N37KON/jr0=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-github.com/swaggo/http-swagger/v2 v2.0.2 h1:FKCdLsl+sFCx60KFsyM0rDarwiUSZ8DqbfSyIKC9OBg=
+golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1 h1:NusfzzA6yGQ+ua51ck7E3omNUX/JuqbFSaRGqU8CcLI=
-github.com/swaggo/http-swagger/v2 v2.0.2/go.mod h1:r7/GBkAWIfK6E/OLnE8fXnviHiDeAHmgIyooa4xm3AQ=
+golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-github.com/swaggo/swag v1.16.6 h1:qBNcx53ZaX+M5dxVyTrgQ0PJ/ACK+NzhwcbieTt+9yI=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-github.com/swaggo/swag v1.16.6/go.mod h1:ngP2etMK5a0P3QBizic5MEwpRmluJZPHjXcMoj4Xesg=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-github.com/twpayne/go-geom v1.6.1 h1:iLE+Opv0Ihm/ABIcvQFGIiFBXd76oBIar9drAwHFhR4=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
-github.com/twpayne/go-geom v1.6.1/go.mod h1:Kr+Nly6BswFsKM5sd31YaoWS5PeDDH2NftJTK7Gd028=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
-github.com/urfave/cli/v2 v2.27.7 h1:bH59vdhbjLv3LAvIu6gd0usJHgoTTPhCFib8qqOwXYU=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
-github.com/urfave/cli/v2 v2.27.7/go.mod h1:CyNAG/xg+iAOg0N4MPGZqVmv2rCoP267496AOXUZjA4=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
-github.com/xrash/smetrics v0.0.0-20250705151800-55b8f293f342 h1:FnBeRrxr7OU4VvAzt5X7s6266i6cSVkkFPS0TuXWbIg=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
-github.com/xrash/smetrics v0.0.0-20250705151800-55b8f293f342/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM=
+google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
-github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU=
+google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
-github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E=
+google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk=
-go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0=
+google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
-go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8=
-go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
-go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
-golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts=
-golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos=
-golang.org/x/mod v0.33.0 h1:tHFzIWbBifEmbwtGz65eaWyGiGZatSrT9prnU8DbVL8=
-golang.org/x/mod v0.33.0/go.mod h1:swjeQEj+6r7fODbD2cqrnje9PnziFuw4bmLbBZFrQ5w=
-golang.org/x/net v0.51.0 h1:94R/GTO7mt3/4wIKpcR5gkGmRLOuE/2hNGeWq/GBIFo=
-golang.org/x/net v0.51.0/go.mod h1:aamm+2QF5ogm02fjy5Bb7CQ0WMt1/WVM7FtyaTLlA9Y=
-golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4=
-golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
-golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k=
-golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
-golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk=
-golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA=
-golang.org/x/tools v0.42.0 h1:uNgphsn75Tdz5Ji2q36v/nsFSfR/9BRFvqhGBaJGd5k=
-golang.org/x/tools v0.42.0/go.mod h1:Ma6lCIwGZvHK6XtgbswSoWroEkhugApmsXyrUmBhfr0=
-google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=
-google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
+gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+rsc.io/goversion v1.2.0 h1:SPn+NLTiAG7w30IRK/DKp1BjvpWabYgxlLp/+kx5J8w=
-sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs=
+rsc.io/goversion v1.2.0/go.mod h1:Eih9y/uIBS3ulggl7KNJ09xGSLcuNaLgmvvqa07sgfo=
-sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4=

internal/api/api.go

@@ -1,25 +0,0 @@
-// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved. This file is part of cc-metric-store.
-// Use of this source code is governed by a MIT-style
-// license that can be found in the LICENSE file.
-
-// Package api provides the REST API for cc-metric-store.
-//
-// @title       cc-metric-store REST API
-// @version     1.0.0
-// @description API for cc-metric-store
-//
-// @contact.name  ClusterCockpit Project
-// @contact.url   https://clustercockpit.org
-// @contact.email support@clustercockpit.org
-//
-// @license.name MIT License
-// @license.url  https://opensource.org/licenses/MIT
-//
-// @host     localhost:8082
-// @basePath /api/
-//
-// @securityDefinitions.apikey ApiKeyAuth
-// @in                         header
-// @name                       X-Auth-Token
-package api

internal/api/authentication.go

@@ -1,61 +0,0 @@
-// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved. This file is part of cc-metric-store.
-// Use of this source code is governed by a MIT-style
-// license that can be found in the LICENSE file.
-
-package api
-
-import (
-	"crypto/ed25519"
-	"errors"
-	"net/http"
-	"strings"
-	"sync"
-
-	"github.com/golang-jwt/jwt/v4"
-)
-
-func authHandler(next http.Handler, publicKey ed25519.PublicKey) http.Handler {
-	cacheLock := sync.RWMutex{}
-	cache := map[string]*jwt.Token{}
-
-	return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
-		authheader := r.Header.Get("Authorization")
-		if authheader == "" || !strings.HasPrefix(authheader, "Bearer ") {
-			http.Error(rw, "Use JWT Authentication", http.StatusUnauthorized)
-			return
-		}
-
-		rawtoken := authheader[len("Bearer "):]
-		cacheLock.RLock()
-		token, ok := cache[rawtoken]
-		cacheLock.RUnlock()
-		if ok && token.Claims.Valid() == nil {
-			next.ServeHTTP(rw, r)
-			return
-		}
-
-		// The actual token is ignored for now.
-		// In case expiration and so on are specified, the Parse function
-		// already returns an error for expired tokens.
-		var err error
-		token, err = jwt.Parse(rawtoken, func(t *jwt.Token) (any, error) {
-			if t.Method != jwt.SigningMethodEdDSA {
-				return nil, errors.New("only Ed25519/EdDSA supported")
-			}
-
-			return publicKey, nil
-		})
-		if err != nil {
-			http.Error(rw, err.Error(), http.StatusUnauthorized)
-			return
-		}
-
-		cacheLock.Lock()
-		cache[rawtoken] = token
-		cacheLock.Unlock()
-
-		// Let request through...
-		next.ServeHTTP(rw, r)
-	})
-}

internal/api/docs.go

@@ -1,480 +0,0 @@
-// Package api Code generated by swaggo/swag. DO NOT EDIT
-package api
-
-import "github.com/swaggo/swag"
-
-const docTemplate = `{
-    "schemes": {{ marshal .Schemes }},
-    "swagger": "2.0",
-    "info": {
-        "description": "{{escape .Description}}",
-        "title": "{{.Title}}",
-        "contact": {
-            "name": "ClusterCockpit Project",
-            "url": "https://clustercockpit.org",
-            "email": "support@clustercockpit.org"
-        },
-        "license": {
-            "name": "MIT License",
-            "url": "https://opensource.org/licenses/MIT"
-        },
-        "version": "{{.Version}}"
-    },
-    "host": "{{.Host}}",
-    "basePath": "{{.BasePath}}",
-    "paths": {
-        "/debug/": {
-            "post": {
-                "description": "This endpoint allows the users to print the content of",
-                "produces": [
-                    "application/json"
-                ],
-                "tags": [
-                    "debug"
-                ],
-                "summary": "Debug endpoint",
-                "parameters": [
-                    {
-                        "type": "string",
-                        "description": "Selector",
-                        "name": "selector",
-                        "in": "query"
-                    }
-                ],
-                "responses": {
-                    "200": {
-                        "description": "Debug dump",
-                        "schema": {
-                            "type": "string"
-                        }
-                    },
-                    "400": {
-                        "description": "Bad Request",
-                        "schema": {
-                            "$ref": "#/definitions/api.ErrorResponse"
-                        }
-                    },
-                    "401": {
-                        "description": "Unauthorized",
-                        "schema": {
-                            "$ref": "#/definitions/api.ErrorResponse"
-                        }
-                    },
-                    "403": {
-                        "description": "Forbidden",
-                        "schema": {
-                            "$ref": "#/definitions/api.ErrorResponse"
-                        }
-                    },
-                    "500": {
-                        "description": "Internal Server Error",
-                        "schema": {
-                            "$ref": "#/definitions/api.ErrorResponse"
-                        }
-                    }
-                },
-                "security": [
-                    {
-                        "ApiKeyAuth": []
-                    }
-                ]
-            }
-        },
-        "/free/": {
-            "post": {
-                "description": "This endpoint allows the users to free the Buffers from the",
-                "produces": [
-                    "application/json"
-                ],
-                "tags": [
-                    "free"
-                ],
-                "parameters": [
-                    {
-                        "type": "string",
-                        "description": "up to timestamp",
-                        "name": "to",
-                        "in": "query"
-                    }
-                ],
-                "responses": {
-                    "200": {
-                        "description": "ok",
-                        "schema": {
-                            "type": "string"
-                        }
-                    },
-                    "400": {
-                        "description": "Bad Request",
-                        "schema": {
-                            "$ref": "#/definitions/api.ErrorResponse"
-                        }
-                    },
-                    "401": {
-                        "description": "Unauthorized",
-                        "schema": {
-                            "$ref": "#/definitions/api.ErrorResponse"
-                        }
-                    },
-                    "403": {
-                        "description": "Forbidden",
-                        "schema": {
-                            "$ref": "#/definitions/api.ErrorResponse"
-                        }
-                    },
-                    "500": {
-                        "description": "Internal Server Error",
-                        "schema": {
-                            "$ref": "#/definitions/api.ErrorResponse"
-                        }
-                    }
-                },
-                "security": [
-                    {
-                        "ApiKeyAuth": []
-                    }
-                ]
-            }
-        },
-        "/healthcheck/": {
-            "get": {
-                "description": "This endpoint allows the users to check if a node is healthy",
-                "produces": [
-                    "application/json"
-                ],
-                "tags": [
-                    "healthcheck"
-                ],
-                "summary": "HealthCheck endpoint",
-                "parameters": [
-                    {
-                        "type": "string",
-                        "description": "Selector",
-                        "name": "selector",
-                        "in": "query"
-                    }
-                ],
-                "responses": {
-                    "200": {
-                        "description": "Debug dump",
-                        "schema": {
-                            "type": "string"
-                        }
-                    },
-                    "400": {
-                        "description": "Bad Request",
-                        "schema": {
-                            "$ref": "#/definitions/api.ErrorResponse"
-                        }
-                    },
-                    "401": {
-                        "description": "Unauthorized",
-                        "schema": {
-                            "$ref": "#/definitions/api.ErrorResponse"
-                        }
-                    },
-                    "403": {
-                        "description": "Forbidden",
-                        "schema": {
-                            "$ref": "#/definitions/api.ErrorResponse"
-                        }
-                    },
-                    "500": {
-                        "description": "Internal Server Error",
-                        "schema": {
-                            "$ref": "#/definitions/api.ErrorResponse"
-                        }
-                    }
-                },
-                "security": [
-                    {
-                        "ApiKeyAuth": []
-                    }
-                ]
-            }
-        },
-        "/query/": {
-            "get": {
-                "description": "This endpoint allows the users to retrieve data from the",
-                "consumes": [
-                    "application/json"
-                ],
-                "produces": [
-                    "application/json"
-                ],
-                "tags": [
-                    "query"
-                ],
-                "summary": "Query metrics",
-                "parameters": [
-                    {
-                        "description": "API query payload object",
-                        "name": "request",
-                        "in": "body",
-                        "required": true,
-                        "schema": {
-                            "$ref": "#/definitions/api.APIQueryRequest"
-                        }
-                    }
-                ],
-                "responses": {
-                    "200": {
-                        "description": "API query response object",
-                        "schema": {
-                            "$ref": "#/definitions/api.APIQueryResponse"
-                        }
-                    },
-                    "400": {
-                        "description": "Bad Request",
-                        "schema": {
-                            "$ref": "#/definitions/api.ErrorResponse"
-                        }
-                    },
-                    "401": {
-                        "description": "Unauthorized",
-                        "schema": {
-                            "$ref": "#/definitions/api.ErrorResponse"
-                        }
-                    },
-                    "403": {
-                        "description": "Forbidden",
-                        "schema": {
-                            "$ref": "#/definitions/api.ErrorResponse"
-                        }
-                    },
-                    "500": {
-                        "description": "Internal Server Error",
-                        "schema": {
-                            "$ref": "#/definitions/api.ErrorResponse"
-                        }
-                    }
-                },
-                "security": [
-                    {
-                        "ApiKeyAuth": []
-                    }
-                ]
-            }
-        },
-        "/write/": {
-            "post": {
-                "consumes": [
-                    "text/plain"
-                ],
-                "produces": [
-                    "application/json"
-                ],
-                "parameters": [
-                    {
-                        "type": "string",
-                        "description": "If the lines in the body do not have a cluster tag, use this value instead.",
-                        "name": "cluster",
-                        "in": "query"
-                    }
-                ],
-                "responses": {
-                    "200": {
-                        "description": "ok",
-                        "schema": {
-                            "type": "string"
-                        }
-                    },
-                    "400": {
-                        "description": "Bad Request",
-                        "schema": {
-                            "$ref": "#/definitions/api.ErrorResponse"
-                        }
-                    },
-                    "401": {
-                        "description": "Unauthorized",
-                        "schema": {
-                            "$ref": "#/definitions/api.ErrorResponse"
-                        }
-                    },
-                    "403": {
-                        "description": "Forbidden",
-                        "schema": {
-                            "$ref": "#/definitions/api.ErrorResponse"
-                        }
-                    },
-                    "500": {
-                        "description": "Internal Server Error",
-                        "schema": {
-                            "$ref": "#/definitions/api.ErrorResponse"
-                        }
-                    }
-                },
-                "security": [
-                    {
-                        "ApiKeyAuth": []
-                    }
-                ]
-            }
-        }
-    },
-    "definitions": {
-        "api.APIMetricData": {
-            "type": "object",
-            "properties": {
-                "avg": {
-                    "type": "number"
-                },
-                "data": {
-                    "type": "array",
-                    "items": {
-                        "type": "number"
-                    }
-                },
-                "error": {
-                    "type": "string"
-                },
-                "from": {
-                    "type": "integer"
-                },
-                "max": {
-                    "type": "number"
-                },
-                "min": {
-                    "type": "number"
-                },
-                "resolution": {
-                    "type": "integer"
-                },
-                "to": {
-                    "type": "integer"
-                }
-            }
-        },
-        "api.APIQuery": {
-            "type": "object",
-            "properties": {
-                "aggreg": {
-                    "type": "boolean"
-                },
-                "host": {
-                    "type": "string"
-                },
-                "metric": {
-                    "type": "string"
-                },
-                "resolution": {
-                    "type": "integer"
-                },
-                "scale-by": {
-                    "type": "number"
-                },
-                "subtype": {
-                    "type": "string"
-                },
-                "subtype-ids": {
-                    "type": "array",
-                    "items": {
-                        "type": "string"
-                    }
-                },
-                "type": {
-                    "type": "string"
-                },
-                "type-ids": {
-                    "type": "array",
-                    "items": {
-                        "type": "string"
-                    }
-                }
-            }
-        },
-        "api.APIQueryRequest": {
-            "type": "object",
-            "properties": {
-                "cluster": {
-                    "type": "string"
-                },
-                "for-all-nodes": {
-                    "type": "array",
-                    "items": {
-                        "type": "string"
-                    }
-                },
-                "from": {
-                    "type": "integer"
-                },
-                "queries": {
-                    "type": "array",
-                    "items": {
-                        "$ref": "#/definitions/api.APIQuery"
-                    }
-                },
-                "to": {
-                    "type": "integer"
-                },
-                "with-data": {
-                    "type": "boolean"
-                },
-                "with-padding": {
-                    "type": "boolean"
-                },
-                "with-stats": {
-                    "type": "boolean"
-                }
-            }
-        },
-        "api.APIQueryResponse": {
-            "type": "object",
-            "properties": {
-                "queries": {
-                    "type": "array",
-                    "items": {
-                        "$ref": "#/definitions/api.APIQuery"
-                    }
-                },
-                "results": {
-                    "type": "array",
-                    "items": {
-                        "type": "array",
-                        "items": {
-                            "$ref": "#/definitions/api.APIMetricData"
-                        }
-                    }
-                }
-            }
-        },
-        "api.ErrorResponse": {
-            "type": "object",
-            "properties": {
-                "error": {
-                    "description": "Error Message",
-                    "type": "string"
-                },
-                "status": {
-                    "description": "Statustext of Errorcode",
-                    "type": "string"
-                }
-            }
-        }
-    },
-    "securityDefinitions": {
-        "ApiKeyAuth": {
-            "type": "apiKey",
-            "name": "X-Auth-Token",
-            "in": "header"
-        }
-    }
-}`
-
-// SwaggerInfo holds exported Swagger Info so clients can modify it
-var SwaggerInfo = &swag.Spec{
-	Version:          "1.0.0",
-	Host:             "localhost:8082",
-	BasePath:         "/api/",
-	Schemes:          []string{},
-	Title:            "cc-metric-store REST API",
-	Description:      "API for cc-metric-store",
-	InfoInstanceName: "swagger",
-	SwaggerTemplate:  docTemplate,
-	LeftDelim:        "{{",
-	RightDelim:       "}}",
-}
-
-func init() {
-	swag.Register(SwaggerInfo.InstanceName(), SwaggerInfo)
-}

internal/api/metricstore.go

@@ -1,469 +0,0 @@
-// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved. This file is part of cc-metric-store.
-// Use of this source code is governed by a MIT-style
-// license that can be found in the LICENSE file.
-
-package api
-
-import (
-	"bufio"
-	"encoding/json"
-	"errors"
-	"fmt"
-	"log"
-	"math"
-	"net/http"
-	"strconv"
-	"strings"
-
-	"github.com/ClusterCockpit/cc-backend/pkg/metricstore"
-	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
-	"github.com/ClusterCockpit/cc-lib/v2/schema"
-	"github.com/ClusterCockpit/cc-lib/v2/util"
-	"github.com/ClusterCockpit/cc-line-protocol/v2/lineprotocol"
-)
-
-// ErrorResponse model
-type ErrorResponse struct {
-	// Statustext of Errorcode
-	Status string `json:"status"`
-	Error  string `json:"error"` // Error Message
-}
-
-// DefaultAPIResponse model
-type DefaultAPIResponse struct {
-	Message string `json:"msg"`
-}
-
-// handleError writes a standardized JSON error response with the given status code.
-// It logs the error at WARN level and ensures proper Content-Type headers are set.
-func handleError(err error, statusCode int, rw http.ResponseWriter) {
-	cclog.Warnf("REST ERROR : %s", err.Error())
-	rw.Header().Add("Content-Type", "application/json")
-	rw.WriteHeader(statusCode)
-	if err := json.NewEncoder(rw).Encode(ErrorResponse{
-		Status: http.StatusText(statusCode),
-		Error:  err.Error(),
-	}); err != nil {
-		cclog.Errorf("Failed to encode error response: %v", err)
-	}
-}
-
-type APIMetricData struct {
-	Error      *string           `json:"error,omitempty"`
-	Data       schema.FloatArray `json:"data,omitempty"       swaggertype:"array,number"`
-	From       int64             `json:"from"`
-	To         int64             `json:"to"`
-	Resolution int64             `json:"resolution"`
-	Avg        schema.Float      `json:"avg"                  swaggertype:"number"`
-	Min        schema.Float      `json:"min"                  swaggertype:"number"`
-	Max        schema.Float      `json:"max"                  swaggertype:"number"`
-}
-
-// TODO: Optimize this, just like the stats endpoint!
-func (data *APIMetricData) AddStats() {
-	n := 0
-	sum, min, max := 0.0, math.MaxFloat64, -math.MaxFloat64
-	for _, x := range data.Data {
-		if x.IsNaN() {
-			continue
-		}
-
-		n += 1
-		sum += float64(x)
-		min = math.Min(min, float64(x))
-		max = math.Max(max, float64(x))
-	}
-
-	if n > 0 {
-		avg := sum / float64(n)
-		data.Avg = schema.Float(avg)
-		data.Min = schema.Float(min)
-		data.Max = schema.Float(max)
-	} else {
-		data.Avg, data.Min, data.Max = schema.NaN, schema.NaN, schema.NaN
-	}
-}
-
-func (data *APIMetricData) ScaleBy(f schema.Float) {
-	if f == 0 || f == 1 {
-		return
-	}
-
-	data.Avg *= f
-	data.Min *= f
-	data.Max *= f
-	for i := 0; i < len(data.Data); i++ {
-		data.Data[i] *= f
-	}
-}
-
-func (data *APIMetricData) PadDataWithNull(ms *metricstore.MemoryStore, from, to int64, metric string) {
-	minfo, ok := ms.Metrics[metric]
-	if !ok {
-		return
-	}
-
-	if (data.From / minfo.Frequency) > (from / minfo.Frequency) {
-		padfront := int((data.From / minfo.Frequency) - (from / minfo.Frequency))
-		ndata := make([]schema.Float, 0, padfront+len(data.Data))
-		for range padfront {
-			ndata = append(ndata, schema.NaN)
-		}
-		for j := 0; j < len(data.Data); j++ {
-			ndata = append(ndata, data.Data[j])
-		}
-		data.Data = ndata
-	}
-}
-
-type APIQueryRequest struct {
-	Cluster     string     `json:"cluster"`
-	Queries     []APIQuery `json:"queries"`
-	ForAllNodes []string   `json:"for-all-nodes"`
-	From        int64      `json:"from"`
-	To          int64      `json:"to"`
-	WithStats   bool       `json:"with-stats"`
-	WithData    bool       `json:"with-data"`
-	WithPadding bool       `json:"with-padding"`
-}
-
-type APIQueryResponse struct {
-	Queries []APIQuery        `json:"queries,omitempty"`
-	Results [][]APIMetricData `json:"results"`
-}
-
-type APIQuery struct {
-	Type        *string      `json:"type,omitempty"`
-	SubType     *string      `json:"subtype,omitempty"`
-	Metric      string       `json:"metric"`
-	Hostname    string       `json:"host"`
-	Resolution  int64        `json:"resolution"`
-	TypeIds     []string     `json:"type-ids,omitempty"`
-	SubTypeIds  []string     `json:"subtype-ids,omitempty"`
-	ScaleFactor schema.Float `json:"scale-by,omitempty" swaggertype:"number"`
-	Aggregate   bool         `json:"aggreg"`
-}
-
-// handleQuery godoc
-// @summary    Query metrics
-// @tags query
-// @description This endpoint allows the users to retrieve data from the
-// in-memory database. The CCMS will return data in JSON format for the
-// specified interval requested by the user
-// @accept      json
-// @produce     json
-// @param       request body     APIQueryRequest  true "API query payload object"
-// @success     200            {object} APIQueryResponse  "API query response object"
-// @failure     400            {object} ErrorResponse       "Bad Request"
-// @failure     401   		   {object} ErrorResponse       "Unauthorized"
-// @failure     403            {object} ErrorResponse       "Forbidden"
-// @failure     500            {object} ErrorResponse       "Internal Server Error"
-// @security    ApiKeyAuth
-// @router      /query/ [get]
-func handleQuery(rw http.ResponseWriter, r *http.Request) {
-	var err error
-	ver := r.URL.Query().Get("version")
-	if ver == "" {
-		ver = "v2"
-	}
-	req := APIQueryRequest{WithStats: true, WithData: true, WithPadding: true}
-	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
-		handleError(err, http.StatusBadRequest, rw)
-		return
-	}
-
-	ms := metricstore.GetMemoryStore()
-
-	response := APIQueryResponse{
-		Results: make([][]APIMetricData, 0, len(req.Queries)),
-	}
-	if req.ForAllNodes != nil {
-		nodes := ms.ListChildren([]string{req.Cluster})
-		for _, node := range nodes {
-			for _, metric := range req.ForAllNodes {
-				q := APIQuery{
-					Metric:   metric,
-					Hostname: node,
-				}
-				req.Queries = append(req.Queries, q)
-				response.Queries = append(response.Queries, q)
-			}
-		}
-	}
-
-	for _, query := range req.Queries {
-		sels := make([]util.Selector, 0, 1)
-		if query.Aggregate || query.Type == nil {
-			sel := util.Selector{{String: req.Cluster}, {String: query.Hostname}}
-			if query.Type != nil {
-				if len(query.TypeIds) == 1 {
-					sel = append(sel, util.SelectorElement{String: *query.Type + query.TypeIds[0]})
-				} else {
-					ids := make([]string, len(query.TypeIds))
-					for i, id := range query.TypeIds {
-						ids[i] = *query.Type + id
-					}
-					sel = append(sel, util.SelectorElement{Group: ids})
-				}
-
-				if query.SubType != nil {
-					if len(query.SubTypeIds) == 1 {
-						sel = append(sel, util.SelectorElement{String: *query.SubType + query.SubTypeIds[0]})
-					} else {
-						ids := make([]string, len(query.SubTypeIds))
-						for i, id := range query.SubTypeIds {
-							ids[i] = *query.SubType + id
-						}
-						sel = append(sel, util.SelectorElement{Group: ids})
-					}
-				}
-			}
-			sels = append(sels, sel)
-		} else {
-			for _, typeID := range query.TypeIds {
-				if query.SubType != nil {
-					for _, subTypeID := range query.SubTypeIds {
-						sels = append(sels, util.Selector{
-							{String: req.Cluster},
-							{String: query.Hostname},
-							{String: *query.Type + typeID},
-							{String: *query.SubType + subTypeID},
-						})
-					}
-				} else {
-					sels = append(sels, util.Selector{
-						{String: req.Cluster},
-						{String: query.Hostname},
-						{String: *query.Type + typeID},
-					})
-				}
-			}
-		}
-
-		// log.Printf("query: %#v\n", query)
-		// log.Printf("sels: %#v\n", sels)
-
-		res := make([]APIMetricData, 0, len(sels))
-		for _, sel := range sels {
-			data := APIMetricData{}
-
-			data.Data, data.From, data.To, data.Resolution, err = ms.Read(sel, query.Metric, req.From, req.To, query.Resolution)
-			if err != nil {
-				// Skip Error If Just Missing Host or Metric, Continue
-				// Empty Return For Metric Handled Gracefully By Frontend
-				if err != metricstore.ErrNoHostOrMetric {
-					msg := err.Error()
-					data.Error = &msg
-					res = append(res, data)
-				} else {
-					cclog.Warnf("failed to fetch '%s' from host '%s' (cluster: %s): %s", query.Metric, query.Hostname, req.Cluster, err.Error())
-				}
-				continue
-			}
-
-			if req.WithStats {
-				data.AddStats()
-			}
-			if query.ScaleFactor != 0 {
-				data.ScaleBy(query.ScaleFactor)
-			}
-			if req.WithPadding {
-				data.PadDataWithNull(ms, req.From, req.To, query.Metric)
-			}
-			if !req.WithData {
-				data.Data = nil
-			}
-			res = append(res, data)
-		}
-		response.Results = append(response.Results, res)
-	}
-
-	rw.Header().Set("Content-Type", "application/json")
-	bw := bufio.NewWriter(rw)
-	defer bw.Flush()
-	if err := json.NewEncoder(bw).Encode(response); err != nil {
-		log.Print(err)
-		return
-	}
-}
-
-// handleFree godoc
-// @summary
-// @tags free
-// @description This endpoint allows the users to free the Buffers from the
-// metric store. This endpoint offers the users to remove then systematically
-// and also allows then to prune the data under node, if they do not want to
-// remove the whole node.
-// @produce     json
-// @param       to        query    string        false  "up to timestamp"
-// @success     200            {string} string  "ok"
-// @failure     400            {object} ErrorResponse       "Bad Request"
-// @failure     401            {object} ErrorResponse       "Unauthorized"
-// @failure     403            {object} ErrorResponse       "Forbidden"
-// @failure     500            {object} ErrorResponse       "Internal Server Error"
-// @security    ApiKeyAuth
-// @router      /free/ [post]
-func freeMetrics(rw http.ResponseWriter, r *http.Request) {
-	rawTo := r.URL.Query().Get("to")
-	if rawTo == "" {
-		handleError(errors.New("'to' is a required query parameter"), http.StatusBadRequest, rw)
-		return
-	}
-
-	to, err := strconv.ParseInt(rawTo, 10, 64)
-	if err != nil {
-		handleError(err, http.StatusInternalServerError, rw)
-		return
-	}
-
-	bodyDec := json.NewDecoder(r.Body)
-	var selectors [][]string
-	err = bodyDec.Decode(&selectors)
-	if err != nil {
-		http.Error(rw, err.Error(), http.StatusBadRequest)
-		return
-	}
-
-	ms := metricstore.GetMemoryStore()
-	n := 0
-	for _, sel := range selectors {
-		bn, err := ms.Free(sel, to)
-		if err != nil {
-			handleError(err, http.StatusInternalServerError, rw)
-			return
-		}
-
-		n += bn
-	}
-
-	rw.WriteHeader(http.StatusOK)
-	fmt.Fprintf(rw, "buffers freed: %d\n", n)
-}
-
-// handleWrite godoc
-// @summary Receive metrics in InfluxDB line-protocol
-// @tags write
-// @description Write data to the in-memory store in the InfluxDB line-protocol using [this format](https://github.com/ClusterCockpit/cc-specifications/blob/master/metrics/lineprotocol_alternative.md)
-
-// @accept      plain
-// @produce     json
-// @param       cluster        query string false "If the lines in the body do not have a cluster tag, use this value instead."
-// @success     200            {string} string  "ok"
-// @failure     400            {object} ErrorResponse       "Bad Request"
-// @failure     401            {object} ErrorResponse       "Unauthorized"
-// @failure     403            {object} ErrorResponse       "Forbidden"
-// @failure     500            {object} ErrorResponse       "Internal Server Error"
-// @security    ApiKeyAuth
-// @router      /write/ [post]
-func writeMetrics(rw http.ResponseWriter, r *http.Request) {
-	rw.Header().Add("Content-Type", "application/json")
-
-	// Extract the "cluster" query parameter without allocating a url.Values map.
-	cluster := queryParam(r.URL.RawQuery, "cluster")
-
-	// Stream directly from the request body instead of copying it into a
-	// temporary buffer via io.ReadAll. The line-protocol decoder supports
-	// io.Reader natively, so this avoids the largest heap allocation.
-	ms := metricstore.GetMemoryStore()
-	dec := lineprotocol.NewDecoder(r.Body)
-	if err := metricstore.DecodeLine(dec, ms, cluster); err != nil {
-		cclog.Errorf("/api/write error: %s", err.Error())
-		handleError(err, http.StatusBadRequest, rw)
-		return
-	}
-	rw.WriteHeader(http.StatusOK)
-}
-
-// handleDebug godoc
-// @summary Debug endpoint
-// @tags debug
-// @description This endpoint allows the users to print the content of
-// nodes/clusters/metrics to review the state of the data.
-// @produce     json
-// @param       selector        query    string            false "Selector"
-// @success     200            {string} string  "Debug dump"
-// @failure     400            {object} ErrorResponse       "Bad Request"
-// @failure     401            {object} ErrorResponse       "Unauthorized"
-// @failure     403            {object} ErrorResponse       "Forbidden"
-// @failure     500            {object} ErrorResponse       "Internal Server Error"
-// @security    ApiKeyAuth
-// @router      /debug/ [post]
-func debugMetrics(rw http.ResponseWriter, r *http.Request) {
-	raw := r.URL.Query().Get("selector")
-	rw.Header().Add("Content-Type", "application/json")
-	selector := []string{}
-	if len(raw) != 0 {
-		selector = strings.Split(raw, ":")
-	}
-
-	ms := metricstore.GetMemoryStore()
-	if err := ms.DebugDump(bufio.NewWriter(rw), selector); err != nil {
-		handleError(err, http.StatusBadRequest, rw)
-		return
-	}
-}
-
-// handleHealthCheck godoc
-// @summary HealthCheck endpoint
-// @tags healthcheck
-// @description This endpoint allows the users to check if a node is healthy
-// @produce     json
-// @param       selector        query    string            false "Selector"
-// @success     200            {string} string  "Debug dump"
-// @failure     400            {object} ErrorResponse       "Bad Request"
-// @failure     401            {object} ErrorResponse       "Unauthorized"
-// @failure     403            {object} ErrorResponse       "Forbidden"
-// @failure     500            {object} ErrorResponse       "Internal Server Error"
-// @security    ApiKeyAuth
-// @router      /healthcheck/ [get]
-func metricsHealth(rw http.ResponseWriter, r *http.Request) {
-	req := metricstore.HealthCheckReq{}
-
-	dec := json.NewDecoder(r.Body)
-	dec.DisallowUnknownFields()
-
-	if err := dec.Decode(&req); err != nil {
-		handleError(fmt.Errorf("parsing request body failed: %w", err),
-			http.StatusBadRequest, rw)
-		return
-	}
-
-	rw.Header().Add("Content-Type", "application/json")
-
-	ms := metricstore.GetMemoryStore()
-	results, err := ms.HealthCheck(req.Cluster, req.Nodes, req.MetricNames)
-	if err != nil {
-		handleError(err, http.StatusBadRequest, rw)
-		return
-	}
-
-	rw.Header().Set("Content-Type", "application/json")
-	bw := bufio.NewWriter(rw)
-	defer bw.Flush()
-	if err := json.NewEncoder(bw).Encode(results); err != nil {
-		log.Print(err)
-		return
-	}
-}
-
-// queryParam extracts a single query parameter value from a raw query string
-// without allocating a url.Values map.
-func queryParam(rawQuery, key string) string {
-	for rawQuery != "" {
-		var kv string
-		if before, after, ok := strings.Cut(rawQuery, "&"); ok {
-			kv, rawQuery = before, after
-		} else {
-			kv, rawQuery = rawQuery, ""
-		}
-		if before, after, ok := strings.Cut(kv, "="); ok {
-			if before == key {
-				return after
-			}
-		} else if kv == key {
-			return ""
-		}
-	}
-	return ""
-}

internal/api/nodeprovider.go

@@ -1,54 +0,0 @@
-// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved. This file is part of cc-metric-store.
-// Use of this source code is governed by a MIT-style
-// license that can be found in the LICENSE file.
-
-package api
-
-import (
-	"encoding/json"
-	"fmt"
-	"net/http"
-	"time"
-)
-
-// BackendNodeProvider implements metricstore.NodeProvider by querying
-// the cc-backend /api/jobs/used_nodes endpoint.
-type BackendNodeProvider struct {
-	backendURL string
-	client     *http.Client
-}
-
-// NewBackendNodeProvider creates a new BackendNodeProvider that queries
-// the given cc-backend URL for used nodes information.
-func NewBackendNodeProvider(backendURL string) *BackendNodeProvider {
-	return &BackendNodeProvider{
-		backendURL: backendURL,
-		client: &http.Client{
-			Timeout: 10 * time.Second,
-		},
-	}
-}
-
-// GetUsedNodes returns a map of cluster names to sorted lists of unique hostnames
-// that are currently in use by jobs that started before the given timestamp.
-func (p *BackendNodeProvider) GetUsedNodes(ts int64) (map[string][]string, error) {
-	url := fmt.Sprintf("%s/api/jobs/used_nodes?ts=%d", p.backendURL, ts)
-
-	resp, err := p.client.Get(url)
-	if err != nil {
-		return nil, fmt.Errorf("querying used nodes from backend: %w", err)
-	}
-	defer resp.Body.Close()
-
-	if resp.StatusCode != http.StatusOK {
-		return nil, fmt.Errorf("backend returned status %d", resp.StatusCode)
-	}
-
-	var result map[string][]string
-	if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
-		return nil, fmt.Errorf("decoding used nodes response: %w", err)
-	}
-
-	return result, nil
-}

internal/api/server.go

@@ -1,50 +0,0 @@
-// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved. This file is part of cc-metric-store.
-// Use of this source code is governed by a MIT-style
-// license that can be found in the LICENSE file.
-
-package api
-
-import (
-	"crypto/ed25519"
-	"encoding/base64"
-	"log"
-	"net/http"
-
-	"github.com/ClusterCockpit/cc-metric-store/internal/config"
-)
-
-func MountRoutes(r *http.ServeMux) {
-	if len(config.Keys.JwtPublicKey) > 0 {
-		buf, err := base64.StdEncoding.DecodeString(config.Keys.JwtPublicKey)
-		if err != nil {
-			log.Fatalf("starting server failed: %v", err)
-		}
-		publicKey := ed25519.PublicKey(buf)
-		// Compatibility
-		r.Handle("POST /api/free", authHandler(http.HandlerFunc(freeMetrics), publicKey))
-		r.Handle("POST /api/write", authHandler(http.HandlerFunc(writeMetrics), publicKey))
-		r.Handle("GET /api/query", authHandler(http.HandlerFunc(handleQuery), publicKey))
-		r.Handle("GET /api/debug", authHandler(http.HandlerFunc(debugMetrics), publicKey))
-		r.Handle("GET /api/healthcheck", authHandler(http.HandlerFunc(metricsHealth), publicKey))
-		// Refactor
-		r.Handle("POST /api/free/", authHandler(http.HandlerFunc(freeMetrics), publicKey))
-		r.Handle("POST /api/write/", authHandler(http.HandlerFunc(writeMetrics), publicKey))
-		r.Handle("GET /api/query/", authHandler(http.HandlerFunc(handleQuery), publicKey))
-		r.Handle("GET /api/debug/", authHandler(http.HandlerFunc(debugMetrics), publicKey))
-		r.Handle("GET /api/healthcheck/", authHandler(http.HandlerFunc(metricsHealth), publicKey))
-	} else {
-		// Compatibility
-		r.HandleFunc("POST /api/free", freeMetrics)
-		r.HandleFunc("POST /api/write", writeMetrics)
-		r.HandleFunc("GET /api/query", handleQuery)
-		r.HandleFunc("GET /api/debug", debugMetrics)
-		r.HandleFunc("GET /api/healthcheck", metricsHealth)
-		// Refactor
-		r.HandleFunc("POST /api/free/", freeMetrics)
-		r.HandleFunc("POST /api/write/", writeMetrics)
-		r.HandleFunc("GET /api/query/", handleQuery)
-		r.HandleFunc("GET /api/debug/", debugMetrics)
-		r.HandleFunc("GET /api/healthcheck/", metricsHealth)
-	}
-}

internal/config/config.go

@@ -1,81 +0,0 @@
-// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved. This file is part of cc-metric-store.
-// Use of this source code is governed by a MIT-style
-// license that can be found in the LICENSE file.
-
-package config
-
-import (
-	"bytes"
-	"encoding/json"
-	"fmt"
-
-	"github.com/ClusterCockpit/cc-backend/pkg/metricstore"
-	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
-)
-
-var metrics map[string]metricstore.MetricConfig
-
-type Config struct {
-	Address    string `json:"addr"`
-	CertFile   string `json:"https-cert-file"`
-	KeyFile    string `json:"https-key-file"`
-	User       string `json:"user"`
-	Group      string `json:"group"`
-	BackendURL string `json:"backend-url"`
-	Debug      struct {
-		DumpToFile string `json:"dump-to-file"`
-		EnableGops bool   `json:"gops"`
-	} `json:"debug"`
-	JwtPublicKey string `json:"jwt-public-key"`
-}
-
-var Keys Config
-
-type metricConfigJSON struct {
-	Frequency   int64  `json:"frequency"`
-	Aggregation string `json:"aggregation"`
-}
-
-func InitMetrics(metricConfig json.RawMessage) {
-	Validate(metricConfigSchema, metricConfig)
-
-	var tempMetrics map[string]metricConfigJSON
-	dec := json.NewDecoder(bytes.NewReader(metricConfig))
-	dec.DisallowUnknownFields()
-	if err := dec.Decode(&tempMetrics); err != nil {
-		cclog.Abortf("Config Init: Could not decode config file '%s'.\nError: %s\n", metricConfig, err.Error())
-	}
-
-	metrics = make(map[string]metricstore.MetricConfig)
-	for name, cfg := range tempMetrics {
-		agg, err := metricstore.AssignAggregationStrategy(cfg.Aggregation)
-		if err != nil {
-			cclog.Warnf("Could not parse aggregation strategy for metric '%s': %s", name, err.Error())
-		}
-		metrics[name] = metricstore.MetricConfig{
-			Frequency:   cfg.Frequency,
-			Aggregation: agg,
-		}
-	}
-}
-
-func Init(mainConfig json.RawMessage) {
-	Validate(configSchema, mainConfig)
-	dec := json.NewDecoder(bytes.NewReader(mainConfig))
-	dec.DisallowUnknownFields()
-	if err := dec.Decode(&Keys); err != nil {
-		cclog.Abortf("Config Init: Could not decode config file '%s'.\nError: %s\n", mainConfig, err.Error())
-	}
-}
-
-func GetMetricFrequency(metricName string) (int64, error) {
-	if metric, ok := metrics[metricName]; ok {
-		return metric.Frequency, nil
-	}
-	return 0, fmt.Errorf("metric %s not found", metricName)
-}
-
-func GetMetrics() map[string]metricstore.MetricConfig {
-	return metrics
-}

internal/config/metricSchema.go

@@ -1,26 +0,0 @@
-// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved. This file is part of cc-metric-store.
-// Use of this source code is governed by a MIT-style
-// license that can be found in the LICENSE file.
-
-package config
-
-var metricConfigSchema = `
-{
-  "type": "object",
-  "description": "Map of metric names to their configuration.",
-  "additionalProperties": {
-    "type": "object",
-    "properties": {
-      "frequency": {
-        "description": "Sampling frequency in seconds.",
-        "type": "integer"
-      },
-      "aggregation": {
-        "description": "Aggregation strategy: 'sum', 'avg', or 'null'.",
-        "type": ["string", "null"]
-      }
-    },
-    "required": ["frequency", "aggregation"]
-  }
-}`

internal/config/schema.go

@@ -1,55 +0,0 @@
-// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved. This file is part of cc-metric-store.
-// Use of this source code is governed by a MIT-style
-// license that can be found in the LICENSE file.
-
-package config
-
-var configSchema = `
-{
-  "type": "object",
-  "properties": {
-    "addr": {
-      "description": "Address where the http (or https) server will listen on (for example: 'localhost:8080').",
-      "type": "string"
-    },
-    "https-cert-file": {
-      "description": "Filepath to SSL certificate. If also https-key-file is set, use HTTPS.",
-      "type": "string"
-    },
-    "https-key-file": {
-      "description": "Filepath to SSL key file. If also https-cert-file is set, use HTTPS.",
-      "type": "string"
-    },
-    "user": {
-      "description": "Drop root permissions once the port was taken. Only applicable if using privileged port.",
-      "type": "string"
-    },
-    "group": {
-      "description": "Drop root permissions once the port was taken. Only applicable if using privileged port.",
-      "type": "string"
-    },
-    "backend-url": {
-      "description": "URL of cc-backend for querying job information (e.g., 'https://localhost:8080').",
-      "type": "string"
-    },
-    "debug": {
-      "description": "Debug options.",
-      "type": "object",
-      "properties": {
-        "dump-to-file": {
-          "description": "Path to file for dumping internal state.",
-          "type": "string"
-        },
-        "gops": {
-          "description": "Enable gops agent for debugging.",
-          "type": "boolean"
-        }
-      }
-    },
-    "jwt-public-key": {
-      "description": "Ed25519 public key for JWT verification.",
-      "type": "string"
-    }
-  }
-}`

internal/config/validate.go

@@ -1,29 +0,0 @@
-// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved. This file is part of cc-metric-store.
-// Use of this source code is governed by a MIT-style
-// license that can be found in the LICENSE file.
-
-package config
-
-import (
-	"encoding/json"
-
-	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
-	"github.com/santhosh-tekuri/jsonschema/v5"
-)
-
-func Validate(schema string, instance json.RawMessage) {
-	sch, err := jsonschema.CompileString("schema.json", schema)
-	if err != nil {
-		cclog.Fatalf("%#v", err)
-	}
-
-	var v any
-	if err := json.Unmarshal([]byte(instance), &v); err != nil {
-		cclog.Fatal(err)
-	}
-
-	if err = sch.Validate(v); err != nil {
-		cclog.Fatalf("%#v", err)
-	}
-}

lineprotocol.go

@@ -0,0 +1,318 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"log"
+	"net"
+	"sync"
+	"time"
+
+	"github.com/influxdata/line-protocol/v2/lineprotocol"
+	"github.com/nats-io/nats.go"
+)
+
+type Metric struct {
+	Name  string
+	Value Float
+	Unit  string
+
+	mc MetricConfig
+}
+
+// Currently unused, could be used to send messages via raw TCP.
+// Each connection is handled in it's own goroutine. This is a blocking function.
+func ReceiveRaw(ctx context.Context, listener net.Listener, handleLine func(*lineprotocol.Decoder, string) error) error {
+	var wg sync.WaitGroup
+
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		<-ctx.Done()
+		if err := listener.Close(); err != nil {
+			log.Printf("listener.Close(): %s", err.Error())
+		}
+	}()
+
+	for {
+		conn, err := listener.Accept()
+		if err != nil {
+			if errors.Is(err, net.ErrClosed) {
+				break
+			}
+
+			log.Printf("listener.Accept(): %s", err.Error())
+		}
+
+		wg.Add(2)
+		go func() {
+			defer wg.Done()
+			defer conn.Close()
+
+			dec := lineprotocol.NewDecoder(conn)
+			connctx, cancel := context.WithCancel(context.Background())
+			defer cancel()
+			go func() {
+				defer wg.Done()
+				select {
+				case <-connctx.Done():
+					conn.Close()
+				case <-ctx.Done():
+					conn.Close()
+				}
+			}()
+
+			if err := handleLine(dec, "default"); err != nil {
+				if errors.Is(err, net.ErrClosed) {
+					return
+				}
+
+				log.Printf("%s: %s", conn.RemoteAddr().String(), err.Error())
+				errmsg := make([]byte, 128)
+				errmsg = append(errmsg, `error: `...)
+				errmsg = append(errmsg, err.Error()...)
+				errmsg = append(errmsg, '\n')
+				conn.Write(errmsg)
+			}
+		}()
+	}
+
+	wg.Wait()
+	return nil
+}
+
+// Connect to a nats server and subscribe to "updates". This is a blocking
+// function. handleLine will be called for each line recieved via nats.
+// Send `true` through the done channel for gracefull termination.
+func ReceiveNats(conf *NatsConfig, handleLine func(*lineprotocol.Decoder, string) error, workers int, ctx context.Context) error {
+	var opts []nats.Option
+	if conf.Username != "" && conf.Password != "" {
+		opts = append(opts, nats.UserInfo(conf.Username, conf.Password))
+	}
+
+	nc, err := nats.Connect(conf.Address, opts...)
+	if err != nil {
+		return err
+	}
+	defer nc.Close()
+
+	var wg sync.WaitGroup
+	var subs []*nats.Subscription
+
+	msgs := make(chan *nats.Msg, workers*2)
+
+	for _, sc := range conf.Subscriptions {
+		clusterTag := sc.ClusterTag
+		var sub *nats.Subscription
+		if workers > 1 {
+			wg.Add(workers)
+
+			for i := 0; i < workers; i++ {
+				go func() {
+					for m := range msgs {
+						dec := lineprotocol.NewDecoderWithBytes(m.Data)
+						if err := handleLine(dec, clusterTag); err != nil {
+							log.Printf("error: %s\n", err.Error())
+						}
+					}
+
+					wg.Done()
+				}()
+			}
+
+			sub, err = nc.Subscribe(sc.SubscribeTo, func(m *nats.Msg) {
+				msgs <- m
+			})
+		} else {
+			sub, err = nc.Subscribe(sc.SubscribeTo, func(m *nats.Msg) {
+				dec := lineprotocol.NewDecoderWithBytes(m.Data)
+				if err := handleLine(dec, clusterTag); err != nil {
+					log.Printf("error: %s\n", err.Error())
+				}
+			})
+		}
+
+		if err != nil {
+			return err
+		}
+		log.Printf("NATS subscription to '%s' on '%s' established\n", sc.SubscribeTo, conf.Address)
+		subs = append(subs, sub)
+	}
+
+	<-ctx.Done()
+	for _, sub := range subs {
+		err = sub.Unsubscribe()
+		if err != nil {
+			log.Printf("NATS unsubscribe failed: %s", err.Error())
+		}
+	}
+	close(msgs)
+	wg.Wait()
+
+	nc.Close()
+	log.Println("NATS connection closed")
+	return nil
+}
+
+// Place `prefix` in front of `buf` but if possible,
+// do that inplace in `buf`.
+func reorder(buf, prefix []byte) []byte {
+	n := len(prefix)
+	m := len(buf)
+	if cap(buf) < m+n {
+		return append(prefix[:n:n], buf...)
+	} else {
+		buf = buf[:n+m]
+		for i := m - 1; i >= 0; i-- {
+			buf[i+n] = buf[i]
+		}
+		for i := 0; i < n; i++ {
+			buf[i] = prefix[i]
+		}
+		return buf
+	}
+}
+
+// Decode lines using dec and make write calls to the MemoryStore.
+// If a line is missing its cluster tag, use clusterDefault as default.
+func decodeLine(dec *lineprotocol.Decoder, clusterDefault string) error {
+	// Reduce allocations in loop:
+	t := time.Now()
+	metric, metricBuf := Metric{}, make([]byte, 0, 16)
+	selector := make([]string, 0, 4)
+	typeBuf, subTypeBuf := make([]byte, 0, 16), make([]byte, 0)
+
+	// Optimize for the case where all lines in a "batch" are about the same
+	// cluster and host. By using `WriteToLevel` (level = host), we do not need
+	// to take the root- and cluster-level lock as often.
+	var lvl *level = nil
+	var prevCluster, prevHost string = "", ""
+
+	var ok bool
+	for dec.Next() {
+		rawmeasurement, err := dec.Measurement()
+		if err != nil {
+			return err
+		}
+
+		// Needs to be copied because another call to dec.* would
+		// invalidate the returned slice.
+		metricBuf = append(metricBuf[:0], rawmeasurement...)
+
+		// The go compiler optimizes map[string(byteslice)] lookups:
+		metric.mc, ok = memoryStore.metrics[string(rawmeasurement)]
+		if !ok {
+			continue
+		}
+
+		typeBuf, subTypeBuf := typeBuf[:0], subTypeBuf[:0]
+		cluster, host, unit := clusterDefault, "", ""
+		for {
+			key, val, err := dec.NextTag()
+			if err != nil {
+				return err
+			}
+			if key == nil {
+				break
+			}
+
+			// The go compiler optimizes string([]byte{...}) == "...":
+			switch string(key) {
+			case "cluster":
+				if string(val) == prevCluster {
+					cluster = prevCluster
+				} else {
+					cluster = string(val)
+					lvl = nil
+				}
+			case "hostname", "host":
+				if string(val) == prevHost {
+					host = prevHost
+				} else {
+					host = string(val)
+					lvl = nil
+				}
+			case "unit":
+				unit = string(val)
+			case "type":
+				if string(val) == "node" {
+					break
+				}
+
+				// We cannot be sure that the "type" tag comes before the "type-id" tag:
+				if len(typeBuf) == 0 {
+					typeBuf = append(typeBuf, val...)
+				} else {
+					typeBuf = reorder(typeBuf, val)
+				}
+			case "type-id":
+				typeBuf = append(typeBuf, val...)
+			case "subtype":
+				// We cannot be sure that the "subtype" tag comes before the "stype-id" tag:
+				if len(subTypeBuf) == 0 {
+					subTypeBuf = append(subTypeBuf, val...)
+				} else {
+					subTypeBuf = reorder(typeBuf, val)
+				}
+			case "stype-id":
+				subTypeBuf = append(subTypeBuf, val...)
+			default:
+				// Ignore unkown tags (cc-metric-collector might send us a unit for example that we do not need)
+				// return fmt.Errorf("unkown tag: '%s' (value: '%s')", string(key), string(val))
+			}
+		}
+
+		// If the cluster or host changed, the lvl was set to nil
+		if lvl == nil {
+			selector = selector[:2]
+			selector[0], selector[1] = cluster, host
+			lvl = memoryStore.GetLevel(selector)
+			prevCluster, prevHost = cluster, host
+		}
+
+		// subtypes:
+		selector = selector[:0]
+		if len(typeBuf) > 0 {
+			selector = append(selector, string(typeBuf)) // <- Allocation :(
+			if len(subTypeBuf) > 0 {
+				selector = append(selector, string(subTypeBuf))
+			}
+		}
+
+		for {
+			key, val, err := dec.NextField()
+			if err != nil {
+				return err
+			}
+
+			if key == nil {
+				break
+			}
+
+			if string(key) != "value" {
+				return fmt.Errorf("unkown field: '%s' (value: %#v)", string(key), val)
+			}
+
+			if val.Kind() == lineprotocol.Float {
+				metric.Value = Float(val.FloatV())
+			} else if val.Kind() == lineprotocol.Int {
+				metric.Value = Float(val.IntV())
+			} else if val.Kind() == lineprotocol.Uint {
+				metric.Value = Float(val.UintV())
+			} else {
+				return fmt.Errorf("unsupported value type in message: %s", val.Kind().String())
+			}
+			metric.Unit = unit
+		}
+
+		if t, err = dec.Time(lineprotocol.Second, t); err != nil {
+			return err
+		}
+
+		if err := memoryStore.WriteToLevel(lvl, selector, t.Unix(), []Metric{metric}); err != nil {
+			return err
+		}
+	}
+	return nil
+}

lineprotocol_test.go

@@ -0,0 +1,144 @@
+package main
+
+import (
+	"bytes"
+	"log"
+	"strconv"
+	"testing"
+
+	"github.com/influxdata/line-protocol/v2/lineprotocol"
+)
+
+const TestDataClassicFormat string = `
+m1,cluster=ctest,hostname=htest1,type=node value=1 123456789
+m2,cluster=ctest,hostname=htest1,type=node value=2 123456789
+m3,hostname=htest2,type=node value=3 123456789
+m4,cluster=ctest,hostname=htest2,type=core,type-id=1 value=4 123456789
+m4,cluster=ctest,hostname=htest2,type-id=2,type=core value=5 123456789
+`
+
+const BenchmarkLineBatch string = `
+nm1,cluster=ctest,hostname=htest1,type=node value=123.0 123456789
+nm2,cluster=ctest,hostname=htest1,type=node value=123.0 123456789
+nm3,cluster=ctest,hostname=htest1,type=node value=123.0 123456789
+nm4,cluster=ctest,hostname=htest1,type=node value=123.0 123456789
+nm5,cluster=ctest,hostname=htest1,type=node value=123.0 123456789
+nm6,cluster=ctest,hostname=htest1,type=node value=123.0 123456789
+nm7,cluster=ctest,hostname=htest1,type=node value=123.0 123456789
+nm8,cluster=ctest,hostname=htest1,type=node value=123.0 123456789
+nm9,cluster=ctest,hostname=htest1,type=node value=123.0 123456789
+cm1,cluster=ctest,hostname=htest1,type=core,type-id=1 value=234.0 123456789
+cm2,cluster=ctest,hostname=htest1,type=core,type-id=1 value=234.0 123456789
+cm3,cluster=ctest,hostname=htest1,type=core,type-id=1 value=234.0 123456789
+cm4,cluster=ctest,hostname=htest1,type=core,type-id=1 value=234.0 123456789
+cm5,cluster=ctest,hostname=htest1,type=core,type-id=1 value=234.0 123456789
+cm6,cluster=ctest,hostname=htest1,type=core,type-id=1 value=234.0 123456789
+cm7,cluster=ctest,hostname=htest1,type=core,type-id=1 value=234.0 123456789
+cm8,cluster=ctest,hostname=htest1,type=core,type-id=1 value=234.0 123456789
+cm9,cluster=ctest,hostname=htest1,type=core,type-id=1 value=234.0 123456789
+cm1,cluster=ctest,hostname=htest1,type=core,type-id=2 value=345.0 123456789
+cm2,cluster=ctest,hostname=htest1,type=core,type-id=2 value=345.0 123456789
+cm3,cluster=ctest,hostname=htest1,type=core,type-id=2 value=345.0 123456789
+cm4,cluster=ctest,hostname=htest1,type=core,type-id=2 value=345.0 123456789
+cm5,cluster=ctest,hostname=htest1,type=core,type-id=2 value=345.0 123456789
+cm6,cluster=ctest,hostname=htest1,type=core,type-id=2 value=345.0 123456789
+cm7,cluster=ctest,hostname=htest1,type=core,type-id=2 value=345.0 123456789
+cm8,cluster=ctest,hostname=htest1,type=core,type-id=2 value=345.0 123456789
+cm9,cluster=ctest,hostname=htest1,type=core,type-id=2 value=345.0 123456789
+cm1,cluster=ctest,hostname=htest1,type=core,type-id=3 value=456.0 123456789
+cm2,cluster=ctest,hostname=htest1,type=core,type-id=3 value=456.0 123456789
+cm3,cluster=ctest,hostname=htest1,type=core,type-id=3 value=456.0 123456789
+cm4,cluster=ctest,hostname=htest1,type=core,type-id=3 value=456.0 123456789
+cm5,cluster=ctest,hostname=htest1,type=core,type-id=3 value=456.0 123456789
+cm6,cluster=ctest,hostname=htest1,type=core,type-id=3 value=456.0 123456789
+cm7,cluster=ctest,hostname=htest1,type=core,type-id=3 value=456.0 123456789
+cm8,cluster=ctest,hostname=htest1,type=core,type-id=3 value=456.0 123456789
+cm9,cluster=ctest,hostname=htest1,type=core,type-id=3 value=456.0 123456789
+cm1,cluster=ctest,hostname=htest1,type=core,type-id=4 value=567.0 123456789
+cm2,cluster=ctest,hostname=htest1,type=core,type-id=4 value=567.0 123456789
+cm3,cluster=ctest,hostname=htest1,type=core,type-id=4 value=567.0 123456789
+cm4,cluster=ctest,hostname=htest1,type=core,type-id=4 value=567.0 123456789
+cm5,cluster=ctest,hostname=htest1,type=core,type-id=4 value=567.0 123456789
+cm6,cluster=ctest,hostname=htest1,type=core,type-id=4 value=567.0 123456789
+cm7,cluster=ctest,hostname=htest1,type=core,type-id=4 value=567.0 123456789
+cm8,cluster=ctest,hostname=htest1,type=core,type-id=4 value=567.0 123456789
+cm9,cluster=ctest,hostname=htest1,type=core,type-id=4 value=567.0 123456789
+`
+
+func TestLineprotocolDecoder(t *testing.T) {
+	prevMemoryStore := memoryStore
+	t.Cleanup(func() {
+		memoryStore = prevMemoryStore
+	})
+
+	memoryStore = NewMemoryStore(map[string]MetricConfig{
+		"m1": {Frequency: 1},
+		"m2": {Frequency: 1},
+		"m3": {Frequency: 1},
+		"m4": {Frequency: 1},
+	})
+
+	dec := lineprotocol.NewDecoderWithBytes([]byte(TestDataClassicFormat))
+	if err := decodeLine(dec, "ctest"); err != nil {
+		log.Fatal(err)
+	}
+
+	// memoryStore.DebugDump(bufio.NewWriter(os.Stderr))
+
+	h1 := memoryStore.GetLevel([]string{"ctest", "htest1"})
+	h1b1 := h1.metrics[memoryStore.metrics["m1"].offset]
+	h1b2 := h1.metrics[memoryStore.metrics["m2"].offset]
+	if h1b1.data[0] != 1.0 || h1b2.data[0] != 2.0 {
+		log.Fatal()
+	}
+
+	h2 := memoryStore.GetLevel([]string{"ctest", "htest2"})
+	h2b3 := h2.metrics[memoryStore.metrics["m3"].offset]
+	if h2b3.data[0] != 3.0 {
+		log.Fatal()
+	}
+
+	h2c1 := memoryStore.GetLevel([]string{"ctest", "htest2", "core1"})
+	h2c1b4 := h2c1.metrics[memoryStore.metrics["m4"].offset]
+	h2c2 := memoryStore.GetLevel([]string{"ctest", "htest2", "core2"})
+	h2c2b4 := h2c2.metrics[memoryStore.metrics["m4"].offset]
+	if h2c1b4.data[0] != 4.0 || h2c2b4.data[0] != 5.0 {
+		log.Fatal()
+	}
+}
+
+func BenchmarkLineprotocolDecoder(b *testing.B) {
+	b.StopTimer()
+	memoryStore = NewMemoryStore(map[string]MetricConfig{
+		"nm1": {Frequency: 1},
+		"nm2": {Frequency: 1},
+		"nm3": {Frequency: 1},
+		"nm4": {Frequency: 1},
+		"nm5": {Frequency: 1},
+		"nm6": {Frequency: 1},
+		"nm7": {Frequency: 1},
+		"nm8": {Frequency: 1},
+		"nm9": {Frequency: 1},
+		"cm1": {Frequency: 1},
+		"cm2": {Frequency: 1},
+		"cm3": {Frequency: 1},
+		"cm4": {Frequency: 1},
+		"cm5": {Frequency: 1},
+		"cm6": {Frequency: 1},
+		"cm7": {Frequency: 1},
+		"cm8": {Frequency: 1},
+		"cm9": {Frequency: 1},
+	})
+
+	for i := 0; i < b.N; i++ {
+		data := []byte(BenchmarkLineBatch)
+		data = bytes.ReplaceAll(data, []byte("123456789"), []byte(strconv.Itoa(i+123456789)))
+		dec := lineprotocol.NewDecoderWithBytes(data)
+
+		b.StartTimer()
+		if err := decodeLine(dec, "ctest"); err != nil {
+			b.Fatal(err)
+		}
+		b.StopTimer()
+	}
+}

memoryStore_test.go.orig

@@ -0,0 +1,504 @@
+package main
+
+import (
+	"fmt"
+	"log"
+	"math"
+	"testing"
+
+	"github.com/ClusterCockpit/cc-metric-store/lineprotocol"
+)
+
+var testMetrics [][]lineprotocol.Metric = [][]lineprotocol.Metric{
+	{{"flops", 100.5}, {"mem_bw", 2088.67}},
+	{{"flops", 180.5}, {"mem_bw", 4078.32}, {"mem_capacity", 1020}},
+	{{"flops", 980.5}, {"mem_bw", 9078.32}, {"mem_capacity", 5010}},
+	{{"flops", 940.5}, {"mem_bw", 9278.32}, {"mem_capacity", 6010}},
+	{{"flops", 930.5}, {"mem_bw", 9378.32}, {"mem_capacity", 7010}},
+	{{"flops", 980.5}, {"mem_bw", 9478.32}, {"mem_capacity", 8010}},
+	{{"flops", 980.5}, {"mem_bw", 9478.32}, {"mem_capacity", 8010}},
+	{{"flops", 980.5}, {"mem_bw", 9478.32}, {"mem_capacity", 8010}},
+	{{"flops", 970.5}, {"mem_bw", 9178.32}, {"mem_capacity", 2010}},
+	{{"flops", 970.5}, {"mem_bw", 9178.32}, {"mem_capacity", 2010}}}
+
+var testMetricsAlt [][]lineprotocol.Metric = [][]lineprotocol.Metric{
+	{{"flops", 120.5}, {"mem_bw", 2080.67}},
+	{{"flops", 130.5}, {"mem_bw", 4071.32}, {"mem_capacity", 1120}},
+	{{"flops", 940.5}, {"mem_bw", 9072.32}, {"mem_capacity", 5210}},
+	{{"flops", 950.5}, {"mem_bw", 9273.32}, {"mem_capacity", 6310}},
+	{{"flops", 960.5}, {"mem_bw", 9374.32}, {"mem_capacity", 7410}},
+	{{"flops", 970.5}, {"mem_bw", 9475.32}, {"mem_capacity", 8510}},
+	{{"flops", 990.5}, {"mem_bw", 9476.32}, {"mem_capacity", 8610}},
+	{{"flops", 910.5}, {"mem_bw", 9477.32}, {"mem_capacity", 8710}},
+	{{"flops", 920.5}, {"mem_bw", 9178.32}, {"mem_capacity", 2810}},
+	{{"flops", 930.5}, {"mem_bw", 9179.32}, {"mem_capacity", 2910}}}
+
+func dumpStoreBuffer(s *storeBuffer) {
+	log.Printf("Start TS %d\n", s.start)
+	ctr := 0
+
+	for _, val := range s.store {
+		fmt.Printf("%f\t", val)
+		ctr++
+
+		if ctr == 10 {
+			fmt.Printf("\n")
+			ctr = 0
+		}
+	}
+}
+
+func printMemStore(m *MemoryStore) {
+	log.Println("########################")
+	log.Printf("Frequency %d, Metrics %d Slots %d\n",
+		m.frequency, m.numMetrics, m.numSlots)
+	log.Println("##Offsets")
+	for key, val := range m.offsets {
+		log.Printf("\t%s = %d\n", key, val)
+	}
+	log.Println("##Containers")
+	for key, c := range m.containers {
+		log.Printf("ID %s\n", key)
+		log.Println("###current")
+		dumpStoreBuffer(c.current)
+		log.Println("###next")
+		dumpStoreBuffer(c.next)
+	}
+	log.Println("########################")
+}
+
+//############################
+//#### Whitebox tests ########
+//############################
+func TestAddMetricSimple(t *testing.T) {
+	key := "m1220"
+	m := newMemoryStore([]string{"flops", "mem_bw", "mem_capacity"}, 10, 60)
+	// printMemStore(m)
+
+	m.AddMetrics(key, 1584022800, testMetrics[0])
+	m.AddMetrics(key, 1584022890, testMetrics[1])
+
+	want := testMetrics[0][0].Value
+	got := m.containers[key].current.store[0]
+	if got != want {
+		t.Errorf("Want %f got %f\n", want, got)
+	}
+
+	want = testMetrics[1][2].Value
+	got = m.containers[key].current.store[21]
+	if got != want {
+		t.Errorf("Want %f got %f\n", want, got)
+	}
+	// printMemStore(m)
+}
+
+func TestAddMetricReplace(t *testing.T) {
+	key := "m1220"
+	m := newMemoryStore([]string{"flops", "mem_bw", "mem_capacity"}, 10, 60)
+	// printMemStore(m)
+
+	m.AddMetrics(key, 1584022800, testMetrics[0])
+	m.AddMetrics(key, 1584022800, testMetrics[1])
+
+	want := testMetrics[1][0].Value
+	got := m.containers[key].current.store[0]
+	if got != want {
+		t.Errorf("Want %f got %f\n", want, got)
+	}
+
+	m.AddMetrics(key, 1584022850, testMetrics[0])
+	want = testMetrics[0][0].Value
+	got = m.containers[key].current.store[0]
+	if got != want {
+		t.Errorf("Want %f got %f\n", want, got)
+	}
+
+	m.AddMetrics(key, 1584022860, testMetrics[1])
+	want = testMetrics[0][0].Value
+	got = m.containers[key].current.store[0]
+	if got != want {
+		t.Errorf("Want %f got %f\n", want, got)
+	}
+	// printMemStore(m)
+}
+
+func TestAddMetricSwitch(t *testing.T) {
+	key := "m1220"
+	m := newMemoryStore([]string{"flops", "mem_bw", "mem_capacity"}, 10, 60)
+	// printMemStore(m)
+
+	m.AddMetrics(key, 1584023000, testMetrics[0])
+	m.AddMetrics(key, 1584023580, testMetrics[1])
+
+	want := testMetrics[1][2].Value
+	got := m.containers[key].current.store[29]
+	if got != want {
+		t.Errorf("Want %f got %f\n", want, got)
+	}
+
+	m.AddMetrics(key, 1584023600, testMetrics[2])
+	want = testMetrics[2][2].Value
+	got = m.containers[key].current.store[20]
+	if got != want {
+		t.Errorf("Want %f got %f\n", want, got)
+	}
+
+	// printMemStore(m)
+}
+
+//############################
+//#### Blackbox tests ########
+//############################
+
+func TestAddMetricOutOfBounds(t *testing.T) {
+	key := "m1220"
+	m := newMemoryStore([]string{"flops", "mem_bw", "mem_capacity"}, 30, 60)
+
+	err := m.AddMetrics(key, 1584023000, testMetrics[0])
+	if err != nil {
+		t.Errorf("Got error 1584023000\n")
+	}
+	err = m.AddMetrics(key, 1584026600, testMetrics[0])
+	if err == nil {
+		t.Errorf("Got no error 1584026600\n")
+	}
+	err = m.AddMetrics(key, 1584021580, testMetrics[1])
+	if err == nil {
+		t.Errorf("Got no error 1584021580\n")
+	}
+	err = m.AddMetrics(key, 1584024580, testMetrics[1])
+	if err != nil {
+		t.Errorf("Got error 1584024580\n")
+	}
+	err = m.AddMetrics(key, 1584091580, testMetrics[1])
+	if err == nil {
+		t.Errorf("Got no error 1584091580\n")
+	}
+	err = m.AddMetrics(key, 1584024780, testMetrics[0])
+	if err != nil {
+		t.Errorf("Got error 1584024780\n")
+	}
+}
+
+func TestGetMetricPlainCurrent(t *testing.T) {
+	key := "m1220"
+	m := newMemoryStore([]string{"flops", "mem_bw", "mem_capacity"}, 10, 60)
+
+	for i := 0; i < len(testMetrics); i++ {
+		m.AddMetrics(key, int64(1584023000+i*60), testMetrics[i])
+	}
+
+	// printMemStore(m)
+	val, tsFrom, err := m.GetMetric(key, "flops", 1584023000, 1584023560)
+
+	if err != nil {
+		t.Errorf("Got error\n")
+	}
+	if tsFrom != 1584023000 {
+		t.Errorf("Start ts differs: %d\n", tsFrom)
+	}
+	if len(val) != 9 {
+		t.Errorf("Want 9. Got %d\n", len(val))
+	}
+	if val[0] != 100.5 {
+		t.Errorf("Want 100.5 Got %f\n", val[0])
+	}
+	if val[8] != 970.5 {
+		t.Errorf("Want 970.5 Got %f\n", val[9])
+	}
+}
+
+func TestGetMetricPlainNext(t *testing.T) {
+	key := "m1220"
+	m := newMemoryStore([]string{"flops", "mem_bw", "mem_capacity"}, 10, 60)
+
+	for i := 0; i < len(testMetrics); i++ {
+		m.AddMetrics(key, int64(1584023000+i*60), testMetrics[i])
+	}
+	for i := 0; i < len(testMetrics); i++ {
+		m.AddMetrics(key, int64(1584023600+i*60), testMetricsAlt[i])
+	}
+
+	// printMemStore(m)
+	val, tsFrom, err := m.GetMetric(key, "flops", 1584023000, 1584023560)
+
+	if err != nil {
+		t.Errorf("Got error\n")
+	}
+	if tsFrom != 1584023000 {
+		t.Errorf("Start ts differs: %d\n", tsFrom)
+	}
+	if len(val) != 9 {
+		t.Errorf("Want 9. Got %d\n", len(val))
+	}
+	if val[0] != 100.5 {
+		t.Errorf("Want 100.5 Got %f\n", val[0])
+	}
+	if val[8] != 970.5 {
+		t.Errorf("Want 970.5 Got %f\n", val[9])
+	}
+}
+
+func TestGetMetricGap(t *testing.T) {
+	key := "m1220"
+	m := newMemoryStore([]string{"flops", "mem_bw", "mem_capacity"}, 10, 60)
+
+	for i := 0; i < len(testMetrics); i++ {
+		m.AddMetrics(key, int64(1584023000+i*120), testMetrics[i])
+	}
+
+	val, tsFrom, err := m.GetMetric(key, "flops", 1584023000, 1584023600)
+
+	if err != nil {
+		t.Errorf("Got error\n")
+	}
+	if tsFrom != 1584023000 {
+		t.Errorf("Start ts differs: %d\n", tsFrom)
+	}
+	if len(val) != 10 {
+		t.Errorf("Want 10. Got %d\n", len(val))
+	}
+	if val[0] != 100.5 {
+		t.Errorf("Want 100.5 Got %f\n", val[0])
+	}
+	if !math.IsNaN(float64(val[1])) {
+		t.Errorf("Want NaN Got %f\n", val[1])
+	}
+	if val[0] != 100.5 {
+		t.Errorf("Want 100.5 Got %f\n", val[0])
+	}
+
+	// fmt.Println(val)
+}
+
+func TestGetMetricSplit(t *testing.T) {
+	key := "m1220"
+	m := newMemoryStore([]string{"flops", "mem_bw", "mem_capacity"}, 10, 60)
+
+	for i := 0; i < len(testMetrics); i++ {
+		m.AddMetrics(key, int64(1584023000+i*60), testMetrics[i])
+	}
+	for i := 0; i < len(testMetrics); i++ {
+		m.AddMetrics(key, int64(1584023600+i*60), testMetricsAlt[i])
+	}
+
+	// printMemStore(m)
+
+	val, tsFrom, err := m.GetMetric(key, "flops", 1584023200, 1584023860)
+
+	if err != nil {
+		t.Errorf("Got error\n")
+	}
+	if tsFrom != 1584023200 {
+		t.Errorf("Start ts differs: %d\n", tsFrom)
+	}
+	if len(val) != 11 {
+		t.Errorf("Want 11. Got %d\n", len(val))
+	}
+	if val[0] != 940.5 {
+		t.Errorf("Want 940.5 Got %f\n", val[0])
+	}
+	if val[10] != 950.5 {
+		t.Errorf("Want 950.5 Got %f\n", val[0])
+	}
+}
+
+func TestGetMetricExceedNext(t *testing.T) {
+	key := "m1220"
+	m := newMemoryStore([]string{"flops", "mem_bw", "mem_capacity"}, 10, 60)
+
+	for i := 0; i < len(testMetrics); i++ {
+		m.AddMetrics(key, int64(1584023000+i*60), testMetrics[i])
+	}
+	for i := 0; i < len(testMetrics); i++ {
+		m.AddMetrics(key, int64(1584023600+i*60), testMetricsAlt[i])
+	}
+
+	// printMemStore(m)
+
+	val, tsFrom, err := m.GetMetric(key, "flops", 1584022800, 1584023400)
+
+	if err != nil {
+		t.Errorf("Got error\n")
+	}
+	if tsFrom != 1584023000 {
+		t.Errorf("Start ts differs: %d\n", tsFrom)
+	}
+	if len(val) != 6 {
+		t.Errorf("Want 6. Got %d\n", len(val))
+	}
+	if val[0] != 100.5 {
+		t.Errorf("Want 100.5 Got %f\n", val[0])
+	}
+	if val[5] != 980.5 {
+		t.Errorf("Want 980.5 Got %f\n", val[5])
+	}
+}
+
+func TestGetMetricExceedNextSplit(t *testing.T) {
+	key := "m1220"
+	m := newMemoryStore([]string{"flops", "mem_bw", "mem_capacity"}, 10, 60)
+
+	for i := 0; i < len(testMetrics); i++ {
+		m.AddMetrics(key, int64(1584023000+i*60), testMetrics[i])
+	}
+	for i := 0; i < len(testMetrics); i++ {
+		m.AddMetrics(key, int64(1584023600+i*60), testMetricsAlt[i])
+	}
+
+	// printMemStore(m)
+
+	val, tsFrom, err := m.GetMetric(key, "flops", 1584022800, 1584023900)
+
+	if err != nil {
+		t.Errorf("Got error\n")
+	}
+	if tsFrom != 1584023000 {
+		t.Errorf("Start ts differs: %d\n", tsFrom)
+	}
+	if len(val) != 15 {
+		t.Errorf("Want 14. Got %d\n", len(val))
+	}
+	if val[0] != 100.5 {
+		t.Errorf("Want 100.5 Got %f\n", val[0])
+	}
+	if val[14] != 960.5 {
+		t.Errorf("Want 960.5 Got %f\n", val[13])
+	}
+}
+
+func TestGetMetricExceedCurrent(t *testing.T) {
+	key := "m1220"
+	m := newMemoryStore([]string{"flops", "mem_bw", "mem_capacity"}, 10, 60)
+
+	for i := 0; i < len(testMetrics); i++ {
+		m.AddMetrics(key, int64(1584023000+i*60), testMetrics[i])
+	}
+	for i := 0; i < len(testMetrics); i++ {
+		m.AddMetrics(key, int64(1584023600+i*60), testMetricsAlt[i])
+	}
+
+	// printMemStore(m)
+
+	val, tsFrom, err := m.GetMetric(key, "flops", 1584023800, 1584027900)
+
+	if err != nil {
+		t.Errorf("Got error\n")
+	}
+	if tsFrom != 1584023800 {
+		t.Errorf("Start ts differs: %d\n", tsFrom)
+	}
+	if len(val) != 7 {
+		t.Errorf("Want 6. Got %d\n", len(val))
+	}
+	if val[0] != 950.5 {
+		t.Errorf("Want 950.5 Got %f\n", val[0])
+	}
+	if val[6] != 930.5 {
+		t.Errorf("Want 930.5 Got %f\n", val[5])
+	}
+}
+
+func TestGetMetricExceedCurrentSplit(t *testing.T) {
+	key := "m1220"
+	m := newMemoryStore([]string{"flops", "mem_bw", "mem_capacity"}, 10, 60)
+
+	for i := 0; i < len(testMetrics); i++ {
+		m.AddMetrics(key, int64(1584023000+i*60), testMetrics[i])
+	}
+	for i := 0; i < len(testMetrics); i++ {
+		m.AddMetrics(key, int64(1584023600+i*60), testMetricsAlt[i])
+	}
+
+	// printMemStore(m)
+
+	val, tsFrom, err := m.GetMetric(key, "flops", 1584023120, 1584027900)
+
+	if err != nil {
+		t.Errorf("Got error\n")
+	}
+	if tsFrom != 1584023120 {
+		t.Errorf("Start ts differs: %d\n", tsFrom)
+	}
+	if len(val) != 18 {
+		t.Errorf("Want 18. Got %d\n", len(val))
+	}
+	if val[0] != 980.5 {
+		t.Errorf("Want 950.5 Got %f\n", val[0])
+	}
+	if val[17] != 930.5 {
+		t.Errorf("Want 930.5 Got %f\n", val[17])
+	}
+}
+
+func TestGetMetricExceedBoth(t *testing.T) {
+	key := "m1220"
+	m := newMemoryStore([]string{"flops", "mem_bw", "mem_capacity"}, 10, 60)
+
+	for i := 0; i < len(testMetrics); i++ {
+		m.AddMetrics(key, int64(1584023000+i*60), testMetrics[i])
+	}
+	for i := 0; i < len(testMetrics); i++ {
+		m.AddMetrics(key, int64(1584023600+i*60), testMetricsAlt[i])
+	}
+
+	// printMemStore(m)
+
+	val, tsFrom, err := m.GetMetric(key, "flops", 1584022800, 1584027900)
+
+	if err != nil {
+		t.Errorf("Got error\n")
+	}
+	if tsFrom != 1584023000 {
+		t.Errorf("Start ts differs: %d\n", tsFrom)
+	}
+	if len(val) != 20 {
+		t.Errorf("Want 20. Got %d\n", len(val))
+	}
+	if val[0] != 100.5 {
+		t.Errorf("Want 950.5 Got %f\n", val[0])
+	}
+	if val[19] != 930.5 {
+		t.Errorf("Want 930.5 Got %f\n", val[17])
+	}
+}
+
+func TestGetMetricOutUpper(t *testing.T) {
+	key := "m1220"
+	m := newMemoryStore([]string{"flops", "mem_bw", "mem_capacity"}, 10, 60)
+
+	for i := 0; i < len(testMetrics); i++ {
+		m.AddMetrics(key, int64(1584023000+i*60), testMetrics[i])
+	}
+	for i := 0; i < len(testMetrics); i++ {
+		m.AddMetrics(key, int64(1584023600+i*60), testMetricsAlt[i])
+	}
+
+	// printMemStore(m)
+
+	_, _, err := m.GetMetric(key, "flops", 1584032800, 1584037900)
+
+	if err == nil {
+		t.Errorf("Got no error\n")
+	}
+}
+
+func TestGetMetricOutLower(t *testing.T) {
+	key := "m1220"
+	m := newMemoryStore([]string{"flops", "mem_bw", "mem_capacity"}, 10, 60)
+
+	for i := 0; i < len(testMetrics); i++ {
+		m.AddMetrics(key, int64(1584023000+i*60), testMetrics[i])
+	}
+	for i := 0; i < len(testMetrics); i++ {
+		m.AddMetrics(key, int64(1584023600+i*60), testMetricsAlt[i])
+	}
+
+	// printMemStore(m)
+
+	_, _, err := m.GetMetric(key, "flops", 1584002800, 1584007900)
+
+	if err == nil {
+		t.Errorf("Got no error\n")
+	}
+}

memstore.go

@@ -0,0 +1,542 @@
+package main
+
+import (
+	"errors"
+	"sync"
+	"unsafe"
+)
+
+// Default buffer capacity.
+// `buffer.data` will only ever grow up to it's capacity and a new link
+// in the buffer chain will be created if needed so that no copying
+// of data or reallocation needs to happen on writes.
+const (
+	BUFFER_CAP int = 512
+)
+
+// So that we can reuse allocations
+var bufferPool sync.Pool = sync.Pool{
+	New: func() interface{} {
+		return &buffer{
+			data: make([]Float, 0, BUFFER_CAP),
+		}
+	},
+}
+
+var (
+	ErrNoData           error = errors.New("no data for this metric/level")
+	ErrDataDoesNotAlign error = errors.New("data from lower granularities does not align")
+)
+
+// Each metric on each level has it's own buffer.
+// This is where the actual values go.
+// If `cap(data)` is reached, a new buffer is created and
+// becomes the new head of a buffer list.
+type buffer struct {
+	frequency  int64   // Time between two "slots"
+	start      int64   // Timestamp of when `data[0]` was written.
+	unit       string  // Unit for the data in this buffer
+	data       []Float // The slice should never reallocacte as `cap(data)` is respected.
+	prev, next *buffer // `prev` contains older data, `next` newer data.
+	archived   bool    // If true, this buffer is already archived
+
+	closed bool
+	/*
+		statisticts struct {
+			samples int
+			min     Float
+			max     Float
+			avg     Float
+		}
+	*/
+}
+
+func newBuffer(ts, freq int64, unit string) *buffer {
+	b := bufferPool.Get().(*buffer)
+	b.frequency = freq
+	b.start = ts - (freq / 2)
+	b.prev = nil
+	b.next = nil
+	b.unit = unit
+	b.archived = false
+	b.closed = false
+	b.data = b.data[:0]
+	return b
+}
+
+// If a new buffer was created, the new head is returnd.
+// Otherwise, the existing buffer is returnd.
+// Normaly, only "newer" data should be written, but if the value would
+// end up in the same buffer anyways it is allowed.
+func (b *buffer) write(ts int64, value Float) (*buffer, error) {
+	if ts < b.start {
+		return nil, errors.New("cannot write value to buffer from past")
+	}
+
+	// idx := int((ts - b.start + (b.frequency / 3)) / b.frequency)
+	idx := int((ts - b.start) / b.frequency)
+	if idx >= cap(b.data) {
+		newbuf := newBuffer(ts, b.frequency, b.unit)
+		newbuf.prev = b
+		b.next = newbuf
+		b.close()
+		b = newbuf
+		idx = 0
+	}
+
+	// Overwriting value or writing value from past
+	if idx < len(b.data) {
+		b.data[idx] = value
+		return b, nil
+	}
+
+	// Fill up unwritten slots with NaN
+	for i := len(b.data); i < idx; i++ {
+		b.data = append(b.data, NaN)
+	}
+
+	b.data = append(b.data, value)
+	return b, nil
+}
+
+func (b *buffer) end() int64 {
+	return b.firstWrite() + int64(len(b.data))*b.frequency
+}
+
+func (b *buffer) firstWrite() int64 {
+	return b.start + (b.frequency / 2)
+}
+
+func (b *buffer) close() {}
+
+/*
+func (b *buffer) close() {
+	if b.closed {
+		return
+	}
+
+	b.closed = true
+	n, sum, min, max := 0, 0., math.MaxFloat64, -math.MaxFloat64
+	for _, x := range b.data {
+		if x.IsNaN() {
+			continue
+		}
+
+		n += 1
+		f := float64(x)
+		sum += f
+		min = math.Min(min, f)
+		max = math.Max(max, f)
+	}
+
+	b.statisticts.samples = n
+	if n > 0 {
+		b.statisticts.avg = Float(sum / float64(n))
+		b.statisticts.min = Float(min)
+		b.statisticts.max = Float(max)
+	} else {
+		b.statisticts.avg = NaN
+		b.statisticts.min = NaN
+		b.statisticts.max = NaN
+	}
+}
+*/
+
+// func interpolate(idx int, data []Float) Float {
+// 	if idx == 0 || idx+1 == len(data) {
+// 		return NaN
+// 	}
+// 	return (data[idx-1] + data[idx+1]) / 2.0
+// }
+
+// Return all known values from `from` to `to`. Gaps of information are represented as NaN.
+// Simple linear interpolation is done between the two neighboring cells if possible.
+// If values at the start or end are missing, instead of NaN values, the second and thrid
+// return values contain the actual `from`/`to`.
+// This function goes back the buffer chain if `from` is older than the currents buffer start.
+// The loaded values are added to `data` and `data` is returned, possibly with a shorter length.
+// If `data` is not long enough to hold all values, this function will panic!
+func (b *buffer) read(from, to int64, data []Float) ([]Float, int64, int64, error) {
+	if from < b.firstWrite() {
+		if b.prev != nil {
+			return b.prev.read(from, to, data)
+		}
+		from = b.firstWrite()
+	}
+
+	var i int = 0
+	var t int64 = from
+	for ; t < to; t += b.frequency {
+		idx := int((t - b.start) / b.frequency)
+		if idx >= cap(b.data) {
+			if b.next == nil {
+				break
+			}
+			b = b.next
+			idx = 0
+		}
+
+		if idx >= len(b.data) {
+			if b.next == nil || to <= b.next.start {
+				break
+			}
+			data[i] += NaN
+		} else if t < b.start {
+			data[i] += NaN
+			// } else if b.data[idx].IsNaN() {
+			// 	data[i] += interpolate(idx, b.data)
+		} else {
+			data[i] += b.data[idx]
+		}
+		i++
+	}
+
+	return data[:i], from, t, nil
+}
+
+// Returns true if this buffer needs to be freed.
+func (b *buffer) free(t int64) (delme bool, n int) {
+	if b.prev != nil {
+		delme, m := b.prev.free(t)
+		n += m
+		if delme {
+			b.prev.next = nil
+			if cap(b.prev.data) == BUFFER_CAP {
+				bufferPool.Put(b.prev)
+			}
+			b.prev = nil
+		}
+	}
+
+	end := b.end()
+	if end < t {
+		return true, n + 1
+	}
+
+	return false, n
+}
+
+// Call `callback` on every buffer that contains data in the range from `from` to `to`.
+func (b *buffer) iterFromTo(from, to int64, callback func(b *buffer) error) error {
+	if b == nil {
+		return nil
+	}
+
+	if err := b.prev.iterFromTo(from, to, callback); err != nil {
+		return err
+	}
+
+	if from <= b.end() && b.start <= to {
+		return callback(b)
+	}
+
+	return nil
+}
+
+func (b *buffer) count() int64 {
+	res := int64(len(b.data))
+	if b.prev != nil {
+		res += b.prev.count()
+	}
+	return res
+}
+
+// Could also be called "node" as this forms a node in a tree structure.
+// Called level because "node" might be confusing here.
+// Can be both a leaf or a inner node. In this tree structue, inner nodes can
+// also hold data (in `metrics`).
+type level struct {
+	lock     sync.RWMutex
+	metrics  []*buffer         // Every level can store metrics.
+	children map[string]*level // Lower levels.
+}
+
+// Find the correct level for the given selector, creating it if
+// it does not exist. Example selector in the context of the
+// ClusterCockpit could be: []string{ "emmy", "host123", "cpu0" }.
+// This function would probably benefit a lot from `level.children` beeing a `sync.Map`?
+func (l *level) findLevelOrCreate(selector []string, nMetrics int) *level {
+	if len(selector) == 0 {
+		return l
+	}
+
+	// Allow concurrent reads:
+	l.lock.RLock()
+	var child *level
+	var ok bool
+	if l.children == nil {
+		// Children map needs to be created...
+		l.lock.RUnlock()
+	} else {
+		child, ok := l.children[selector[0]]
+		l.lock.RUnlock()
+		if ok {
+			return child.findLevelOrCreate(selector[1:], nMetrics)
+		}
+	}
+
+	// The level does not exist, take write lock for unqiue access:
+	l.lock.Lock()
+	// While this thread waited for the write lock, another thread
+	// could have created the child node.
+	if l.children != nil {
+		child, ok = l.children[selector[0]]
+		if ok {
+			l.lock.Unlock()
+			return child.findLevelOrCreate(selector[1:], nMetrics)
+		}
+	}
+
+	child = &level{
+		metrics:  make([]*buffer, nMetrics),
+		children: nil,
+	}
+
+	if l.children != nil {
+		l.children[selector[0]] = child
+	} else {
+		l.children = map[string]*level{selector[0]: child}
+	}
+	l.lock.Unlock()
+	return child.findLevelOrCreate(selector[1:], nMetrics)
+}
+
+func (l *level) free(t int64) (int, error) {
+	l.lock.Lock()
+	defer l.lock.Unlock()
+
+	n := 0
+	for i, b := range l.metrics {
+		if b != nil {
+			delme, m := b.free(t)
+			n += m
+			if delme {
+				if cap(b.data) == BUFFER_CAP {
+					bufferPool.Put(b)
+				}
+				l.metrics[i] = nil
+			}
+		}
+	}
+
+	for _, l := range l.children {
+		m, err := l.free(t)
+		n += m
+		if err != nil {
+			return n, err
+		}
+	}
+
+	return n, nil
+}
+
+func (l *level) sizeInBytes() int64 {
+	l.lock.RLock()
+	defer l.lock.RUnlock()
+	size := int64(0)
+
+	for _, b := range l.metrics {
+		if b != nil {
+			size += b.count() * int64(unsafe.Sizeof(Float(0)))
+		}
+	}
+
+	for _, child := range l.children {
+		size += child.sizeInBytes()
+	}
+
+	return size
+}
+
+type MemoryStore struct {
+	root    level // root of the tree structure
+	metrics map[string]MetricConfig
+}
+
+// Return a new, initialized instance of a MemoryStore.
+// Will panic if values in the metric configurations are invalid.
+func NewMemoryStore(metrics map[string]MetricConfig) *MemoryStore {
+	offset := 0
+	for key, config := range metrics {
+		if config.Frequency == 0 {
+			panic("invalid frequency")
+		}
+
+		metrics[key] = MetricConfig{
+			Frequency:   config.Frequency,
+			Aggregation: config.Aggregation,
+			offset:      offset,
+		}
+		offset += 1
+	}
+
+	return &MemoryStore{
+		root: level{
+			metrics:  make([]*buffer, len(metrics)),
+			children: make(map[string]*level),
+		},
+		metrics: metrics,
+	}
+}
+
+// Write all values in `metrics` to the level specified by `selector` for time `ts`.
+// Look at `findLevelOrCreate` for how selectors work.
+func (m *MemoryStore) Write(selector []string, ts int64, metrics []Metric) error {
+	var ok bool
+	for i, metric := range metrics {
+		if metric.mc.Frequency == 0 {
+			metric.mc, ok = m.metrics[metric.Name]
+			if !ok {
+				metric.mc.Frequency = 0
+			}
+			metrics[i] = metric
+		}
+	}
+
+	return m.WriteToLevel(&m.root, selector, ts, metrics)
+}
+
+func (m *MemoryStore) GetLevel(selector []string) *level {
+	return m.root.findLevelOrCreate(selector, len(m.metrics))
+}
+
+// Assumes that `minfo` in `metrics` is filled in!
+func (m *MemoryStore) WriteToLevel(l *level, selector []string, ts int64, metrics []Metric) error {
+	l = l.findLevelOrCreate(selector, len(m.metrics))
+	l.lock.Lock()
+	defer l.lock.Unlock()
+
+	for _, metric := range metrics {
+		if metric.mc.Frequency == 0 {
+			continue
+		}
+
+		b := l.metrics[metric.mc.offset]
+		if b == nil {
+			// First write to this metric and level
+			b = newBuffer(ts, metric.mc.Frequency, metric.Unit)
+			l.metrics[metric.mc.offset] = b
+		}
+
+		nb, err := b.write(ts, metric.Value)
+		if err != nil {
+			return err
+		}
+
+		// Last write created a new buffer...
+		if b != nb {
+			l.metrics[metric.mc.offset] = nb
+		}
+	}
+	return nil
+}
+
+// Returns all values for metric `metric` from `from` to `to` for the selected level(s).
+// If the level does not hold the metric itself, the data will be aggregated recursively from the children.
+// The second and third return value are the actual from/to for the data. Those can be different from
+// the range asked for if no data was available.
+func (m *MemoryStore) Read(selector Selector, metric string, from, to int64) ([]Float, int64, int64, string, error) {
+	var unit string = ""
+	if from > to {
+		return nil, 0, 0, "", errors.New("invalid time range")
+	}
+
+	minfo, ok := m.metrics[metric]
+	if !ok {
+		return nil, 0, 0, "", errors.New("unkown metric: " + metric)
+	}
+
+	n, data := 0, make([]Float, (to-from)/minfo.Frequency+1)
+	err := m.root.findBuffers(selector, minfo.offset, func(b *buffer) error {
+		cdata, cfrom, cto, err := b.read(from, to, data)
+		if err != nil {
+			return err
+		}
+		unit = b.unit
+
+		if n == 0 {
+			from, to = cfrom, cto
+		} else if from != cfrom || to != cto || len(data) != len(cdata) {
+			missingfront, missingback := int((from-cfrom)/minfo.Frequency), int((to-cto)/minfo.Frequency)
+			if missingfront != 0 {
+				return ErrDataDoesNotAlign
+			}
+
+			newlen := len(cdata) - missingback
+			if newlen < 1 {
+				return ErrDataDoesNotAlign
+			}
+			cdata = cdata[0:newlen]
+			if len(cdata) != len(data) {
+				return ErrDataDoesNotAlign
+			}
+
+			from, to = cfrom, cto
+		}
+
+		data = cdata
+		n += 1
+		return nil
+	})
+
+	if err != nil {
+		return nil, 0, 0, "", err
+	} else if n == 0 {
+		return nil, 0, 0, "", errors.New("metric or host not found")
+	} else if n > 1 {
+		if minfo.Aggregation == AvgAggregation {
+			normalize := 1. / Float(n)
+			for i := 0; i < len(data); i++ {
+				data[i] *= normalize
+			}
+		} else if minfo.Aggregation != SumAggregation {
+			return nil, 0, 0, "", errors.New("invalid aggregation")
+		}
+	}
+
+	return data, from, to, unit, nil
+}
+
+// Release all buffers for the selected level and all its children that contain only
+// values older than `t`.
+func (m *MemoryStore) Free(selector []string, t int64) (int, error) {
+	return m.GetLevel(selector).free(t)
+}
+
+func (m *MemoryStore) FreeAll() error {
+	for k := range m.root.children {
+		delete(m.root.children, k)
+	}
+
+	return nil
+}
+
+func (m *MemoryStore) SizeInBytes() int64 {
+	return m.root.sizeInBytes()
+}
+
+// Given a selector, return a list of all children of the level selected.
+func (m *MemoryStore) ListChildren(selector []string) []string {
+	lvl := &m.root
+	for lvl != nil && len(selector) != 0 {
+		lvl.lock.RLock()
+		next := lvl.children[selector[0]]
+		lvl.lock.RUnlock()
+		lvl = next
+		selector = selector[1:]
+	}
+
+	if lvl == nil {
+		return nil
+	}
+
+	lvl.lock.RLock()
+	defer lvl.lock.RUnlock()
+
+	children := make([]string, 0, len(lvl.children))
+	for child := range lvl.children {
+		children = append(children, child)
+	}
+
+	return children
+}

memstore_test.go

@@ -0,0 +1,512 @@
+package main
+
+import (
+	"fmt"
+	"math"
+	"math/rand"
+	"sync"
+	"testing"
+)
+
+func TestMemoryStoreBasics(t *testing.T) {
+	frequency := int64(10)
+	start, count := int64(100), int64(5000)
+	store := NewMemoryStore(map[string]MetricConfig{
+		"a": {Frequency: frequency},
+		"b": {Frequency: frequency * 2},
+	})
+
+	for i := int64(0); i < count; i++ {
+		err := store.Write([]string{"testhost"}, start+i*frequency, []Metric{
+			{Name: "a", Value: Float(i)},
+			{Name: "b", Value: Float(i / 2)},
+		})
+		if err != nil {
+			t.Error(err)
+			return
+		}
+	}
+
+	sel := Selector{{String: "testhost"}}
+	adata, from, to, unit, err := store.Read(sel, "a", start, start+count*frequency)
+	if err != nil || from != start || to != start+count*frequency || unit != "" {
+		t.Error(err)
+		return
+	}
+	bdata, _, _, unit, err := store.Read(sel, "b", start, start+count*frequency)
+	if err != nil {
+		t.Error(err)
+		return
+	}
+
+	if len(adata) != int(count) || len(bdata) != int(count/2) {
+		t.Error("unexpected count of returned values")
+		return
+	}
+
+	for i := 0; i < int(count); i++ {
+		if adata[i] != Float(i) {
+			t.Errorf("incorrect value for metric a (%f vs. %f)", adata[i], Float(i))
+			return
+		}
+	}
+
+	for i := 0; i < int(count/2); i++ {
+		if bdata[i] != Float(i) && bdata[i] != Float(i-1) {
+			t.Errorf("incorrect value for metric b (%f) at index %d", bdata[i], i)
+			return
+		}
+	}
+
+}
+
+func TestMemoryStoreTooMuchWrites(t *testing.T) {
+	frequency := int64(10)
+	count := BUFFER_CAP*3 + 10
+	store := NewMemoryStore(map[string]MetricConfig{
+		"a": {Frequency: frequency},
+		"b": {Frequency: frequency * 2},
+		"c": {Frequency: frequency / 2},
+		"d": {Frequency: frequency * 3},
+	})
+
+	start := int64(100)
+	for i := 0; i < count; i++ {
+		if err := store.Write([]string{"test"}, start+int64(i)*frequency, []Metric{
+			{Name: "a", Value: Float(i)},
+			{Name: "b", Value: Float(i / 2)},
+			{Name: "c", Value: Float(i * 2)},
+			{Name: "d", Value: Float(i / 3)},
+		}); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	end := start + int64(count)*frequency
+	data, from, to, unit, err := store.Read(Selector{{String: "test"}}, "a", start, end)
+	if len(data) != count || from != start || to != end || err != nil || unit != "" {
+		t.Fatalf("a: err=%#v, from=%d, to=%d, data=%#v\n", err, from, to, data)
+	}
+
+	data, from, to, unit, err = store.Read(Selector{{String: "test"}}, "b", start, end)
+	if len(data) != count/2 || from != start || to != end || err != nil || unit != "" {
+		t.Fatalf("b: err=%#v, from=%d, to=%d, data=%#v\n", err, from, to, data)
+	}
+
+	data, from, to, unit, err = store.Read(Selector{{String: "test"}}, "c", start, end)
+	if len(data) != count*2-1 || from != start || to != end-frequency/2 || err != nil || unit != "" {
+		t.Fatalf("c: err=%#v, from=%d, to=%d, data=%#v\n", err, from, to, data)
+	}
+
+	data, from, to, unit, err = store.Read(Selector{{String: "test"}}, "d", start, end)
+	if len(data) != count/3+1 || from != start || to != end+frequency*2 || err != nil || unit != "" {
+		t.Errorf("expected: err=nil, from=%d, to=%d, len(data)=%d\n", start, end+frequency*2, count/3)
+		t.Fatalf("d: err=%#v, from=%d, to=%d, data=%#v\n", err, from, to, data)
+	}
+}
+
+func TestMemoryStoreOutOfBounds(t *testing.T) {
+	count := 2000
+	toffset := 1000
+	store := NewMemoryStore(map[string]MetricConfig{
+		"a": {Frequency: 60},
+	})
+
+	for i := 0; i < count; i++ {
+		if err := store.Write([]string{"cluster", "host", "cpu"}, int64(toffset+i*60), []Metric{
+			{Name: "a", Value: Float(i)},
+		}); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	sel := Selector{{String: "cluster"}, {String: "host"}, {String: "cpu"}}
+	data, from, to, unit, err := store.Read(sel, "a", 500, int64(toffset+count*60+500))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if from/60 != int64(toffset)/60 || to/60 != int64(toffset+count*60)/60 {
+		t.Fatalf("Got %d-%d, expected %d-%d", from, to, toffset, toffset+count*60)
+	}
+
+	if len(data) != count || data[0] != 0 || data[len(data)-1] != Float((count-1)) {
+		t.Fatalf("Wrong data (got: %d, %f, %f, expected: %d, %f, %f)",
+			len(data), data[0], data[len(data)-1], count, 0., Float(count-1))
+	}
+
+	testfrom, testlen := int64(100000000), int64(10000)
+	data, from, to, unit, err = store.Read(sel, "a", testfrom, testfrom+testlen)
+	if len(data) != 0 || from != testfrom || to != testfrom || err != nil || unit != "" {
+		t.Fatal("Unexpected data returned when reading range after valid data")
+	}
+
+	testfrom, testlen = 0, 10
+	data, from, to, unit, err = store.Read(sel, "a", testfrom, testfrom+testlen)
+	if len(data) != 0 || from/60 != int64(toffset)/60 || to/60 != int64(toffset)/60 || err != nil || unit != "" {
+		t.Fatal("Unexpected data returned when reading range before valid data")
+	}
+}
+
+func TestMemoryStoreMissingDatapoints(t *testing.T) {
+	count := 3000
+	store := NewMemoryStore(map[string]MetricConfig{
+		"a": {Frequency: 1},
+	})
+
+	for i := 0; i < count; i++ {
+		if i%3 != 0 {
+			continue
+		}
+
+		err := store.Write([]string{"testhost"}, int64(i), []Metric{
+			{Name: "a", Value: Float(i)},
+		})
+		if err != nil {
+			t.Error(err)
+			return
+		}
+	}
+
+	sel := Selector{{String: "testhost"}}
+	adata, _, _, _, err := store.Read(sel, "a", 0, int64(count))
+	if err != nil {
+		t.Error(err)
+		return
+	}
+
+	if len(adata) != count-2 {
+		t.Error("unexpected len")
+		return
+	}
+
+	for i := 0; i < count-2; i++ {
+		if i%3 == 0 {
+			if adata[i] != Float(i) {
+				t.Error("unexpected value")
+				return
+			}
+		} else {
+			if !math.IsNaN(float64(adata[i])) {
+				t.Errorf("NaN expected (i = %d, value = %f)\n", i, adata[i])
+				return
+			}
+		}
+	}
+}
+
+func TestMemoryStoreAggregation(t *testing.T) {
+	count := 3000
+	store := NewMemoryStore(map[string]MetricConfig{
+		"a": {Frequency: 1, Aggregation: SumAggregation},
+	})
+
+	for i := 0; i < count; i++ {
+		err := store.Write([]string{"host0", "cpu0"}, int64(i), []Metric{
+			{Name: "a", Value: Float(i) / 2.},
+		})
+		if err != nil {
+			t.Error(err)
+			return
+		}
+
+		err = store.Write([]string{"host0", "cpu1"}, int64(i), []Metric{
+			{Name: "a", Value: Float(i) * 2.},
+		})
+		if err != nil {
+			t.Error(err)
+			return
+		}
+	}
+
+	adata, from, to, _, err := store.Read(Selector{{String: "host0"}}, "a", int64(0), int64(count))
+	if err != nil {
+		t.Error(err)
+		return
+	}
+
+	if len(adata) != count || from != 0 || to != int64(count) {
+		t.Error("unexpected length or time range of returned data")
+		return
+	}
+
+	for i := 0; i < count; i++ {
+		expected := Float(i)/2. + Float(i)*2.
+		if adata[i] != expected {
+			t.Errorf("expected: %f, got: %f", expected, adata[i])
+			return
+		}
+	}
+}
+
+func TestMemoryStoreStats(t *testing.T) {
+	count := 3000
+	store := NewMemoryStore(map[string]MetricConfig{
+		"a": {Frequency: 1},
+		"b": {Frequency: 1, Aggregation: AvgAggregation},
+	})
+
+	sel1 := []string{"cluster", "host1"}
+	sel2 := []string{"cluster", "host2", "left"}
+	sel3 := []string{"cluster", "host2", "right"}
+
+	samples := 0
+	asum, amin, amax := 0., math.MaxFloat32, -math.MaxFloat32
+	bsum, bmin, bmax := 0., math.MaxFloat32, -math.MaxFloat32
+
+	for i := 0; i < count; i++ {
+		if i%5 == 0 {
+			// Skip some writes, test if samples is calculated correctly
+			continue
+		}
+
+		samples += 1
+		a := float64(rand.Int()%100 - 50)
+		asum += a
+		amin = math.Min(amin, a)
+		amax = math.Max(amax, a)
+		b := float64(rand.Int()%100 - 50)
+		bsum += b * 2
+		bmin = math.Min(bmin, b)
+		bmax = math.Max(bmax, b)
+
+		store.Write(sel1, int64(i), []Metric{
+			{Name: "a", Value: Float(a)},
+		})
+		store.Write(sel2, int64(i), []Metric{
+			{Name: "b", Value: Float(b)},
+		})
+		store.Write(sel3, int64(i), []Metric{
+			{Name: "b", Value: Float(b)},
+		})
+	}
+
+	stats, from, to, err := store.Stats(Selector{{String: "cluster"}, {String: "host1"}}, "a", 0, int64(count))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if from != 1 || to != int64(count) || stats.Samples != samples {
+		t.Fatalf("unexpected: from=%d, to=%d, stats.Samples=%d (expected samples=%d)\n", from, to, stats.Samples, samples)
+	}
+
+	if stats.Avg != Float(asum/float64(samples)) || stats.Min != Float(amin) || stats.Max != Float(amax) {
+		t.Fatalf("wrong stats: %#v\n", stats)
+	}
+
+	stats, from, to, err = store.Stats(Selector{{String: "cluster"}, {String: "host2"}}, "b", 0, int64(count))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if from != 1 || to != int64(count) || stats.Samples != samples*2 {
+		t.Fatalf("unexpected: from=%d, to=%d, stats.Samples=%d (expected samples=%d)\n", from, to, stats.Samples, samples*2)
+	}
+
+	if stats.Avg != Float(bsum/float64(samples*2)) || stats.Min != Float(bmin) || stats.Max != Float(bmax) {
+		t.Fatalf("wrong stats: %#v (expected: avg=%f, min=%f, max=%f)\n", stats, bsum/float64(samples*2), bmin, bmax)
+	}
+}
+
+func TestMemoryStoreArchive(t *testing.T) {
+	store1 := NewMemoryStore(map[string]MetricConfig{
+		"a": {Frequency: 1},
+		"b": {Frequency: 1},
+	})
+
+	count := 2000
+	for i := 0; i < count; i++ {
+		err := store1.Write([]string{"cluster", "host", "cpu0"}, 100+int64(i), []Metric{
+			{Name: "a", Value: Float(i)},
+			{Name: "b", Value: Float(i * 2)},
+		})
+		if err != nil {
+			t.Error(err)
+			return
+		}
+	}
+
+	// store1.DebugDump(bufio.NewWriter(os.Stdout))
+
+	archiveRoot := t.TempDir()
+	_, err := store1.ToCheckpoint(archiveRoot, 100, 100+int64(count/2))
+	if err != nil {
+		t.Error(err)
+		return
+	}
+
+	_, err = store1.ToCheckpoint(archiveRoot, 100+int64(count/2), 100+int64(count))
+	if err != nil {
+		t.Error(err)
+		return
+	}
+
+	store2 := NewMemoryStore(map[string]MetricConfig{
+		"a": {Frequency: 1},
+		"b": {Frequency: 1},
+	})
+	n, err := store2.FromCheckpoint(archiveRoot, 100)
+	if err != nil {
+		t.Error(err)
+		return
+	}
+
+	sel := Selector{{String: "cluster"}, {String: "host"}, {String: "cpu0"}}
+	adata, from, to, _, err := store2.Read(sel, "a", 100, int64(100+count))
+	if err != nil {
+		t.Error(err)
+		return
+	}
+
+	if n != 2 || len(adata) != count || from != 100 || to != int64(100+count) {
+		t.Errorf("unexpected: n=%d, len=%d, from=%d, to=%d\n", n, len(adata), from, to)
+		return
+	}
+
+	for i := 0; i < count; i++ {
+		expected := Float(i)
+		if adata[i] != expected {
+			t.Errorf("expected: %f, got: %f", expected, adata[i])
+		}
+	}
+}
+
+func TestMemoryStoreFree(t *testing.T) {
+	store := NewMemoryStore(map[string]MetricConfig{
+		"a": {Frequency: 1},
+		"b": {Frequency: 2},
+	})
+
+	count := 3000
+	sel := []string{"cluster", "host", "1"}
+	for i := 0; i < count; i++ {
+		err := store.Write(sel, int64(i), []Metric{
+			{Name: "a", Value: Float(i)},
+			{Name: "b", Value: Float(i)},
+		})
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	n, err := store.Free([]string{"cluster", "host"}, int64(BUFFER_CAP*2)+100)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if n != 3 {
+		t.Fatal("two buffers expected to be released")
+	}
+
+	adata, from, to, _, err := store.Read(Selector{{String: "cluster"}, {String: "host"}, {String: "1"}}, "a", 0, int64(count))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if from != int64(BUFFER_CAP*2) || to != int64(count) || len(adata) != count-2*BUFFER_CAP {
+		t.Fatalf("unexpected values from call to `Read`: from=%d, to=%d, len=%d", from, to, len(adata))
+	}
+
+	// bdata, from, to, err := store.Read(Selector{{String: "cluster"}, {String: "host"}, {String: "1"}}, "b", 0, int64(count))
+	// if err != nil {
+	// 	t.Fatal(err)
+	// }
+
+	// if from != int64(BUFFER_CAP*2) || to != int64(count) || len(bdata) != (count-2*BUFFER_CAP)/2 {
+	// 	t.Fatalf("unexpected values from call to `Read`: from=%d (expected: %d), to=%d (expected: %d), len=%d (expected: %d)",
+	// 		from, BUFFER_CAP*2, to, count, len(bdata), (count-2*BUFFER_CAP)/2)
+	// }
+
+	if adata[0] != Float(BUFFER_CAP*2) || adata[len(adata)-1] != Float(count-1) {
+		t.Fatal("wrong values")
+	}
+}
+
+func BenchmarkMemoryStoreConcurrentWrites(b *testing.B) {
+	frequency := int64(5)
+	count := b.N
+	goroutines := 4
+	store := NewMemoryStore(map[string]MetricConfig{
+		"a": {Frequency: frequency},
+	})
+
+	var wg sync.WaitGroup
+	wg.Add(goroutines)
+
+	for g := 0; g < goroutines; g++ {
+		go func(g int) {
+			host := fmt.Sprintf("host%d", g)
+			for i := 0; i < count; i++ {
+				store.Write([]string{"cluster", host, "cpu0"}, int64(i)*frequency, []Metric{
+					{Name: "a", Value: Float(i)},
+				})
+			}
+			wg.Done()
+		}(g)
+	}
+
+	wg.Wait()
+	b.StopTimer()
+
+	for g := 0; g < goroutines; g++ {
+		host := fmt.Sprintf("host%d", g)
+		sel := Selector{{String: "cluster"}, {String: host}, {String: "cpu0"}}
+		adata, _, _, _, err := store.Read(sel, "a", 0, int64(count)*frequency)
+		if err != nil {
+			b.Error(err)
+			return
+		}
+
+		if len(adata) != count {
+			b.Error("unexpected count")
+			return
+		}
+
+		for i := 0; i < count; i++ {
+			expected := Float(i)
+			if adata[i] != expected {
+				b.Error("incorrect value for metric a")
+				return
+			}
+		}
+	}
+}
+
+func BenchmarkMemoryStoreAggregation(b *testing.B) {
+	b.StopTimer()
+	count := 2000
+	store := NewMemoryStore(map[string]MetricConfig{
+		"flops_any": {Frequency: 1, Aggregation: AvgAggregation},
+	})
+
+	sel := []string{"testcluster", "host123", "cpu0"}
+	for i := 0; i < count; i++ {
+		sel[2] = "cpu0"
+		err := store.Write(sel, int64(i), []Metric{
+			{Name: "flops_any", Value: Float(i)},
+		})
+		if err != nil {
+			b.Fatal(err)
+		}
+
+		sel[2] = "cpu1"
+		err = store.Write(sel, int64(i), []Metric{
+			{Name: "flops_any", Value: Float(i)},
+		})
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+
+	b.StartTimer()
+	for n := 0; n < b.N; n++ {
+		data, from, to, _, err := store.Read(Selector{{String: "testcluster"}, {String: "host123"}}, "flops_any", 0, int64(count))
+		if err != nil {
+			b.Fatal(err)
+		}
+
+		if len(data) != count || from != 0 || to != int64(count) {
+			b.Fatal()
+		}
+	}
+}

openapi.yaml

@@ -0,0 +1,148 @@
+# OpenAPI spec describing a subset of the HTTP REST API for the cc-metric-store.
+
+openapi: 3.0.3
+info:
+  title: 'cc-metric-store REST API'
+  description: 'In-memory time series database for hpc metrics to be used with the [ClusterCockpit](https://github.com/ClusterCockpit) toolsuite'
+  version: 0.1.0
+paths:
+  '/api/write':
+    post:
+      operationId: 'writeMetrics'
+      description: 'Recieves metrics in the influx line-protocol using [this format](https://github.com/ClusterCockpit/cc-specifications/blob/master/metrics/lineprotocol_alternative.md)'
+      parameters:
+        - name: cluster
+          in: query
+          schema: { type: string }
+          description: "If the lines in the body do not have a cluster tag, use this value instead."
+      requestBody:
+        required: true
+        content:
+          'text/plain':
+            example:
+              'flops_any,cluster=emmy,hostname=e1001,type=cpu,type-id=0 value=42.0'
+      responses:
+        200:
+          description: 'Everything went fine'
+        400:
+          description: 'Bad Request'
+  '/api/query':
+    post:
+      operationId: 'queryMetrics'
+      description: 'Query metrics'
+      requestBody:
+        required: true
+        content:
+          'application/json':
+            schema:
+              type: object
+              required: [cluster, from, to]
+              properties:
+                cluster:
+                  type: string
+                from:
+                  type: integer
+                to:
+                  type: integer
+                with-stats:
+                  type: boolean
+                  default: true
+                with-data:
+                  type: boolean
+                  default: true
+                queries:
+                  type: array
+                  items:
+                    $ref: '#/components/schemas/ApiQuery'
+                for-all-nodes:
+                  description: 'If not null, add a new query for every known host on that cluster and every metric (at node-scope) specified in this array to the request. This can be used to get a metric for every host in a cluster without knowing the name of every host.'
+                  type: array
+                  items:
+                    type: string
+      responses:
+        200:
+          description: 'Requested data and stats as JSON'
+          content:
+            'application/json':
+              schema:
+                type: object
+                properties:
+                  queries:
+                    description: 'Only if for-all-nodes was used, this property exists.'
+                  results:
+                    type: array
+                    description: 'Array where each element is a response to the query at that same index in the request'
+                    items:
+                      description: 'If `aggreg` is true, only ever has one element.'
+                      type: array
+                      items:
+                        type: object
+                        properties:
+                          error:
+                            description: 'If not null or undefined, an error happend processing that query'
+                            type: string
+                            nullable: true
+                          data:
+                            type: array
+                            items:
+                              type: number
+                              nullable: true
+                          avg: { type: number }
+                          min: { type: number }
+                          max: { type: number }
+        400:
+          description: 'Bad Request'
+  '/api/free':
+    post:
+      operationId: 'freeBuffers'
+      description: 'Allow all buffers containing only data older than `to`'
+      parameters:
+        - name: to
+          in: query
+          description: 'Unix Timestamp'
+          required: true
+          schema:
+            type: integer
+      requestBody:
+        required: true
+        content:
+          'application/json':
+            schema:
+              type: array
+              items:
+                type: array
+                items:
+                  type: string
+      responses:
+        200:
+          description: 'Everything went fine'
+        400:
+          description: 'Bad Request'
+components:
+  schemas:
+    ApiQuery:
+      description: 'A single query for a specific metric resulting in one series'
+      type: object
+      required: [metric, hostname, aggreg]
+      properties:
+        metirc:
+          type: string
+        hostname:
+          type: string
+        type:
+          description: 'Not required for node-level requests. Usually something like socket, cpu or hwthread.'
+          type: string
+        type-ids:
+          type: array
+          items:
+            type: string
+        aggreg:
+          type: boolean
+          description: 'If true, every query result will have exactly one element. Otherwise, the data for every requested type-id/sub-type-id is provided seperately'
+  securitySchemes:
+    bearerAuth:
+      type: http
+      scheme: bearer
+      bearerFormat: JWT
+security:
+  - bearerAuth: [] # Applies `bearerAuth` globally

scripts/cc-metric-store.config

@@ -0,0 +1,17 @@
+CC_USER=clustercockpit
+
+CC_GROUP=clustercockpit
+
+CC_HOME=/tmp
+
+LOG_DIR=/var/log
+
+DATA_DIR=/var/run/cc-metric-store
+
+MAX_OPEN_FILES=10000
+
+CONF_DIR=/etc/cc-metric-store
+
+CONF_FILE=/etc/cc-metric-store/cc-metric-store.json
+
+RESTART_ON_UPGRADE=true

scripts/cc-metric-store.deb.control

@@ -0,0 +1,12 @@
+Package: cc-metric-store
+Version: {VERSION}
+Installed-Size: {INSTALLED_SIZE}
+Architecture: {ARCH}
+Maintainer: thomas.gruber@fau.de
+Depends: libc6 (>= 2.2.1)
+Build-Depends: debhelper-compat (= 13), git, golang-go
+Description: In-memory metric store daemon from the ClusterCockpit suite
+Homepage: https://github.com/ClusterCockpit/cc-metric-store
+Source: cc-metric-store
+Rules-Requires-Root: no
+

scripts/cc-metric-store.init

@@ -0,0 +1,141 @@
+#! /usr/bin/env bash
+
+# chkconfig: 2345 80 05
+# description: ClusterCockpit metric store
+# processname: cc-metric-store
+# config: /etc/default/cc-metric-store
+# pidfile: /var/run/cc-metric-store.pid
+
+### BEGIN INIT INFO
+# Provides:          cc-metric-store
+# Required-Start:    $all
+# Required-Stop:     $remote_fs $syslog
+# Default-Start:     2 3 4 5
+# Default-Stop:      0 1 6
+# Short-Description: Start ClusterCockpit metric store at boot time
+### END INIT INFO
+
+
+PATH=/bin:/usr/bin:/sbin:/usr/sbin
+NAME=cc-metric-store
+DESC="ClusterCockpit metric store"
+DEFAULT=/etc/default/${NAME}.json
+
+CC_USER=clustercockpit
+CC_GROUP=clustercockpit
+CONF_DIR=/etc/cc-metric-store
+PID_FILE=/var/run/$NAME.pid
+DAEMON=/usr/sbin/$NAME
+CONF_FILE=${CONF_DIR}/cc-metric-store.json
+
+umask 0027
+
+if [ ! -x $DAEMON ]; then
+  echo "Program not installed or not executable"
+  exit 5
+fi
+
+. /lib/lsb/init-functions
+
+if [ -r /etc/default/rcS ]; then
+	. /etc/default/rcS
+fi
+
+# overwrite settings from default file
+if [ -f "$DEFAULT" ]; then
+	. "$DEFAULT"
+fi
+
+CC_OPTS="--config=${CONF_FILE}"
+
+function checkUser() {
+  if [ `id -u` -ne 0 ]; then
+  	echo "You need root privileges to run this script"
+  	exit 4
+  fi
+}
+
+case "$1" in
+  start)
+  checkUser
+	log_daemon_msg "Starting $DESC"
+
+	pid=`pidofproc -p $PID_FILE $NAME`
+	if [ -n "$pid" ] ; then
+		log_begin_msg "Already running."
+		log_end_msg 0
+		exit 0
+	fi
+
+	# Prepare environment
+	touch "$PID_FILE" && chown "$CC_USER":"$CC_GROUP" "$PID_FILE"
+
+  if [ -n "$MAX_OPEN_FILES" ]; then
+		ulimit -n $MAX_OPEN_FILES
+	fi
+
+	# Start Daemon
+	start-stop-daemon --start -b --chdir "$WORK_DIR" --user "$CC_USER" -c "$CC_USER" --pidfile "$PID_FILE" --exec $DAEMON -- $DAEMON_OPTS
+	return=$?
+	if [ $return -eq 0 ]
+	then
+	  sleep 1
+
+    # check if pid file has been written to
+	  if ! [[ -s $PID_FILE ]]; then
+	    log_end_msg 1
+	    exit 1
+	  fi
+
+		i=0
+		timeout=10
+		# Wait for the process to be properly started before exiting
+		until { cat "$PID_FILE" | xargs kill -0; } >/dev/null 2>&1
+		do
+			sleep 1
+			i=$(($i + 1))
+      if [ $i -gt $timeout ]; then
+			  log_end_msg 1
+			  exit 1
+			fi
+		done
+  fi
+  log_end_msg $return
+	;;
+  stop)
+  checkUser
+	log_daemon_msg "Stopping $DESC"
+
+	if [ -f "$PID_FILE" ]; then
+		start-stop-daemon --stop --pidfile "$PID_FILE" \
+			--user "$CC_USER" \
+			--retry=TERM/20/KILL/5 >/dev/null
+		if [ $? -eq 1 ]; then
+			log_progress_msg "$DESC is not running but pid file exists, cleaning up"
+		elif [ $? -eq 3 ]; then
+			PID="`cat $PID_FILE`"
+			log_failure_msg "Failed to stop $DESC (pid $PID)"
+			exit 1
+		fi
+		rm -f "$PID_FILE"
+	else
+		log_progress_msg "(not running)"
+	fi
+	log_end_msg 0
+	;;
+  status)
+	status_of_proc -p $PID_FILE $NAME $NAME && exit 0 || exit $?
+    ;;
+  restart|force-reload)
+	if [ -f "$PID_FILE" ]; then
+		$0 stop
+		sleep 1
+	fi
+	$0 start
+	;;
+  *)
+	log_success_msg "Usage: $0 {start|stop|restart|force-reload|status}"
+	exit 3
+	;;
+esac
+

scripts/cc-metric-store.spec

@@ -0,0 +1,62 @@
+Name:           cc-metric-store
+Version:        %{VERS}
+Release:        1%{?dist}
+Summary:        In-memory metric database from the ClusterCockpit suite
+
+License:        MIT
+Source0:        %{name}-%{version}.tar.gz
+
+BuildRequires:  go-toolset
+BuildRequires:  systemd-rpm-macros
+
+Provides:       %{name} = %{version}
+
+%description
+In-memory metric database from the ClusterCockpit suite
+
+%global debug_package %{nil}
+
+%prep
+%autosetup
+
+
+%build
+make
+
+
+%install
+# Install cc-metric-store
+make PREFIX=%{buildroot} install
+# Integrate into system
+install -Dpm 0644 scripts/%{name}.service %{buildroot}%{_unitdir}/%{name}.service
+install -Dpm 0600 scripts/%{name}.config %{buildroot}%{_sysconfdir}/default/%{name}
+install -Dpm 0644 scripts/%{name}.sysusers %{buildroot}%{_sysusersdir}/%{name}.conf
+
+
+%check
+# go test should be here... :)
+
+%pre
+%sysusers_create_package scripts/%{name}.sysusers
+
+%post
+%systemd_post %{name}.service
+
+%preun
+%systemd_preun %{name}.service
+
+%files
+# Binary
+%attr(-,clustercockpit,clustercockpit) %{_bindir}/%{name}
+# Config
+%dir %{_sysconfdir}/%{name}
+%attr(0600,clustercockpit,clustercockpit) %config(noreplace) %{_sysconfdir}/%{name}/%{name}.json
+# Systemd
+%{_unitdir}/%{name}.service
+%{_sysconfdir}/default/%{name}
+%{_sysusersdir}/%{name}.conf
+
+%changelog
+* Mon Mar 07 2022 Thomas Gruber - 0.1
+- Initial metric store implementation
+

scripts/cc-metric-store.sysusers

@@ -0,0 +1,2 @@
+#Type Name           ID GECOS                     Home directory           Shell
+u     clustercockpit -  "User for ClusterCockpit" /run/cc-metric-collector /sbin/nologin

scripts/send-fake-data.go

@@ -0,0 +1,105 @@
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"log"
+	"math"
+	"math/rand"
+	"net/http"
+	"time"
+)
+
+const token = "eyJ0eXAiOiJKV1QiLCJhbGciOiJFZERTQSJ9.eyJ1c2VyIjoiYWRtaW4iLCJyb2xlcyI6WyJST0xFX0FETUlOIiwiUk9MRV9BTkFMWVNUIiwiUk9MRV9VU0VSIl19.d-3_3FZTsadPjDEdsWrrQ7nS0edMAR4zjl-eK7rJU3HziNBfI9PDHDIpJVHTNN5E5SlLGLFXctWyKAkwhXL-Dw"
+const ccmsurl = "http://localhost:8081/api/write"
+const cluster = "fakedev"
+const sockets = 2
+const cpus = 8
+const freq = 15 * time.Second
+
+var hosts = []string{"fake001", "fake002", "fake003", "fake004", "fake005"}
+var metrics = []struct {
+	Name     string
+	Type     string
+	AvgValue float64
+}{
+	{"flops_any", "cpu", 10.0},
+	{"mem_bw", "socket", 50.0},
+	{"ipc", "cpu", 1.25},
+	{"cpu_load", "node", 4},
+	{"mem_used", "node", 20},
+}
+
+var states = make([]float64, 0)
+
+func send(client *http.Client, t int64) {
+	msg := &bytes.Buffer{}
+
+	i := 0
+	for _, host := range hosts {
+		for _, metric := range metrics {
+			n := 1
+			if metric.Type == "socket" {
+				n = sockets
+			} else if metric.Type == "cpu" {
+				n = cpus
+			}
+
+			for j := 0; j < n; j++ {
+				fmt.Fprintf(msg, "%s,cluster=%s,host=%s,type=%s", metric.Name, cluster, host, metric.Type)
+				if metric.Type == "socket" {
+					fmt.Fprintf(msg, ",type-id=%d", j)
+				} else if metric.Type == "cpu" {
+					fmt.Fprintf(msg, ",type-id=%d", j)
+				}
+
+				x := metric.AvgValue + math.Sin(states[i])*(metric.AvgValue/10.)
+				states[i] += 0.1
+				fmt.Fprintf(msg, " value=%f ", x)
+
+				fmt.Fprintf(msg, "%d\n", t)
+				i++
+			}
+		}
+	}
+
+	req, _ := http.NewRequest(http.MethodPost, ccmsurl, msg)
+	req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token))
+	res, err := client.Do(req)
+	if err != nil {
+		log.Print(err)
+		return
+	}
+	if res.StatusCode != http.StatusOK {
+		body, _ := io.ReadAll(res.Body)
+		log.Printf("%s: %s", res.Status, string(body))
+	}
+}
+
+func main() {
+	for range hosts {
+		for _, m := range metrics {
+			n := 1
+			if m.Type == "socket" {
+				n = sockets
+			} else if m.Type == "cpu" {
+				n = cpus
+			}
+
+			for i := 0; i < n; i++ {
+				states = append(states, rand.Float64()*100)
+			}
+		}
+	}
+
+	client := &http.Client{}
+
+	i := 0
+	for t := range time.Tick(freq) {
+		log.Printf("tick... (#%d)", i)
+		i++
+
+		send(client, t.Unix())
+	}
+}

selector.go

@@ -0,0 +1,123 @@
+package main
+
+import (
+	"encoding/json"
+	"errors"
+)
+
+type SelectorElement struct {
+	Any    bool
+	String string
+	Group  []string
+}
+
+func (se *SelectorElement) UnmarshalJSON(input []byte) error {
+	if input[0] == '"' {
+		if err := json.Unmarshal(input, &se.String); err != nil {
+			return err
+		}
+
+		if se.String == "*" {
+			se.Any = true
+			se.String = ""
+		}
+
+		return nil
+	}
+
+	if input[0] == '[' {
+		return json.Unmarshal(input, &se.Group)
+	}
+
+	return errors.New("the Go SelectorElement type can only be a string or an array of strings")
+}
+
+func (se *SelectorElement) MarshalJSON() ([]byte, error) {
+	if se.Any {
+		return []byte("\"*\""), nil
+	}
+
+	if se.String != "" {
+		return json.Marshal(se.String)
+	}
+
+	if se.Group != nil {
+		return json.Marshal(se.Group)
+	}
+
+	return nil, errors.New("a Go Selector must be a non-empty string or a non-empty slice of strings")
+}
+
+type Selector []SelectorElement
+
+func (l *level) findLevel(selector []string) *level {
+	if len(selector) == 0 {
+		return l
+	}
+
+	l.lock.RLock()
+	defer l.lock.RUnlock()
+
+	lvl := l.children[selector[0]]
+	if lvl == nil {
+		return nil
+	}
+
+	return lvl.findLevel(selector[1:])
+}
+
+func (l *level) findBuffers(selector Selector, offset int, f func(b *buffer) error) error {
+	l.lock.RLock()
+	defer l.lock.RUnlock()
+
+	if len(selector) == 0 {
+		b := l.metrics[offset]
+		if b != nil {
+			return f(b)
+		}
+
+		for _, lvl := range l.children {
+			err := lvl.findBuffers(nil, offset, f)
+			if err != nil {
+				return err
+			}
+		}
+		return nil
+	}
+
+	sel := selector[0]
+	if len(sel.String) != 0 && l.children != nil {
+		lvl, ok := l.children[sel.String]
+		if ok {
+			err := lvl.findBuffers(selector[1:], offset, f)
+			if err != nil {
+				return err
+			}
+		}
+		return nil
+	}
+
+	if sel.Group != nil && l.children != nil {
+		for _, key := range sel.Group {
+			lvl, ok := l.children[key]
+			if ok {
+				err := lvl.findBuffers(selector[1:], offset, f)
+				if err != nil {
+					return err
+				}
+			}
+		}
+		return nil
+	}
+
+	if sel.Any && l.children != nil {
+		for _, lvl := range l.children {
+			if err := lvl.findBuffers(selector[1:], offset, f); err != nil {
+				return err
+			}
+		}
+		return nil
+	}
+
+	return nil
+}

stats.go

@@ -0,0 +1,118 @@
+package main
+
+import (
+	"errors"
+	"math"
+)
+
+type Stats struct {
+	Samples int
+	Avg     Float
+	Min     Float
+	Max     Float
+}
+
+func (b *buffer) stats(from, to int64) (Stats, int64, int64, error) {
+	if from < b.start {
+		if b.prev != nil {
+			return b.prev.stats(from, to)
+		}
+		from = b.start
+	}
+
+	// TODO: Check if b.closed and if so and the full buffer is queried,
+	// use b.statistics instead of iterating over the buffer.
+
+	samples := 0
+	sum, min, max := 0.0, math.MaxFloat32, -math.MaxFloat32
+
+	var t int64
+	for t = from; t < to; t += b.frequency {
+		idx := int((t - b.start) / b.frequency)
+		if idx >= cap(b.data) {
+			b = b.next
+			if b == nil {
+				break
+			}
+			idx = 0
+		}
+
+		if t < b.start || idx >= len(b.data) {
+			continue
+		}
+
+		xf := float64(b.data[idx])
+		if math.IsNaN(xf) {
+			continue
+		}
+
+		samples += 1
+		sum += xf
+		min = math.Min(min, xf)
+		max = math.Max(max, xf)
+	}
+
+	return Stats{
+		Samples: samples,
+		Avg:     Float(sum) / Float(samples),
+		Min:     Float(min),
+		Max:     Float(max),
+	}, from, t, nil
+}
+
+// Returns statistics for the requested metric on the selected node/level.
+// Data is aggregated to the selected level the same way as in `MemoryStore.Read`.
+// If `Stats.Samples` is zero, the statistics should not be considered as valid.
+func (m *MemoryStore) Stats(selector Selector, metric string, from, to int64) (*Stats, int64, int64, error) {
+	if from > to {
+		return nil, 0, 0, errors.New("invalid time range")
+	}
+
+	minfo, ok := m.metrics[metric]
+	if !ok {
+		return nil, 0, 0, errors.New("unkown metric: " + metric)
+	}
+
+	n, samples := 0, 0
+	avg, min, max := Float(0), math.MaxFloat32, -math.MaxFloat32
+	err := m.root.findBuffers(selector, minfo.offset, func(b *buffer) error {
+		stats, cfrom, cto, err := b.stats(from, to)
+		if err != nil {
+			return err
+		}
+
+		if n == 0 {
+			from, to = cfrom, cto
+		} else if from != cfrom || to != cto {
+			return ErrDataDoesNotAlign
+		}
+
+		samples += stats.Samples
+		avg += stats.Avg
+		min = math.Min(min, float64(stats.Min))
+		max = math.Max(max, float64(stats.Max))
+		n += 1
+		return nil
+	})
+
+	if err != nil {
+		return nil, 0, 0, err
+	}
+
+	if n == 0 {
+		return nil, 0, 0, ErrNoData
+	}
+
+	if minfo.Aggregation == AvgAggregation {
+		avg /= Float(n)
+	} else if n > 1 && minfo.Aggregation != SumAggregation {
+		return nil, 0, 0, errors.New("invalid aggregation")
+	}
+
+	return &Stats{
+		Samples: samples,
+		Avg:     avg,
+		Min:     Float(min),
+		Max:     Float(max),
+	}, from, to, nil
+}

testdata/checkpoints/fritz/f0131/1692628930.json

@@ -0,0 +1 @@
+{"from":1692628930,"to":1692629003,"metrics":{},"children":{"hwthread0":{"from":1692628930,"to":1692629003,"metrics":{"cpi":{"frequency":5,"start":1692628930,"unit":"","data":[2.2,0.7,2.1,1.6,2.3,0.7,0.9,1.8,0.7,1.7,1.5,1.8,0.9,1.6]},"flops_any":{"frequency":5,"start":1692628930,"unit":"MFLOP/s","data":[0.3,0.0,0.0,0.0,0.0,28.2,5.1,4.7,0.1,0.3,0.0,0.2,0.0,0.0]},"flops_dp":{"frequency":5,"start":1692628931,"unit":"MFLOP/s","data":[0.0,0.0,0.0,0.0,0.0,0.5,0.1,0.0,0.0,0.0,0.0,0.0,0.0,0.0]},"flops_sp":{"frequency":5,"start":1692628930,"unit":"MFLOP/s","data":[0.3,0.0,0.0,0.0,0.0,27.2,4.8,4.7,0.0,0.3,0.0,0.2,0.0,0.0]}},"children":{}},"hwthread1":{"from":1692628930,"to":1692629003,"metrics":{"cpi":{"frequency":5,"start":1692628930,"unit":"","data":[0.9,4.2,4.2,3.8,3.1,0.7,1.3,0.7,1.8,1.6,5.0,3.0,4.9,5.2]},"flops_any":{"frequency":5,"start":1692628930,"unit":"MFLOP/s","data":[0.5,0.0,0.0,0.0,0.0,49.8,5.0,0.1,0.0,0.0,0.0,0.0,0.0,0.0]},"flops_dp":{"frequency":5,"start":1692628931,"unit":"MFLOP/s","data":[0.0,0.0,0.0,0.0,0.0,0.4,0.1,0.0,0.0,0.0,0.0,0.0,0.0,0.0]},"flops_sp":{"frequency":5,"start":1692628930,"unit":"MFLOP/s","data":[0.5,0.0,0.0,0.0,0.0,49.1,4.9,0.1,0.0,0.0,0.0,0.0,0.0,0.0]}},"children":{}},"hwthread2":{"from":1692628930,"to":1692629003,"metrics":{"cpi":{"frequency":5,"start":1692628930,"unit":"","data":[1.8,0.9,2.2,2.1,4.3,1.4,0.8,1.0,1.1,1.6,3.9,4.9,2.7,5.1]},"flops_any":{"frequency":5,"start":1692628930,"unit":"MFLOP/s","data":[0.0,0.0,0.0,0.0,0.0,16.2,4.4,0.0,0.1,0.0,0.0,0.0,0.0,0.0]},"flops_dp":{"frequency":5,"start":1692628931,"unit":"MFLOP/s","data":[0.0,0.0,0.0,0.0,0.0,0.1,0.2,0.0,0.0,0.0,0.0,0.0,0.0,0.0]},"flops_sp":{"frequency":5,"start":1692628930,"unit":"MFLOP/s","data":[0.0,0.0,0.0,0.0,0.0,16.1,3.9,0.0,0.1,0.0,0.0,0.0,0.0,0.0]}},"children":{}},"hwthread3":{"from":1692628930,"to":1692629003,"metrics":{"cpi":{"frequency":5,"start":1692628930,"unit":"","data":[2.0,0.6,6.0,3.5,2.3,0.9,1.2,1.2,1.1,1.0,2.0,0.7,1.1,3.6]},"flops_any":{"frequency":5,"start":1692628930,"unit":"MFLOP/s","data":[0.0,0.0,0.0,0.0,0.0,1.6,11.5,0.0,0.0,0.0,0.0,0.2,0.0,0.0]},"flops_dp":{"frequency":5,"start":1692628931,"unit":"MFLOP/s","data":[0.0,0.0,0.0,0.0,0.0,0.1,0.4,0.0,0.0,0.0,0.0,0.1,0.0,0.0]},"flops_sp":{"frequency":5,"start":1692628930,"unit":"MFLOP/s","data":[0.0,0.0,0.0,0.0,0.0,1.4,10.6,0.0,0.0,0.0,0.0,0.0,0.0,0.0]}},"children":{}},"hwthread4":{"from":1692628930,"to":1692629003,"metrics":{"cpi":{"frequency":5,"start":1692628930,"unit":"","data":[1.0,1.5,1.3,0.9,0.8,0.8,1.2,1.2,1.4,1.0,0.8,2.3,2.7,0.8]},"flops_any":{"frequency":5,"start":1692628930,"unit":"MFLOP/s","data":[0.8,0.0,0.0,0.0,0.0,3.4,1.1,2.2,0.4,0.0,0.0,0.0,0.0,0.0]},"flops_dp":{"frequency":5,"start":1692628931,"unit":"MFLOP/s","data":[0.0,0.0,0.0,0.0,0.0,0.7,0.3,0.2,0.0,0.0,0.0,0.0,0.0,0.0]},"flops_sp":{"frequency":5,"start":1692628930,"unit":"MFLOP/s","data":[0.8,0.0,0.0,0.0,0.0,2.0,0.5,1.8,0.4,0.0,0.0,0.0,0.0,0.0]}},"children":{}},"hwthread5":{"from":1692628930,"to":1692629003,"metrics":{"cpi":{"frequency":5,"start":1692628930,"unit":"","data":[0.9,2.3,1.1,0.7,0.8,1.5,1.0,0.9,0.9,1.0,0.8,1.1,1.0,2.0]},"flops_any":{"frequency":5,"start":1692628930,"unit":"MFLOP/s","data":[0.5,0.0,1.9,0.1,0.3,3.0,13.8,0.0,1.7,1.1,0.0,0.0,0.5,1.0]},"flops_dp":{"frequency":5,"start":1692628931,"unit":"MFLOP/s","data":[0.3,0.0,0.1,0.0,0.2,0.1,0.9,0.0,0.8,0.6,0.0,0.0,0.0,0.0]},"flops_sp":{"frequency":5,"start":1692628930,"unit":"MFLOP/s","data":[0.0,0.0,1.7,0.0,0.0,2.9,12.0,0.0,0.0,0.0,0.0,0.0,0.5,1.0]}},"children":{}},"hwthread6":{"from":1692628930,"to":1692629003,"metrics":{"cpi":{"frequency":5,"start":1692628930,"unit":"","data":[1.4,0.8,0.8,1.2,1.4,0.8,1.2,1.1,1.5,0.9,1.5,1.5,1.4,0.8]},"flops_any":{"frequency":5,"start":1692628930,"unit":"MFLOP/s","data":[0.0,0.1,0.0,3.9,10.6,3.0,1.4,7.0,0.0,1.0,0.2,0.0,0.0,0.0]},"flops_dp":{"frequency":5,"start":1692628931,"unit":"MFLOP/s","data":[0.0,0.1,0.0,0.0,0.0,0.1,0.1,0.2,0.0,0.4,0.0,0.0,0.0,0.0]},"flops_sp":{"frequency":5,"start":1692628930,"unit":"MFLOP/s","data":[0.0,0.0,0.0,3.9,10.6,2.9,1.2,6.6,0.0,0.1,0.2,0.0,0.0,0.0]}},"children":{}},"hwthread7":{"from":1692628930,"to":1692629003,"metrics":{"cpi":{"frequency":5,"start":1692628930,"unit":"","data":[1.2,1.2,1.3,0.8,1.2,1.5,1.0,0.7,1.7,1.8,3.5,0.7,1.5,4.6]},"flops_any":{"frequency":5,"start":1692628930,"unit":"MFLOP/s","data":[0.0,0.0,0.0,0.3,0.0,2.3,4.7,0.0,12.5,0.7,0.2,0.2,0.0,0.0]},"flops_dp":{"frequency":5,"start":1692628931,"unit":"MFLOP/s","data":[0.0,0.0,0.0,0.0,0.0,0.1,0.9,0.0,0.0,0.0,0.0,0.0,0.0,0.0]},"flops_sp":{"frequency":5,"start":1692628930,"unit":"MFLOP/s","data":[0.0,0.0,0.0,0.3,0.0,2.2,2.8,0.0,12.5,0.7,0.2,0.2,0.0,0.0]}},"children":{}}}}

tools/tools.go

@@ -1,5 +0,0 @@
-//go:build tools
-
-package tools
-
-import _ "github.com/swaggo/swag/cmd/swag"