Merge pull request #18 from ClusterCockpit/refactor

Refactor

.github/workflows/Release.yml

@@ -1,314 +0,0 @@
-# See: https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions
-
-# Workflow name
-name: Release
-
-# Run on tag push
-on:
- push:
-  tags:
-  - '**'
-
-jobs:
-
-  #
-  # Build on AlmaLinux 8.5 using golang-1.18.2
-  #
-  AlmaLinux-RPM-build:
-    runs-on: ubuntu-latest
-    # See: https://hub.docker.com/_/almalinux
-    container: almalinux:8.5
-    # The job outputs link to the outputs of the 'rpmrename' step
-    # Only job outputs can be used in child jobs
-    outputs:
-      rpm : ${{steps.rpmrename.outputs.RPM}}
-      srpm : ${{steps.rpmrename.outputs.SRPM}}
-    steps:
-
-    # Use dnf to install development packages
-    - name: Install development packages
-      run: |
-          dnf --assumeyes group install "Development Tools" "RPM Development Tools"
-          dnf --assumeyes install wget openssl-devel diffutils delve which
-
-    # Checkout git repository and submodules
-    # fetch-depth must be 0 to use git describe
-    # See: https://github.com/marketplace/actions/checkout
-    - name: Checkout
-      uses: actions/checkout@v2
-      with:
-        submodules: recursive
-        fetch-depth: 0
-
-    # Use dnf to install build dependencies
-    - name: Install build dependencies
-      run: |
-          wget -q http://mirror.centos.org/centos/8-stream/AppStream/x86_64/os/Packages/golang-1.18.2-1.module_el8.7.0+1173+5d37c0fd.x86_64.rpm \
-                  http://mirror.centos.org/centos/8-stream/AppStream/x86_64/os/Packages/golang-bin-1.18.2-1.module_el8.7.0+1173+5d37c0fd.x86_64.rpm \
-                  http://mirror.centos.org/centos/8-stream/AppStream/x86_64/os/Packages/golang-src-1.18.2-1.module_el8.7.0+1173+5d37c0fd.noarch.rpm \
-                  http://mirror.centos.org/centos/8-stream/AppStream/x86_64/os/Packages/go-toolset-1.18.2-1.module_el8.7.0+1173+5d37c0fd.x86_64.rpm
-          rpm -i go*.rpm
-
-    - name: RPM build MetricStore
-      id: rpmbuild
-      run: make RPM
-
-    # AlmaLinux 8.5 is a derivate of RedHat Enterprise Linux 8 (UBI8),
-    # so the created RPM both contain the substring 'el8' in the RPM file names
-    # This step replaces the substring 'el8' to 'alma85'. It uses the move operation
-    # because it is unclear whether the default AlmaLinux 8.5 container contains the 
-    # 'rename' command. This way we also get the new names for output.
-    - name: Rename RPMs (s/el8/alma85/)
-      id: rpmrename
-      run: |
-        OLD_RPM="${{steps.rpmbuild.outputs.RPM}}"
-        OLD_SRPM="${{steps.rpmbuild.outputs.SRPM}}"
-        NEW_RPM="${OLD_RPM/el8/alma85}"
-        NEW_SRPM=${OLD_SRPM/el8/alma85}
-        mv "${OLD_RPM}" "${NEW_RPM}"
-        mv "${OLD_SRPM}" "${NEW_SRPM}"
-        echo "::set-output name=SRPM::${NEW_SRPM}"
-        echo "::set-output name=RPM::${NEW_RPM}"
-
-    # See: https://github.com/actions/upload-artifact
-    - name: Save RPM as artifact
-      uses: actions/upload-artifact@v2
-      with:
-        name: cc-metric-store RPM for AlmaLinux 8.5
-        path: ${{ steps.rpmrename.outputs.RPM }}
-    - name: Save SRPM as artifact
-      uses: actions/upload-artifact@v2
-      with:
-        name: cc-metric-store SRPM for AlmaLinux 8.5
-        path: ${{ steps.rpmrename.outputs.SRPM }}
-
-  #
-  # Build on UBI 8 using golang-1.18.2
-  #
-  UBI-8-RPM-build:
-    runs-on: ubuntu-latest
-    # See: https://catalog.redhat.com/software/containers/ubi8/ubi/5c359854d70cc534b3a3784e?container-tabs=gti
-    container: registry.access.redhat.com/ubi8/ubi:8.5-226.1645809065
-    # The job outputs link to the outputs of the 'rpmbuild' step
-    outputs:
-      rpm : ${{steps.rpmbuild.outputs.RPM}}
-      srpm : ${{steps.rpmbuild.outputs.SRPM}}
-    steps:
-
-    # Use dnf to install development packages
-    - name: Install development packages
-      run: dnf --assumeyes --disableplugin=subscription-manager install rpm-build go-srpm-macros rpm-build-libs rpm-libs gcc make python38 git wget openssl-devel diffutils delve which
-
-    # Checkout git repository and submodules
-    # fetch-depth must be 0 to use git describe
-    # See: https://github.com/marketplace/actions/checkout
-    - name: Checkout
-      uses: actions/checkout@v2
-      with:
-        submodules: recursive
-        fetch-depth: 0
-
-    # Use dnf to install build dependencies
-    - name: Install build dependencies
-      run: |
-          wget -q http://mirror.centos.org/centos/8-stream/AppStream/x86_64/os/Packages/golang-1.18.2-1.module_el8.7.0+1173+5d37c0fd.x86_64.rpm \
-                  http://mirror.centos.org/centos/8-stream/AppStream/x86_64/os/Packages/golang-bin-1.18.2-1.module_el8.7.0+1173+5d37c0fd.x86_64.rpm \
-                  http://mirror.centos.org/centos/8-stream/AppStream/x86_64/os/Packages/golang-src-1.18.2-1.module_el8.7.0+1173+5d37c0fd.noarch.rpm \
-                  http://mirror.centos.org/centos/8-stream/AppStream/x86_64/os/Packages/go-toolset-1.18.2-1.module_el8.7.0+1173+5d37c0fd.x86_64.rpm
-          rpm -i go*.rpm
-
-    - name: RPM build MetricStore
-      id: rpmbuild
-      run: make RPM
-
-    # See: https://github.com/actions/upload-artifact
-    - name: Save RPM as artifact
-      uses: actions/upload-artifact@v2
-      with:
-        name: cc-metric-store RPM for UBI 8
-        path: ${{ steps.rpmbuild.outputs.RPM }}
-    - name: Save SRPM as artifact
-      uses: actions/upload-artifact@v2
-      with:
-        name: cc-metric-store SRPM for UBI 8
-        path: ${{ steps.rpmbuild.outputs.SRPM }}
-
-  #
-  # Build on Ubuntu 20.04 using official go 1.19.1 package
-  #
-  Ubuntu-focal-build:
-    runs-on: ubuntu-latest
-    container: ubuntu:20.04
-    # The job outputs link to the outputs of the 'debrename' step
-    # Only job outputs can be used in child jobs
-    outputs:
-      deb : ${{steps.debrename.outputs.DEB}}
-    steps:
-    # Use apt to install development packages
-    - name: Install development packages
-      run: |
-          apt update && apt --assume-yes upgrade
-          apt --assume-yes install build-essential sed git wget bash
-    # Checkout git repository and submodules
-    # fetch-depth must be 0 to use git describe
-    # See: https://github.com/marketplace/actions/checkout
-    - name: Checkout
-      uses: actions/checkout@v2
-      with:
-        submodules: recursive
-        fetch-depth: 0
-    # Use official golang package
-    - name: Install Golang
-      run: |
-          wget -q https://go.dev/dl/go1.19.1.linux-amd64.tar.gz
-          tar -C /usr/local -xzf go1.19.1.linux-amd64.tar.gz
-          export PATH=/usr/local/go/bin:/usr/local/go/pkg/tool/linux_amd64:$PATH
-          go version
-    - name: DEB build MetricStore
-      id: dpkg-build
-      run: |
-          export PATH=/usr/local/go/bin:/usr/local/go/pkg/tool/linux_amd64:$PATH
-          make DEB
-    - name: Rename DEB (add '_ubuntu20.04')
-      id: debrename
-      run: |
-        OLD_DEB_NAME=$(echo "${{steps.dpkg-build.outputs.DEB}}" | rev | cut -d '.' -f 2- | rev)
-        NEW_DEB_FILE="${OLD_DEB_NAME}_ubuntu20.04.deb"
-        mv "${{steps.dpkg-build.outputs.DEB}}" "${NEW_DEB_FILE}"
-        echo "::set-output name=DEB::${NEW_DEB_FILE}"
-    # See: https://github.com/actions/upload-artifact
-    - name: Save DEB as artifact
-      uses: actions/upload-artifact@v2
-      with:
-        name: cc-metric-store DEB for Ubuntu 20.04
-        path: ${{ steps.debrename.outputs.DEB }}
-
-  #
-  # Build on Ubuntu 20.04 using official go 1.19.1 package
-  #
-  Ubuntu-jammy-build:
-    runs-on: ubuntu-latest
-    container: ubuntu:22.04
-    # The job outputs link to the outputs of the 'debrename' step
-    # Only job outputs can be used in child jobs
-    outputs:
-      deb : ${{steps.debrename.outputs.DEB}}
-    steps:
-    # Use apt to install development packages
-    - name: Install development packages
-      run: |
-          apt update && apt --assume-yes upgrade
-          apt --assume-yes install build-essential sed git wget bash
-    # Checkout git repository and submodules
-    # fetch-depth must be 0 to use git describe
-    # See: https://github.com/marketplace/actions/checkout
-    - name: Checkout
-      uses: actions/checkout@v2
-      with:
-        submodules: recursive
-        fetch-depth: 0
-    # Use official golang package
-    - name: Install Golang
-      run: |
-          wget -q https://go.dev/dl/go1.19.1.linux-amd64.tar.gz
-          tar -C /usr/local -xzf go1.19.1.linux-amd64.tar.gz
-          export PATH=/usr/local/go/bin:/usr/local/go/pkg/tool/linux_amd64:$PATH
-          go version
-    - name: DEB build MetricStore
-      id: dpkg-build
-      run: |
-          export PATH=/usr/local/go/bin:/usr/local/go/pkg/tool/linux_amd64:$PATH
-          make DEB
-    - name: Rename DEB (add '_ubuntu22.04')
-      id: debrename
-      run: |
-        OLD_DEB_NAME=$(echo "${{steps.dpkg-build.outputs.DEB}}" | rev | cut -d '.' -f 2- | rev)
-        NEW_DEB_FILE="${OLD_DEB_NAME}_ubuntu22.04.deb"
-        mv "${{steps.dpkg-build.outputs.DEB}}" "${NEW_DEB_FILE}"
-        echo "::set-output name=DEB::${NEW_DEB_FILE}"
-    # See: https://github.com/actions/upload-artifact
-    - name: Save DEB as artifact
-      uses: actions/upload-artifact@v2
-      with:
-        name: cc-metric-store DEB for Ubuntu 22.04
-        path: ${{ steps.debrename.outputs.DEB }}
-
-  #
-  # Create release with fresh RPMs
-  #
-  Release:
-    runs-on: ubuntu-latest
-    # We need the RPMs, so add dependency
-    needs: [AlmaLinux-RPM-build, UBI-8-RPM-build, Ubuntu-focal-build, Ubuntu-jammy-build]
-
-    steps:
-    # See: https://github.com/actions/download-artifact
-    - name: Download AlmaLinux 8.5 RPM
-      uses: actions/download-artifact@v2
-      with:
-        name: cc-metric-store RPM for AlmaLinux 8.5
-    - name: Download AlmaLinux 8.5 SRPM
-      uses: actions/download-artifact@v2
-      with:
-        name: cc-metric-store SRPM for AlmaLinux 8.5
-    
-    - name: Download UBI 8 RPM
-      uses: actions/download-artifact@v2
-      with:
-        name: cc-metric-store RPM for UBI 8
-    - name: Download UBI 8 SRPM
-      uses: actions/download-artifact@v2
-      with:
-        name: cc-metric-store SRPM for UBI 8
-
-    - name: Download Ubuntu 20.04 DEB
-      uses: actions/download-artifact@v2
-      with:
-        name: cc-metric-store DEB for Ubuntu 20.04
-
-    - name: Download Ubuntu 22.04 DEB
-      uses: actions/download-artifact@v2
-      with:
-        name: cc-metric-store DEB for Ubuntu 22.04
-
-    # The download actions do not publish the name of the downloaded file,
-    # so we re-use the job outputs of the parent jobs. The files are all
-    # downloaded to the current folder.
-    # The gh-release action afterwards does not accept file lists but all
-    # files have to be listed at 'files'. The step creates one output per
-    # RPM package (2 per distro)
-    - name: Set RPM variables
-      id: files
-      run: |
-        ALMA_85_RPM=$(basename "${{ needs.AlmaLinux-RPM-build.outputs.rpm}}")
-        ALMA_85_SRPM=$(basename "${{ needs.AlmaLinux-RPM-build.outputs.srpm}}")
-        UBI_8_RPM=$(basename "${{ needs.UBI-8-RPM-build.outputs.rpm}}")
-        UBI_8_SRPM=$(basename "${{ needs.UBI-8-RPM-build.outputs.srpm}}")
-        U_2004_DEB=$(basename "${{ needs.Ubuntu-focal-build.outputs.deb}}")
-        U_2204_DEB=$(basename "${{ needs.Ubuntu-jammy-build.outputs.deb}}")
-        echo "ALMA_85_RPM::${ALMA_85_RPM}"
-        echo "ALMA_85_SRPM::${ALMA_85_SRPM}"
-        echo "UBI_8_RPM::${UBI_8_RPM}"
-        echo "UBI_8_SRPM::${UBI_8_SRPM}"
-        echo "U_2004_DEB::${U_2004_DEB}"
-        echo "U_2204_DEB::${U_2204_DEB}"
-        echo "::set-output name=ALMA_85_RPM::${ALMA_85_RPM}"
-        echo "::set-output name=ALMA_85_SRPM::${ALMA_85_SRPM}"
-        echo "::set-output name=UBI_8_RPM::${UBI_8_RPM}"
-        echo "::set-output name=UBI_8_SRPM::${UBI_8_SRPM}"
-        echo "::set-output name=U_2004_DEB::${U_2004_DEB}"
-        echo "::set-output name=U_2204_DEB::${U_2204_DEB}"
-
-    # See: https://github.com/softprops/action-gh-release
-    - name: Release
-      uses: softprops/action-gh-release@v1
-      if: startsWith(github.ref, 'refs/tags/')
-      with:
-        name: cc-metric-store-${{github.ref_name}}
-        files: |
-         ${{ steps.files.outputs.ALMA_85_RPM }}
-         ${{ steps.files.outputs.ALMA_85_SRPM }}
-         ${{ steps.files.outputs.UBI_8_RPM }}
-         ${{ steps.files.outputs.UBI_8_SRPM }}
-         ${{ steps.files.outputs.U_2004_DEB }}
-         ${{ steps.files.outputs.U_2204_DEB }}

Makefile

@@ -1,38 +1,21 @@
+TARGET = ./cc-metric-store
+VERSION = 1.3.0
+GIT_HASH := $(shell git rev-parse --short HEAD || echo 'development')
+CURRENT_TIME = $(shell date +"%Y-%m-%d:T%H:%M:%S")
+LD_FLAGS = '-s -X main.date=${CURRENT_TIME} -X main.version=${VERSION} -X main.commit=${GIT_HASH}'
 
-APP = cc-metric-store
+.PHONY: clean test tags $(TARGET)
-GOSRC_APP        := cc-metric-store.go
-GOSRC_FILES      := api.go \
-		    memstore.go \
-		    archive.go \
-		    debug.go \
-		    float.go \
-		    lineprotocol.go \
-		    selector.go \
-		    stats.go
 
+.NOTPARALLEL:
 
+$(TARGET):
+	$(info ===>  BUILD cc-metric-store)
+	@go build -ldflags=${LD_FLAGS} ./cmd/cc-metric-store
 
-BINDIR ?= bin
-
-
-.PHONY: all
-all: $(APP)
-
-$(APP): $(GOSRC)
-	go get
-	go build -o $(APP) $(GOSRC_APP) $(GOSRC_FILES)
-
-install: $(APP)
-	@WORKSPACE=$(PREFIX)
-	@if [ -z "$${WORKSPACE}" ]; then exit 1; fi
-	@mkdir --parents --verbose $${WORKSPACE}/usr/$(BINDIR)
-	@install -Dpm 755 $(APP) $${WORKSPACE}/usr/$(BINDIR)/$(APP)
-	@install -Dpm 600 config.json $${WORKSPACE}/etc/$(APP)/$(APP).json
-
-.PHONY: clean test fmt
-.ONESHELL:
 clean:
-	rm -f $(APP)
+	$(info ===>  CLEAN)
+	@go clean
+	@rm -f $(TARGET)
 
 test:
 	$(info ===>  TESTING)
@@ -41,74 +24,6 @@ test:
 	@go vet ./...
 	@go test ./...
 
-fmt:
+tags:
-	go fmt $(GOSRC_APP)
+	$(info ===>  TAGS)
-
+	@ctags -R
-# Run linter for the Go programming language.
-# Using static analysis, it finds bugs and performance issues, offers simplifications, and enforces style rules
-.PHONY: staticcheck
-staticcheck:
-	go install honnef.co/go/tools/cmd/staticcheck@latest
-	$$(go env GOPATH)/bin/staticcheck ./...
-
-.ONESHELL:
-.PHONY: RPM
-RPM: scripts/cc-metric-store.spec
-	@WORKSPACE="$${PWD}"
-	@SPECFILE="$${WORKSPACE}/scripts/cc-metric-store.spec"
-	# Setup RPM build tree
-	@eval $$(rpm --eval "ARCH='%{_arch}' RPMDIR='%{_rpmdir}' SOURCEDIR='%{_sourcedir}' SPECDIR='%{_specdir}' SRPMDIR='%{_srcrpmdir}' BUILDDIR='%{_builddir}'")
-	@mkdir --parents --verbose "$${RPMDIR}" "$${SOURCEDIR}" "$${SPECDIR}" "$${SRPMDIR}" "$${BUILDDIR}"
-	# Create source tarball
-	@COMMITISH="HEAD"
-	@VERS=$$(git describe --tags $${COMMITISH})
-	@VERS=$${VERS#v}
-	@VERS=$$(echo $$VERS | sed -e s+'-'+'_'+g)
-	@if [ "$${VERS}" = "" ]; then VERS="0.0.1"; fi
-	@eval $$(rpmspec --query --queryformat "NAME='%{name}' VERSION='%{version}' RELEASE='%{release}' NVR='%{NVR}' NVRA='%{NVRA}'" --define="VERS $${VERS}" "$${SPECFILE}")
-	@PREFIX="$${NAME}-$${VERSION}"
-	@FORMAT="tar.gz"
-	@SRCFILE="$${SOURCEDIR}/$${PREFIX}.$${FORMAT}"
-	@git archive --verbose --format "$${FORMAT}" --prefix="$${PREFIX}/" --output="$${SRCFILE}" $${COMMITISH}
-	# Build RPM and SRPM
-	@rpmbuild -ba --define="VERS $${VERS}" --rmsource --clean "$${SPECFILE}"
-	# Report RPMs and SRPMs when in GitHub Workflow
-	@if [[ "$${GITHUB_ACTIONS}" == true ]]; then
-	@     RPMFILE="$${RPMDIR}/$${ARCH}/$${NVRA}.rpm"
-	@     SRPMFILE="$${SRPMDIR}/$${NVR}.src.rpm"
-	@     echo "RPM: $${RPMFILE}"
-	@     echo "SRPM: $${SRPMFILE}"
-	@     echo "::set-output name=SRPM::$${SRPMFILE}"
-	@     echo "::set-output name=RPM::$${RPMFILE}"
-	@fi
-
-.ONESHELL:
-.PHONY: DEB
-DEB: scripts/cc-metric-store.deb.control $(APP)
-	@BASEDIR=$${PWD}
-	@WORKSPACE=$${PWD}/.dpkgbuild
-	@DEBIANDIR=$${WORKSPACE}/debian
-	@DEBIANBINDIR=$${WORKSPACE}/DEBIAN
-	@mkdir --parents --verbose $$WORKSPACE $$DEBIANBINDIR
-	#@mkdir --parents --verbose $$DEBIANDIR
-	@CONTROLFILE="$${BASEDIR}/scripts/cc-metric-store.deb.control"
-	@COMMITISH="HEAD"
-	@VERS=$$(git describe --tags --abbrev=0 $${COMMITISH})
-	@VERS=$${VERS#v}
-	@VERS=$$(echo $$VERS | sed -e s+'-'+'_'+g)
-	@if [ "$${VERS}" = "" ]; then VERS="0.0.1"; fi
-	@ARCH=$$(uname -m)
-	@ARCH=$$(echo $$ARCH | sed -e s+'_'+'-'+g)
-	@if [ "$${ARCH}" = "x86-64" ]; then ARCH=amd64; fi
-	@PREFIX="$${NAME}-$${VERSION}_$${ARCH}"
-	@SIZE_BYTES=$$(du -bcs --exclude=.dpkgbuild "$$WORKSPACE"/ | awk '{print $$1}' | head -1 | sed -e 's/^0\+//')
-	@SIZE="$$(awk -v size="$$SIZE_BYTES" 'BEGIN {print (size/1024)+1}' | awk '{print int($$0)}')"
-	#@sed -e s+"{VERSION}"+"$$VERS"+g -e s+"{INSTALLED_SIZE}"+"$$SIZE"+g -e s+"{ARCH}"+"$$ARCH"+g $$CONTROLFILE > $${DEBIANDIR}/control
-	@sed -e s+"{VERSION}"+"$$VERS"+g -e s+"{INSTALLED_SIZE}"+"$$SIZE"+g -e s+"{ARCH}"+"$$ARCH"+g $$CONTROLFILE > $${DEBIANBINDIR}/control
-	@make PREFIX=$${WORKSPACE} install
-	@DEB_FILE="cc-metric-store_$${VERS}_$${ARCH}.deb"
-	@dpkg-deb -b $${WORKSPACE} "$$DEB_FILE"
-	@rm -r "$${WORKSPACE}"
-	@if [ "$${GITHUB_ACTIONS}" = "true" ]; then
-	@     echo "::set-output name=DEB::$${DEB_FILE}"
-	@fi

Makefile.orig

@@ -0,0 +1,126 @@
+TARGET = ./cc-metric-store
+VERSION = 1.3.0
+GIT_HASH := $(shell git rev-parse --short HEAD || echo 'development')
+CURRENT_TIME = $(shell date +"%Y-%m-%d:T%H:%M:%S")
+LD_FLAGS = '-s -X main.date=${CURRENT_TIME} -X main.version=${VERSION} -X main.commit=${GIT_HASH}'
+
+.PHONY: clean test tags $(TARGET)
+
+.NOTPARALLEL:
+
+$(TARGET):
+	$(info ===>  BUILD cc-backend)
+	@go build -ldflags=${LD_FLAGS} ./cmd/cc-metric-store
+
+<<<<<<< HEAD
+=======
+BINDIR ?= bin
+
+
+.PHONY: all
+all: $(APP)
+
+$(APP): $(GOSRC)
+	go get
+	go build -o $(APP) $(GOSRC_APP) $(GOSRC_FILES)
+
+install: $(APP)
+	@WORKSPACE=$(PREFIX)
+	@if [ -z "$${WORKSPACE}" ]; then exit 1; fi
+	@mkdir --parents --verbose $${WORKSPACE}/usr/$(BINDIR)
+	@install -Dpm 755 $(APP) $${WORKSPACE}/usr/$(BINDIR)/$(APP)
+	@install -Dpm 600 config.json $${WORKSPACE}/etc/$(APP)/$(APP).json
+
+.PHONY: clean test fmt
+.ONESHELL:
+>>>>>>> main
+clean:
+	$(info ===>  CLEAN)
+	@go clean
+	@rm -f $(TARGET)
+
+test:
+	$(info ===>  TESTING)
+	@go clean -testcache
+	@go build ./...
+	@go vet ./...
+	@go test ./...
+<<<<<<< HEAD
+
+tags:
+	$(info ===>  TAGS)
+	@ctags -R
+=======
+
+fmt:
+	go fmt $(GOSRC_APP)
+
+# Run linter for the Go programming language.
+# Using static analysis, it finds bugs and performance issues, offers simplifications, and enforces style rules
+.PHONY: staticcheck
+staticcheck:
+	go install honnef.co/go/tools/cmd/staticcheck@latest
+	$$(go env GOPATH)/bin/staticcheck ./...
+
+.ONESHELL:
+.PHONY: RPM
+RPM: scripts/cc-metric-store.spec
+	@WORKSPACE="$${PWD}"
+	@SPECFILE="$${WORKSPACE}/scripts/cc-metric-store.spec"
+	# Setup RPM build tree
+	@eval $$(rpm --eval "ARCH='%{_arch}' RPMDIR='%{_rpmdir}' SOURCEDIR='%{_sourcedir}' SPECDIR='%{_specdir}' SRPMDIR='%{_srcrpmdir}' BUILDDIR='%{_builddir}'")
+	@mkdir --parents --verbose "$${RPMDIR}" "$${SOURCEDIR}" "$${SPECDIR}" "$${SRPMDIR}" "$${BUILDDIR}"
+	# Create source tarball
+	@COMMITISH="HEAD"
+	@VERS=$$(git describe --tags $${COMMITISH})
+	@VERS=$${VERS#v}
+	@VERS=$$(echo $$VERS | sed -e s+'-'+'_'+g)
+	@if [ "$${VERS}" = "" ]; then VERS="0.0.1"; fi
+	@eval $$(rpmspec --query --queryformat "NAME='%{name}' VERSION='%{version}' RELEASE='%{release}' NVR='%{NVR}' NVRA='%{NVRA}'" --define="VERS $${VERS}" "$${SPECFILE}")
+	@PREFIX="$${NAME}-$${VERSION}"
+	@FORMAT="tar.gz"
+	@SRCFILE="$${SOURCEDIR}/$${PREFIX}.$${FORMAT}"
+	@git archive --verbose --format "$${FORMAT}" --prefix="$${PREFIX}/" --output="$${SRCFILE}" $${COMMITISH}
+	# Build RPM and SRPM
+	@rpmbuild -ba --define="VERS $${VERS}" --rmsource --clean "$${SPECFILE}"
+	# Report RPMs and SRPMs when in GitHub Workflow
+	@if [[ "$${GITHUB_ACTIONS}" == true ]]; then
+	@     RPMFILE="$${RPMDIR}/$${ARCH}/$${NVRA}.rpm"
+	@     SRPMFILE="$${SRPMDIR}/$${NVR}.src.rpm"
+	@     echo "RPM: $${RPMFILE}"
+	@     echo "SRPM: $${SRPMFILE}"
+	@     echo "::set-output name=SRPM::$${SRPMFILE}"
+	@     echo "::set-output name=RPM::$${RPMFILE}"
+	@fi
+
+.ONESHELL:
+.PHONY: DEB
+DEB: scripts/cc-metric-store.deb.control $(APP)
+	@BASEDIR=$${PWD}
+	@WORKSPACE=$${PWD}/.dpkgbuild
+	@DEBIANDIR=$${WORKSPACE}/debian
+	@DEBIANBINDIR=$${WORKSPACE}/DEBIAN
+	@mkdir --parents --verbose $$WORKSPACE $$DEBIANBINDIR
+	#@mkdir --parents --verbose $$DEBIANDIR
+	@CONTROLFILE="$${BASEDIR}/scripts/cc-metric-store.deb.control"
+	@COMMITISH="HEAD"
+	@VERS=$$(git describe --tags --abbrev=0 $${COMMITISH})
+	@VERS=$${VERS#v}
+	@VERS=$$(echo $$VERS | sed -e s+'-'+'_'+g)
+	@if [ "$${VERS}" = "" ]; then VERS="0.0.1"; fi
+	@ARCH=$$(uname -m)
+	@ARCH=$$(echo $$ARCH | sed -e s+'_'+'-'+g)
+	@if [ "$${ARCH}" = "x86-64" ]; then ARCH=amd64; fi
+	@PREFIX="$${NAME}-$${VERSION}_$${ARCH}"
+	@SIZE_BYTES=$$(du -bcs --exclude=.dpkgbuild "$$WORKSPACE"/ | awk '{print $$1}' | head -1 | sed -e 's/^0\+//')
+	@SIZE="$$(awk -v size="$$SIZE_BYTES" 'BEGIN {print (size/1024)+1}' | awk '{print int($$0)}')"
+	#@sed -e s+"{VERSION}"+"$$VERS"+g -e s+"{INSTALLED_SIZE}"+"$$SIZE"+g -e s+"{ARCH}"+"$$ARCH"+g $$CONTROLFILE > $${DEBIANDIR}/control
+	@sed -e s+"{VERSION}"+"$$VERS"+g -e s+"{INSTALLED_SIZE}"+"$$SIZE"+g -e s+"{ARCH}"+"$$ARCH"+g $$CONTROLFILE > $${DEBIANBINDIR}/control
+	@make PREFIX=$${WORKSPACE} install
+	@DEB_FILE="cc-metric-store_$${VERS}_$${ARCH}.deb"
+	@dpkg-deb -b $${WORKSPACE} "$$DEB_FILE"
+	@rm -r "$${WORKSPACE}"
+	@if [ "$${GITHUB_ACTIONS}" = "true" ]; then
+	@     echo "::set-output name=DEB::$${DEB_FILE}"
+	@fi
+>>>>>>> main

cc-metric-store.go

@@ -1,349 +0,0 @@
-package main
-
-import (
-	"bufio"
-	"context"
-	"encoding/json"
-	"flag"
-	"fmt"
-	"io"
-	"log"
-	"os"
-	"os/signal"
-	"runtime"
-	"runtime/debug"
-	"sync"
-	"syscall"
-	"time"
-
-	"github.com/google/gops/agent"
-)
-
-// For aggregation over multiple values at different cpus/sockets/..., not time!
-type AggregationStrategy int
-
-const (
-	NoAggregation AggregationStrategy = iota
-	SumAggregation
-	AvgAggregation
-)
-
-func (as *AggregationStrategy) UnmarshalJSON(data []byte) error {
-	var str string
-	if err := json.Unmarshal(data, &str); err != nil {
-		return err
-	}
-
-	switch str {
-	case "":
-		*as = NoAggregation
-	case "sum":
-		*as = SumAggregation
-	case "avg":
-		*as = AvgAggregation
-	default:
-		return fmt.Errorf("invalid aggregation strategy: %#v", str)
-	}
-	return nil
-}
-
-type MetricConfig struct {
-	// Interval in seconds at which measurements will arive.
-	Frequency int64 `json:"frequency"`
-
-	// Can be 'sum', 'avg' or null. Describes how to aggregate metrics from the same timestep over the hierarchy.
-	Aggregation AggregationStrategy `json:"aggregation"`
-
-	// Private, used internally...
-	offset int
-}
-
-type HttpConfig struct {
-	// Address to bind to, for example "0.0.0.0:8081"
-	Address string `json:"address"`
-
-	// If not the empty string, use https with this as the certificate file
-	CertFile string `json:"https-cert-file"`
-
-	// If not the empty string, use https with this as the key file
-	KeyFile string `json:"https-key-file"`
-}
-
-type NatsConfig struct {
-	// Address of the nats server
-	Address string `json:"address"`
-
-	// Username/Password, optional
-	Username string `json:"username"`
-	Password string `json:"password"`
-
-	Subscriptions []struct {
-		// Channel name
-		SubscribeTo string `json:"subscribe-to"`
-
-		// Allow lines without a cluster tag, use this as default, optional
-		ClusterTag string `json:"cluster-tag"`
-	} `json:"subscriptions"`
-}
-
-type Config struct {
-	Metrics           map[string]MetricConfig `json:"metrics"`
-	RetentionInMemory string                  `json:"retention-in-memory"`
-	Nats              []*NatsConfig           `json:"nats"`
-	JwtPublicKey      string                  `json:"jwt-public-key"`
-	HttpConfig        *HttpConfig             `json:"http-api"`
-	Checkpoints       struct {
-		Interval string `json:"interval"`
-		RootDir  string `json:"directory"`
-		Restore  string `json:"restore"`
-	} `json:"checkpoints"`
-	Archive struct {
-		Interval      string `json:"interval"`
-		RootDir       string `json:"directory"`
-		DeleteInstead bool   `json:"delete-instead"`
-	} `json:"archive"`
-	Debug struct {
-		EnableGops bool   `json:"gops"`
-		DumpToFile string `json:"dump-to-file"`
-	} `json:"debug"`
-}
-
-var conf Config
-var memoryStore *MemoryStore = nil
-var lastCheckpoint time.Time
-
-var debugDumpLock sync.Mutex
-var debugDump io.Writer = io.Discard
-
-func loadConfiguration(file string) Config {
-	var config Config
-	configFile, err := os.Open(file)
-	if err != nil {
-		log.Fatal(err)
-	}
-	defer configFile.Close()
-	dec := json.NewDecoder(configFile)
-	dec.DisallowUnknownFields()
-	if err := dec.Decode(&config); err != nil {
-		log.Fatal(err)
-	}
-	return config
-}
-
-func intervals(wg *sync.WaitGroup, ctx context.Context) {
-	wg.Add(3)
-	// go func() {
-	// 	defer wg.Done()
-	// 	ticks := time.Tick(30 * time.Minute)
-	// 	for {
-	// 		select {
-	// 		case <-ctx.Done():
-	// 			return
-	// 		case <-ticks:
-	// 			runtime.GC()
-	// 		}
-	// 	}
-	// }()
-
-	go func() {
-		defer wg.Done()
-		d, err := time.ParseDuration(conf.RetentionInMemory)
-		if err != nil {
-			log.Fatal(err)
-		}
-		if d <= 0 {
-			return
-		}
-
-		ticks := time.Tick(d / 2)
-		for {
-			select {
-			case <-ctx.Done():
-				return
-			case <-ticks:
-				t := time.Now().Add(-d)
-				log.Printf("start freeing buffers (older than %s)...\n", t.Format(time.RFC3339))
-				freed, err := memoryStore.Free(nil, t.Unix())
-				if err != nil {
-					log.Printf("freeing up buffers failed: %s\n", err.Error())
-				} else {
-					log.Printf("done: %d buffers freed\n", freed)
-				}
-			}
-		}
-	}()
-
-	lastCheckpoint = time.Now()
-	go func() {
-		defer wg.Done()
-		d, err := time.ParseDuration(conf.Checkpoints.Interval)
-		if err != nil {
-			log.Fatal(err)
-		}
-		if d <= 0 {
-			return
-		}
-
-		ticks := time.Tick(d)
-		for {
-			select {
-			case <-ctx.Done():
-				return
-			case <-ticks:
-				log.Printf("start checkpointing (starting at %s)...\n", lastCheckpoint.Format(time.RFC3339))
-				now := time.Now()
-				n, err := memoryStore.ToCheckpoint(conf.Checkpoints.RootDir,
-					lastCheckpoint.Unix(), now.Unix())
-				if err != nil {
-					log.Printf("checkpointing failed: %s\n", err.Error())
-				} else {
-					log.Printf("done: %d checkpoint files created\n", n)
-					lastCheckpoint = now
-				}
-			}
-		}
-	}()
-
-	go func() {
-		defer wg.Done()
-		d, err := time.ParseDuration(conf.Archive.Interval)
-		if err != nil {
-			log.Fatal(err)
-		}
-		if d <= 0 {
-			return
-		}
-
-		ticks := time.Tick(d)
-		for {
-			select {
-			case <-ctx.Done():
-				return
-			case <-ticks:
-				t := time.Now().Add(-d)
-				log.Printf("start archiving checkpoints (older than %s)...\n", t.Format(time.RFC3339))
-				n, err := ArchiveCheckpoints(conf.Checkpoints.RootDir, conf.Archive.RootDir, t.Unix(), conf.Archive.DeleteInstead)
-				if err != nil {
-					log.Printf("archiving failed: %s\n", err.Error())
-				} else {
-					log.Printf("done: %d files zipped and moved to archive\n", n)
-				}
-			}
-		}
-	}()
-}
-
-func main() {
-	var configFile string
-	var enableGopsAgent bool
-	flag.StringVar(&configFile, "config", "./config.json", "configuration file")
-	flag.BoolVar(&enableGopsAgent, "gops", false, "Listen via github.com/google/gops/agent")
-	flag.Parse()
-
-	startupTime := time.Now()
-	conf = loadConfiguration(configFile)
-	memoryStore = NewMemoryStore(conf.Metrics)
-
-	if enableGopsAgent || conf.Debug.EnableGops {
-		if err := agent.Listen(agent.Options{}); err != nil {
-			log.Fatal(err)
-		}
-	}
-
-	if conf.Debug.DumpToFile != "" {
-		f, err := os.Create(conf.Debug.DumpToFile)
-		if err != nil {
-			log.Fatal(err)
-		}
-
-		debugDump = f
-	}
-
-	d, err := time.ParseDuration(conf.Checkpoints.Restore)
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	restoreFrom := startupTime.Add(-d)
-	log.Printf("Loading checkpoints newer than %s\n", restoreFrom.Format(time.RFC3339))
-	files, err := memoryStore.FromCheckpoint(conf.Checkpoints.RootDir, restoreFrom.Unix())
-	loadedData := memoryStore.SizeInBytes() / 1024 / 1024 // In MB
-	if err != nil {
-		log.Fatalf("Loading checkpoints failed: %s\n", err.Error())
-	} else {
-		log.Printf("Checkpoints loaded (%d files, %d MB, that took %fs)\n", files, loadedData, time.Since(startupTime).Seconds())
-	}
-
-	// Try to use less memory by forcing a GC run here and then
-	// lowering the target percentage. The default of 100 means
-	// that only once the ratio of new allocations execeds the
-	// previously active heap, a GC is triggered.
-	// Forcing a GC here will set the "previously active heap"
-	// to a minumum.
-	runtime.GC()
-	if loadedData > 1000 && os.Getenv("GOGC") == "" {
-		debug.SetGCPercent(10)
-	}
-
-	ctx, shutdown := context.WithCancel(context.Background())
-
-	var wg sync.WaitGroup
-	sigs := make(chan os.Signal, 1)
-	signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM, syscall.SIGUSR1)
-	go func() {
-		for {
-			sig := <-sigs
-			if sig == syscall.SIGUSR1 {
-				memoryStore.DebugDump(bufio.NewWriter(os.Stdout), nil)
-				continue
-			}
-
-			log.Println("Shuting down...")
-			shutdown()
-		}
-	}()
-
-	intervals(&wg, ctx)
-
-	wg.Add(1)
-
-	go func() {
-		err := StartApiServer(ctx, conf.HttpConfig)
-		if err != nil {
-			log.Fatal(err)
-		}
-		wg.Done()
-	}()
-
-	if conf.Nats != nil {
-		for _, natsConf := range conf.Nats {
-			// TODO: When multiple nats configs share a URL, do a single connect.
-			wg.Add(1)
-			nc := natsConf
-			go func() {
-				// err := ReceiveNats(conf.Nats, decodeLine, runtime.NumCPU()-1, ctx)
-				err := ReceiveNats(nc, decodeLine, 1, ctx)
-
-				if err != nil {
-					log.Fatal(err)
-				}
-				wg.Done()
-			}()
-		}
-	}
-
-	wg.Wait()
-
-	log.Printf("Writing to '%s'...\n", conf.Checkpoints.RootDir)
-	files, err = memoryStore.ToCheckpoint(conf.Checkpoints.RootDir, lastCheckpoint.Unix(), time.Now().Unix())
-	if err != nil {
-		log.Printf("Writing checkpoint failed: %s\n", err.Error())
-	}
-	log.Printf("Done! (%d files written)\n", files)
-
-	if closer, ok := debugDump.(io.Closer); ok {
-		if err := closer.Close(); err != nil {
-			log.Printf("error: %s", err.Error())
-		}
-	}
-}

cmd/cc-metric-store/main.go

@@ -0,0 +1,133 @@
+package main
+
+import (
+	"bufio"
+	"context"
+	"flag"
+	"fmt"
+	"log"
+	"os"
+	"os/signal"
+	"runtime"
+	"runtime/debug"
+	"sync"
+	"syscall"
+	"time"
+
+	"github.com/ClusterCockpit/cc-metric-store/internal/api"
+	"github.com/ClusterCockpit/cc-metric-store/internal/config"
+	"github.com/ClusterCockpit/cc-metric-store/internal/memorystore"
+	"github.com/google/gops/agent"
+)
+
+var (
+	date    string
+	commit  string
+	version string
+)
+
+func main() {
+	var configFile string
+	var enableGopsAgent, flagVersion bool
+	flag.StringVar(&configFile, "config", "./config.json", "configuration file")
+	flag.BoolVar(&enableGopsAgent, "gops", false, "Listen via github.com/google/gops/agent")
+	flag.BoolVar(&flagVersion, "version", false, "Show version information and exit")
+	flag.Parse()
+
+	if flagVersion {
+		fmt.Printf("Version:\t%s\n", version)
+		fmt.Printf("Git hash:\t%s\n", commit)
+		fmt.Printf("Build time:\t%s\n", date)
+		os.Exit(0)
+	}
+
+	startupTime := time.Now()
+	config.Init(configFile)
+	memorystore.Init(config.Keys.Metrics)
+	ms := memorystore.GetMemoryStore()
+
+	if enableGopsAgent || config.Keys.Debug.EnableGops {
+		if err := agent.Listen(agent.Options{}); err != nil {
+			log.Fatal(err)
+		}
+	}
+
+	d, err := time.ParseDuration(config.Keys.Checkpoints.Restore)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	restoreFrom := startupTime.Add(-d)
+	log.Printf("Loading checkpoints newer than %s\n", restoreFrom.Format(time.RFC3339))
+	files, err := ms.FromCheckpoint(config.Keys.Checkpoints.RootDir, restoreFrom.Unix())
+	loadedData := ms.SizeInBytes() / 1024 / 1024 // In MB
+	if err != nil {
+		log.Fatalf("Loading checkpoints failed: %s\n", err.Error())
+	} else {
+		log.Printf("Checkpoints loaded (%d files, %d MB, that took %fs)\n", files, loadedData, time.Since(startupTime).Seconds())
+	}
+
+	// Try to use less memory by forcing a GC run here and then
+	// lowering the target percentage. The default of 100 means
+	// that only once the ratio of new allocations execeds the
+	// previously active heap, a GC is triggered.
+	// Forcing a GC here will set the "previously active heap"
+	// to a minumum.
+	runtime.GC()
+	if loadedData > 1000 && os.Getenv("GOGC") == "" {
+		debug.SetGCPercent(10)
+	}
+
+	ctx, shutdown := context.WithCancel(context.Background())
+
+	var wg sync.WaitGroup
+	sigs := make(chan os.Signal, 1)
+	signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM, syscall.SIGUSR1)
+	go func() {
+		for {
+			sig := <-sigs
+			if sig == syscall.SIGUSR1 {
+				ms.DebugDump(bufio.NewWriter(os.Stdout), nil)
+				continue
+			}
+
+			log.Println("Shutting down...")
+			shutdown()
+		}
+	}()
+
+	wg.Add(3)
+
+	memorystore.Retention(&wg, ctx)
+	memorystore.Checkpointing(&wg, ctx)
+	memorystore.Archiving(&wg, ctx)
+
+	wg.Add(1)
+
+	go func() {
+		err := api.StartApiServer(ctx, config.Keys.HttpConfig)
+		if err != nil {
+			log.Fatal(err)
+		}
+		wg.Done()
+	}()
+
+	if config.Keys.Nats != nil {
+		for _, natsConf := range config.Keys.Nats {
+			// TODO: When multiple nats configs share a URL, do a single connect.
+			wg.Add(1)
+			nc := natsConf
+			go func() {
+				// err := ReceiveNats(conf.Nats, decodeLine, runtime.NumCPU()-1, ctx)
+				err := api.ReceiveNats(nc, ms, 1, ctx)
+				if err != nil {
+					log.Fatal(err)
+				}
+				wg.Done()
+			}()
+		}
+	}
+
+	wg.Wait()
+	memorystore.Shutdown()
+}

go.mod

@@ -3,7 +3,7 @@ module github.com/ClusterCockpit/cc-metric-store
 go 1.19
 
 require (
-	github.com/golang-jwt/jwt/v5 v5.2.1
+	github.com/golang-jwt/jwt/v4 v4.5.0
 	github.com/google/gops v0.3.28
 	github.com/gorilla/mux v1.8.1
 	github.com/influxdata/line-protocol/v2 v2.2.1

go.mod.orig

@@ -0,0 +1,37 @@
+module github.com/ClusterCockpit/cc-metric-store
+
+go 1.19
+
+require (
+<<<<<<< HEAD
+	github.com/golang-jwt/jwt/v4 v4.0.0
+	github.com/google/gops v0.3.22
+	github.com/gorilla/mux v1.8.0
+	github.com/influxdata/line-protocol/v2 v2.2.0
+	github.com/nats-io/nats.go v1.11.0
+=======
+	github.com/golang-jwt/jwt/v5 v5.2.1
+	github.com/google/gops v0.3.28
+	github.com/gorilla/mux v1.8.1
+	github.com/influxdata/line-protocol/v2 v2.2.1
+	github.com/nats-io/nats.go v1.33.1
+)
+
+require (
+	github.com/klauspost/compress v1.17.7 // indirect
+	github.com/nats-io/nkeys v0.4.7 // indirect
+	github.com/nats-io/nuid v1.0.1 // indirect
+	golang.org/x/crypto v0.21.0 // indirect
+	golang.org/x/sys v0.18.0 // indirect
+>>>>>>> main
+)
+
+require (
+	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/nats-io/nats-server/v2 v2.2.6 // indirect
+	github.com/nats-io/nkeys v0.3.0 // indirect
+	github.com/nats-io/nuid v1.0.1 // indirect
+	golang.org/x/crypto v0.0.0-20210314154223-e6e6c4f2bb5b // indirect
+	golang.org/x/sys v0.0.0-20210902050250-f475640dd07b // indirect
+	google.golang.org/protobuf v1.26.0 // indirect
+)

go.sum

@@ -3,8 +3,8 @@ github.com/frankban/quicktest v1.11.0/go.mod h1:K+q6oSqb0W0Ininfk863uOk1lMy69l/P
 github.com/frankban/quicktest v1.11.2/go.mod h1:K+q6oSqb0W0Ininfk863uOk1lMy69l/P6txr3mVT54s=
 github.com/frankban/quicktest v1.13.0 h1:yNZif1OkDfNoDfb9zZa9aXIpejNR4F23Wely0c+Qdqk=
 github.com/frankban/quicktest v1.13.0/go.mod h1:qLE0fzW0VuyUAJgPU19zByoIr0HtCHN/r/VLSOOIySU=
-github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk=
+github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg=
-github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
+github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
 github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=

go.sum.orig

@@ -0,0 +1,110 @@
+<<<<<<< HEAD
+github.com/StackExchange/wmi v1.2.1/go.mod h1:rcmrprowKIVzvc+NUiLncP2uuArMWLCbu9SBzvHz7e8=
+=======
+>>>>>>> main
+github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
+github.com/frankban/quicktest v1.11.0/go.mod h1:K+q6oSqb0W0Ininfk863uOk1lMy69l/P6txr3mVT54s=
+github.com/frankban/quicktest v1.11.2/go.mod h1:K+q6oSqb0W0Ininfk863uOk1lMy69l/P6txr3mVT54s=
+github.com/frankban/quicktest v1.13.0 h1:yNZif1OkDfNoDfb9zZa9aXIpejNR4F23Wely0c+Qdqk=
+github.com/frankban/quicktest v1.13.0/go.mod h1:qLE0fzW0VuyUAJgPU19zByoIr0HtCHN/r/VLSOOIySU=
+<<<<<<< HEAD
+github.com/go-ole/go-ole v1.2.5/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
+github.com/go-ole/go-ole v1.2.6-0.20210915003542-8b1f7f90f6b1/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
+github.com/golang-jwt/jwt/v4 v4.0.0 h1:RAqyYixv1p7uEnocuy8P1nru5wprCh/MH2BIlW5z5/o=
+github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg=
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
+github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
+github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+=======
+github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk=
+github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
+>>>>>>> main
+github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
+github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/gops v0.3.28 h1:2Xr57tqKAmQYRAfG12E+yLcoa2Y42UJo2lOrUFL9ark=
+github.com/google/gops v0.3.28/go.mod h1:6f6+Nl8LcHrzJwi8+p0ii+vmBFSlB4f8cOOkTJ7sk4c=
+github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY=
+github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ=
+github.com/influxdata/line-protocol-corpus v0.0.0-20210519164801-ca6fa5da0184/go.mod h1:03nmhxzZ7Xk2pdG+lmMd7mHDfeVOYFyhOgwO61qWU98=
+github.com/influxdata/line-protocol-corpus v0.0.0-20210922080147-aa28ccfb8937 h1:MHJNQ+p99hFATQm6ORoLmpUCF7ovjwEFshs/NHzAbig=
+github.com/influxdata/line-protocol-corpus v0.0.0-20210922080147-aa28ccfb8937/go.mod h1:BKR9c0uHSmRgM/se9JhFHtTT7JTO67X23MtKMHtZcpo=
+github.com/influxdata/line-protocol/v2 v2.0.0-20210312151457-c52fdecb625a/go.mod h1:6+9Xt5Sq1rWx+glMgxhcg2c0DUaehK+5TDcPZ76GypY=
+github.com/influxdata/line-protocol/v2 v2.1.0/go.mod h1:QKw43hdUBg3GTk2iC3iyCxksNj7PX9aUSeYOYE/ceHY=
+<<<<<<< HEAD
+github.com/influxdata/line-protocol/v2 v2.2.0 h1:UPmAqE15Hw5zu9E10SYhoXVLWnEJkWnuCbaCiRsA3c0=
+github.com/influxdata/line-protocol/v2 v2.2.0/go.mod h1:DmB3Cnh+3oxmG6LOBIxce4oaL4CPj3OmMPgvauXh+tM=
+github.com/keybase/go-ps v0.0.0-20190827175125-91aafc93ba19/go.mod h1:hY+WOq6m2FpbvyrI93sMaypsttvaIL5nhVR92dTMUcQ=
+github.com/klauspost/compress v1.11.12 h1:famVnQVu7QwryBN4jNseQdUKES71ZAOnB6UQQJPZvqk=
+github.com/klauspost/compress v1.11.12/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
+=======
+github.com/influxdata/line-protocol/v2 v2.2.1 h1:EAPkqJ9Km4uAxtMRgUubJyqAr6zgWM0dznKMLRauQRE=
+github.com/influxdata/line-protocol/v2 v2.2.1/go.mod h1:DmB3Cnh+3oxmG6LOBIxce4oaL4CPj3OmMPgvauXh+tM=
+github.com/klauspost/compress v1.17.7 h1:ehO88t2UGzQK66LMdE8tibEd1ErmzZjNEqWkjLAKQQg=
+github.com/klauspost/compress v1.17.7/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
+>>>>>>> main
+github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
+github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/nats-io/nats.go v1.33.1 h1:8TxLZZ/seeEfR97qV0/Bl939tpDnt2Z2fK3HkPypj70=
+github.com/nats-io/nats.go v1.33.1/go.mod h1:Ubdu4Nh9exXdSz0RVWRFBbRfrbSxOYd26oF0wkWclB8=
+github.com/nats-io/nkeys v0.4.7 h1:RwNJbbIdYCoClSDNY7QVKZlyb/wfT6ugvFCiKy6vDvI=
+github.com/nats-io/nkeys v0.4.7/go.mod h1:kqXRgRDPlGy7nGaEDMuYzmiJCIAAWDK0IMBtDmGD0nc=
+github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
+github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
+github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
+<<<<<<< HEAD
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/shirou/gopsutil/v3 v3.21.9/go.mod h1:YWp/H8Qs5fVmf17v7JNZzA0mPJ+mS2e9JdiUF9LlKzQ=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/tklauser/go-sysconf v0.3.9/go.mod h1:11DU/5sG7UexIrp/O6g35hrWzu0JxlwQ3LSFUzyeuhs=
+github.com/tklauser/numcpus v0.3.0/go.mod h1:yFGUr7TUHQRAhyqBcEg0Ge34zDBAsIvJJcyE6boqnA8=
+github.com/xlab/treeprint v1.1.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20210314154223-e6e6c4f2bb5b h1:wSOdpTq0/eI46Ez/LkDwIsAKA71YP2SRKBODiRWM0as=
+golang.org/x/crypto v0.0.0-20210314154223-e6e6c4f2bb5b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/sys v0.0.0-20190130150945-aca44879d564/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210816074244-15123e1e1f71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210902050250-f475640dd07b h1:S7hKs0Flbq0bbc9xgYt4stIEG1zNDFqyrPwAX2Wj/sE=
+golang.org/x/sys v0.0.0-20210902050250-f475640dd07b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1 h1:NusfzzA6yGQ+ua51ck7E3omNUX/JuqbFSaRGqU8CcLI=
+golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+=======
+golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA=
+golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
+golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
+golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+>>>>>>> main
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+<<<<<<< HEAD
+rsc.io/goversion v1.2.0/go.mod h1:Eih9y/uIBS3ulggl7KNJ09xGSLcuNaLgmvvqa07sgfo=
+=======
+>>>>>>> main

internal/api/api.go

@@ -0,0 +1,368 @@
+package api
+
+import (
+	"bufio"
+	"context"
+	"crypto/ed25519"
+	"encoding/base64"
+	"encoding/json"
+	"fmt"
+	"io"
+	"log"
+	"math"
+	"net/http"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/ClusterCockpit/cc-metric-store/internal/config"
+	"github.com/ClusterCockpit/cc-metric-store/internal/memorystore"
+	"github.com/ClusterCockpit/cc-metric-store/internal/util"
+	"github.com/gorilla/mux"
+	"github.com/influxdata/line-protocol/v2/lineprotocol"
+)
+
+type ApiMetricData struct {
+	Error *string    `json:"error,omitempty"`
+	Data  util.FloatArray `json:"data,omitempty"`
+	From  int64      `json:"from"`
+	To    int64      `json:"to"`
+	Avg   util.Float      `json:"avg"`
+	Min   util.Float      `json:"min"`
+	Max   util.Float      `json:"max"`
+}
+
+// TODO: Optimize this, just like the stats endpoint!
+func (data *ApiMetricData) AddStats() {
+	n := 0
+	sum, min, max := 0.0, math.MaxFloat64, -math.MaxFloat64
+	for _, x := range data.Data {
+		if x.IsNaN() {
+			continue
+		}
+
+		n += 1
+		sum += float64(x)
+		min = math.Min(min, float64(x))
+		max = math.Max(max, float64(x))
+	}
+
+	if n > 0 {
+		avg := sum / float64(n)
+		data.Avg = util.Float(avg)
+		data.Min = util.Float(min)
+		data.Max = util.Float(max)
+	} else {
+		data.Avg, data.Min, data.Max = util.NaN, util.NaN, util.NaN
+	}
+}
+
+func (data *ApiMetricData) ScaleBy(f util.Float) {
+	if f == 0 || f == 1 {
+		return
+	}
+
+	data.Avg *= f
+	data.Min *= f
+	data.Max *= f
+	for i := 0; i < len(data.Data); i++ {
+		data.Data[i] *= f
+	}
+}
+
+func (data *ApiMetricData) PadDataWithNull(ms *memorystore.MemoryStore, from, to int64, metric string) {
+	minfo, ok := ms.Metrics[metric]
+	if !ok {
+		return
+	}
+
+	if (data.From / minfo.Frequency) > (from / minfo.Frequency) {
+		padfront := int((data.From / minfo.Frequency) - (from / minfo.Frequency))
+		ndata := make([]util.Float, 0, padfront+len(data.Data))
+		for i := 0; i < padfront; i++ {
+			ndata = append(ndata, util.NaN)
+		}
+		for j := 0; j < len(data.Data); j++ {
+			ndata = append(ndata, data.Data[j])
+		}
+		data.Data = ndata
+	}
+}
+
+func handleFree(rw http.ResponseWriter, r *http.Request) {
+	rawTo := r.URL.Query().Get("to")
+	if rawTo == "" {
+		http.Error(rw, "'to' is a required query parameter", http.StatusBadRequest)
+		return
+	}
+
+	to, err := strconv.ParseInt(rawTo, 10, 64)
+	if err != nil {
+		http.Error(rw, err.Error(), http.StatusBadRequest)
+		return
+	}
+
+	// // TODO: lastCheckpoint might be modified by different go-routines.
+	// // Load it using the sync/atomic package?
+	// freeUpTo := lastCheckpoint.Unix()
+	// if to < freeUpTo {
+	// 	freeUpTo = to
+	// }
+
+	if r.Method != http.MethodPost {
+		http.Error(rw, "Method Not Allowed", http.StatusMethodNotAllowed)
+		return
+	}
+
+	bodyDec := json.NewDecoder(r.Body)
+	var selectors [][]string
+	err = bodyDec.Decode(&selectors)
+	if err != nil {
+		http.Error(rw, err.Error(), http.StatusBadRequest)
+		return
+	}
+
+	ms := memorystore.GetMemoryStore()
+	n := 0
+	for _, sel := range selectors {
+		bn, err := ms.Free(sel, to)
+		if err != nil {
+			http.Error(rw, err.Error(), http.StatusInternalServerError)
+			return
+		}
+
+		n += bn
+	}
+
+	rw.WriteHeader(http.StatusOK)
+	fmt.Fprintf(rw, "buffers freed: %d\n", n)
+}
+
+func handleWrite(rw http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodPost {
+		http.Error(rw, "Method Not Allowed", http.StatusMethodNotAllowed)
+		return
+	}
+
+	bytes, err := io.ReadAll(r.Body)
+	if err != nil {
+		log.Printf("error while reading request body: %s", err.Error())
+		http.Error(rw, err.Error(), http.StatusInternalServerError)
+		return
+	}
+
+	ms := memorystore.GetMemoryStore()
+	dec := lineprotocol.NewDecoderWithBytes(bytes)
+	if err := decodeLine(dec, ms, r.URL.Query().Get("cluster")); err != nil {
+		log.Printf("/api/write error: %s", err.Error())
+		http.Error(rw, err.Error(), http.StatusBadRequest)
+		return
+	}
+	rw.WriteHeader(http.StatusOK)
+}
+
+type ApiQueryRequest struct {
+	Cluster     string     `json:"cluster"`
+	Queries     []ApiQuery `json:"queries"`
+	ForAllNodes []string   `json:"for-all-nodes"`
+	From        int64      `json:"from"`
+	To          int64      `json:"to"`
+	WithStats   bool       `json:"with-stats"`
+	WithData    bool       `json:"with-data"`
+	WithPadding bool       `json:"with-padding"`
+}
+
+type ApiQueryResponse struct {
+	Queries []ApiQuery        `json:"queries,omitempty"`
+	Results [][]ApiMetricData `json:"results"`
+}
+
+type ApiQuery struct {
+	Type        *string    `json:"type,omitempty"`
+	SubType     *string    `json:"subtype,omitempty"`
+	Metric      string   `json:"metric"`
+	Hostname    string   `json:"host"`
+	TypeIds     []string `json:"type-ids,omitempty"`
+	SubTypeIds  []string `json:"subtype-ids,omitempty"`
+	ScaleFactor util.Float `json:"scale-by,omitempty"`
+	Aggregate   bool       `json:"aggreg"`
+}
+
+func handleQuery(rw http.ResponseWriter, r *http.Request) {
+	var err error
+	req := ApiQueryRequest{WithStats: true, WithData: true, WithPadding: true}
+	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
+		http.Error(rw, err.Error(), http.StatusBadRequest)
+		return
+	}
+
+	ms := memorystore.GetMemoryStore()
+
+	response := ApiQueryResponse{
+		Results: make([][]ApiMetricData, 0, len(req.Queries)),
+	}
+	if req.ForAllNodes != nil {
+		nodes := ms.ListChildren([]string{req.Cluster})
+		for _, node := range nodes {
+			for _, metric := range req.ForAllNodes {
+				q := ApiQuery{
+					Metric:   metric,
+					Hostname: node,
+				}
+				req.Queries = append(req.Queries, q)
+				response.Queries = append(response.Queries, q)
+			}
+		}
+	}
+
+	for _, query := range req.Queries {
+		sels := make([]util.Selector, 0, 1)
+		if query.Aggregate || query.Type == nil {
+			sel := util.Selector{{String: req.Cluster}, {String: query.Hostname}}
+			if query.Type != nil {
+				if len(query.TypeIds) == 1 {
+					sel = append(sel, util.SelectorElement{String: *query.Type + query.TypeIds[0]})
+				} else {
+					ids := make([]string, len(query.TypeIds))
+					for i, id := range query.TypeIds {
+						ids[i] = *query.Type + id
+					}
+					sel = append(sel, util.SelectorElement{Group: ids})
+				}
+
+				if query.SubType != nil {
+					if len(query.SubTypeIds) == 1 {
+						sel = append(sel, util.SelectorElement{String: *query.SubType + query.SubTypeIds[0]})
+					} else {
+						ids := make([]string, len(query.SubTypeIds))
+						for i, id := range query.SubTypeIds {
+							ids[i] = *query.SubType + id
+						}
+						sel = append(sel, util.SelectorElement{Group: ids})
+					}
+				}
+			}
+			sels = append(sels, sel)
+		} else {
+			for _, typeId := range query.TypeIds {
+				if query.SubType != nil {
+					for _, subTypeId := range query.SubTypeIds {
+						sels = append(sels, util.Selector{
+							{String: req.Cluster},
+							{String: query.Hostname},
+							{String: *query.Type + typeId},
+							{String: *query.SubType + subTypeId},
+						})
+					}
+				} else {
+					sels = append(sels, util.Selector{
+						{String: req.Cluster},
+						{String: query.Hostname},
+						{String: *query.Type + typeId},
+					})
+				}
+			}
+		}
+
+		// log.Printf("query: %#v\n", query)
+		// log.Printf("sels: %#v\n", sels)
+
+		res := make([]ApiMetricData, 0, len(sels))
+		for _, sel := range sels {
+			data := ApiMetricData{}
+			data.Data, data.From, data.To, err = ms.Read(sel, query.Metric, req.From, req.To)
+			// log.Printf("data: %#v, %#v, %#v, %#v", data.Data, data.From, data.To, err)
+			if err != nil {
+				msg := err.Error()
+				data.Error = &msg
+				res = append(res, data)
+				continue
+			}
+
+			if req.WithStats {
+				data.AddStats()
+			}
+			if query.ScaleFactor != 0 {
+				data.ScaleBy(query.ScaleFactor)
+			}
+			if req.WithPadding {
+				data.PadDataWithNull(ms, req.From, req.To, query.Metric)
+			}
+			if !req.WithData {
+				data.Data = nil
+			}
+			res = append(res, data)
+		}
+		response.Results = append(response.Results, res)
+	}
+
+	rw.Header().Set("Content-Type", "application/json")
+	bw := bufio.NewWriter(rw)
+	defer bw.Flush()
+	if err := json.NewEncoder(bw).Encode(response); err != nil {
+		log.Print(err)
+		return
+	}
+}
+
+func handleDebug(rw http.ResponseWriter, r *http.Request) {
+	raw := r.URL.Query().Get("selector")
+	selector := []string{}
+	if len(raw) != 0 {
+		selector = strings.Split(raw, ":")
+		}
+
+	ms := memorystore.GetMemoryStore()
+	if err := ms.DebugDump(bufio.NewWriter(rw), selector); err != nil {
+		rw.WriteHeader(http.StatusBadRequest)
+		rw.Write([]byte(err.Error()))
+		}
+}
+
+func StartApiServer(ctx context.Context, httpConfig *config.HttpConfig) error {
+	r := mux.NewRouter()
+
+	r.HandleFunc("/api/free", handleFree)
+	r.HandleFunc("/api/write", handleWrite)
+	r.HandleFunc("/api/query", handleQuery)
+	r.HandleFunc("/api/debug", handleDebug)
+
+	server := &http.Server{
+		Handler:      r,
+		Addr:         httpConfig.Address,
+		WriteTimeout: 30 * time.Second,
+		ReadTimeout:  30 * time.Second,
+	}
+
+	if len(config.Keys.JwtPublicKey) > 0 {
+		buf, err := base64.StdEncoding.DecodeString(config.Keys.JwtPublicKey)
+		if err != nil {
+			return err
+		}
+		publicKey := ed25519.PublicKey(buf)
+		server.Handler = authentication(server.Handler, publicKey)
+	}
+
+	go func() {
+		if httpConfig.CertFile != "" && httpConfig.KeyFile != "" {
+			log.Printf("API https endpoint listening on '%s'\n", httpConfig.Address)
+			err := server.ListenAndServeTLS(httpConfig.CertFile, httpConfig.KeyFile)
+			if err != nil && err != http.ErrServerClosed {
+				log.Println(err)
+			}
+		} else {
+			log.Printf("API http endpoint listening on '%s'\n", httpConfig.Address)
+			err := server.ListenAndServe()
+			if err != nil && err != http.ErrServerClosed {
+				log.Println(err)
+			}
+		}
+	}()
+
+	for {
+		<-ctx.Done()
+		err := server.Shutdown(context.Background())
+		log.Println("API server shut down")
+		return err
+	}
+}

internal/api/api.go.orig

@@ -1,4 +1,4 @@
-package main
+package api
 
 import (
 	"bufio"
@@ -6,7 +6,6 @@ import (
 	"crypto/ed25519"
 	"encoding/base64"
 	"encoding/json"
-	"errors"
 	"fmt"
 	"io"
 	"log"
@@ -14,15 +13,29 @@ import (
 	"net/http"
 	"strconv"
 	"strings"
-	"sync"
 	"time"
 
+<<<<<<< HEAD:internal/api/api.go
+	"github.com/ClusterCockpit/cc-metric-store/internal/config"
+	"github.com/ClusterCockpit/cc-metric-store/internal/memorystore"
+	"github.com/ClusterCockpit/cc-metric-store/internal/util"
+=======
 	"github.com/golang-jwt/jwt/v5"
+>>>>>>> main:api.go
 	"github.com/gorilla/mux"
 	"github.com/influxdata/line-protocol/v2/lineprotocol"
 )
 
 type ApiMetricData struct {
+<<<<<<< HEAD:internal/api/api.go
+	Error *string         `json:"error,omitempty"`
+	Data  util.FloatArray `json:"data,omitempty"`
+	From  int64           `json:"from"`
+	To    int64           `json:"to"`
+	Avg   util.Float      `json:"avg"`
+	Min   util.Float      `json:"min"`
+	Max   util.Float      `json:"max"`
+=======
 	Error *string    `json:"error,omitempty"`
 	Data  FloatArray `json:"data,omitempty"`
 	From  int64      `json:"from"`
@@ -30,6 +43,7 @@ type ApiMetricData struct {
 	Avg   Float      `json:"avg"`
 	Min   Float      `json:"min"`
 	Max   Float      `json:"max"`
+>>>>>>> main:api.go
 }
 
 // TODO: Optimize this, just like the stats endpoint!
@@ -49,15 +63,15 @@ func (data *ApiMetricData) AddStats() {
 
 	if n > 0 {
 		avg := sum / float64(n)
-		data.Avg = Float(avg)
+		data.Avg = util.Float(avg)
-		data.Min = Float(min)
+		data.Min = util.Float(min)
-		data.Max = Float(max)
+		data.Max = util.Float(max)
 	} else {
-		data.Avg, data.Min, data.Max = NaN, NaN, NaN
+		data.Avg, data.Min, data.Max = util.NaN, util.NaN, util.NaN
 	}
 }
 
-func (data *ApiMetricData) ScaleBy(f Float) {
+func (data *ApiMetricData) ScaleBy(f util.Float) {
 	if f == 0 || f == 1 {
 		return
 	}
@@ -70,17 +84,17 @@ func (data *ApiMetricData) ScaleBy(f Float) {
 	}
 }
 
-func (data *ApiMetricData) PadDataWithNull(from, to int64, metric string) {
+func (data *ApiMetricData) PadDataWithNull(ms *memorystore.MemoryStore, from, to int64, metric string) {
-	minfo, ok := memoryStore.metrics[metric]
+	minfo, ok := ms.Metrics[metric]
 	if !ok {
 		return
 	}
 
 	if (data.From / minfo.Frequency) > (from / minfo.Frequency) {
 		padfront := int((data.From / minfo.Frequency) - (from / minfo.Frequency))
-		ndata := make([]Float, 0, padfront+len(data.Data))
+		ndata := make([]util.Float, 0, padfront+len(data.Data))
 		for i := 0; i < padfront; i++ {
-			ndata = append(ndata, NaN)
+			ndata = append(ndata, util.NaN)
 		}
 		for j := 0; j < len(data.Data); j++ {
 			ndata = append(ndata, data.Data[j])
@@ -102,12 +116,12 @@ func handleFree(rw http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	// TODO: lastCheckpoint might be modified by different go-routines.
+	// // TODO: lastCheckpoint might be modified by different go-routines.
-	// Load it using the sync/atomic package?
+	// // Load it using the sync/atomic package?
-	freeUpTo := lastCheckpoint.Unix()
+	// freeUpTo := lastCheckpoint.Unix()
-	if to < freeUpTo {
+	// if to < freeUpTo {
-		freeUpTo = to
+	// 	freeUpTo = to
-	}
+	// }
 
 	if r.Method != http.MethodPost {
 		http.Error(rw, "Method Not Allowed", http.StatusMethodNotAllowed)
@@ -122,9 +136,10 @@ func handleFree(rw http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	ms := memorystore.GetMemoryStore()
 	n := 0
 	for _, sel := range selectors {
-		bn, err := memoryStore.Free(sel, freeUpTo)
+		bn, err := ms.Free(sel, to)
 		if err != nil {
 			http.Error(rw, err.Error(), http.StatusInternalServerError)
 			return
@@ -150,26 +165,9 @@ func handleWrite(rw http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	if debugDump != io.Discard {
+	ms := memorystore.GetMemoryStore()
-		now := time.Now()
-		msg := make([]byte, 0, 512)
-		msg = append(msg, "\n--- local unix time: "...)
-		msg = strconv.AppendInt(msg, now.Unix(), 10)
-		msg = append(msg, " ---\n"...)
-
-		debugDumpLock.Lock()
-		defer debugDumpLock.Unlock()
-		if _, err := debugDump.Write(msg); err != nil {
-			log.Printf("error while writing to debug dump: %s", err.Error())
-		}
-		if _, err := debugDump.Write(bytes); err != nil {
-			log.Printf("error while writing to debug dump: %s", err.Error())
-		}
-		return
-	}
-
 	dec := lineprotocol.NewDecoderWithBytes(bytes)
-	if err := decodeLine(dec, r.URL.Query().Get("cluster")); err != nil {
+	if err := decodeLine(dec, ms, r.URL.Query().Get("cluster")); err != nil {
 		log.Printf("/api/write error: %s", err.Error())
 		http.Error(rw, err.Error(), http.StatusBadRequest)
 		return
@@ -194,6 +192,16 @@ type ApiQueryResponse struct {
 }
 
 type ApiQuery struct {
+<<<<<<< HEAD:internal/api/api.go
+	Type        *string    `json:"type,omitempty"`
+	SubType     *string    `json:"subtype,omitempty"`
+	Metric      string     `json:"metric"`
+	Hostname    string     `json:"host"`
+	TypeIds     []string   `json:"type-ids,omitempty"`
+	SubTypeIds  []string   `json:"subtype-ids,omitempty"`
+	ScaleFactor util.Float `json:"scale-by,omitempty"`
+	Aggregate   bool       `json:"aggreg"`
+=======
 	Type        *string  `json:"type,omitempty"`
 	SubType     *string  `json:"subtype,omitempty"`
 	Metric      string   `json:"metric"`
@@ -202,21 +210,28 @@ type ApiQuery struct {
 	SubTypeIds  []string `json:"subtype-ids,omitempty"`
 	ScaleFactor Float    `json:"scale-by,omitempty"`
 	Aggregate   bool     `json:"aggreg"`
+>>>>>>> main:api.go
 }
 
 func handleQuery(rw http.ResponseWriter, r *http.Request) {
 	var err error
 	req := ApiQueryRequest{WithStats: true, WithData: true, WithPadding: true}
+<<<<<<< HEAD:internal/api/api.go
+	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
+=======
 	if err = json.NewDecoder(r.Body).Decode(&req); err != nil {
+>>>>>>> main:api.go
 		http.Error(rw, err.Error(), http.StatusBadRequest)
 		return
 	}
 
+	ms := memorystore.GetMemoryStore()
+
 	response := ApiQueryResponse{
 		Results: make([][]ApiMetricData, 0, len(req.Queries)),
 	}
 	if req.ForAllNodes != nil {
-		nodes := memoryStore.ListChildren([]string{req.Cluster})
+		nodes := ms.ListChildren([]string{req.Cluster})
 		for _, node := range nodes {
 			for _, metric := range req.ForAllNodes {
 				q := ApiQuery{
@@ -230,29 +245,29 @@ func handleQuery(rw http.ResponseWriter, r *http.Request) {
 	}
 
 	for _, query := range req.Queries {
-		sels := make([]Selector, 0, 1)
+		sels := make([]util.Selector, 0, 1)
 		if query.Aggregate || query.Type == nil {
-			sel := Selector{{String: req.Cluster}, {String: query.Hostname}}
+			sel := util.Selector{{String: req.Cluster}, {String: query.Hostname}}
 			if query.Type != nil {
 				if len(query.TypeIds) == 1 {
-					sel = append(sel, SelectorElement{String: *query.Type + query.TypeIds[0]})
+					sel = append(sel, util.SelectorElement{String: *query.Type + query.TypeIds[0]})
 				} else {
 					ids := make([]string, len(query.TypeIds))
 					for i, id := range query.TypeIds {
 						ids[i] = *query.Type + id
 					}
-					sel = append(sel, SelectorElement{Group: ids})
+					sel = append(sel, util.SelectorElement{Group: ids})
 				}
 
 				if query.SubType != nil {
 					if len(query.SubTypeIds) == 1 {
-						sel = append(sel, SelectorElement{String: *query.SubType + query.SubTypeIds[0]})
+						sel = append(sel, util.SelectorElement{String: *query.SubType + query.SubTypeIds[0]})
 					} else {
 						ids := make([]string, len(query.SubTypeIds))
 						for i, id := range query.SubTypeIds {
 							ids[i] = *query.SubType + id
 						}
-						sel = append(sel, SelectorElement{Group: ids})
+						sel = append(sel, util.SelectorElement{Group: ids})
 					}
 				}
 			}
@@ -261,7 +276,11 @@ func handleQuery(rw http.ResponseWriter, r *http.Request) {
 			for _, typeId := range query.TypeIds {
 				if query.SubType != nil {
 					for _, subTypeId := range query.SubTypeIds {
+<<<<<<< HEAD:internal/api/api.go
+						sels = append(sels, util.Selector{
+=======
 						sels = append(sels, Selector{
+>>>>>>> main:api.go
 							{String: req.Cluster},
 							{String: query.Hostname},
 							{String: *query.Type + typeId},
@@ -269,7 +288,7 @@ func handleQuery(rw http.ResponseWriter, r *http.Request) {
 						})
 					}
 				} else {
-					sels = append(sels, Selector{
+					sels = append(sels, util.Selector{
 						{String: req.Cluster},
 						{String: query.Hostname},
 						{String: *query.Type + typeId},
@@ -284,7 +303,7 @@ func handleQuery(rw http.ResponseWriter, r *http.Request) {
 		res := make([]ApiMetricData, 0, len(sels))
 		for _, sel := range sels {
 			data := ApiMetricData{}
-			data.Data, data.From, data.To, err = memoryStore.Read(sel, query.Metric, req.From, req.To)
+			data.Data, data.From, data.To, err = ms.Read(sel, query.Metric, req.From, req.To)
 			// log.Printf("data: %#v, %#v, %#v, %#v", data.Data, data.From, data.To, err)
 			if err != nil {
 				msg := err.Error()
@@ -300,7 +319,7 @@ func handleQuery(rw http.ResponseWriter, r *http.Request) {
 				data.ScaleBy(query.ScaleFactor)
 			}
 			if req.WithPadding {
-				data.PadDataWithNull(req.From, req.To, query.Metric)
+				data.PadDataWithNull(ms, req.From, req.To, query.Metric)
 			}
 			if !req.WithData {
 				data.Data = nil
@@ -319,10 +338,20 @@ func handleQuery(rw http.ResponseWriter, r *http.Request) {
 	}
 }
 
-func authentication(next http.Handler, publicKey ed25519.PublicKey) http.Handler {
+func handleDebug(rw http.ResponseWriter, r *http.Request) {
-	cacheLock := sync.RWMutex{}
+	raw := r.URL.Query().Get("selector")
-	cache := map[string]*jwt.Token{}
+	selector := []string{}
+	if len(raw) != 0 {
+		selector = strings.Split(raw, ":")
+	}
 
+<<<<<<< HEAD:internal/api/api.go
+	ms := memorystore.GetMemoryStore()
+	if err := ms.DebugDump(bufio.NewWriter(rw), selector); err != nil {
+		rw.WriteHeader(http.StatusBadRequest)
+		rw.Write([]byte(err.Error()))
+	}
+=======
 	return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
 		authheader := r.Header.Get("Authorization")
 		if authheader == "" || !strings.HasPrefix(authheader, "Bearer ") {
@@ -373,26 +402,16 @@ func authentication(next http.Handler, publicKey ed25519.PublicKey) http.Handler
 		// Let request through...
 		next.ServeHTTP(rw, r)
 	})
+>>>>>>> main:api.go
 }
 
-func StartApiServer(ctx context.Context, httpConfig *HttpConfig) error {
+func StartApiServer(ctx context.Context, httpConfig *config.HttpConfig) error {
 	r := mux.NewRouter()
 
 	r.HandleFunc("/api/free", handleFree)
 	r.HandleFunc("/api/write", handleWrite)
 	r.HandleFunc("/api/query", handleQuery)
-	r.HandleFunc("/api/debug", func(rw http.ResponseWriter, r *http.Request) {
+	r.HandleFunc("/api/debug", handleDebug)
-		raw := r.URL.Query().Get("selector")
-		selector := []string{}
-		if len(raw) != 0 {
-			selector = strings.Split(raw, ":")
-		}
-
-		if err := memoryStore.DebugDump(bufio.NewWriter(rw), selector); err != nil {
-			rw.WriteHeader(http.StatusBadRequest)
-			rw.Write([]byte(err.Error()))
-		}
-	})
 
 	server := &http.Server{
 		Handler:      r,
@@ -401,8 +420,8 @@ func StartApiServer(ctx context.Context, httpConfig *HttpConfig) error {
 		ReadTimeout:  30 * time.Second,
 	}
 
-	if len(conf.JwtPublicKey) > 0 {
+	if len(config.Keys.JwtPublicKey) > 0 {
-		buf, err := base64.StdEncoding.DecodeString(conf.JwtPublicKey)
+		buf, err := base64.StdEncoding.DecodeString(config.Keys.JwtPublicKey)
 		if err != nil {
 			return err
 		}

internal/api/authentication.go

@@ -0,0 +1,56 @@
+package api
+
+import (
+	"crypto/ed25519"
+	"errors"
+	"net/http"
+	"strings"
+	"sync"
+
+	"github.com/golang-jwt/jwt/v4"
+)
+
+func authentication(next http.Handler, publicKey ed25519.PublicKey) http.Handler {
+	cacheLock := sync.RWMutex{}
+	cache := map[string]*jwt.Token{}
+
+	return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
+		authheader := r.Header.Get("Authorization")
+		if authheader == "" || !strings.HasPrefix(authheader, "Bearer ") {
+			http.Error(rw, "Use JWT Authentication", http.StatusUnauthorized)
+			return
+		}
+
+		rawtoken := authheader[len("Bearer "):]
+		cacheLock.RLock()
+		token, ok := cache[rawtoken]
+		cacheLock.RUnlock()
+		if ok && token.Claims.Valid() == nil {
+			next.ServeHTTP(rw, r)
+			return
+		}
+
+		// The actual token is ignored for now.
+		// In case expiration and so on are specified, the Parse function
+		// already returns an error for expired tokens.
+		var err error
+		token, err = jwt.Parse(rawtoken, func(t *jwt.Token) (interface{}, error) {
+			if t.Method != jwt.SigningMethodEdDSA {
+				return nil, errors.New("only Ed25519/EdDSA supported")
+			}
+
+			return publicKey, nil
+		})
+		if err != nil {
+			http.Error(rw, err.Error(), http.StatusUnauthorized)
+			return
+		}
+
+		cacheLock.Lock()
+		cache[rawtoken] = token
+		cacheLock.Unlock()
+
+		// Let request through...
+		next.ServeHTTP(rw, r)
+	})
+}

internal/api/lineprotocol.go

@@ -1,4 +1,4 @@
-package main
+package api
 
 import (
 	"context"
@@ -9,20 +9,18 @@ import (
 	"sync"
 	"time"
 
+	"github.com/ClusterCockpit/cc-metric-store/internal/config"
+	"github.com/ClusterCockpit/cc-metric-store/internal/memorystore"
+	"github.com/ClusterCockpit/cc-metric-store/internal/util"
 	"github.com/influxdata/line-protocol/v2/lineprotocol"
 	"github.com/nats-io/nats.go"
 )
 
-type Metric struct {
-	Name  string
-	Value Float
-
-	mc MetricConfig
-}
-
-// Currently unused, could be used to send messages via raw TCP.
 // Each connection is handled in it's own goroutine. This is a blocking function.
-func ReceiveRaw(ctx context.Context, listener net.Listener, handleLine func(*lineprotocol.Decoder, string) error) error {
+func ReceiveRaw(ctx context.Context,
+	listener net.Listener,
+	handleLine func(*lineprotocol.Decoder, string) error,
+) error {
 	var wg sync.WaitGroup
 
 	wg.Add(1)
@@ -84,7 +82,11 @@ func ReceiveRaw(ctx context.Context, listener net.Listener, handleLine func(*lin
 // Connect to a nats server and subscribe to "updates". This is a blocking
 // function. handleLine will be called for each line recieved via nats.
 // Send `true` through the done channel for gracefull termination.
-func ReceiveNats(conf *NatsConfig, handleLine func(*lineprotocol.Decoder, string) error, workers int, ctx context.Context) error {
+func ReceiveNats(conf *config.NatsConfig,
+	ms *memorystore.MemoryStore,
+	workers int,
+	ctx context.Context,
+) error {
 	var opts []nats.Option
 	if conf.Username != "" && conf.Password != "" {
 		opts = append(opts, nats.UserInfo(conf.Username, conf.Password))
@@ -111,7 +113,7 @@ func ReceiveNats(conf *NatsConfig, handleLine func(*lineprotocol.Decoder, string
 				go func() {
 					for m := range msgs {
 						dec := lineprotocol.NewDecoderWithBytes(m.Data)
-						if err := handleLine(dec, clusterTag); err != nil {
+						if err := decodeLine(dec, ms, clusterTag); err != nil {
 							log.Printf("error: %s\n", err.Error())
 						}
 					}
@@ -126,7 +128,7 @@ func ReceiveNats(conf *NatsConfig, handleLine func(*lineprotocol.Decoder, string
 		} else {
 			sub, err = nc.Subscribe(sc.SubscribeTo, func(m *nats.Msg) {
 				dec := lineprotocol.NewDecoderWithBytes(m.Data)
-				if err := handleLine(dec, clusterTag); err != nil {
+				if err := decodeLine(dec, ms, clusterTag); err != nil {
 					log.Printf("error: %s\n", err.Error())
 				}
 			})
@@ -175,18 +177,21 @@ func reorder(buf, prefix []byte) []byte {
 
 // Decode lines using dec and make write calls to the MemoryStore.
 // If a line is missing its cluster tag, use clusterDefault as default.
-func decodeLine(dec *lineprotocol.Decoder, clusterDefault string) error {
+func decodeLine(dec *lineprotocol.Decoder,
+	ms *memorystore.MemoryStore,
+	clusterDefault string,
+) error {
 	// Reduce allocations in loop:
 	t := time.Now()
-	metric, metricBuf := Metric{}, make([]byte, 0, 16)
+	metric, metricBuf := memorystore.Metric{}, make([]byte, 0, 16)
 	selector := make([]string, 0, 4)
 	typeBuf, subTypeBuf := make([]byte, 0, 16), make([]byte, 0)
 
 	// Optimize for the case where all lines in a "batch" are about the same
 	// cluster and host. By using `WriteToLevel` (level = host), we do not need
 	// to take the root- and cluster-level lock as often.
-	var lvl *level = nil
+	var lvl *memorystore.Level = nil
-	var prevCluster, prevHost string = "", ""
+	prevCluster, prevHost := "", ""
 
 	var ok bool
 	for dec.Next() {
@@ -200,7 +205,7 @@ func decodeLine(dec *lineprotocol.Decoder, clusterDefault string) error {
 		metricBuf = append(metricBuf[:0], rawmeasurement...)
 
 		// The go compiler optimizes map[string(byteslice)] lookups:
-		metric.mc, ok = memoryStore.metrics[string(rawmeasurement)]
+		metric.MetricConfig, ok = ms.Metrics[string(rawmeasurement)]
 		if !ok {
 			continue
 		}
@@ -264,7 +269,7 @@ func decodeLine(dec *lineprotocol.Decoder, clusterDefault string) error {
 		if lvl == nil {
 			selector = selector[:2]
 			selector[0], selector[1] = cluster, host
-			lvl = memoryStore.GetLevel(selector)
+			lvl = ms.GetLevel(selector)
 			prevCluster, prevHost = cluster, host
 		}
 
@@ -292,11 +297,11 @@ func decodeLine(dec *lineprotocol.Decoder, clusterDefault string) error {
 			}
 
 			if val.Kind() == lineprotocol.Float {
-				metric.Value = Float(val.FloatV())
+				metric.Value = util.Float(val.FloatV())
 			} else if val.Kind() == lineprotocol.Int {
-				metric.Value = Float(val.IntV())
+				metric.Value = util.Float(val.IntV())
 			} else if val.Kind() == lineprotocol.Uint {
-				metric.Value = Float(val.UintV())
+				metric.Value = util.Float(val.UintV())
 			} else {
 				return fmt.Errorf("unsupported value type in message: %s", val.Kind().String())
 			}
@@ -306,7 +311,7 @@ func decodeLine(dec *lineprotocol.Decoder, clusterDefault string) error {
 			return err
 		}
 
-		if err := memoryStore.WriteToLevel(lvl, selector, t.Unix(), []Metric{metric}); err != nil {
+		if err := ms.WriteToLevel(lvl, selector, t.Unix(), []memorystore.Metric{metric}); err != nil {
 			return err
 		}
 	}

internal/config/config.go

@@ -0,0 +1,112 @@
+package config
+
+import (
+	"encoding/json"
+	"fmt"
+	"log"
+	"os"
+)
+
+// For aggregation over multiple values at different cpus/sockets/..., not time!
+type AggregationStrategy int
+
+const (
+	NoAggregation AggregationStrategy = iota
+	SumAggregation
+	AvgAggregation
+)
+
+func (as *AggregationStrategy) UnmarshalJSON(data []byte) error {
+	var str string
+	if err := json.Unmarshal(data, &str); err != nil {
+		return err
+	}
+
+	switch str {
+	case "":
+		*as = NoAggregation
+	case "sum":
+		*as = SumAggregation
+	case "avg":
+		*as = AvgAggregation
+	default:
+		return fmt.Errorf("invalid aggregation strategy: %#v", str)
+	}
+	return nil
+}
+
+type MetricConfig struct {
+	// Interval in seconds at which measurements will arive.
+	Frequency int64 `json:"frequency"`
+
+	// Can be 'sum', 'avg' or null. Describes how to aggregate metrics from the same timestep over the hierarchy.
+	Aggregation AggregationStrategy `json:"aggregation"`
+
+	// Private, used internally...
+	Offset int
+}
+
+type HttpConfig struct {
+	// Address to bind to, for example "0.0.0.0:8081"
+	Address string `json:"address"`
+
+	// If not the empty string, use https with this as the certificate file
+	CertFile string `json:"https-cert-file"`
+
+	// If not the empty string, use https with this as the key file
+	KeyFile string `json:"https-key-file"`
+}
+
+type NatsConfig struct {
+	// Address of the nats server
+	Address string `json:"address"`
+
+	// Username/Password, optional
+	Username string `json:"username"`
+	Password string `json:"password"`
+
+	Subscriptions []struct {
+		// Channel name
+		SubscribeTo string `json:"subscribe-to"`
+
+		// Allow lines without a cluster tag, use this as default, optional
+		ClusterTag string `json:"cluster-tag"`
+	} `json:"subscriptions"`
+}
+
+type Config struct {
+	Metrics     map[string]MetricConfig `json:"metrics"`
+	HttpConfig  *HttpConfig             `json:"http-api"`
+	Checkpoints struct {
+		Interval string `json:"interval"`
+		RootDir  string `json:"directory"`
+		Restore  string `json:"restore"`
+	} `json:"checkpoints"`
+	Debug struct {
+		DumpToFile string `json:"dump-to-file"`
+		EnableGops bool   `json:"gops"`
+	} `json:"debug"`
+	RetentionInMemory string `json:"retention-in-memory"`
+	JwtPublicKey      string `json:"jwt-public-key"`
+	Archive           struct {
+		Interval      string `json:"interval"`
+		RootDir       string `json:"directory"`
+		DeleteInstead bool   `json:"delete-instead"`
+	} `json:"archive"`
+	Nats []*NatsConfig `json:"nats"`
+}
+
+var Keys Config
+
+func Init(file string) {
+	configFile, err := os.Open(file)
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer configFile.Close()
+	dec := json.NewDecoder(configFile)
+	dec.DisallowUnknownFields()
+	if err := dec.Decode(&Keys); err != nil {
+		log.Fatal(err)
+	}
+}

internal/memorystore/archive.go

@@ -0,0 +1,184 @@
+package memorystore
+
+import (
+	"archive/zip"
+	"bufio"
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"log"
+	"os"
+	"path/filepath"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/ClusterCockpit/cc-metric-store/internal/config"
+)
+
+func Archiving(wg *sync.WaitGroup, ctx context.Context) {
+	go func() {
+		defer wg.Done()
+		d, err := time.ParseDuration(config.Keys.Archive.Interval)
+		if err != nil {
+			log.Fatal(err)
+		}
+		if d <= 0 {
+			return
+		}
+
+		ticks := func() <-chan time.Time {
+			if d <= 0 {
+				return nil
+			}
+			return time.NewTicker(d).C
+		}()
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			case <-ticks:
+				t := time.Now().Add(-d)
+				log.Printf("start archiving checkpoints (older than %s)...\n", t.Format(time.RFC3339))
+				n, err := ArchiveCheckpoints(config.Keys.Checkpoints.RootDir, config.Keys.Archive.RootDir, t.Unix(), config.Keys.Archive.DeleteInstead)
+				if err != nil {
+					log.Printf("archiving failed: %s\n", err.Error())
+				} else {
+					log.Printf("done: %d files zipped and moved to archive\n", n)
+				}
+			}
+		}
+	}()
+}
+
+var ErrNoNewData error = errors.New("all data already archived")
+
+// ZIP all checkpoint files older than `from` together and write them to the `archiveDir`,
+// deleting them from the `checkpointsDir`.
+func ArchiveCheckpoints(checkpointsDir, archiveDir string, from int64, deleteInstead bool) (int, error) {
+	entries1, err := os.ReadDir(checkpointsDir)
+	if err != nil {
+		return 0, err
+	}
+
+	type workItem struct {
+		cdir, adir    string
+		cluster, host string
+	}
+
+	var wg sync.WaitGroup
+	n, errs := int32(0), int32(0)
+	work := make(chan workItem, NumWorkers)
+
+	wg.Add(NumWorkers)
+	for worker := 0; worker < NumWorkers; worker++ {
+		go func() {
+			defer wg.Done()
+			for workItem := range work {
+				m, err := archiveCheckpoints(workItem.cdir, workItem.adir, from, deleteInstead)
+				if err != nil {
+					log.Printf("error while archiving %s/%s: %s", workItem.cluster, workItem.host, err.Error())
+					atomic.AddInt32(&errs, 1)
+				}
+				atomic.AddInt32(&n, int32(m))
+			}
+		}()
+	}
+
+	for _, de1 := range entries1 {
+		entries2, e := os.ReadDir(filepath.Join(checkpointsDir, de1.Name()))
+		if e != nil {
+			err = e
+		}
+
+		for _, de2 := range entries2 {
+			cdir := filepath.Join(checkpointsDir, de1.Name(), de2.Name())
+			adir := filepath.Join(archiveDir, de1.Name(), de2.Name())
+			work <- workItem{
+				adir: adir, cdir: cdir,
+				cluster: de1.Name(), host: de2.Name(),
+			}
+		}
+	}
+
+	close(work)
+	wg.Wait()
+
+	if err != nil {
+		return int(n), err
+	}
+
+	if errs > 0 {
+		return int(n), fmt.Errorf("%d errors happend while archiving (%d successes)", errs, n)
+	}
+	return int(n), nil
+}
+
+// Helper function for `ArchiveCheckpoints`.
+func archiveCheckpoints(dir string, archiveDir string, from int64, deleteInstead bool) (int, error) {
+	entries, err := os.ReadDir(dir)
+	if err != nil {
+		return 0, err
+	}
+
+	files, err := findFiles(entries, from, false)
+	if err != nil {
+		return 0, err
+	}
+
+	if deleteInstead {
+		n := 0
+		for _, checkpoint := range files {
+			filename := filepath.Join(dir, checkpoint)
+			if err = os.Remove(filename); err != nil {
+				return n, err
+			}
+			n += 1
+		}
+		return n, nil
+	}
+
+	filename := filepath.Join(archiveDir, fmt.Sprintf("%d.zip", from))
+	f, err := os.OpenFile(filename, os.O_CREATE|os.O_WRONLY, 0o644)
+	if err != nil && os.IsNotExist(err) {
+		err = os.MkdirAll(archiveDir, 0o755)
+		if err == nil {
+			f, err = os.OpenFile(filename, os.O_CREATE|os.O_WRONLY, 0o644)
+		}
+	}
+	if err != nil {
+		return 0, err
+	}
+	defer f.Close()
+	bw := bufio.NewWriter(f)
+	defer bw.Flush()
+	zw := zip.NewWriter(bw)
+	defer zw.Close()
+
+	n := 0
+	for _, checkpoint := range files {
+		filename := filepath.Join(dir, checkpoint)
+		r, err := os.Open(filename)
+		if err != nil {
+			return n, err
+		}
+		defer r.Close()
+
+		w, err := zw.Create(checkpoint)
+		if err != nil {
+			return n, err
+		}
+
+		if _, err = io.Copy(w, r); err != nil {
+			return n, err
+		}
+
+		if err = os.Remove(filename); err != nil {
+			return n, err
+		}
+		n += 1
+	}
+
+	return n, nil
+}

internal/memorystore/buffer.go

@@ -0,0 +1,233 @@
+package memorystore
+
+import (
+	"errors"
+	"sync"
+
+	"github.com/ClusterCockpit/cc-metric-store/internal/util"
+)
+
+// Default buffer capacity.
+// `buffer.data` will only ever grow up to it's capacity and a new link
+// in the buffer chain will be created if needed so that no copying
+// of data or reallocation needs to happen on writes.
+const (
+	BUFFER_CAP int = 512
+)
+
+// So that we can reuse allocations
+var bufferPool sync.Pool = sync.Pool{
+	New: func() interface{} {
+		return &buffer{
+			data: make([]util.Float, 0, BUFFER_CAP),
+		}
+	},
+}
+
+var (
+	ErrNoData           error = errors.New("no data for this metric/level")
+	ErrDataDoesNotAlign error = errors.New("data from lower granularities does not align")
+)
+
+// Each metric on each level has it's own buffer.
+// This is where the actual values go.
+// If `cap(data)` is reached, a new buffer is created and
+// becomes the new head of a buffer list.
+type buffer struct {
+	prev      *buffer
+	next      *buffer
+	data      []util.Float
+	frequency int64
+	start     int64
+	archived  bool
+	closed    bool
+}
+
+func newBuffer(ts, freq int64) *buffer {
+	b := bufferPool.Get().(*buffer)
+	b.frequency = freq
+	b.start = ts - (freq / 2)
+	b.prev = nil
+	b.next = nil
+	b.archived = false
+	b.closed = false
+	b.data = b.data[:0]
+	return b
+}
+
+// If a new buffer was created, the new head is returnd.
+// Otherwise, the existing buffer is returnd.
+// Normaly, only "newer" data should be written, but if the value would
+// end up in the same buffer anyways it is allowed.
+func (b *buffer) write(ts int64, value util.Float) (*buffer, error) {
+	if ts < b.start {
+		return nil, errors.New("cannot write value to buffer from past")
+	}
+
+	// idx := int((ts - b.start + (b.frequency / 3)) / b.frequency)
+	idx := int((ts - b.start) / b.frequency)
+	if idx >= cap(b.data) {
+		newbuf := newBuffer(ts, b.frequency)
+		newbuf.prev = b
+		b.next = newbuf
+		b.close()
+		b = newbuf
+		idx = 0
+	}
+
+	// Overwriting value or writing value from past
+	if idx < len(b.data) {
+		b.data[idx] = value
+		return b, nil
+	}
+
+	// Fill up unwritten slots with NaN
+	for i := len(b.data); i < idx; i++ {
+		b.data = append(b.data, util.NaN)
+	}
+
+	b.data = append(b.data, value)
+	return b, nil
+}
+
+func (b *buffer) end() int64 {
+	return b.firstWrite() + int64(len(b.data))*b.frequency
+}
+
+func (b *buffer) firstWrite() int64 {
+	return b.start + (b.frequency / 2)
+}
+
+func (b *buffer) close() {}
+
+/*
+func (b *buffer) close() {
+	if b.closed {
+		return
+	}
+
+	b.closed = true
+	n, sum, min, max := 0, 0., math.MaxFloat64, -math.MaxFloat64
+	for _, x := range b.data {
+		if x.IsNaN() {
+			continue
+		}
+
+		n += 1
+		f := float64(x)
+		sum += f
+		min = math.Min(min, f)
+		max = math.Max(max, f)
+	}
+
+	b.statisticts.samples = n
+	if n > 0 {
+		b.statisticts.avg = Float(sum / float64(n))
+		b.statisticts.min = Float(min)
+		b.statisticts.max = Float(max)
+	} else {
+		b.statisticts.avg = NaN
+		b.statisticts.min = NaN
+		b.statisticts.max = NaN
+	}
+}
+*/
+
+// func interpolate(idx int, data []Float) Float {
+// 	if idx == 0 || idx+1 == len(data) {
+// 		return NaN
+// 	}
+// 	return (data[idx-1] + data[idx+1]) / 2.0
+// }
+
+// Return all known values from `from` to `to`. Gaps of information are represented as NaN.
+// Simple linear interpolation is done between the two neighboring cells if possible.
+// If values at the start or end are missing, instead of NaN values, the second and thrid
+// return values contain the actual `from`/`to`.
+// This function goes back the buffer chain if `from` is older than the currents buffer start.
+// The loaded values are added to `data` and `data` is returned, possibly with a shorter length.
+// If `data` is not long enough to hold all values, this function will panic!
+func (b *buffer) read(from, to int64, data []util.Float) ([]util.Float, int64, int64, error) {
+	if from < b.firstWrite() {
+		if b.prev != nil {
+			return b.prev.read(from, to, data)
+		}
+		from = b.firstWrite()
+	}
+
+	i := 0
+	t := from
+	for ; t < to; t += b.frequency {
+		idx := int((t - b.start) / b.frequency)
+		if idx >= cap(b.data) {
+			if b.next == nil {
+				break
+			}
+			b = b.next
+			idx = 0
+		}
+
+		if idx >= len(b.data) {
+			if b.next == nil || to <= b.next.start {
+				break
+			}
+			data[i] += util.NaN
+		} else if t < b.start {
+			data[i] += util.NaN
+			// } else if b.data[idx].IsNaN() {
+			// 	data[i] += interpolate(idx, b.data)
+		} else {
+			data[i] += b.data[idx]
+		}
+		i++
+	}
+
+	return data[:i], from, t, nil
+}
+
+// Returns true if this buffer needs to be freed.
+func (b *buffer) free(t int64) (delme bool, n int) {
+	if b.prev != nil {
+		delme, m := b.prev.free(t)
+		n += m
+		if delme {
+			b.prev.next = nil
+			if cap(b.prev.data) == BUFFER_CAP {
+				bufferPool.Put(b.prev)
+			}
+			b.prev = nil
+		}
+	}
+
+	end := b.end()
+	if end < t {
+		return true, n + 1
+	}
+
+	return false, n
+}
+
+// Call `callback` on every buffer that contains data in the range from `from` to `to`.
+func (b *buffer) iterFromTo(from, to int64, callback func(b *buffer) error) error {
+	if b == nil {
+		return nil
+	}
+
+	if err := b.prev.iterFromTo(from, to, callback); err != nil {
+		return err
+	}
+
+	if from <= b.end() && b.start <= to {
+		return callback(b)
+	}
+
+	return nil
+}
+
+func (b *buffer) count() int64 {
+	res := int64(len(b.data))
+	if b.prev != nil {
+		res += b.prev.count()
+	}
+	return res
+}

internal/memorystore/checkpoint.go

@@ -1,12 +1,11 @@
-package main
+package memorystore
 
 import (
-	"archive/zip"
 	"bufio"
+	"context"
 	"encoding/json"
 	"errors"
 	"fmt"
-	"io"
 	"io/fs"
 	"log"
 	"os"
@@ -18,13 +17,66 @@ import (
 	"strings"
 	"sync"
 	"sync/atomic"
+	"time"
+
+	"github.com/ClusterCockpit/cc-metric-store/internal/config"
+	"github.com/ClusterCockpit/cc-metric-store/internal/util"
 )
 
 // Whenever changed, update MarshalJSON as well!
 type CheckpointMetrics struct {
-	Frequency int64   `json:"frequency"`
+	Data      []util.Float `json:"data"`
-	Start     int64   `json:"start"`
+	Frequency int64        `json:"frequency"`
-	Data      []Float `json:"data"`
+	Start     int64        `json:"start"`
+}
+
+type CheckpointFile struct {
+	Metrics  map[string]*CheckpointMetrics `json:"metrics"`
+	Children map[string]*CheckpointFile    `json:"children"`
+	From     int64                         `json:"from"`
+	To       int64                         `json:"to"`
+}
+
+var lastCheckpoint time.Time
+
+func Checkpointing(wg *sync.WaitGroup, ctx context.Context) {
+	lastCheckpoint = time.Now()
+	ms := GetMemoryStore()
+
+	go func() {
+		defer wg.Done()
+		d, err := time.ParseDuration(config.Keys.Checkpoints.Interval)
+		if err != nil {
+			log.Fatal(err)
+		}
+		if d <= 0 {
+			return
+		}
+
+		ticks := func() <-chan time.Time {
+			if d <= 0 {
+				return nil
+			}
+			return time.NewTicker(d).C
+		}()
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			case <-ticks:
+				log.Printf("start checkpointing (starting at %s)...\n", lastCheckpoint.Format(time.RFC3339))
+				now := time.Now()
+				n, err := ms.ToCheckpoint(config.Keys.Checkpoints.RootDir,
+					lastCheckpoint.Unix(), now.Unix())
+				if err != nil {
+					log.Printf("checkpointing failed: %s\n", err.Error())
+				} else {
+					log.Printf("done: %d checkpoint files created\n", n)
+					lastCheckpoint = now
+				}
+			}
+		}
+	}()
 }
 
 // As `Float` implements a custom MarshalJSON() function,
@@ -51,31 +103,12 @@ func (cm *CheckpointMetrics) MarshalJSON() ([]byte, error) {
 	return buf, nil
 }
 
-type CheckpointFile struct {
-	From     int64                         `json:"from"`
-	To       int64                         `json:"to"`
-	Metrics  map[string]*CheckpointMetrics `json:"metrics"`
-	Children map[string]*CheckpointFile    `json:"children"`
-}
-
-var ErrNoNewData error = errors.New("all data already archived")
-
-var NumWorkers int = 4
-
-func init() {
-	maxWorkers := 10
-	NumWorkers = runtime.NumCPU()/2 + 1
-	if NumWorkers > maxWorkers {
-		NumWorkers = maxWorkers
-	}
-}
-
 // Metrics stored at the lowest 2 levels are not stored away (root and cluster)!
 // On a per-host basis a new JSON file is created. I have no idea if this will scale.
 // The good thing: Only a host at a time is locked, so this function can run
 // in parallel to writes/reads.
 func (m *MemoryStore) ToCheckpoint(dir string, from, to int64) (int, error) {
-	levels := make([]*level, 0)
+	levels := make([]*Level, 0)
 	selectors := make([][]string, 0)
 	m.root.lock.RLock()
 	for sel1, l1 := range m.root.children {
@@ -89,7 +122,7 @@ func (m *MemoryStore) ToCheckpoint(dir string, from, to int64) (int, error) {
 	m.root.lock.RUnlock()
 
 	type workItem struct {
-		level    *level
+		level    *Level
 		dir      string
 		selector []string
 	}
@@ -136,7 +169,7 @@ func (m *MemoryStore) ToCheckpoint(dir string, from, to int64) (int, error) {
 	return int(n), nil
 }
 
-func (l *level) toCheckpointFile(from, to int64, m *MemoryStore) (*CheckpointFile, error) {
+func (l *Level) toCheckpointFile(from, to int64, m *MemoryStore) (*CheckpointFile, error) {
 	l.lock.RLock()
 	defer l.lock.RUnlock()
 
@@ -147,8 +180,8 @@ func (l *level) toCheckpointFile(from, to int64, m *MemoryStore) (*CheckpointFil
 		Children: make(map[string]*CheckpointFile),
 	}
 
-	for metric, minfo := range m.metrics {
+	for metric, minfo := range m.Metrics {
-		b := l.metrics[minfo.offset]
+		b := l.metrics[minfo.Offset]
 		if b == nil {
 			continue
 		}
@@ -165,14 +198,14 @@ func (l *level) toCheckpointFile(from, to int64, m *MemoryStore) (*CheckpointFil
 			continue
 		}
 
-		data := make([]Float, (to-from)/b.frequency+1)
+		data := make([]util.Float, (to-from)/b.frequency+1)
 		data, start, end, err := b.read(from, to, data)
 		if err != nil {
 			return nil, err
 		}
 
 		for i := int((end - start) / b.frequency); i < len(data); i++ {
-			data[i] = NaN
+			data[i] = util.NaN
 		}
 
 		retval.Metrics[metric] = &CheckpointMetrics{
@@ -200,7 +233,7 @@ func (l *level) toCheckpointFile(from, to int64, m *MemoryStore) (*CheckpointFil
 	return retval, nil
 }
 
-func (l *level) toCheckpoint(dir string, from, to int64, m *MemoryStore) error {
+func (l *Level) toCheckpoint(dir string, from, to int64, m *MemoryStore) error {
 	cf, err := l.toCheckpointFile(from, to, m)
 	if err != nil {
 		return err
@@ -211,11 +244,11 @@ func (l *level) toCheckpoint(dir string, from, to int64, m *MemoryStore) error {
 	}
 
 	filepath := path.Join(dir, fmt.Sprintf("%d.json", from))
-	f, err := os.OpenFile(filepath, os.O_CREATE|os.O_WRONLY, 0644)
+	f, err := os.OpenFile(filepath, os.O_CREATE|os.O_WRONLY, 0o644)
 	if err != nil && os.IsNotExist(err) {
-		err = os.MkdirAll(dir, 0755)
+		err = os.MkdirAll(dir, 0o755)
 		if err == nil {
-			f, err = os.OpenFile(filepath, os.O_CREATE|os.O_WRONLY, 0644)
+			f, err = os.OpenFile(filepath, os.O_CREATE|os.O_WRONLY, 0o644)
 		}
 	}
 	if err != nil {
@@ -244,7 +277,7 @@ func (m *MemoryStore) FromCheckpoint(dir string, from int64) (int, error) {
 		go func() {
 			defer wg.Done()
 			for host := range work {
-				lvl := m.root.findLevelOrCreate(host[:], len(m.metrics))
+				lvl := m.root.findLevelOrCreate(host[:], len(m.Metrics))
 				nn, err := lvl.fromCheckpoint(filepath.Join(dir, host[0], host[1]), from, m)
 				if err != nil {
 					log.Fatalf("error while loading checkpoints: %s", err.Error())
@@ -302,7 +335,7 @@ done:
 	return int(n), nil
 }
 
-func (l *level) loadFile(cf *CheckpointFile, m *MemoryStore) error {
+func (l *Level) loadFile(cf *CheckpointFile, m *MemoryStore) error {
 	for name, metric := range cf.Metrics {
 		n := len(metric.Data)
 		b := &buffer{
@@ -315,15 +348,15 @@ func (l *level) loadFile(cf *CheckpointFile, m *MemoryStore) error {
 		}
 		b.close()
 
-		minfo, ok := m.metrics[name]
+		minfo, ok := m.Metrics[name]
 		if !ok {
 			continue
 			// return errors.New("Unkown metric: " + name)
 		}
 
-		prev := l.metrics[minfo.offset]
+		prev := l.metrics[minfo.Offset]
 		if prev == nil {
-			l.metrics[minfo.offset] = b
+			l.metrics[minfo.Offset] = b
 		} else {
 			if prev.start > b.start {
 				return errors.New("wooops")
@@ -332,18 +365,18 @@ func (l *level) loadFile(cf *CheckpointFile, m *MemoryStore) error {
 			b.prev = prev
 			prev.next = b
 		}
-		l.metrics[minfo.offset] = b
+		l.metrics[minfo.Offset] = b
 	}
 
 	if len(cf.Children) > 0 && l.children == nil {
-		l.children = make(map[string]*level)
+		l.children = make(map[string]*Level)
 	}
 
 	for sel, childCf := range cf.Children {
 		child, ok := l.children[sel]
 		if !ok {
-			child = &level{
+			child = &Level{
-				metrics:  make([]*buffer, len(m.metrics)),
+				metrics:  make([]*buffer, len(m.Metrics)),
 				children: nil,
 			}
 			l.children[sel] = child
@@ -357,7 +390,7 @@ func (l *level) loadFile(cf *CheckpointFile, m *MemoryStore) error {
 	return nil
 }
 
-func (l *level) fromCheckpoint(dir string, from int64, m *MemoryStore) (int, error) {
+func (l *Level) fromCheckpoint(dir string, from int64, m *MemoryStore) (int, error) {
 	direntries, err := os.ReadDir(dir)
 	if err != nil {
 		if os.IsNotExist(err) {
@@ -371,9 +404,9 @@ func (l *level) fromCheckpoint(dir string, from int64, m *MemoryStore) (int, err
 	filesLoaded := 0
 	for _, e := range direntries {
 		if e.IsDir() {
-			child := &level{
+			child := &Level{
-				metrics:  make([]*buffer, len(m.metrics)),
+				metrics:  make([]*buffer, len(m.Metrics)),
-				children: make(map[string]*level),
+				children: make(map[string]*Level),
 			}
 
 			files, err := child.fromCheckpoint(path.Join(dir, e.Name()), from, m)
@@ -466,132 +499,3 @@ func findFiles(direntries []fs.DirEntry, t int64, findMoreRecentFiles bool) ([]s
 
 	return filenames, nil
 }
-
-// ZIP all checkpoint files older than `from` together and write them to the `archiveDir`,
-// deleting them from the `checkpointsDir`.
-func ArchiveCheckpoints(checkpointsDir, archiveDir string, from int64, deleteInstead bool) (int, error) {
-	entries1, err := os.ReadDir(checkpointsDir)
-	if err != nil {
-		return 0, err
-	}
-
-	type workItem struct {
-		cdir, adir    string
-		cluster, host string
-	}
-
-	var wg sync.WaitGroup
-	n, errs := int32(0), int32(0)
-	work := make(chan workItem, NumWorkers)
-
-	wg.Add(NumWorkers)
-	for worker := 0; worker < NumWorkers; worker++ {
-		go func() {
-			defer wg.Done()
-			for workItem := range work {
-				m, err := archiveCheckpoints(workItem.cdir, workItem.adir, from, deleteInstead)
-				if err != nil {
-					log.Printf("error while archiving %s/%s: %s", workItem.cluster, workItem.host, err.Error())
-					atomic.AddInt32(&errs, 1)
-				}
-				atomic.AddInt32(&n, int32(m))
-			}
-		}()
-	}
-
-	for _, de1 := range entries1 {
-		entries2, e := os.ReadDir(filepath.Join(checkpointsDir, de1.Name()))
-		if e != nil {
-			err = e
-		}
-
-		for _, de2 := range entries2 {
-			cdir := filepath.Join(checkpointsDir, de1.Name(), de2.Name())
-			adir := filepath.Join(archiveDir, de1.Name(), de2.Name())
-			work <- workItem{
-				adir: adir, cdir: cdir,
-				cluster: de1.Name(), host: de2.Name(),
-			}
-		}
-	}
-
-	close(work)
-	wg.Wait()
-
-	if err != nil {
-		return int(n), err
-	}
-
-	if errs > 0 {
-		return int(n), fmt.Errorf("%d errors happend while archiving (%d successes)", errs, n)
-	}
-	return int(n), nil
-}
-
-// Helper function for `ArchiveCheckpoints`.
-func archiveCheckpoints(dir string, archiveDir string, from int64, deleteInstead bool) (int, error) {
-	entries, err := os.ReadDir(dir)
-	if err != nil {
-		return 0, err
-	}
-
-	files, err := findFiles(entries, from, false)
-	if err != nil {
-		return 0, err
-	}
-
-	if deleteInstead {
-		n := 0
-		for _, checkpoint := range files {
-			filename := filepath.Join(dir, checkpoint)
-			if err = os.Remove(filename); err != nil {
-				return n, err
-			}
-			n += 1
-		}
-		return n, nil
-	}
-
-	filename := filepath.Join(archiveDir, fmt.Sprintf("%d.zip", from))
-	f, err := os.OpenFile(filename, os.O_CREATE|os.O_WRONLY, 0644)
-	if err != nil && os.IsNotExist(err) {
-		err = os.MkdirAll(archiveDir, 0755)
-		if err == nil {
-			f, err = os.OpenFile(filename, os.O_CREATE|os.O_WRONLY, 0644)
-		}
-	}
-	if err != nil {
-		return 0, err
-	}
-	defer f.Close()
-	bw := bufio.NewWriter(f)
-	defer bw.Flush()
-	zw := zip.NewWriter(bw)
-	defer zw.Close()
-
-	n := 0
-	for _, checkpoint := range files {
-		filename := filepath.Join(dir, checkpoint)
-		r, err := os.Open(filename)
-		if err != nil {
-			return n, err
-		}
-		defer r.Close()
-
-		w, err := zw.Create(checkpoint)
-		if err != nil {
-			return n, err
-		}
-
-		if _, err = io.Copy(w, r); err != nil {
-			return n, err
-		}
-
-		if err = os.Remove(filename); err != nil {
-			return n, err
-		}
-		n += 1
-	}
-
-	return n, nil
-}

internal/memorystore/debug.go

@@ -1,4 +1,4 @@
-package main
+package memorystore
 
 import (
 	"bufio"
@@ -29,7 +29,7 @@ func (b *buffer) debugDump(buf []byte) []byte {
 	return buf
 }
 
-func (l *level) debugDump(m *MemoryStore, w *bufio.Writer, lvlname string, buf []byte, depth int) ([]byte, error) {
+func (l *Level) debugDump(m *MemoryStore, w *bufio.Writer, lvlname string, buf []byte, depth int) ([]byte, error) {
 	l.lock.RLock()
 	defer l.lock.RUnlock()
 	for i := 0; i < depth; i++ {
@@ -40,8 +40,8 @@ func (l *level) debugDump(m *MemoryStore, w *bufio.Writer, lvlname string, buf [
 	buf = append(buf, "\":{\n"...)
 	depth += 1
 	objitems := 0
-	for name, mc := range m.metrics {
+	for name, mc := range m.Metrics {
-		if b := l.metrics[mc.offset]; b != nil {
+		if b := l.metrics[mc.Offset]; b != nil {
 			for i := 0; i < depth; i++ {
 				buf = append(buf, '\t')
 			}

internal/memorystore/level.go

@@ -0,0 +1,183 @@
+package memorystore
+
+import (
+	"sync"
+	"unsafe"
+
+	"github.com/ClusterCockpit/cc-metric-store/internal/util"
+)
+
+// Could also be called "node" as this forms a node in a tree structure.
+// Called Level because "node" might be confusing here.
+// Can be both a leaf or a inner node. In this tree structue, inner nodes can
+// also hold data (in `metrics`).
+type Level struct {
+	children map[string]*Level
+	metrics  []*buffer
+	lock     sync.RWMutex
+}
+
+// Find the correct level for the given selector, creating it if
+// it does not exist. Example selector in the context of the
+// ClusterCockpit could be: []string{ "emmy", "host123", "cpu0" }.
+// This function would probably benefit a lot from `level.children` beeing a `sync.Map`?
+func (l *Level) findLevelOrCreate(selector []string, nMetrics int) *Level {
+	if len(selector) == 0 {
+		return l
+	}
+
+	// Allow concurrent reads:
+	l.lock.RLock()
+	var child *Level
+	var ok bool
+	if l.children == nil {
+		// Children map needs to be created...
+		l.lock.RUnlock()
+	} else {
+		child, ok := l.children[selector[0]]
+		l.lock.RUnlock()
+		if ok {
+			return child.findLevelOrCreate(selector[1:], nMetrics)
+		}
+	}
+
+	// The level does not exist, take write lock for unqiue access:
+	l.lock.Lock()
+	// While this thread waited for the write lock, another thread
+	// could have created the child node.
+	if l.children != nil {
+		child, ok = l.children[selector[0]]
+		if ok {
+			l.lock.Unlock()
+			return child.findLevelOrCreate(selector[1:], nMetrics)
+		}
+	}
+
+	child = &Level{
+		metrics:  make([]*buffer, nMetrics),
+		children: nil,
+	}
+
+	if l.children != nil {
+		l.children[selector[0]] = child
+	} else {
+		l.children = map[string]*Level{selector[0]: child}
+	}
+	l.lock.Unlock()
+	return child.findLevelOrCreate(selector[1:], nMetrics)
+}
+
+func (l *Level) free(t int64) (int, error) {
+	l.lock.Lock()
+	defer l.lock.Unlock()
+
+	n := 0
+	for i, b := range l.metrics {
+		if b != nil {
+			delme, m := b.free(t)
+			n += m
+			if delme {
+				if cap(b.data) == BUFFER_CAP {
+					bufferPool.Put(b)
+				}
+				l.metrics[i] = nil
+			}
+		}
+	}
+
+	for _, l := range l.children {
+		m, err := l.free(t)
+		n += m
+		if err != nil {
+			return n, err
+		}
+	}
+
+	return n, nil
+}
+
+func (l *Level) sizeInBytes() int64 {
+	l.lock.RLock()
+	defer l.lock.RUnlock()
+	size := int64(0)
+
+	for _, b := range l.metrics {
+		if b != nil {
+			size += b.count() * int64(unsafe.Sizeof(util.Float(0)))
+		}
+	}
+
+	return size
+}
+
+func (l *Level) findLevel(selector []string) *Level {
+	if len(selector) == 0 {
+		return l
+	}
+
+	l.lock.RLock()
+	defer l.lock.RUnlock()
+
+	lvl := l.children[selector[0]]
+	if lvl == nil {
+		return nil
+	}
+
+	return lvl.findLevel(selector[1:])
+}
+
+func (l *Level) findBuffers(selector util.Selector, offset int, f func(b *buffer) error) error {
+	l.lock.RLock()
+	defer l.lock.RUnlock()
+
+	if len(selector) == 0 {
+		b := l.metrics[offset]
+		if b != nil {
+			return f(b)
+		}
+
+		for _, lvl := range l.children {
+			err := lvl.findBuffers(nil, offset, f)
+			if err != nil {
+				return err
+			}
+		}
+		return nil
+	}
+
+	sel := selector[0]
+	if len(sel.String) != 0 && l.children != nil {
+		lvl, ok := l.children[sel.String]
+		if ok {
+			err := lvl.findBuffers(selector[1:], offset, f)
+			if err != nil {
+				return err
+			}
+		}
+		return nil
+	}
+
+	if sel.Group != nil && l.children != nil {
+		for _, key := range sel.Group {
+			lvl, ok := l.children[key]
+			if ok {
+				err := lvl.findBuffers(selector[1:], offset, f)
+				if err != nil {
+					return err
+				}
+			}
+		}
+		return nil
+	}
+
+	if sel.Any && l.children != nil {
+		for _, lvl := range l.children {
+			if err := lvl.findBuffers(selector[1:], offset, f); err != nil {
+				return err
+			}
+		}
+		return nil
+	}
+
+	return nil
+}

internal/memorystore/memorystore.go

@@ -0,0 +1,283 @@
+package memorystore
+
+import (
+	"context"
+	"errors"
+	"log"
+	"runtime"
+	"sync"
+	"time"
+
+	"github.com/ClusterCockpit/cc-metric-store/internal/config"
+	"github.com/ClusterCockpit/cc-metric-store/internal/util"
+)
+
+var (
+	singleton  sync.Once
+	msInstance *MemoryStore
+)
+
+var NumWorkers int = 4
+
+func init() {
+	maxWorkers := 10
+	NumWorkers = runtime.NumCPU()/2 + 1
+	if NumWorkers > maxWorkers {
+		NumWorkers = maxWorkers
+	}
+}
+
+type Metric struct {
+	Name         string
+	Value        util.Float
+	MetricConfig config.MetricConfig
+}
+
+type MemoryStore struct {
+	Metrics map[string]config.MetricConfig
+	root    Level
+}
+
+// Create a new, initialized instance of a MemoryStore.
+// Will panic if values in the metric configurations are invalid.
+func Init(metrics map[string]config.MetricConfig) {
+	singleton.Do(func() {
+		offset := 0
+		for key, cfg := range metrics {
+			if cfg.Frequency == 0 {
+				panic("invalid frequency")
+			}
+
+			metrics[key] = config.MetricConfig{
+				Frequency:   cfg.Frequency,
+				Aggregation: cfg.Aggregation,
+				Offset:      offset,
+			}
+			offset += 1
+		}
+
+		msInstance = &MemoryStore{
+			root: Level{
+				metrics:  make([]*buffer, len(metrics)),
+				children: make(map[string]*Level),
+			},
+			Metrics: metrics,
+		}
+	})
+}
+
+func GetMemoryStore() *MemoryStore {
+	if msInstance == nil {
+		log.Fatalf("MemoryStore not initialized!")
+	}
+
+	return msInstance
+}
+
+func Shutdown() {
+	ms := GetMemoryStore()
+	log.Printf("Writing to '%s'...\n", config.Keys.Checkpoints.RootDir)
+	files, err := ms.ToCheckpoint(config.Keys.Checkpoints.RootDir, lastCheckpoint.Unix(), time.Now().Unix())
+	if err != nil {
+		log.Printf("Writing checkpoint failed: %s\n", err.Error())
+	}
+	log.Printf("Done! (%d files written)\n", files)
+}
+
+func Retention(wg *sync.WaitGroup, ctx context.Context) {
+	ms := GetMemoryStore()
+
+	go func() {
+		defer wg.Done()
+		d, err := time.ParseDuration(config.Keys.RetentionInMemory)
+		if err != nil {
+			log.Fatal(err)
+		}
+		if d <= 0 {
+			return
+		}
+
+		ticks := func() <-chan time.Time {
+			d := d / 2
+			if d <= 0 {
+				return nil
+			}
+			return time.NewTicker(d).C
+		}()
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			case <-ticks:
+				t := time.Now().Add(-d)
+				log.Printf("start freeing buffers (older than %s)...\n", t.Format(time.RFC3339))
+				freed, err := ms.Free(nil, t.Unix())
+				if err != nil {
+					log.Printf("freeing up buffers failed: %s\n", err.Error())
+				} else {
+					log.Printf("done: %d buffers freed\n", freed)
+				}
+			}
+		}
+	}()
+}
+
+// Write all values in `metrics` to the level specified by `selector` for time `ts`.
+// Look at `findLevelOrCreate` for how selectors work.
+func (m *MemoryStore) Write(selector []string, ts int64, metrics []Metric) error {
+	var ok bool
+	for i, metric := range metrics {
+		if metric.MetricConfig.Frequency == 0 {
+			metric.MetricConfig, ok = m.Metrics[metric.Name]
+			if !ok {
+				metric.MetricConfig.Frequency = 0
+			}
+			metrics[i] = metric
+		}
+	}
+
+	return m.WriteToLevel(&m.root, selector, ts, metrics)
+}
+
+func (m *MemoryStore) GetLevel(selector []string) *Level {
+	return m.root.findLevelOrCreate(selector, len(m.Metrics))
+}
+
+// Assumes that `minfo` in `metrics` is filled in!
+func (m *MemoryStore) WriteToLevel(l *Level, selector []string, ts int64, metrics []Metric) error {
+	l = l.findLevelOrCreate(selector, len(m.Metrics))
+	l.lock.Lock()
+	defer l.lock.Unlock()
+
+	for _, metric := range metrics {
+		if metric.MetricConfig.Frequency == 0 {
+			continue
+		}
+
+		b := l.metrics[metric.MetricConfig.Offset]
+		if b == nil {
+			// First write to this metric and level
+			b = newBuffer(ts, metric.MetricConfig.Frequency)
+			l.metrics[metric.MetricConfig.Offset] = b
+		}
+
+		nb, err := b.write(ts, metric.Value)
+		if err != nil {
+			return err
+		}
+
+		// Last write created a new buffer...
+		if b != nb {
+			l.metrics[metric.MetricConfig.Offset] = nb
+		}
+	}
+	return nil
+}
+
+// Returns all values for metric `metric` from `from` to `to` for the selected level(s).
+// If the level does not hold the metric itself, the data will be aggregated recursively from the children.
+// The second and third return value are the actual from/to for the data. Those can be different from
+// the range asked for if no data was available.
+func (m *MemoryStore) Read(selector util.Selector, metric string, from, to int64) ([]util.Float, int64, int64, error) {
+	if from > to {
+		return nil, 0, 0, errors.New("invalid time range")
+	}
+
+	minfo, ok := m.Metrics[metric]
+	if !ok {
+		return nil, 0, 0, errors.New("unkown metric: " + metric)
+	}
+
+	n, data := 0, make([]util.Float, (to-from)/minfo.Frequency+1)
+	err := m.root.findBuffers(selector, minfo.Offset, func(b *buffer) error {
+		cdata, cfrom, cto, err := b.read(from, to, data)
+		if err != nil {
+			return err
+		}
+
+		if n == 0 {
+			from, to = cfrom, cto
+		} else if from != cfrom || to != cto || len(data) != len(cdata) {
+			missingfront, missingback := int((from-cfrom)/minfo.Frequency), int((to-cto)/minfo.Frequency)
+			if missingfront != 0 {
+				return ErrDataDoesNotAlign
+			}
+
+			newlen := len(cdata) - missingback
+			if newlen < 1 {
+				return ErrDataDoesNotAlign
+			}
+			cdata = cdata[0:newlen]
+			if len(cdata) != len(data) {
+				return ErrDataDoesNotAlign
+			}
+
+			from, to = cfrom, cto
+		}
+
+		data = cdata
+		n += 1
+		return nil
+	})
+
+	if err != nil {
+		return nil, 0, 0, err
+	} else if n == 0 {
+		return nil, 0, 0, errors.New("metric or host not found")
+	} else if n > 1 {
+		if minfo.Aggregation == config.AvgAggregation {
+			normalize := 1. / util.Float(n)
+			for i := 0; i < len(data); i++ {
+				data[i] *= normalize
+			}
+		} else if minfo.Aggregation != config.SumAggregation {
+			return nil, 0, 0, errors.New("invalid aggregation")
+		}
+	}
+
+	return data, from, to, nil
+}
+
+// Release all buffers for the selected level and all its children that contain only
+// values older than `t`.
+func (m *MemoryStore) Free(selector []string, t int64) (int, error) {
+	return m.GetLevel(selector).free(t)
+}
+
+func (m *MemoryStore) FreeAll() error {
+	for k := range m.root.children {
+		delete(m.root.children, k)
+	}
+
+	return nil
+}
+
+func (m *MemoryStore) SizeInBytes() int64 {
+	return m.root.sizeInBytes()
+}
+
+// Given a selector, return a list of all children of the level selected.
+func (m *MemoryStore) ListChildren(selector []string) []string {
+	lvl := &m.root
+	for lvl != nil && len(selector) != 0 {
+		lvl.lock.RLock()
+		next := lvl.children[selector[0]]
+		lvl.lock.RUnlock()
+		lvl = next
+		selector = selector[1:]
+	}
+
+	if lvl == nil {
+		return nil
+	}
+
+	lvl.lock.RLock()
+	defer lvl.lock.RUnlock()
+
+	children := make([]string, 0, len(lvl.children))
+	for child := range lvl.children {
+		children = append(children, child)
+	}
+
+	return children
+}

internal/memorystore/stats.go

@@ -1,15 +1,18 @@
-package main
+package memorystore
 
 import (
 	"errors"
 	"math"
+
+	"github.com/ClusterCockpit/cc-metric-store/internal/config"
+	"github.com/ClusterCockpit/cc-metric-store/internal/util"
 )
 
 type Stats struct {
 	Samples int
-	Avg     Float
+	Avg     util.Float
-	Min     Float
+	Min     util.Float
-	Max     Float
+	Max     util.Float
 }
 
 func (b *buffer) stats(from, to int64) (Stats, int64, int64, error) {
@@ -54,28 +57,28 @@ func (b *buffer) stats(from, to int64) (Stats, int64, int64, error) {
 
 	return Stats{
 		Samples: samples,
-		Avg:     Float(sum) / Float(samples),
+		Avg:     util.Float(sum) / util.Float(samples),
-		Min:     Float(min),
+		Min:     util.Float(min),
-		Max:     Float(max),
+		Max:     util.Float(max),
 	}, from, t, nil
 }
 
 // Returns statistics for the requested metric on the selected node/level.
 // Data is aggregated to the selected level the same way as in `MemoryStore.Read`.
 // If `Stats.Samples` is zero, the statistics should not be considered as valid.
-func (m *MemoryStore) Stats(selector Selector, metric string, from, to int64) (*Stats, int64, int64, error) {
+func (m *MemoryStore) Stats(selector util.Selector, metric string, from, to int64) (*Stats, int64, int64, error) {
 	if from > to {
 		return nil, 0, 0, errors.New("invalid time range")
 	}
 
-	minfo, ok := m.metrics[metric]
+	minfo, ok := m.Metrics[metric]
 	if !ok {
 		return nil, 0, 0, errors.New("unkown metric: " + metric)
 	}
 
 	n, samples := 0, 0
-	avg, min, max := Float(0), math.MaxFloat32, -math.MaxFloat32
+	avg, min, max := util.Float(0), math.MaxFloat32, -math.MaxFloat32
-	err := m.root.findBuffers(selector, minfo.offset, func(b *buffer) error {
+	err := m.root.findBuffers(selector, minfo.Offset, func(b *buffer) error {
 		stats, cfrom, cto, err := b.stats(from, to)
 		if err != nil {
 			return err
@@ -94,7 +97,6 @@ func (m *MemoryStore) Stats(selector Selector, metric string, from, to int64) (*
 		n += 1
 		return nil
 	})
-
 	if err != nil {
 		return nil, 0, 0, err
 	}
@@ -103,16 +105,16 @@ func (m *MemoryStore) Stats(selector Selector, metric string, from, to int64) (*
 		return nil, 0, 0, ErrNoData
 	}
 
-	if minfo.Aggregation == AvgAggregation {
+	if minfo.Aggregation == config.AvgAggregation {
-		avg /= Float(n)
+		avg /= util.Float(n)
-	} else if n > 1 && minfo.Aggregation != SumAggregation {
+	} else if n > 1 && minfo.Aggregation != config.SumAggregation {
 		return nil, 0, 0, errors.New("invalid aggregation")
 	}
 
 	return &Stats{
 		Samples: samples,
 		Avg:     avg,
-		Min:     Float(min),
+		Min:     util.Float(min),
-		Max:     Float(max),
+		Max:     util.Float(max),
 	}, from, to, nil
 }

internal/util/float.go

@@ -1,4 +1,4 @@
-package main
+package util
 
 import (
 	"math"
@@ -11,8 +11,10 @@ import (
 // we have to use our own type which implements encoding/json.Marshaler itself.
 type Float float64
 
-var NaN Float = Float(math.NaN())
+var (
-var nullAsBytes []byte = []byte("null")
+	NaN         Float  = Float(math.NaN())
+	nullAsBytes []byte = []byte("null")
+)
 
 func (f Float) IsNaN() bool {
 	return math.IsNaN(float64(f))
@@ -55,7 +57,6 @@ func (fa FloatArray) MarshalJSON() ([]byte, error) {
 			buf = append(buf, `null`...)
 		} else {
 			buf = strconv.AppendFloat(buf, float64(fa[i]), 'f', 3, 64)
-
 		}
 	}
 	buf = append(buf, ']')

internal/util/selector.go

@@ -0,0 +1,51 @@
+package util
+
+import (
+	"encoding/json"
+	"errors"
+)
+
+type SelectorElement struct {
+	String string
+	Group  []string
+	Any    bool
+}
+
+func (se *SelectorElement) UnmarshalJSON(input []byte) error {
+	if input[0] == '"' {
+		if err := json.Unmarshal(input, &se.String); err != nil {
+			return err
+		}
+
+		if se.String == "*" {
+			se.Any = true
+			se.String = ""
+		}
+
+		return nil
+	}
+
+	if input[0] == '[' {
+		return json.Unmarshal(input, &se.Group)
+	}
+
+	return errors.New("the Go SelectorElement type can only be a string or an array of strings")
+}
+
+func (se *SelectorElement) MarshalJSON() ([]byte, error) {
+	if se.Any {
+		return []byte("\"*\""), nil
+	}
+
+	if se.String != "" {
+		return json.Marshal(se.String)
+	}
+
+	if se.Group != nil {
+		return json.Marshal(se.Group)
+	}
+
+	return nil, errors.New("a Go Selector must be a non-empty string or a non-empty slice of strings")
+}
+
+type Selector []SelectorElement

lineprotocol_test.go

@@ -1,144 +0,0 @@
-package main
-
-import (
-	"bytes"
-	"log"
-	"strconv"
-	"testing"
-
-	"github.com/influxdata/line-protocol/v2/lineprotocol"
-)
-
-const TestDataClassicFormat string = `
-m1,cluster=ctest,hostname=htest1,type=node value=1 123456789
-m2,cluster=ctest,hostname=htest1,type=node value=2 123456789
-m3,hostname=htest2,type=node value=3 123456789
-m4,cluster=ctest,hostname=htest2,type=core,type-id=1 value=4 123456789
-m4,cluster=ctest,hostname=htest2,type-id=2,type=core value=5 123456789
-`
-
-const BenchmarkLineBatch string = `
-nm1,cluster=ctest,hostname=htest1,type=node value=123.0 123456789
-nm2,cluster=ctest,hostname=htest1,type=node value=123.0 123456789
-nm3,cluster=ctest,hostname=htest1,type=node value=123.0 123456789
-nm4,cluster=ctest,hostname=htest1,type=node value=123.0 123456789
-nm5,cluster=ctest,hostname=htest1,type=node value=123.0 123456789
-nm6,cluster=ctest,hostname=htest1,type=node value=123.0 123456789
-nm7,cluster=ctest,hostname=htest1,type=node value=123.0 123456789
-nm8,cluster=ctest,hostname=htest1,type=node value=123.0 123456789
-nm9,cluster=ctest,hostname=htest1,type=node value=123.0 123456789
-cm1,cluster=ctest,hostname=htest1,type=core,type-id=1 value=234.0 123456789
-cm2,cluster=ctest,hostname=htest1,type=core,type-id=1 value=234.0 123456789
-cm3,cluster=ctest,hostname=htest1,type=core,type-id=1 value=234.0 123456789
-cm4,cluster=ctest,hostname=htest1,type=core,type-id=1 value=234.0 123456789
-cm5,cluster=ctest,hostname=htest1,type=core,type-id=1 value=234.0 123456789
-cm6,cluster=ctest,hostname=htest1,type=core,type-id=1 value=234.0 123456789
-cm7,cluster=ctest,hostname=htest1,type=core,type-id=1 value=234.0 123456789
-cm8,cluster=ctest,hostname=htest1,type=core,type-id=1 value=234.0 123456789
-cm9,cluster=ctest,hostname=htest1,type=core,type-id=1 value=234.0 123456789
-cm1,cluster=ctest,hostname=htest1,type=core,type-id=2 value=345.0 123456789
-cm2,cluster=ctest,hostname=htest1,type=core,type-id=2 value=345.0 123456789
-cm3,cluster=ctest,hostname=htest1,type=core,type-id=2 value=345.0 123456789
-cm4,cluster=ctest,hostname=htest1,type=core,type-id=2 value=345.0 123456789
-cm5,cluster=ctest,hostname=htest1,type=core,type-id=2 value=345.0 123456789
-cm6,cluster=ctest,hostname=htest1,type=core,type-id=2 value=345.0 123456789
-cm7,cluster=ctest,hostname=htest1,type=core,type-id=2 value=345.0 123456789
-cm8,cluster=ctest,hostname=htest1,type=core,type-id=2 value=345.0 123456789
-cm9,cluster=ctest,hostname=htest1,type=core,type-id=2 value=345.0 123456789
-cm1,cluster=ctest,hostname=htest1,type=core,type-id=3 value=456.0 123456789
-cm2,cluster=ctest,hostname=htest1,type=core,type-id=3 value=456.0 123456789
-cm3,cluster=ctest,hostname=htest1,type=core,type-id=3 value=456.0 123456789
-cm4,cluster=ctest,hostname=htest1,type=core,type-id=3 value=456.0 123456789
-cm5,cluster=ctest,hostname=htest1,type=core,type-id=3 value=456.0 123456789
-cm6,cluster=ctest,hostname=htest1,type=core,type-id=3 value=456.0 123456789
-cm7,cluster=ctest,hostname=htest1,type=core,type-id=3 value=456.0 123456789
-cm8,cluster=ctest,hostname=htest1,type=core,type-id=3 value=456.0 123456789
-cm9,cluster=ctest,hostname=htest1,type=core,type-id=3 value=456.0 123456789
-cm1,cluster=ctest,hostname=htest1,type=core,type-id=4 value=567.0 123456789
-cm2,cluster=ctest,hostname=htest1,type=core,type-id=4 value=567.0 123456789
-cm3,cluster=ctest,hostname=htest1,type=core,type-id=4 value=567.0 123456789
-cm4,cluster=ctest,hostname=htest1,type=core,type-id=4 value=567.0 123456789
-cm5,cluster=ctest,hostname=htest1,type=core,type-id=4 value=567.0 123456789
-cm6,cluster=ctest,hostname=htest1,type=core,type-id=4 value=567.0 123456789
-cm7,cluster=ctest,hostname=htest1,type=core,type-id=4 value=567.0 123456789
-cm8,cluster=ctest,hostname=htest1,type=core,type-id=4 value=567.0 123456789
-cm9,cluster=ctest,hostname=htest1,type=core,type-id=4 value=567.0 123456789
-`
-
-func TestLineprotocolDecoder(t *testing.T) {
-	prevMemoryStore := memoryStore
-	t.Cleanup(func() {
-		memoryStore = prevMemoryStore
-	})
-
-	memoryStore = NewMemoryStore(map[string]MetricConfig{
-		"m1": {Frequency: 1},
-		"m2": {Frequency: 1},
-		"m3": {Frequency: 1},
-		"m4": {Frequency: 1},
-	})
-
-	dec := lineprotocol.NewDecoderWithBytes([]byte(TestDataClassicFormat))
-	if err := decodeLine(dec, "ctest"); err != nil {
-		log.Fatal(err)
-	}
-
-	// memoryStore.DebugDump(bufio.NewWriter(os.Stderr))
-
-	h1 := memoryStore.GetLevel([]string{"ctest", "htest1"})
-	h1b1 := h1.metrics[memoryStore.metrics["m1"].offset]
-	h1b2 := h1.metrics[memoryStore.metrics["m2"].offset]
-	if h1b1.data[0] != 1.0 || h1b2.data[0] != 2.0 {
-		log.Fatal()
-	}
-
-	h2 := memoryStore.GetLevel([]string{"ctest", "htest2"})
-	h2b3 := h2.metrics[memoryStore.metrics["m3"].offset]
-	if h2b3.data[0] != 3.0 {
-		log.Fatal()
-	}
-
-	h2c1 := memoryStore.GetLevel([]string{"ctest", "htest2", "core1"})
-	h2c1b4 := h2c1.metrics[memoryStore.metrics["m4"].offset]
-	h2c2 := memoryStore.GetLevel([]string{"ctest", "htest2", "core2"})
-	h2c2b4 := h2c2.metrics[memoryStore.metrics["m4"].offset]
-	if h2c1b4.data[0] != 4.0 || h2c2b4.data[0] != 5.0 {
-		log.Fatal()
-	}
-}
-
-func BenchmarkLineprotocolDecoder(b *testing.B) {
-	b.StopTimer()
-	memoryStore = NewMemoryStore(map[string]MetricConfig{
-		"nm1": {Frequency: 1},
-		"nm2": {Frequency: 1},
-		"nm3": {Frequency: 1},
-		"nm4": {Frequency: 1},
-		"nm5": {Frequency: 1},
-		"nm6": {Frequency: 1},
-		"nm7": {Frequency: 1},
-		"nm8": {Frequency: 1},
-		"nm9": {Frequency: 1},
-		"cm1": {Frequency: 1},
-		"cm2": {Frequency: 1},
-		"cm3": {Frequency: 1},
-		"cm4": {Frequency: 1},
-		"cm5": {Frequency: 1},
-		"cm6": {Frequency: 1},
-		"cm7": {Frequency: 1},
-		"cm8": {Frequency: 1},
-		"cm9": {Frequency: 1},
-	})
-
-	for i := 0; i < b.N; i++ {
-		data := []byte(BenchmarkLineBatch)
-		data = bytes.ReplaceAll(data, []byte("123456789"), []byte(strconv.Itoa(i+123456789)))
-		dec := lineprotocol.NewDecoderWithBytes(data)
-
-		b.StartTimer()
-		if err := decodeLine(dec, "ctest"); err != nil {
-			b.Fatal(err)
-		}
-		b.StopTimer()
-	}
-}

memoryStore_test.go.orig

@@ -1,504 +0,0 @@
-package main
-
-import (
-	"fmt"
-	"log"
-	"math"
-	"testing"
-
-	"github.com/ClusterCockpit/cc-metric-store/lineprotocol"
-)
-
-var testMetrics [][]lineprotocol.Metric = [][]lineprotocol.Metric{
-	{{"flops", 100.5}, {"mem_bw", 2088.67}},
-	{{"flops", 180.5}, {"mem_bw", 4078.32}, {"mem_capacity", 1020}},
-	{{"flops", 980.5}, {"mem_bw", 9078.32}, {"mem_capacity", 5010}},
-	{{"flops", 940.5}, {"mem_bw", 9278.32}, {"mem_capacity", 6010}},
-	{{"flops", 930.5}, {"mem_bw", 9378.32}, {"mem_capacity", 7010}},
-	{{"flops", 980.5}, {"mem_bw", 9478.32}, {"mem_capacity", 8010}},
-	{{"flops", 980.5}, {"mem_bw", 9478.32}, {"mem_capacity", 8010}},
-	{{"flops", 980.5}, {"mem_bw", 9478.32}, {"mem_capacity", 8010}},
-	{{"flops", 970.5}, {"mem_bw", 9178.32}, {"mem_capacity", 2010}},
-	{{"flops", 970.5}, {"mem_bw", 9178.32}, {"mem_capacity", 2010}}}
-
-var testMetricsAlt [][]lineprotocol.Metric = [][]lineprotocol.Metric{
-	{{"flops", 120.5}, {"mem_bw", 2080.67}},
-	{{"flops", 130.5}, {"mem_bw", 4071.32}, {"mem_capacity", 1120}},
-	{{"flops", 940.5}, {"mem_bw", 9072.32}, {"mem_capacity", 5210}},
-	{{"flops", 950.5}, {"mem_bw", 9273.32}, {"mem_capacity", 6310}},
-	{{"flops", 960.5}, {"mem_bw", 9374.32}, {"mem_capacity", 7410}},
-	{{"flops", 970.5}, {"mem_bw", 9475.32}, {"mem_capacity", 8510}},
-	{{"flops", 990.5}, {"mem_bw", 9476.32}, {"mem_capacity", 8610}},
-	{{"flops", 910.5}, {"mem_bw", 9477.32}, {"mem_capacity", 8710}},
-	{{"flops", 920.5}, {"mem_bw", 9178.32}, {"mem_capacity", 2810}},
-	{{"flops", 930.5}, {"mem_bw", 9179.32}, {"mem_capacity", 2910}}}
-
-func dumpStoreBuffer(s *storeBuffer) {
-	log.Printf("Start TS %d\n", s.start)
-	ctr := 0
-
-	for _, val := range s.store {
-		fmt.Printf("%f\t", val)
-		ctr++
-
-		if ctr == 10 {
-			fmt.Printf("\n")
-			ctr = 0
-		}
-	}
-}
-
-func printMemStore(m *MemoryStore) {
-	log.Println("########################")
-	log.Printf("Frequency %d, Metrics %d Slots %d\n",
-		m.frequency, m.numMetrics, m.numSlots)
-	log.Println("##Offsets")
-	for key, val := range m.offsets {
-		log.Printf("\t%s = %d\n", key, val)
-	}
-	log.Println("##Containers")
-	for key, c := range m.containers {
-		log.Printf("ID %s\n", key)
-		log.Println("###current")
-		dumpStoreBuffer(c.current)
-		log.Println("###next")
-		dumpStoreBuffer(c.next)
-	}
-	log.Println("########################")
-}
-
-//############################
-//#### Whitebox tests ########
-//############################
-func TestAddMetricSimple(t *testing.T) {
-	key := "m1220"
-	m := newMemoryStore([]string{"flops", "mem_bw", "mem_capacity"}, 10, 60)
-	// printMemStore(m)
-
-	m.AddMetrics(key, 1584022800, testMetrics[0])
-	m.AddMetrics(key, 1584022890, testMetrics[1])
-
-	want := testMetrics[0][0].Value
-	got := m.containers[key].current.store[0]
-	if got != want {
-		t.Errorf("Want %f got %f\n", want, got)
-	}
-
-	want = testMetrics[1][2].Value
-	got = m.containers[key].current.store[21]
-	if got != want {
-		t.Errorf("Want %f got %f\n", want, got)
-	}
-	// printMemStore(m)
-}
-
-func TestAddMetricReplace(t *testing.T) {
-	key := "m1220"
-	m := newMemoryStore([]string{"flops", "mem_bw", "mem_capacity"}, 10, 60)
-	// printMemStore(m)
-
-	m.AddMetrics(key, 1584022800, testMetrics[0])
-	m.AddMetrics(key, 1584022800, testMetrics[1])
-
-	want := testMetrics[1][0].Value
-	got := m.containers[key].current.store[0]
-	if got != want {
-		t.Errorf("Want %f got %f\n", want, got)
-	}
-
-	m.AddMetrics(key, 1584022850, testMetrics[0])
-	want = testMetrics[0][0].Value
-	got = m.containers[key].current.store[0]
-	if got != want {
-		t.Errorf("Want %f got %f\n", want, got)
-	}
-
-	m.AddMetrics(key, 1584022860, testMetrics[1])
-	want = testMetrics[0][0].Value
-	got = m.containers[key].current.store[0]
-	if got != want {
-		t.Errorf("Want %f got %f\n", want, got)
-	}
-	// printMemStore(m)
-}
-
-func TestAddMetricSwitch(t *testing.T) {
-	key := "m1220"
-	m := newMemoryStore([]string{"flops", "mem_bw", "mem_capacity"}, 10, 60)
-	// printMemStore(m)
-
-	m.AddMetrics(key, 1584023000, testMetrics[0])
-	m.AddMetrics(key, 1584023580, testMetrics[1])
-
-	want := testMetrics[1][2].Value
-	got := m.containers[key].current.store[29]
-	if got != want {
-		t.Errorf("Want %f got %f\n", want, got)
-	}
-
-	m.AddMetrics(key, 1584023600, testMetrics[2])
-	want = testMetrics[2][2].Value
-	got = m.containers[key].current.store[20]
-	if got != want {
-		t.Errorf("Want %f got %f\n", want, got)
-	}
-
-	// printMemStore(m)
-}
-
-//############################
-//#### Blackbox tests ########
-//############################
-
-func TestAddMetricOutOfBounds(t *testing.T) {
-	key := "m1220"
-	m := newMemoryStore([]string{"flops", "mem_bw", "mem_capacity"}, 30, 60)
-
-	err := m.AddMetrics(key, 1584023000, testMetrics[0])
-	if err != nil {
-		t.Errorf("Got error 1584023000\n")
-	}
-	err = m.AddMetrics(key, 1584026600, testMetrics[0])
-	if err == nil {
-		t.Errorf("Got no error 1584026600\n")
-	}
-	err = m.AddMetrics(key, 1584021580, testMetrics[1])
-	if err == nil {
-		t.Errorf("Got no error 1584021580\n")
-	}
-	err = m.AddMetrics(key, 1584024580, testMetrics[1])
-	if err != nil {
-		t.Errorf("Got error 1584024580\n")
-	}
-	err = m.AddMetrics(key, 1584091580, testMetrics[1])
-	if err == nil {
-		t.Errorf("Got no error 1584091580\n")
-	}
-	err = m.AddMetrics(key, 1584024780, testMetrics[0])
-	if err != nil {
-		t.Errorf("Got error 1584024780\n")
-	}
-}
-
-func TestGetMetricPlainCurrent(t *testing.T) {
-	key := "m1220"
-	m := newMemoryStore([]string{"flops", "mem_bw", "mem_capacity"}, 10, 60)
-
-	for i := 0; i < len(testMetrics); i++ {
-		m.AddMetrics(key, int64(1584023000+i*60), testMetrics[i])
-	}
-
-	// printMemStore(m)
-	val, tsFrom, err := m.GetMetric(key, "flops", 1584023000, 1584023560)
-
-	if err != nil {
-		t.Errorf("Got error\n")
-	}
-	if tsFrom != 1584023000 {
-		t.Errorf("Start ts differs: %d\n", tsFrom)
-	}
-	if len(val) != 9 {
-		t.Errorf("Want 9. Got %d\n", len(val))
-	}
-	if val[0] != 100.5 {
-		t.Errorf("Want 100.5 Got %f\n", val[0])
-	}
-	if val[8] != 970.5 {
-		t.Errorf("Want 970.5 Got %f\n", val[9])
-	}
-}
-
-func TestGetMetricPlainNext(t *testing.T) {
-	key := "m1220"
-	m := newMemoryStore([]string{"flops", "mem_bw", "mem_capacity"}, 10, 60)
-
-	for i := 0; i < len(testMetrics); i++ {
-		m.AddMetrics(key, int64(1584023000+i*60), testMetrics[i])
-	}
-	for i := 0; i < len(testMetrics); i++ {
-		m.AddMetrics(key, int64(1584023600+i*60), testMetricsAlt[i])
-	}
-
-	// printMemStore(m)
-	val, tsFrom, err := m.GetMetric(key, "flops", 1584023000, 1584023560)
-
-	if err != nil {
-		t.Errorf("Got error\n")
-	}
-	if tsFrom != 1584023000 {
-		t.Errorf("Start ts differs: %d\n", tsFrom)
-	}
-	if len(val) != 9 {
-		t.Errorf("Want 9. Got %d\n", len(val))
-	}
-	if val[0] != 100.5 {
-		t.Errorf("Want 100.5 Got %f\n", val[0])
-	}
-	if val[8] != 970.5 {
-		t.Errorf("Want 970.5 Got %f\n", val[9])
-	}
-}
-
-func TestGetMetricGap(t *testing.T) {
-	key := "m1220"
-	m := newMemoryStore([]string{"flops", "mem_bw", "mem_capacity"}, 10, 60)
-
-	for i := 0; i < len(testMetrics); i++ {
-		m.AddMetrics(key, int64(1584023000+i*120), testMetrics[i])
-	}
-
-	val, tsFrom, err := m.GetMetric(key, "flops", 1584023000, 1584023600)
-
-	if err != nil {
-		t.Errorf("Got error\n")
-	}
-	if tsFrom != 1584023000 {
-		t.Errorf("Start ts differs: %d\n", tsFrom)
-	}
-	if len(val) != 10 {
-		t.Errorf("Want 10. Got %d\n", len(val))
-	}
-	if val[0] != 100.5 {
-		t.Errorf("Want 100.5 Got %f\n", val[0])
-	}
-	if !math.IsNaN(float64(val[1])) {
-		t.Errorf("Want NaN Got %f\n", val[1])
-	}
-	if val[0] != 100.5 {
-		t.Errorf("Want 100.5 Got %f\n", val[0])
-	}
-
-	// fmt.Println(val)
-}
-
-func TestGetMetricSplit(t *testing.T) {
-	key := "m1220"
-	m := newMemoryStore([]string{"flops", "mem_bw", "mem_capacity"}, 10, 60)
-
-	for i := 0; i < len(testMetrics); i++ {
-		m.AddMetrics(key, int64(1584023000+i*60), testMetrics[i])
-	}
-	for i := 0; i < len(testMetrics); i++ {
-		m.AddMetrics(key, int64(1584023600+i*60), testMetricsAlt[i])
-	}
-
-	// printMemStore(m)
-
-	val, tsFrom, err := m.GetMetric(key, "flops", 1584023200, 1584023860)
-
-	if err != nil {
-		t.Errorf("Got error\n")
-	}
-	if tsFrom != 1584023200 {
-		t.Errorf("Start ts differs: %d\n", tsFrom)
-	}
-	if len(val) != 11 {
-		t.Errorf("Want 11. Got %d\n", len(val))
-	}
-	if val[0] != 940.5 {
-		t.Errorf("Want 940.5 Got %f\n", val[0])
-	}
-	if val[10] != 950.5 {
-		t.Errorf("Want 950.5 Got %f\n", val[0])
-	}
-}
-
-func TestGetMetricExceedNext(t *testing.T) {
-	key := "m1220"
-	m := newMemoryStore([]string{"flops", "mem_bw", "mem_capacity"}, 10, 60)
-
-	for i := 0; i < len(testMetrics); i++ {
-		m.AddMetrics(key, int64(1584023000+i*60), testMetrics[i])
-	}
-	for i := 0; i < len(testMetrics); i++ {
-		m.AddMetrics(key, int64(1584023600+i*60), testMetricsAlt[i])
-	}
-
-	// printMemStore(m)
-
-	val, tsFrom, err := m.GetMetric(key, "flops", 1584022800, 1584023400)
-
-	if err != nil {
-		t.Errorf("Got error\n")
-	}
-	if tsFrom != 1584023000 {
-		t.Errorf("Start ts differs: %d\n", tsFrom)
-	}
-	if len(val) != 6 {
-		t.Errorf("Want 6. Got %d\n", len(val))
-	}
-	if val[0] != 100.5 {
-		t.Errorf("Want 100.5 Got %f\n", val[0])
-	}
-	if val[5] != 980.5 {
-		t.Errorf("Want 980.5 Got %f\n", val[5])
-	}
-}
-
-func TestGetMetricExceedNextSplit(t *testing.T) {
-	key := "m1220"
-	m := newMemoryStore([]string{"flops", "mem_bw", "mem_capacity"}, 10, 60)
-
-	for i := 0; i < len(testMetrics); i++ {
-		m.AddMetrics(key, int64(1584023000+i*60), testMetrics[i])
-	}
-	for i := 0; i < len(testMetrics); i++ {
-		m.AddMetrics(key, int64(1584023600+i*60), testMetricsAlt[i])
-	}
-
-	// printMemStore(m)
-
-	val, tsFrom, err := m.GetMetric(key, "flops", 1584022800, 1584023900)
-
-	if err != nil {
-		t.Errorf("Got error\n")
-	}
-	if tsFrom != 1584023000 {
-		t.Errorf("Start ts differs: %d\n", tsFrom)
-	}
-	if len(val) != 15 {
-		t.Errorf("Want 14. Got %d\n", len(val))
-	}
-	if val[0] != 100.5 {
-		t.Errorf("Want 100.5 Got %f\n", val[0])
-	}
-	if val[14] != 960.5 {
-		t.Errorf("Want 960.5 Got %f\n", val[13])
-	}
-}
-
-func TestGetMetricExceedCurrent(t *testing.T) {
-	key := "m1220"
-	m := newMemoryStore([]string{"flops", "mem_bw", "mem_capacity"}, 10, 60)
-
-	for i := 0; i < len(testMetrics); i++ {
-		m.AddMetrics(key, int64(1584023000+i*60), testMetrics[i])
-	}
-	for i := 0; i < len(testMetrics); i++ {
-		m.AddMetrics(key, int64(1584023600+i*60), testMetricsAlt[i])
-	}
-
-	// printMemStore(m)
-
-	val, tsFrom, err := m.GetMetric(key, "flops", 1584023800, 1584027900)
-
-	if err != nil {
-		t.Errorf("Got error\n")
-	}
-	if tsFrom != 1584023800 {
-		t.Errorf("Start ts differs: %d\n", tsFrom)
-	}
-	if len(val) != 7 {
-		t.Errorf("Want 6. Got %d\n", len(val))
-	}
-	if val[0] != 950.5 {
-		t.Errorf("Want 950.5 Got %f\n", val[0])
-	}
-	if val[6] != 930.5 {
-		t.Errorf("Want 930.5 Got %f\n", val[5])
-	}
-}
-
-func TestGetMetricExceedCurrentSplit(t *testing.T) {
-	key := "m1220"
-	m := newMemoryStore([]string{"flops", "mem_bw", "mem_capacity"}, 10, 60)
-
-	for i := 0; i < len(testMetrics); i++ {
-		m.AddMetrics(key, int64(1584023000+i*60), testMetrics[i])
-	}
-	for i := 0; i < len(testMetrics); i++ {
-		m.AddMetrics(key, int64(1584023600+i*60), testMetricsAlt[i])
-	}
-
-	// printMemStore(m)
-
-	val, tsFrom, err := m.GetMetric(key, "flops", 1584023120, 1584027900)
-
-	if err != nil {
-		t.Errorf("Got error\n")
-	}
-	if tsFrom != 1584023120 {
-		t.Errorf("Start ts differs: %d\n", tsFrom)
-	}
-	if len(val) != 18 {
-		t.Errorf("Want 18. Got %d\n", len(val))
-	}
-	if val[0] != 980.5 {
-		t.Errorf("Want 950.5 Got %f\n", val[0])
-	}
-	if val[17] != 930.5 {
-		t.Errorf("Want 930.5 Got %f\n", val[17])
-	}
-}
-
-func TestGetMetricExceedBoth(t *testing.T) {
-	key := "m1220"
-	m := newMemoryStore([]string{"flops", "mem_bw", "mem_capacity"}, 10, 60)
-
-	for i := 0; i < len(testMetrics); i++ {
-		m.AddMetrics(key, int64(1584023000+i*60), testMetrics[i])
-	}
-	for i := 0; i < len(testMetrics); i++ {
-		m.AddMetrics(key, int64(1584023600+i*60), testMetricsAlt[i])
-	}
-
-	// printMemStore(m)
-
-	val, tsFrom, err := m.GetMetric(key, "flops", 1584022800, 1584027900)
-
-	if err != nil {
-		t.Errorf("Got error\n")
-	}
-	if tsFrom != 1584023000 {
-		t.Errorf("Start ts differs: %d\n", tsFrom)
-	}
-	if len(val) != 20 {
-		t.Errorf("Want 20. Got %d\n", len(val))
-	}
-	if val[0] != 100.5 {
-		t.Errorf("Want 950.5 Got %f\n", val[0])
-	}
-	if val[19] != 930.5 {
-		t.Errorf("Want 930.5 Got %f\n", val[17])
-	}
-}
-
-func TestGetMetricOutUpper(t *testing.T) {
-	key := "m1220"
-	m := newMemoryStore([]string{"flops", "mem_bw", "mem_capacity"}, 10, 60)
-
-	for i := 0; i < len(testMetrics); i++ {
-		m.AddMetrics(key, int64(1584023000+i*60), testMetrics[i])
-	}
-	for i := 0; i < len(testMetrics); i++ {
-		m.AddMetrics(key, int64(1584023600+i*60), testMetricsAlt[i])
-	}
-
-	// printMemStore(m)
-
-	_, _, err := m.GetMetric(key, "flops", 1584032800, 1584037900)
-
-	if err == nil {
-		t.Errorf("Got no error\n")
-	}
-}
-
-func TestGetMetricOutLower(t *testing.T) {
-	key := "m1220"
-	m := newMemoryStore([]string{"flops", "mem_bw", "mem_capacity"}, 10, 60)
-
-	for i := 0; i < len(testMetrics); i++ {
-		m.AddMetrics(key, int64(1584023000+i*60), testMetrics[i])
-	}
-	for i := 0; i < len(testMetrics); i++ {
-		m.AddMetrics(key, int64(1584023600+i*60), testMetricsAlt[i])
-	}
-
-	// printMemStore(m)
-
-	_, _, err := m.GetMetric(key, "flops", 1584002800, 1584007900)
-
-	if err == nil {
-		t.Errorf("Got no error\n")
-	}
-}

memstore.go

@@ -1,538 +0,0 @@
-package main
-
-import (
-	"errors"
-	"sync"
-	"unsafe"
-)
-
-// Default buffer capacity.
-// `buffer.data` will only ever grow up to it's capacity and a new link
-// in the buffer chain will be created if needed so that no copying
-// of data or reallocation needs to happen on writes.
-const (
-	BUFFER_CAP int = 512
-)
-
-// So that we can reuse allocations
-var bufferPool sync.Pool = sync.Pool{
-	New: func() interface{} {
-		return &buffer{
-			data: make([]Float, 0, BUFFER_CAP),
-		}
-	},
-}
-
-var (
-	ErrNoData           error = errors.New("no data for this metric/level")
-	ErrDataDoesNotAlign error = errors.New("data from lower granularities does not align")
-)
-
-// Each metric on each level has it's own buffer.
-// This is where the actual values go.
-// If `cap(data)` is reached, a new buffer is created and
-// becomes the new head of a buffer list.
-type buffer struct {
-	frequency  int64   // Time between two "slots"
-	start      int64   // Timestamp of when `data[0]` was written.
-	data       []Float // The slice should never reallocacte as `cap(data)` is respected.
-	prev, next *buffer // `prev` contains older data, `next` newer data.
-	archived   bool    // If true, this buffer is already archived
-
-	closed bool
-	/*
-		statisticts struct {
-			samples int
-			min     Float
-			max     Float
-			avg     Float
-		}
-	*/
-}
-
-func newBuffer(ts, freq int64) *buffer {
-	b := bufferPool.Get().(*buffer)
-	b.frequency = freq
-	b.start = ts - (freq / 2)
-	b.prev = nil
-	b.next = nil
-	b.archived = false
-	b.closed = false
-	b.data = b.data[:0]
-	return b
-}
-
-// If a new buffer was created, the new head is returnd.
-// Otherwise, the existing buffer is returnd.
-// Normaly, only "newer" data should be written, but if the value would
-// end up in the same buffer anyways it is allowed.
-func (b *buffer) write(ts int64, value Float) (*buffer, error) {
-	if ts < b.start {
-		return nil, errors.New("cannot write value to buffer from past")
-	}
-
-	// idx := int((ts - b.start + (b.frequency / 3)) / b.frequency)
-	idx := int((ts - b.start) / b.frequency)
-	if idx >= cap(b.data) {
-		newbuf := newBuffer(ts, b.frequency)
-		newbuf.prev = b
-		b.next = newbuf
-		b.close()
-		b = newbuf
-		idx = 0
-	}
-
-	// Overwriting value or writing value from past
-	if idx < len(b.data) {
-		b.data[idx] = value
-		return b, nil
-	}
-
-	// Fill up unwritten slots with NaN
-	for i := len(b.data); i < idx; i++ {
-		b.data = append(b.data, NaN)
-	}
-
-	b.data = append(b.data, value)
-	return b, nil
-}
-
-func (b *buffer) end() int64 {
-	return b.firstWrite() + int64(len(b.data))*b.frequency
-}
-
-func (b *buffer) firstWrite() int64 {
-	return b.start + (b.frequency / 2)
-}
-
-func (b *buffer) close() {}
-
-/*
-func (b *buffer) close() {
-	if b.closed {
-		return
-	}
-
-	b.closed = true
-	n, sum, min, max := 0, 0., math.MaxFloat64, -math.MaxFloat64
-	for _, x := range b.data {
-		if x.IsNaN() {
-			continue
-		}
-
-		n += 1
-		f := float64(x)
-		sum += f
-		min = math.Min(min, f)
-		max = math.Max(max, f)
-	}
-
-	b.statisticts.samples = n
-	if n > 0 {
-		b.statisticts.avg = Float(sum / float64(n))
-		b.statisticts.min = Float(min)
-		b.statisticts.max = Float(max)
-	} else {
-		b.statisticts.avg = NaN
-		b.statisticts.min = NaN
-		b.statisticts.max = NaN
-	}
-}
-*/
-
-// func interpolate(idx int, data []Float) Float {
-// 	if idx == 0 || idx+1 == len(data) {
-// 		return NaN
-// 	}
-// 	return (data[idx-1] + data[idx+1]) / 2.0
-// }
-
-// Return all known values from `from` to `to`. Gaps of information are represented as NaN.
-// Simple linear interpolation is done between the two neighboring cells if possible.
-// If values at the start or end are missing, instead of NaN values, the second and thrid
-// return values contain the actual `from`/`to`.
-// This function goes back the buffer chain if `from` is older than the currents buffer start.
-// The loaded values are added to `data` and `data` is returned, possibly with a shorter length.
-// If `data` is not long enough to hold all values, this function will panic!
-func (b *buffer) read(from, to int64, data []Float) ([]Float, int64, int64, error) {
-	if from < b.firstWrite() {
-		if b.prev != nil {
-			return b.prev.read(from, to, data)
-		}
-		from = b.firstWrite()
-	}
-
-	var i int = 0
-	var t int64 = from
-	for ; t < to; t += b.frequency {
-		idx := int((t - b.start) / b.frequency)
-		if idx >= cap(b.data) {
-			if b.next == nil {
-				break
-			}
-			b = b.next
-			idx = 0
-		}
-
-		if idx >= len(b.data) {
-			if b.next == nil || to <= b.next.start {
-				break
-			}
-			data[i] += NaN
-		} else if t < b.start {
-			data[i] += NaN
-			// } else if b.data[idx].IsNaN() {
-			// 	data[i] += interpolate(idx, b.data)
-		} else {
-			data[i] += b.data[idx]
-		}
-		i++
-	}
-
-	return data[:i], from, t, nil
-}
-
-// Returns true if this buffer needs to be freed.
-func (b *buffer) free(t int64) (delme bool, n int) {
-	if b.prev != nil {
-		delme, m := b.prev.free(t)
-		n += m
-		if delme {
-			b.prev.next = nil
-			if cap(b.prev.data) == BUFFER_CAP {
-				bufferPool.Put(b.prev)
-			}
-			b.prev = nil
-		}
-	}
-
-	end := b.end()
-	if end < t {
-		return true, n + 1
-	}
-
-	return false, n
-}
-
-// Call `callback` on every buffer that contains data in the range from `from` to `to`.
-func (b *buffer) iterFromTo(from, to int64, callback func(b *buffer) error) error {
-	if b == nil {
-		return nil
-	}
-
-	if err := b.prev.iterFromTo(from, to, callback); err != nil {
-		return err
-	}
-
-	if from <= b.end() && b.start <= to {
-		return callback(b)
-	}
-
-	return nil
-}
-
-func (b *buffer) count() int64 {
-	res := int64(len(b.data))
-	if b.prev != nil {
-		res += b.prev.count()
-	}
-	return res
-}
-
-// Could also be called "node" as this forms a node in a tree structure.
-// Called level because "node" might be confusing here.
-// Can be both a leaf or a inner node. In this tree structue, inner nodes can
-// also hold data (in `metrics`).
-type level struct {
-	lock     sync.RWMutex
-	metrics  []*buffer         // Every level can store metrics.
-	children map[string]*level // Lower levels.
-}
-
-// Find the correct level for the given selector, creating it if
-// it does not exist. Example selector in the context of the
-// ClusterCockpit could be: []string{ "emmy", "host123", "cpu0" }.
-// This function would probably benefit a lot from `level.children` beeing a `sync.Map`?
-func (l *level) findLevelOrCreate(selector []string, nMetrics int) *level {
-	if len(selector) == 0 {
-		return l
-	}
-
-	// Allow concurrent reads:
-	l.lock.RLock()
-	var child *level
-	var ok bool
-	if l.children == nil {
-		// Children map needs to be created...
-		l.lock.RUnlock()
-	} else {
-		child, ok := l.children[selector[0]]
-		l.lock.RUnlock()
-		if ok {
-			return child.findLevelOrCreate(selector[1:], nMetrics)
-		}
-	}
-
-	// The level does not exist, take write lock for unqiue access:
-	l.lock.Lock()
-	// While this thread waited for the write lock, another thread
-	// could have created the child node.
-	if l.children != nil {
-		child, ok = l.children[selector[0]]
-		if ok {
-			l.lock.Unlock()
-			return child.findLevelOrCreate(selector[1:], nMetrics)
-		}
-	}
-
-	child = &level{
-		metrics:  make([]*buffer, nMetrics),
-		children: nil,
-	}
-
-	if l.children != nil {
-		l.children[selector[0]] = child
-	} else {
-		l.children = map[string]*level{selector[0]: child}
-	}
-	l.lock.Unlock()
-	return child.findLevelOrCreate(selector[1:], nMetrics)
-}
-
-func (l *level) free(t int64) (int, error) {
-	l.lock.Lock()
-	defer l.lock.Unlock()
-
-	n := 0
-	for i, b := range l.metrics {
-		if b != nil {
-			delme, m := b.free(t)
-			n += m
-			if delme {
-				if cap(b.data) == BUFFER_CAP {
-					bufferPool.Put(b)
-				}
-				l.metrics[i] = nil
-			}
-		}
-	}
-
-	for _, l := range l.children {
-		m, err := l.free(t)
-		n += m
-		if err != nil {
-			return n, err
-		}
-	}
-
-	return n, nil
-}
-
-func (l *level) sizeInBytes() int64 {
-	l.lock.RLock()
-	defer l.lock.RUnlock()
-	size := int64(0)
-
-	for _, b := range l.metrics {
-		if b != nil {
-			size += b.count() * int64(unsafe.Sizeof(Float(0)))
-		}
-	}
-
-	for _, child := range l.children {
-		size += child.sizeInBytes()
-	}
-
-	return size
-}
-
-type MemoryStore struct {
-	root    level // root of the tree structure
-	metrics map[string]MetricConfig
-}
-
-// Return a new, initialized instance of a MemoryStore.
-// Will panic if values in the metric configurations are invalid.
-func NewMemoryStore(metrics map[string]MetricConfig) *MemoryStore {
-	offset := 0
-	for key, config := range metrics {
-		if config.Frequency == 0 {
-			panic("invalid frequency")
-		}
-
-		metrics[key] = MetricConfig{
-			Frequency:   config.Frequency,
-			Aggregation: config.Aggregation,
-			offset:      offset,
-		}
-		offset += 1
-	}
-
-	return &MemoryStore{
-		root: level{
-			metrics:  make([]*buffer, len(metrics)),
-			children: make(map[string]*level),
-		},
-		metrics: metrics,
-	}
-}
-
-// Write all values in `metrics` to the level specified by `selector` for time `ts`.
-// Look at `findLevelOrCreate` for how selectors work.
-func (m *MemoryStore) Write(selector []string, ts int64, metrics []Metric) error {
-	var ok bool
-	for i, metric := range metrics {
-		if metric.mc.Frequency == 0 {
-			metric.mc, ok = m.metrics[metric.Name]
-			if !ok {
-				metric.mc.Frequency = 0
-			}
-			metrics[i] = metric
-		}
-	}
-
-	return m.WriteToLevel(&m.root, selector, ts, metrics)
-}
-
-func (m *MemoryStore) GetLevel(selector []string) *level {
-	return m.root.findLevelOrCreate(selector, len(m.metrics))
-}
-
-// Assumes that `minfo` in `metrics` is filled in!
-func (m *MemoryStore) WriteToLevel(l *level, selector []string, ts int64, metrics []Metric) error {
-	l = l.findLevelOrCreate(selector, len(m.metrics))
-	l.lock.Lock()
-	defer l.lock.Unlock()
-
-	for _, metric := range metrics {
-		if metric.mc.Frequency == 0 {
-			continue
-		}
-
-		b := l.metrics[metric.mc.offset]
-		if b == nil {
-			// First write to this metric and level
-			b = newBuffer(ts, metric.mc.Frequency)
-			l.metrics[metric.mc.offset] = b
-		}
-
-		nb, err := b.write(ts, metric.Value)
-		if err != nil {
-			return err
-		}
-
-		// Last write created a new buffer...
-		if b != nb {
-			l.metrics[metric.mc.offset] = nb
-		}
-	}
-	return nil
-}
-
-// Returns all values for metric `metric` from `from` to `to` for the selected level(s).
-// If the level does not hold the metric itself, the data will be aggregated recursively from the children.
-// The second and third return value are the actual from/to for the data. Those can be different from
-// the range asked for if no data was available.
-func (m *MemoryStore) Read(selector Selector, metric string, from, to int64) ([]Float, int64, int64, error) {
-	if from > to {
-		return nil, 0, 0, errors.New("invalid time range")
-	}
-
-	minfo, ok := m.metrics[metric]
-	if !ok {
-		return nil, 0, 0, errors.New("unkown metric: " + metric)
-	}
-
-	n, data := 0, make([]Float, (to-from)/minfo.Frequency+1)
-	err := m.root.findBuffers(selector, minfo.offset, func(b *buffer) error {
-		cdata, cfrom, cto, err := b.read(from, to, data)
-		if err != nil {
-			return err
-		}
-
-		if n == 0 {
-			from, to = cfrom, cto
-		} else if from != cfrom || to != cto || len(data) != len(cdata) {
-			missingfront, missingback := int((from-cfrom)/minfo.Frequency), int((to-cto)/minfo.Frequency)
-			if missingfront != 0 {
-				return ErrDataDoesNotAlign
-			}
-
-			newlen := len(cdata) - missingback
-			if newlen < 1 {
-				return ErrDataDoesNotAlign
-			}
-			cdata = cdata[0:newlen]
-			if len(cdata) != len(data) {
-				return ErrDataDoesNotAlign
-			}
-
-			from, to = cfrom, cto
-		}
-
-		data = cdata
-		n += 1
-		return nil
-	})
-
-	if err != nil {
-		return nil, 0, 0, err
-	} else if n == 0 {
-		return nil, 0, 0, errors.New("metric or host not found")
-	} else if n > 1 {
-		if minfo.Aggregation == AvgAggregation {
-			normalize := 1. / Float(n)
-			for i := 0; i < len(data); i++ {
-				data[i] *= normalize
-			}
-		} else if minfo.Aggregation != SumAggregation {
-			return nil, 0, 0, errors.New("invalid aggregation")
-		}
-	}
-
-	return data, from, to, nil
-}
-
-// Release all buffers for the selected level and all its children that contain only
-// values older than `t`.
-func (m *MemoryStore) Free(selector []string, t int64) (int, error) {
-	return m.GetLevel(selector).free(t)
-}
-
-func (m *MemoryStore) FreeAll() error {
-	for k := range m.root.children {
-		delete(m.root.children, k)
-	}
-
-	return nil
-}
-
-func (m *MemoryStore) SizeInBytes() int64 {
-	return m.root.sizeInBytes()
-}
-
-// Given a selector, return a list of all children of the level selected.
-func (m *MemoryStore) ListChildren(selector []string) []string {
-	lvl := &m.root
-	for lvl != nil && len(selector) != 0 {
-		lvl.lock.RLock()
-		next := lvl.children[selector[0]]
-		lvl.lock.RUnlock()
-		lvl = next
-		selector = selector[1:]
-	}
-
-	if lvl == nil {
-		return nil
-	}
-
-	lvl.lock.RLock()
-	defer lvl.lock.RUnlock()
-
-	children := make([]string, 0, len(lvl.children))
-	for child := range lvl.children {
-		children = append(children, child)
-	}
-
-	return children
-}

memstore_test.go

@@ -1,512 +0,0 @@
-package main
-
-import (
-	"fmt"
-	"math"
-	"math/rand"
-	"sync"
-	"testing"
-)
-
-func TestMemoryStoreBasics(t *testing.T) {
-	frequency := int64(10)
-	start, count := int64(100), int64(5000)
-	store := NewMemoryStore(map[string]MetricConfig{
-		"a": {Frequency: frequency},
-		"b": {Frequency: frequency * 2},
-	})
-
-	for i := int64(0); i < count; i++ {
-		err := store.Write([]string{"testhost"}, start+i*frequency, []Metric{
-			{Name: "a", Value: Float(i)},
-			{Name: "b", Value: Float(i / 2)},
-		})
-		if err != nil {
-			t.Error(err)
-			return
-		}
-	}
-
-	sel := Selector{{String: "testhost"}}
-	adata, from, to, err := store.Read(sel, "a", start, start+count*frequency)
-	if err != nil || from != start || to != start+count*frequency {
-		t.Error(err)
-		return
-	}
-	bdata, _, _, err := store.Read(sel, "b", start, start+count*frequency)
-	if err != nil {
-		t.Error(err)
-		return
-	}
-
-	if len(adata) != int(count) || len(bdata) != int(count/2) {
-		t.Error("unexpected count of returned values")
-		return
-	}
-
-	for i := 0; i < int(count); i++ {
-		if adata[i] != Float(i) {
-			t.Errorf("incorrect value for metric a (%f vs. %f)", adata[i], Float(i))
-			return
-		}
-	}
-
-	for i := 0; i < int(count/2); i++ {
-		if bdata[i] != Float(i) && bdata[i] != Float(i-1) {
-			t.Errorf("incorrect value for metric b (%f) at index %d", bdata[i], i)
-			return
-		}
-	}
-
-}
-
-func TestMemoryStoreTooMuchWrites(t *testing.T) {
-	frequency := int64(10)
-	count := BUFFER_CAP*3 + 10
-	store := NewMemoryStore(map[string]MetricConfig{
-		"a": {Frequency: frequency},
-		"b": {Frequency: frequency * 2},
-		"c": {Frequency: frequency / 2},
-		"d": {Frequency: frequency * 3},
-	})
-
-	start := int64(100)
-	for i := 0; i < count; i++ {
-		if err := store.Write([]string{"test"}, start+int64(i)*frequency, []Metric{
-			{Name: "a", Value: Float(i)},
-			{Name: "b", Value: Float(i / 2)},
-			{Name: "c", Value: Float(i * 2)},
-			{Name: "d", Value: Float(i / 3)},
-		}); err != nil {
-			t.Fatal(err)
-		}
-	}
-
-	end := start + int64(count)*frequency
-	data, from, to, err := store.Read(Selector{{String: "test"}}, "a", start, end)
-	if len(data) != count || from != start || to != end || err != nil {
-		t.Fatalf("a: err=%#v, from=%d, to=%d, data=%#v\n", err, from, to, data)
-	}
-
-	data, from, to, err = store.Read(Selector{{String: "test"}}, "b", start, end)
-	if len(data) != count/2 || from != start || to != end || err != nil {
-		t.Fatalf("b: err=%#v, from=%d, to=%d, data=%#v\n", err, from, to, data)
-	}
-
-	data, from, to, err = store.Read(Selector{{String: "test"}}, "c", start, end)
-	if len(data) != count*2-1 || from != start || to != end-frequency/2 || err != nil {
-		t.Fatalf("c: err=%#v, from=%d, to=%d, data=%#v\n", err, from, to, data)
-	}
-
-	data, from, to, err = store.Read(Selector{{String: "test"}}, "d", start, end)
-	if len(data) != count/3+1 || from != start || to != end+frequency*2 || err != nil {
-		t.Errorf("expected: err=nil, from=%d, to=%d, len(data)=%d\n", start, end+frequency*2, count/3)
-		t.Fatalf("d: err=%#v, from=%d, to=%d, data=%#v\n", err, from, to, data)
-	}
-}
-
-func TestMemoryStoreOutOfBounds(t *testing.T) {
-	count := 2000
-	toffset := 1000
-	store := NewMemoryStore(map[string]MetricConfig{
-		"a": {Frequency: 60},
-	})
-
-	for i := 0; i < count; i++ {
-		if err := store.Write([]string{"cluster", "host", "cpu"}, int64(toffset+i*60), []Metric{
-			{Name: "a", Value: Float(i)},
-		}); err != nil {
-			t.Fatal(err)
-		}
-	}
-
-	sel := Selector{{String: "cluster"}, {String: "host"}, {String: "cpu"}}
-	data, from, to, err := store.Read(sel, "a", 500, int64(toffset+count*60+500))
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	if from/60 != int64(toffset)/60 || to/60 != int64(toffset+count*60)/60 {
-		t.Fatalf("Got %d-%d, expected %d-%d", from, to, toffset, toffset+count*60)
-	}
-
-	if len(data) != count || data[0] != 0 || data[len(data)-1] != Float((count-1)) {
-		t.Fatalf("Wrong data (got: %d, %f, %f, expected: %d, %f, %f)",
-			len(data), data[0], data[len(data)-1], count, 0., Float(count-1))
-	}
-
-	testfrom, testlen := int64(100000000), int64(10000)
-	data, from, to, err = store.Read(sel, "a", testfrom, testfrom+testlen)
-	if len(data) != 0 || from != testfrom || to != testfrom || err != nil {
-		t.Fatal("Unexpected data returned when reading range after valid data")
-	}
-
-	testfrom, testlen = 0, 10
-	data, from, to, err = store.Read(sel, "a", testfrom, testfrom+testlen)
-	if len(data) != 0 || from/60 != int64(toffset)/60 || to/60 != int64(toffset)/60 || err != nil {
-		t.Fatal("Unexpected data returned when reading range before valid data")
-	}
-}
-
-func TestMemoryStoreMissingDatapoints(t *testing.T) {
-	count := 3000
-	store := NewMemoryStore(map[string]MetricConfig{
-		"a": {Frequency: 1},
-	})
-
-	for i := 0; i < count; i++ {
-		if i%3 != 0 {
-			continue
-		}
-
-		err := store.Write([]string{"testhost"}, int64(i), []Metric{
-			{Name: "a", Value: Float(i)},
-		})
-		if err != nil {
-			t.Error(err)
-			return
-		}
-	}
-
-	sel := Selector{{String: "testhost"}}
-	adata, _, _, err := store.Read(sel, "a", 0, int64(count))
-	if err != nil {
-		t.Error(err)
-		return
-	}
-
-	if len(adata) != count-2 {
-		t.Error("unexpected len")
-		return
-	}
-
-	for i := 0; i < count-2; i++ {
-		if i%3 == 0 {
-			if adata[i] != Float(i) {
-				t.Error("unexpected value")
-				return
-			}
-		} else {
-			if !math.IsNaN(float64(adata[i])) {
-				t.Errorf("NaN expected (i = %d, value = %f)\n", i, adata[i])
-				return
-			}
-		}
-	}
-}
-
-func TestMemoryStoreAggregation(t *testing.T) {
-	count := 3000
-	store := NewMemoryStore(map[string]MetricConfig{
-		"a": {Frequency: 1, Aggregation: SumAggregation},
-	})
-
-	for i := 0; i < count; i++ {
-		err := store.Write([]string{"host0", "cpu0"}, int64(i), []Metric{
-			{Name: "a", Value: Float(i) / 2.},
-		})
-		if err != nil {
-			t.Error(err)
-			return
-		}
-
-		err = store.Write([]string{"host0", "cpu1"}, int64(i), []Metric{
-			{Name: "a", Value: Float(i) * 2.},
-		})
-		if err != nil {
-			t.Error(err)
-			return
-		}
-	}
-
-	adata, from, to, err := store.Read(Selector{{String: "host0"}}, "a", int64(0), int64(count))
-	if err != nil {
-		t.Error(err)
-		return
-	}
-
-	if len(adata) != count || from != 0 || to != int64(count) {
-		t.Error("unexpected length or time range of returned data")
-		return
-	}
-
-	for i := 0; i < count; i++ {
-		expected := Float(i)/2. + Float(i)*2.
-		if adata[i] != expected {
-			t.Errorf("expected: %f, got: %f", expected, adata[i])
-			return
-		}
-	}
-}
-
-func TestMemoryStoreStats(t *testing.T) {
-	count := 3000
-	store := NewMemoryStore(map[string]MetricConfig{
-		"a": {Frequency: 1},
-		"b": {Frequency: 1, Aggregation: AvgAggregation},
-	})
-
-	sel1 := []string{"cluster", "host1"}
-	sel2 := []string{"cluster", "host2", "left"}
-	sel3 := []string{"cluster", "host2", "right"}
-
-	samples := 0
-	asum, amin, amax := 0., math.MaxFloat32, -math.MaxFloat32
-	bsum, bmin, bmax := 0., math.MaxFloat32, -math.MaxFloat32
-
-	for i := 0; i < count; i++ {
-		if i%5 == 0 {
-			// Skip some writes, test if samples is calculated correctly
-			continue
-		}
-
-		samples += 1
-		a := float64(rand.Int()%100 - 50)
-		asum += a
-		amin = math.Min(amin, a)
-		amax = math.Max(amax, a)
-		b := float64(rand.Int()%100 - 50)
-		bsum += b * 2
-		bmin = math.Min(bmin, b)
-		bmax = math.Max(bmax, b)
-
-		store.Write(sel1, int64(i), []Metric{
-			{Name: "a", Value: Float(a)},
-		})
-		store.Write(sel2, int64(i), []Metric{
-			{Name: "b", Value: Float(b)},
-		})
-		store.Write(sel3, int64(i), []Metric{
-			{Name: "b", Value: Float(b)},
-		})
-	}
-
-	stats, from, to, err := store.Stats(Selector{{String: "cluster"}, {String: "host1"}}, "a", 0, int64(count))
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	if from != 1 || to != int64(count) || stats.Samples != samples {
-		t.Fatalf("unexpected: from=%d, to=%d, stats.Samples=%d (expected samples=%d)\n", from, to, stats.Samples, samples)
-	}
-
-	if stats.Avg != Float(asum/float64(samples)) || stats.Min != Float(amin) || stats.Max != Float(amax) {
-		t.Fatalf("wrong stats: %#v\n", stats)
-	}
-
-	stats, from, to, err = store.Stats(Selector{{String: "cluster"}, {String: "host2"}}, "b", 0, int64(count))
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	if from != 1 || to != int64(count) || stats.Samples != samples*2 {
-		t.Fatalf("unexpected: from=%d, to=%d, stats.Samples=%d (expected samples=%d)\n", from, to, stats.Samples, samples*2)
-	}
-
-	if stats.Avg != Float(bsum/float64(samples*2)) || stats.Min != Float(bmin) || stats.Max != Float(bmax) {
-		t.Fatalf("wrong stats: %#v (expected: avg=%f, min=%f, max=%f)\n", stats, bsum/float64(samples*2), bmin, bmax)
-	}
-}
-
-func TestMemoryStoreArchive(t *testing.T) {
-	store1 := NewMemoryStore(map[string]MetricConfig{
-		"a": {Frequency: 1},
-		"b": {Frequency: 1},
-	})
-
-	count := 2000
-	for i := 0; i < count; i++ {
-		err := store1.Write([]string{"cluster", "host", "cpu0"}, 100+int64(i), []Metric{
-			{Name: "a", Value: Float(i)},
-			{Name: "b", Value: Float(i * 2)},
-		})
-		if err != nil {
-			t.Error(err)
-			return
-		}
-	}
-
-	// store1.DebugDump(bufio.NewWriter(os.Stdout))
-
-	archiveRoot := t.TempDir()
-	_, err := store1.ToCheckpoint(archiveRoot, 100, 100+int64(count/2))
-	if err != nil {
-		t.Error(err)
-		return
-	}
-
-	_, err = store1.ToCheckpoint(archiveRoot, 100+int64(count/2), 100+int64(count))
-	if err != nil {
-		t.Error(err)
-		return
-	}
-
-	store2 := NewMemoryStore(map[string]MetricConfig{
-		"a": {Frequency: 1},
-		"b": {Frequency: 1},
-	})
-	n, err := store2.FromCheckpoint(archiveRoot, 100)
-	if err != nil {
-		t.Error(err)
-		return
-	}
-
-	sel := Selector{{String: "cluster"}, {String: "host"}, {String: "cpu0"}}
-	adata, from, to, err := store2.Read(sel, "a", 100, int64(100+count))
-	if err != nil {
-		t.Error(err)
-		return
-	}
-
-	if n != 2 || len(adata) != count || from != 100 || to != int64(100+count) {
-		t.Errorf("unexpected: n=%d, len=%d, from=%d, to=%d\n", n, len(adata), from, to)
-		return
-	}
-
-	for i := 0; i < count; i++ {
-		expected := Float(i)
-		if adata[i] != expected {
-			t.Errorf("expected: %f, got: %f", expected, adata[i])
-		}
-	}
-}
-
-func TestMemoryStoreFree(t *testing.T) {
-	store := NewMemoryStore(map[string]MetricConfig{
-		"a": {Frequency: 1},
-		"b": {Frequency: 2},
-	})
-
-	count := 3000
-	sel := []string{"cluster", "host", "1"}
-	for i := 0; i < count; i++ {
-		err := store.Write(sel, int64(i), []Metric{
-			{Name: "a", Value: Float(i)},
-			{Name: "b", Value: Float(i)},
-		})
-		if err != nil {
-			t.Fatal(err)
-		}
-	}
-
-	n, err := store.Free([]string{"cluster", "host"}, int64(BUFFER_CAP*2)+100)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	if n != 3 {
-		t.Fatal("two buffers expected to be released")
-	}
-
-	adata, from, to, err := store.Read(Selector{{String: "cluster"}, {String: "host"}, {String: "1"}}, "a", 0, int64(count))
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	if from != int64(BUFFER_CAP*2) || to != int64(count) || len(adata) != count-2*BUFFER_CAP {
-		t.Fatalf("unexpected values from call to `Read`: from=%d, to=%d, len=%d", from, to, len(adata))
-	}
-
-	// bdata, from, to, err := store.Read(Selector{{String: "cluster"}, {String: "host"}, {String: "1"}}, "b", 0, int64(count))
-	// if err != nil {
-	// 	t.Fatal(err)
-	// }
-
-	// if from != int64(BUFFER_CAP*2) || to != int64(count) || len(bdata) != (count-2*BUFFER_CAP)/2 {
-	// 	t.Fatalf("unexpected values from call to `Read`: from=%d (expected: %d), to=%d (expected: %d), len=%d (expected: %d)",
-	// 		from, BUFFER_CAP*2, to, count, len(bdata), (count-2*BUFFER_CAP)/2)
-	// }
-
-	if adata[0] != Float(BUFFER_CAP*2) || adata[len(adata)-1] != Float(count-1) {
-		t.Fatal("wrong values")
-	}
-}
-
-func BenchmarkMemoryStoreConcurrentWrites(b *testing.B) {
-	frequency := int64(5)
-	count := b.N
-	goroutines := 4
-	store := NewMemoryStore(map[string]MetricConfig{
-		"a": {Frequency: frequency},
-	})
-
-	var wg sync.WaitGroup
-	wg.Add(goroutines)
-
-	for g := 0; g < goroutines; g++ {
-		go func(g int) {
-			host := fmt.Sprintf("host%d", g)
-			for i := 0; i < count; i++ {
-				store.Write([]string{"cluster", host, "cpu0"}, int64(i)*frequency, []Metric{
-					{Name: "a", Value: Float(i)},
-				})
-			}
-			wg.Done()
-		}(g)
-	}
-
-	wg.Wait()
-	b.StopTimer()
-
-	for g := 0; g < goroutines; g++ {
-		host := fmt.Sprintf("host%d", g)
-		sel := Selector{{String: "cluster"}, {String: host}, {String: "cpu0"}}
-		adata, _, _, err := store.Read(sel, "a", 0, int64(count)*frequency)
-		if err != nil {
-			b.Error(err)
-			return
-		}
-
-		if len(adata) != count {
-			b.Error("unexpected count")
-			return
-		}
-
-		for i := 0; i < count; i++ {
-			expected := Float(i)
-			if adata[i] != expected {
-				b.Error("incorrect value for metric a")
-				return
-			}
-		}
-	}
-}
-
-func BenchmarkMemoryStoreAggregation(b *testing.B) {
-	b.StopTimer()
-	count := 2000
-	store := NewMemoryStore(map[string]MetricConfig{
-		"flops_any": {Frequency: 1, Aggregation: AvgAggregation},
-	})
-
-	sel := []string{"testcluster", "host123", "cpu0"}
-	for i := 0; i < count; i++ {
-		sel[2] = "cpu0"
-		err := store.Write(sel, int64(i), []Metric{
-			{Name: "flops_any", Value: Float(i)},
-		})
-		if err != nil {
-			b.Fatal(err)
-		}
-
-		sel[2] = "cpu1"
-		err = store.Write(sel, int64(i), []Metric{
-			{Name: "flops_any", Value: Float(i)},
-		})
-		if err != nil {
-			b.Fatal(err)
-		}
-	}
-
-	b.StartTimer()
-	for n := 0; n < b.N; n++ {
-		data, from, to, err := store.Read(Selector{{String: "testcluster"}, {String: "host123"}}, "flops_any", 0, int64(count))
-		if err != nil {
-			b.Fatal(err)
-		}
-
-		if len(data) != count || from != 0 || to != int64(count) {
-			b.Fatal()
-		}
-	}
-}

scripts/cc-metric-store.config

@@ -1,17 +0,0 @@
-CC_USER=clustercockpit
-
-CC_GROUP=clustercockpit
-
-CC_HOME=/tmp
-
-LOG_DIR=/var/log
-
-DATA_DIR=/var/run/cc-metric-store
-
-MAX_OPEN_FILES=10000
-
-CONF_DIR=/etc/cc-metric-store
-
-CONF_FILE=/etc/cc-metric-store/cc-metric-store.json
-
-RESTART_ON_UPGRADE=true

scripts/cc-metric-store.deb.control

@@ -1,12 +0,0 @@
-Package: cc-metric-store
-Version: {VERSION}
-Installed-Size: {INSTALLED_SIZE}
-Architecture: {ARCH}
-Maintainer: thomas.gruber@fau.de
-Depends: libc6 (>= 2.2.1)
-Build-Depends: debhelper-compat (= 13), git, golang-go
-Description: In-memory metric store daemon from the ClusterCockpit suite
-Homepage: https://github.com/ClusterCockpit/cc-metric-store
-Source: cc-metric-store
-Rules-Requires-Root: no
-

scripts/cc-metric-store.init

@@ -1,141 +0,0 @@
-#! /usr/bin/env bash
-
-# chkconfig: 2345 80 05
-# description: ClusterCockpit metric store
-# processname: cc-metric-store
-# config: /etc/default/cc-metric-store
-# pidfile: /var/run/cc-metric-store.pid
-
-### BEGIN INIT INFO
-# Provides:          cc-metric-store
-# Required-Start:    $all
-# Required-Stop:     $remote_fs $syslog
-# Default-Start:     2 3 4 5
-# Default-Stop:      0 1 6
-# Short-Description: Start ClusterCockpit metric store at boot time
-### END INIT INFO
-
-
-PATH=/bin:/usr/bin:/sbin:/usr/sbin
-NAME=cc-metric-store
-DESC="ClusterCockpit metric store"
-DEFAULT=/etc/default/${NAME}.json
-
-CC_USER=clustercockpit
-CC_GROUP=clustercockpit
-CONF_DIR=/etc/cc-metric-store
-PID_FILE=/var/run/$NAME.pid
-DAEMON=/usr/sbin/$NAME
-CONF_FILE=${CONF_DIR}/cc-metric-store.json
-
-umask 0027
-
-if [ ! -x $DAEMON ]; then
-  echo "Program not installed or not executable"
-  exit 5
-fi
-
-. /lib/lsb/init-functions
-
-if [ -r /etc/default/rcS ]; then
-	. /etc/default/rcS
-fi
-
-# overwrite settings from default file
-if [ -f "$DEFAULT" ]; then
-	. "$DEFAULT"
-fi
-
-CC_OPTS="--config=${CONF_FILE}"
-
-function checkUser() {
-  if [ `id -u` -ne 0 ]; then
-  	echo "You need root privileges to run this script"
-  	exit 4
-  fi
-}
-
-case "$1" in
-  start)
-  checkUser
-	log_daemon_msg "Starting $DESC"
-
-	pid=`pidofproc -p $PID_FILE $NAME`
-	if [ -n "$pid" ] ; then
-		log_begin_msg "Already running."
-		log_end_msg 0
-		exit 0
-	fi
-
-	# Prepare environment
-	touch "$PID_FILE" && chown "$CC_USER":"$CC_GROUP" "$PID_FILE"
-
-  if [ -n "$MAX_OPEN_FILES" ]; then
-		ulimit -n $MAX_OPEN_FILES
-	fi
-
-	# Start Daemon
-	start-stop-daemon --start -b --chdir "$WORK_DIR" --user "$CC_USER" -c "$CC_USER" --pidfile "$PID_FILE" --exec $DAEMON -- $DAEMON_OPTS
-	return=$?
-	if [ $return -eq 0 ]
-	then
-	  sleep 1
-
-    # check if pid file has been written to
-	  if ! [[ -s $PID_FILE ]]; then
-	    log_end_msg 1
-	    exit 1
-	  fi
-
-		i=0
-		timeout=10
-		# Wait for the process to be properly started before exiting
-		until { cat "$PID_FILE" | xargs kill -0; } >/dev/null 2>&1
-		do
-			sleep 1
-			i=$(($i + 1))
-      if [ $i -gt $timeout ]; then
-			  log_end_msg 1
-			  exit 1
-			fi
-		done
-  fi
-  log_end_msg $return
-	;;
-  stop)
-  checkUser
-	log_daemon_msg "Stopping $DESC"
-
-	if [ -f "$PID_FILE" ]; then
-		start-stop-daemon --stop --pidfile "$PID_FILE" \
-			--user "$CC_USER" \
-			--retry=TERM/20/KILL/5 >/dev/null
-		if [ $? -eq 1 ]; then
-			log_progress_msg "$DESC is not running but pid file exists, cleaning up"
-		elif [ $? -eq 3 ]; then
-			PID="`cat $PID_FILE`"
-			log_failure_msg "Failed to stop $DESC (pid $PID)"
-			exit 1
-		fi
-		rm -f "$PID_FILE"
-	else
-		log_progress_msg "(not running)"
-	fi
-	log_end_msg 0
-	;;
-  status)
-	status_of_proc -p $PID_FILE $NAME $NAME && exit 0 || exit $?
-    ;;
-  restart|force-reload)
-	if [ -f "$PID_FILE" ]; then
-		$0 stop
-		sleep 1
-	fi
-	$0 start
-	;;
-  *)
-	log_success_msg "Usage: $0 {start|stop|restart|force-reload|status}"
-	exit 3
-	;;
-esac
-

scripts/cc-metric-store.spec

@@ -1,62 +0,0 @@
-Name:           cc-metric-store
-Version:        %{VERS}
-Release:        1%{?dist}
-Summary:        In-memory metric database from the ClusterCockpit suite
-
-License:        MIT
-Source0:        %{name}-%{version}.tar.gz
-
-BuildRequires:  go-toolset
-BuildRequires:  systemd-rpm-macros
-
-Provides:       %{name} = %{version}
-
-%description
-In-memory metric database from the ClusterCockpit suite
-
-%global debug_package %{nil}
-
-%prep
-%autosetup
-
-
-%build
-make
-
-
-%install
-# Install cc-metric-store
-make PREFIX=%{buildroot} install
-# Integrate into system
-install -Dpm 0644 scripts/%{name}.service %{buildroot}%{_unitdir}/%{name}.service
-install -Dpm 0600 scripts/%{name}.config %{buildroot}%{_sysconfdir}/default/%{name}
-install -Dpm 0644 scripts/%{name}.sysusers %{buildroot}%{_sysusersdir}/%{name}.conf
-
-
-%check
-# go test should be here... :)
-
-%pre
-%sysusers_create_package scripts/%{name}.sysusers
-
-%post
-%systemd_post %{name}.service
-
-%preun
-%systemd_preun %{name}.service
-
-%files
-# Binary
-%attr(-,clustercockpit,clustercockpit) %{_bindir}/%{name}
-# Config
-%dir %{_sysconfdir}/%{name}
-%attr(0600,clustercockpit,clustercockpit) %config(noreplace) %{_sysconfdir}/%{name}/%{name}.json
-# Systemd
-%{_unitdir}/%{name}.service
-%{_sysconfdir}/default/%{name}
-%{_sysusersdir}/%{name}.conf
-
-%changelog
-* Mon Mar 07 2022 Thomas Gruber - 0.1
-- Initial metric store implementation
-

scripts/cc-metric-store.sysusers

@@ -1,2 +0,0 @@
-#Type Name           ID GECOS                     Home directory           Shell
-u     clustercockpit -  "User for ClusterCockpit" /run/cc-metric-collector /sbin/nologin

scripts/send-fake-data.go

@@ -1,105 +0,0 @@
-package main
-
-import (
-	"bytes"
-	"fmt"
-	"io"
-	"log"
-	"math"
-	"math/rand"
-	"net/http"
-	"time"
-)
-
-const token = "eyJ0eXAiOiJKV1QiLCJhbGciOiJFZERTQSJ9.eyJ1c2VyIjoiYWRtaW4iLCJyb2xlcyI6WyJST0xFX0FETUlOIiwiUk9MRV9BTkFMWVNUIiwiUk9MRV9VU0VSIl19.d-3_3FZTsadPjDEdsWrrQ7nS0edMAR4zjl-eK7rJU3HziNBfI9PDHDIpJVHTNN5E5SlLGLFXctWyKAkwhXL-Dw"
-const ccmsurl = "http://localhost:8081/api/write"
-const cluster = "fakedev"
-const sockets = 2
-const cpus = 8
-const freq = 15 * time.Second
-
-var hosts = []string{"fake001", "fake002", "fake003", "fake004", "fake005"}
-var metrics = []struct {
-	Name     string
-	Type     string
-	AvgValue float64
-}{
-	{"flops_any", "cpu", 10.0},
-	{"mem_bw", "socket", 50.0},
-	{"ipc", "cpu", 1.25},
-	{"cpu_load", "node", 4},
-	{"mem_used", "node", 20},
-}
-
-var states = make([]float64, 0)
-
-func send(client *http.Client, t int64) {
-	msg := &bytes.Buffer{}
-
-	i := 0
-	for _, host := range hosts {
-		for _, metric := range metrics {
-			n := 1
-			if metric.Type == "socket" {
-				n = sockets
-			} else if metric.Type == "cpu" {
-				n = cpus
-			}
-
-			for j := 0; j < n; j++ {
-				fmt.Fprintf(msg, "%s,cluster=%s,host=%s,type=%s", metric.Name, cluster, host, metric.Type)
-				if metric.Type == "socket" {
-					fmt.Fprintf(msg, ",type-id=%d", j)
-				} else if metric.Type == "cpu" {
-					fmt.Fprintf(msg, ",type-id=%d", j)
-				}
-
-				x := metric.AvgValue + math.Sin(states[i])*(metric.AvgValue/10.)
-				states[i] += 0.1
-				fmt.Fprintf(msg, " value=%f ", x)
-
-				fmt.Fprintf(msg, "%d\n", t)
-				i++
-			}
-		}
-	}
-
-	req, _ := http.NewRequest(http.MethodPost, ccmsurl, msg)
-	req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token))
-	res, err := client.Do(req)
-	if err != nil {
-		log.Print(err)
-		return
-	}
-	if res.StatusCode != http.StatusOK {
-		body, _ := io.ReadAll(res.Body)
-		log.Printf("%s: %s", res.Status, string(body))
-	}
-}
-
-func main() {
-	for range hosts {
-		for _, m := range metrics {
-			n := 1
-			if m.Type == "socket" {
-				n = sockets
-			} else if m.Type == "cpu" {
-				n = cpus
-			}
-
-			for i := 0; i < n; i++ {
-				states = append(states, rand.Float64()*100)
-			}
-		}
-	}
-
-	client := &http.Client{}
-
-	i := 0
-	for t := range time.Tick(freq) {
-		log.Printf("tick... (#%d)", i)
-		i++
-
-		send(client, t.Unix())
-	}
-}

selector.go

@@ -1,123 +0,0 @@
-package main
-
-import (
-	"encoding/json"
-	"errors"
-)
-
-type SelectorElement struct {
-	Any    bool
-	String string
-	Group  []string
-}
-
-func (se *SelectorElement) UnmarshalJSON(input []byte) error {
-	if input[0] == '"' {
-		if err := json.Unmarshal(input, &se.String); err != nil {
-			return err
-		}
-
-		if se.String == "*" {
-			se.Any = true
-			se.String = ""
-		}
-
-		return nil
-	}
-
-	if input[0] == '[' {
-		return json.Unmarshal(input, &se.Group)
-	}
-
-	return errors.New("the Go SelectorElement type can only be a string or an array of strings")
-}
-
-func (se *SelectorElement) MarshalJSON() ([]byte, error) {
-	if se.Any {
-		return []byte("\"*\""), nil
-	}
-
-	if se.String != "" {
-		return json.Marshal(se.String)
-	}
-
-	if se.Group != nil {
-		return json.Marshal(se.Group)
-	}
-
-	return nil, errors.New("a Go Selector must be a non-empty string or a non-empty slice of strings")
-}
-
-type Selector []SelectorElement
-
-func (l *level) findLevel(selector []string) *level {
-	if len(selector) == 0 {
-		return l
-	}
-
-	l.lock.RLock()
-	defer l.lock.RUnlock()
-
-	lvl := l.children[selector[0]]
-	if lvl == nil {
-		return nil
-	}
-
-	return lvl.findLevel(selector[1:])
-}
-
-func (l *level) findBuffers(selector Selector, offset int, f func(b *buffer) error) error {
-	l.lock.RLock()
-	defer l.lock.RUnlock()
-
-	if len(selector) == 0 {
-		b := l.metrics[offset]
-		if b != nil {
-			return f(b)
-		}
-
-		for _, lvl := range l.children {
-			err := lvl.findBuffers(nil, offset, f)
-			if err != nil {
-				return err
-			}
-		}
-		return nil
-	}
-
-	sel := selector[0]
-	if len(sel.String) != 0 && l.children != nil {
-		lvl, ok := l.children[sel.String]
-		if ok {
-			err := lvl.findBuffers(selector[1:], offset, f)
-			if err != nil {
-				return err
-			}
-		}
-		return nil
-	}
-
-	if sel.Group != nil && l.children != nil {
-		for _, key := range sel.Group {
-			lvl, ok := l.children[key]
-			if ok {
-				err := lvl.findBuffers(selector[1:], offset, f)
-				if err != nil {
-					return err
-				}
-			}
-		}
-		return nil
-	}
-
-	if sel.Any && l.children != nil {
-		for _, lvl := range l.children {
-			if err := lvl.findBuffers(selector[1:], offset, f); err != nil {
-				return err
-			}
-		}
-		return nil
-	}
-
-	return nil
-}