diff --git a/.github/workflows/Release.yml b/.github/workflows/Release.yml deleted file mode 100644 index d3fbe9a..0000000 --- a/.github/workflows/Release.yml +++ /dev/null @@ -1,314 +0,0 @@ -# See: https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions - -# Workflow name -name: Release - -# Run on tag push -on: - push: - tags: - - '**' - -jobs: - - # - # Build on AlmaLinux 8.5 using golang-1.18.2 - # - AlmaLinux-RPM-build: - runs-on: ubuntu-latest - # See: https://hub.docker.com/_/almalinux - container: almalinux:8.5 - # The job outputs link to the outputs of the 'rpmrename' step - # Only job outputs can be used in child jobs - outputs: - rpm : ${{steps.rpmrename.outputs.RPM}} - srpm : ${{steps.rpmrename.outputs.SRPM}} - steps: - - # Use dnf to install development packages - - name: Install development packages - run: | - dnf --assumeyes group install "Development Tools" "RPM Development Tools" - dnf --assumeyes install wget openssl-devel diffutils delve which - - # Checkout git repository and submodules - # fetch-depth must be 0 to use git describe - # See: https://github.com/marketplace/actions/checkout - - name: Checkout - uses: actions/checkout@v2 - with: - submodules: recursive - fetch-depth: 0 - - # Use dnf to install build dependencies - - name: Install build dependencies - run: | - wget -q http://mirror.centos.org/centos/8-stream/AppStream/x86_64/os/Packages/golang-1.18.2-1.module_el8.7.0+1173+5d37c0fd.x86_64.rpm \ - http://mirror.centos.org/centos/8-stream/AppStream/x86_64/os/Packages/golang-bin-1.18.2-1.module_el8.7.0+1173+5d37c0fd.x86_64.rpm \ - http://mirror.centos.org/centos/8-stream/AppStream/x86_64/os/Packages/golang-src-1.18.2-1.module_el8.7.0+1173+5d37c0fd.noarch.rpm \ - http://mirror.centos.org/centos/8-stream/AppStream/x86_64/os/Packages/go-toolset-1.18.2-1.module_el8.7.0+1173+5d37c0fd.x86_64.rpm - rpm -i go*.rpm - - - name: RPM build MetricStore - id: rpmbuild - run: make RPM - - # AlmaLinux 8.5 is a derivate of RedHat Enterprise Linux 8 (UBI8), - # so the created RPM both contain the substring 'el8' in the RPM file names - # This step replaces the substring 'el8' to 'alma85'. It uses the move operation - # because it is unclear whether the default AlmaLinux 8.5 container contains the - # 'rename' command. This way we also get the new names for output. - - name: Rename RPMs (s/el8/alma85/) - id: rpmrename - run: | - OLD_RPM="${{steps.rpmbuild.outputs.RPM}}" - OLD_SRPM="${{steps.rpmbuild.outputs.SRPM}}" - NEW_RPM="${OLD_RPM/el8/alma85}" - NEW_SRPM=${OLD_SRPM/el8/alma85} - mv "${OLD_RPM}" "${NEW_RPM}" - mv "${OLD_SRPM}" "${NEW_SRPM}" - echo "::set-output name=SRPM::${NEW_SRPM}" - echo "::set-output name=RPM::${NEW_RPM}" - - # See: https://github.com/actions/upload-artifact - - name: Save RPM as artifact - uses: actions/upload-artifact@v2 - with: - name: cc-metric-store RPM for AlmaLinux 8.5 - path: ${{ steps.rpmrename.outputs.RPM }} - - name: Save SRPM as artifact - uses: actions/upload-artifact@v2 - with: - name: cc-metric-store SRPM for AlmaLinux 8.5 - path: ${{ steps.rpmrename.outputs.SRPM }} - - # - # Build on UBI 8 using golang-1.18.2 - # - UBI-8-RPM-build: - runs-on: ubuntu-latest - # See: https://catalog.redhat.com/software/containers/ubi8/ubi/5c359854d70cc534b3a3784e?container-tabs=gti - container: registry.access.redhat.com/ubi8/ubi:8.5-226.1645809065 - # The job outputs link to the outputs of the 'rpmbuild' step - outputs: - rpm : ${{steps.rpmbuild.outputs.RPM}} - srpm : ${{steps.rpmbuild.outputs.SRPM}} - steps: - - # Use dnf to install development packages - - name: Install development packages - run: dnf --assumeyes --disableplugin=subscription-manager install rpm-build go-srpm-macros rpm-build-libs rpm-libs gcc make python38 git wget openssl-devel diffutils delve which - - # Checkout git repository and submodules - # fetch-depth must be 0 to use git describe - # See: https://github.com/marketplace/actions/checkout - - name: Checkout - uses: actions/checkout@v2 - with: - submodules: recursive - fetch-depth: 0 - - # Use dnf to install build dependencies - - name: Install build dependencies - run: | - wget -q http://mirror.centos.org/centos/8-stream/AppStream/x86_64/os/Packages/golang-1.18.2-1.module_el8.7.0+1173+5d37c0fd.x86_64.rpm \ - http://mirror.centos.org/centos/8-stream/AppStream/x86_64/os/Packages/golang-bin-1.18.2-1.module_el8.7.0+1173+5d37c0fd.x86_64.rpm \ - http://mirror.centos.org/centos/8-stream/AppStream/x86_64/os/Packages/golang-src-1.18.2-1.module_el8.7.0+1173+5d37c0fd.noarch.rpm \ - http://mirror.centos.org/centos/8-stream/AppStream/x86_64/os/Packages/go-toolset-1.18.2-1.module_el8.7.0+1173+5d37c0fd.x86_64.rpm - rpm -i go*.rpm - - - name: RPM build MetricStore - id: rpmbuild - run: make RPM - - # See: https://github.com/actions/upload-artifact - - name: Save RPM as artifact - uses: actions/upload-artifact@v2 - with: - name: cc-metric-store RPM for UBI 8 - path: ${{ steps.rpmbuild.outputs.RPM }} - - name: Save SRPM as artifact - uses: actions/upload-artifact@v2 - with: - name: cc-metric-store SRPM for UBI 8 - path: ${{ steps.rpmbuild.outputs.SRPM }} - - # - # Build on Ubuntu 20.04 using official go 1.19.1 package - # - Ubuntu-focal-build: - runs-on: ubuntu-latest - container: ubuntu:20.04 - # The job outputs link to the outputs of the 'debrename' step - # Only job outputs can be used in child jobs - outputs: - deb : ${{steps.debrename.outputs.DEB}} - steps: - # Use apt to install development packages - - name: Install development packages - run: | - apt update && apt --assume-yes upgrade - apt --assume-yes install build-essential sed git wget bash - # Checkout git repository and submodules - # fetch-depth must be 0 to use git describe - # See: https://github.com/marketplace/actions/checkout - - name: Checkout - uses: actions/checkout@v2 - with: - submodules: recursive - fetch-depth: 0 - # Use official golang package - - name: Install Golang - run: | - wget -q https://go.dev/dl/go1.19.1.linux-amd64.tar.gz - tar -C /usr/local -xzf go1.19.1.linux-amd64.tar.gz - export PATH=/usr/local/go/bin:/usr/local/go/pkg/tool/linux_amd64:$PATH - go version - - name: DEB build MetricStore - id: dpkg-build - run: | - export PATH=/usr/local/go/bin:/usr/local/go/pkg/tool/linux_amd64:$PATH - make DEB - - name: Rename DEB (add '_ubuntu20.04') - id: debrename - run: | - OLD_DEB_NAME=$(echo "${{steps.dpkg-build.outputs.DEB}}" | rev | cut -d '.' -f 2- | rev) - NEW_DEB_FILE="${OLD_DEB_NAME}_ubuntu20.04.deb" - mv "${{steps.dpkg-build.outputs.DEB}}" "${NEW_DEB_FILE}" - echo "::set-output name=DEB::${NEW_DEB_FILE}" - # See: https://github.com/actions/upload-artifact - - name: Save DEB as artifact - uses: actions/upload-artifact@v2 - with: - name: cc-metric-store DEB for Ubuntu 20.04 - path: ${{ steps.debrename.outputs.DEB }} - - # - # Build on Ubuntu 20.04 using official go 1.19.1 package - # - Ubuntu-jammy-build: - runs-on: ubuntu-latest - container: ubuntu:22.04 - # The job outputs link to the outputs of the 'debrename' step - # Only job outputs can be used in child jobs - outputs: - deb : ${{steps.debrename.outputs.DEB}} - steps: - # Use apt to install development packages - - name: Install development packages - run: | - apt update && apt --assume-yes upgrade - apt --assume-yes install build-essential sed git wget bash - # Checkout git repository and submodules - # fetch-depth must be 0 to use git describe - # See: https://github.com/marketplace/actions/checkout - - name: Checkout - uses: actions/checkout@v2 - with: - submodules: recursive - fetch-depth: 0 - # Use official golang package - - name: Install Golang - run: | - wget -q https://go.dev/dl/go1.19.1.linux-amd64.tar.gz - tar -C /usr/local -xzf go1.19.1.linux-amd64.tar.gz - export PATH=/usr/local/go/bin:/usr/local/go/pkg/tool/linux_amd64:$PATH - go version - - name: DEB build MetricStore - id: dpkg-build - run: | - export PATH=/usr/local/go/bin:/usr/local/go/pkg/tool/linux_amd64:$PATH - make DEB - - name: Rename DEB (add '_ubuntu22.04') - id: debrename - run: | - OLD_DEB_NAME=$(echo "${{steps.dpkg-build.outputs.DEB}}" | rev | cut -d '.' -f 2- | rev) - NEW_DEB_FILE="${OLD_DEB_NAME}_ubuntu22.04.deb" - mv "${{steps.dpkg-build.outputs.DEB}}" "${NEW_DEB_FILE}" - echo "::set-output name=DEB::${NEW_DEB_FILE}" - # See: https://github.com/actions/upload-artifact - - name: Save DEB as artifact - uses: actions/upload-artifact@v2 - with: - name: cc-metric-store DEB for Ubuntu 22.04 - path: ${{ steps.debrename.outputs.DEB }} - - # - # Create release with fresh RPMs - # - Release: - runs-on: ubuntu-latest - # We need the RPMs, so add dependency - needs: [AlmaLinux-RPM-build, UBI-8-RPM-build, Ubuntu-focal-build, Ubuntu-jammy-build] - - steps: - # See: https://github.com/actions/download-artifact - - name: Download AlmaLinux 8.5 RPM - uses: actions/download-artifact@v2 - with: - name: cc-metric-store RPM for AlmaLinux 8.5 - - name: Download AlmaLinux 8.5 SRPM - uses: actions/download-artifact@v2 - with: - name: cc-metric-store SRPM for AlmaLinux 8.5 - - - name: Download UBI 8 RPM - uses: actions/download-artifact@v2 - with: - name: cc-metric-store RPM for UBI 8 - - name: Download UBI 8 SRPM - uses: actions/download-artifact@v2 - with: - name: cc-metric-store SRPM for UBI 8 - - - name: Download Ubuntu 20.04 DEB - uses: actions/download-artifact@v2 - with: - name: cc-metric-store DEB for Ubuntu 20.04 - - - name: Download Ubuntu 22.04 DEB - uses: actions/download-artifact@v2 - with: - name: cc-metric-store DEB for Ubuntu 22.04 - - # The download actions do not publish the name of the downloaded file, - # so we re-use the job outputs of the parent jobs. The files are all - # downloaded to the current folder. - # The gh-release action afterwards does not accept file lists but all - # files have to be listed at 'files'. The step creates one output per - # RPM package (2 per distro) - - name: Set RPM variables - id: files - run: | - ALMA_85_RPM=$(basename "${{ needs.AlmaLinux-RPM-build.outputs.rpm}}") - ALMA_85_SRPM=$(basename "${{ needs.AlmaLinux-RPM-build.outputs.srpm}}") - UBI_8_RPM=$(basename "${{ needs.UBI-8-RPM-build.outputs.rpm}}") - UBI_8_SRPM=$(basename "${{ needs.UBI-8-RPM-build.outputs.srpm}}") - U_2004_DEB=$(basename "${{ needs.Ubuntu-focal-build.outputs.deb}}") - U_2204_DEB=$(basename "${{ needs.Ubuntu-jammy-build.outputs.deb}}") - echo "ALMA_85_RPM::${ALMA_85_RPM}" - echo "ALMA_85_SRPM::${ALMA_85_SRPM}" - echo "UBI_8_RPM::${UBI_8_RPM}" - echo "UBI_8_SRPM::${UBI_8_SRPM}" - echo "U_2004_DEB::${U_2004_DEB}" - echo "U_2204_DEB::${U_2204_DEB}" - echo "::set-output name=ALMA_85_RPM::${ALMA_85_RPM}" - echo "::set-output name=ALMA_85_SRPM::${ALMA_85_SRPM}" - echo "::set-output name=UBI_8_RPM::${UBI_8_RPM}" - echo "::set-output name=UBI_8_SRPM::${UBI_8_SRPM}" - echo "::set-output name=U_2004_DEB::${U_2004_DEB}" - echo "::set-output name=U_2204_DEB::${U_2204_DEB}" - - # See: https://github.com/softprops/action-gh-release - - name: Release - uses: softprops/action-gh-release@v1 - if: startsWith(github.ref, 'refs/tags/') - with: - name: cc-metric-store-${{github.ref_name}} - files: | - ${{ steps.files.outputs.ALMA_85_RPM }} - ${{ steps.files.outputs.ALMA_85_SRPM }} - ${{ steps.files.outputs.UBI_8_RPM }} - ${{ steps.files.outputs.UBI_8_SRPM }} - ${{ steps.files.outputs.U_2004_DEB }} - ${{ steps.files.outputs.U_2204_DEB }} diff --git a/Makefile b/Makefile index 3b2edbc..1f61367 100644 --- a/Makefile +++ b/Makefile @@ -1,38 +1,21 @@ +TARGET = ./cc-metric-store +VERSION = 1.3.0 +GIT_HASH := $(shell git rev-parse --short HEAD || echo 'development') +CURRENT_TIME = $(shell date +"%Y-%m-%d:T%H:%M:%S") +LD_FLAGS = '-s -X main.date=${CURRENT_TIME} -X main.version=${VERSION} -X main.commit=${GIT_HASH}' -APP = cc-metric-store -GOSRC_APP := cc-metric-store.go -GOSRC_FILES := api.go \ - memstore.go \ - archive.go \ - debug.go \ - float.go \ - lineprotocol.go \ - selector.go \ - stats.go +.PHONY: clean test tags $(TARGET) +.NOTPARALLEL: +$(TARGET): + $(info ===> BUILD cc-metric-store) + @go build -ldflags=${LD_FLAGS} ./cmd/cc-metric-store -BINDIR ?= bin - - -.PHONY: all -all: $(APP) - -$(APP): $(GOSRC) - go get - go build -o $(APP) $(GOSRC_APP) $(GOSRC_FILES) - -install: $(APP) - @WORKSPACE=$(PREFIX) - @if [ -z "$${WORKSPACE}" ]; then exit 1; fi - @mkdir --parents --verbose $${WORKSPACE}/usr/$(BINDIR) - @install -Dpm 755 $(APP) $${WORKSPACE}/usr/$(BINDIR)/$(APP) - @install -Dpm 600 config.json $${WORKSPACE}/etc/$(APP)/$(APP).json - -.PHONY: clean test fmt -.ONESHELL: clean: - rm -f $(APP) + $(info ===> CLEAN) + @go clean + @rm -f $(TARGET) test: $(info ===> TESTING) @@ -41,74 +24,6 @@ test: @go vet ./... @go test ./... -fmt: - go fmt $(GOSRC_APP) - -# Run linter for the Go programming language. -# Using static analysis, it finds bugs and performance issues, offers simplifications, and enforces style rules -.PHONY: staticcheck -staticcheck: - go install honnef.co/go/tools/cmd/staticcheck@latest - $$(go env GOPATH)/bin/staticcheck ./... - -.ONESHELL: -.PHONY: RPM -RPM: scripts/cc-metric-store.spec - @WORKSPACE="$${PWD}" - @SPECFILE="$${WORKSPACE}/scripts/cc-metric-store.spec" - # Setup RPM build tree - @eval $$(rpm --eval "ARCH='%{_arch}' RPMDIR='%{_rpmdir}' SOURCEDIR='%{_sourcedir}' SPECDIR='%{_specdir}' SRPMDIR='%{_srcrpmdir}' BUILDDIR='%{_builddir}'") - @mkdir --parents --verbose "$${RPMDIR}" "$${SOURCEDIR}" "$${SPECDIR}" "$${SRPMDIR}" "$${BUILDDIR}" - # Create source tarball - @COMMITISH="HEAD" - @VERS=$$(git describe --tags $${COMMITISH}) - @VERS=$${VERS#v} - @VERS=$$(echo $$VERS | sed -e s+'-'+'_'+g) - @if [ "$${VERS}" = "" ]; then VERS="0.0.1"; fi - @eval $$(rpmspec --query --queryformat "NAME='%{name}' VERSION='%{version}' RELEASE='%{release}' NVR='%{NVR}' NVRA='%{NVRA}'" --define="VERS $${VERS}" "$${SPECFILE}") - @PREFIX="$${NAME}-$${VERSION}" - @FORMAT="tar.gz" - @SRCFILE="$${SOURCEDIR}/$${PREFIX}.$${FORMAT}" - @git archive --verbose --format "$${FORMAT}" --prefix="$${PREFIX}/" --output="$${SRCFILE}" $${COMMITISH} - # Build RPM and SRPM - @rpmbuild -ba --define="VERS $${VERS}" --rmsource --clean "$${SPECFILE}" - # Report RPMs and SRPMs when in GitHub Workflow - @if [[ "$${GITHUB_ACTIONS}" == true ]]; then - @ RPMFILE="$${RPMDIR}/$${ARCH}/$${NVRA}.rpm" - @ SRPMFILE="$${SRPMDIR}/$${NVR}.src.rpm" - @ echo "RPM: $${RPMFILE}" - @ echo "SRPM: $${SRPMFILE}" - @ echo "::set-output name=SRPM::$${SRPMFILE}" - @ echo "::set-output name=RPM::$${RPMFILE}" - @fi - -.ONESHELL: -.PHONY: DEB -DEB: scripts/cc-metric-store.deb.control $(APP) - @BASEDIR=$${PWD} - @WORKSPACE=$${PWD}/.dpkgbuild - @DEBIANDIR=$${WORKSPACE}/debian - @DEBIANBINDIR=$${WORKSPACE}/DEBIAN - @mkdir --parents --verbose $$WORKSPACE $$DEBIANBINDIR - #@mkdir --parents --verbose $$DEBIANDIR - @CONTROLFILE="$${BASEDIR}/scripts/cc-metric-store.deb.control" - @COMMITISH="HEAD" - @VERS=$$(git describe --tags --abbrev=0 $${COMMITISH}) - @VERS=$${VERS#v} - @VERS=$$(echo $$VERS | sed -e s+'-'+'_'+g) - @if [ "$${VERS}" = "" ]; then VERS="0.0.1"; fi - @ARCH=$$(uname -m) - @ARCH=$$(echo $$ARCH | sed -e s+'_'+'-'+g) - @if [ "$${ARCH}" = "x86-64" ]; then ARCH=amd64; fi - @PREFIX="$${NAME}-$${VERSION}_$${ARCH}" - @SIZE_BYTES=$$(du -bcs --exclude=.dpkgbuild "$$WORKSPACE"/ | awk '{print $$1}' | head -1 | sed -e 's/^0\+//') - @SIZE="$$(awk -v size="$$SIZE_BYTES" 'BEGIN {print (size/1024)+1}' | awk '{print int($$0)}')" - #@sed -e s+"{VERSION}"+"$$VERS"+g -e s+"{INSTALLED_SIZE}"+"$$SIZE"+g -e s+"{ARCH}"+"$$ARCH"+g $$CONTROLFILE > $${DEBIANDIR}/control - @sed -e s+"{VERSION}"+"$$VERS"+g -e s+"{INSTALLED_SIZE}"+"$$SIZE"+g -e s+"{ARCH}"+"$$ARCH"+g $$CONTROLFILE > $${DEBIANBINDIR}/control - @make PREFIX=$${WORKSPACE} install - @DEB_FILE="cc-metric-store_$${VERS}_$${ARCH}.deb" - @dpkg-deb -b $${WORKSPACE} "$$DEB_FILE" - @rm -r "$${WORKSPACE}" - @if [ "$${GITHUB_ACTIONS}" = "true" ]; then - @ echo "::set-output name=DEB::$${DEB_FILE}" - @fi +tags: + $(info ===> TAGS) + @ctags -R diff --git a/Makefile.orig b/Makefile.orig new file mode 100644 index 0000000..e796920 --- /dev/null +++ b/Makefile.orig @@ -0,0 +1,126 @@ +TARGET = ./cc-metric-store +VERSION = 1.3.0 +GIT_HASH := $(shell git rev-parse --short HEAD || echo 'development') +CURRENT_TIME = $(shell date +"%Y-%m-%d:T%H:%M:%S") +LD_FLAGS = '-s -X main.date=${CURRENT_TIME} -X main.version=${VERSION} -X main.commit=${GIT_HASH}' + +.PHONY: clean test tags $(TARGET) + +.NOTPARALLEL: + +$(TARGET): + $(info ===> BUILD cc-backend) + @go build -ldflags=${LD_FLAGS} ./cmd/cc-metric-store + +<<<<<<< HEAD +======= +BINDIR ?= bin + + +.PHONY: all +all: $(APP) + +$(APP): $(GOSRC) + go get + go build -o $(APP) $(GOSRC_APP) $(GOSRC_FILES) + +install: $(APP) + @WORKSPACE=$(PREFIX) + @if [ -z "$${WORKSPACE}" ]; then exit 1; fi + @mkdir --parents --verbose $${WORKSPACE}/usr/$(BINDIR) + @install -Dpm 755 $(APP) $${WORKSPACE}/usr/$(BINDIR)/$(APP) + @install -Dpm 600 config.json $${WORKSPACE}/etc/$(APP)/$(APP).json + +.PHONY: clean test fmt +.ONESHELL: +>>>>>>> main +clean: + $(info ===> CLEAN) + @go clean + @rm -f $(TARGET) + +test: + $(info ===> TESTING) + @go clean -testcache + @go build ./... + @go vet ./... + @go test ./... +<<<<<<< HEAD + +tags: + $(info ===> TAGS) + @ctags -R +======= + +fmt: + go fmt $(GOSRC_APP) + +# Run linter for the Go programming language. +# Using static analysis, it finds bugs and performance issues, offers simplifications, and enforces style rules +.PHONY: staticcheck +staticcheck: + go install honnef.co/go/tools/cmd/staticcheck@latest + $$(go env GOPATH)/bin/staticcheck ./... + +.ONESHELL: +.PHONY: RPM +RPM: scripts/cc-metric-store.spec + @WORKSPACE="$${PWD}" + @SPECFILE="$${WORKSPACE}/scripts/cc-metric-store.spec" + # Setup RPM build tree + @eval $$(rpm --eval "ARCH='%{_arch}' RPMDIR='%{_rpmdir}' SOURCEDIR='%{_sourcedir}' SPECDIR='%{_specdir}' SRPMDIR='%{_srcrpmdir}' BUILDDIR='%{_builddir}'") + @mkdir --parents --verbose "$${RPMDIR}" "$${SOURCEDIR}" "$${SPECDIR}" "$${SRPMDIR}" "$${BUILDDIR}" + # Create source tarball + @COMMITISH="HEAD" + @VERS=$$(git describe --tags $${COMMITISH}) + @VERS=$${VERS#v} + @VERS=$$(echo $$VERS | sed -e s+'-'+'_'+g) + @if [ "$${VERS}" = "" ]; then VERS="0.0.1"; fi + @eval $$(rpmspec --query --queryformat "NAME='%{name}' VERSION='%{version}' RELEASE='%{release}' NVR='%{NVR}' NVRA='%{NVRA}'" --define="VERS $${VERS}" "$${SPECFILE}") + @PREFIX="$${NAME}-$${VERSION}" + @FORMAT="tar.gz" + @SRCFILE="$${SOURCEDIR}/$${PREFIX}.$${FORMAT}" + @git archive --verbose --format "$${FORMAT}" --prefix="$${PREFIX}/" --output="$${SRCFILE}" $${COMMITISH} + # Build RPM and SRPM + @rpmbuild -ba --define="VERS $${VERS}" --rmsource --clean "$${SPECFILE}" + # Report RPMs and SRPMs when in GitHub Workflow + @if [[ "$${GITHUB_ACTIONS}" == true ]]; then + @ RPMFILE="$${RPMDIR}/$${ARCH}/$${NVRA}.rpm" + @ SRPMFILE="$${SRPMDIR}/$${NVR}.src.rpm" + @ echo "RPM: $${RPMFILE}" + @ echo "SRPM: $${SRPMFILE}" + @ echo "::set-output name=SRPM::$${SRPMFILE}" + @ echo "::set-output name=RPM::$${RPMFILE}" + @fi + +.ONESHELL: +.PHONY: DEB +DEB: scripts/cc-metric-store.deb.control $(APP) + @BASEDIR=$${PWD} + @WORKSPACE=$${PWD}/.dpkgbuild + @DEBIANDIR=$${WORKSPACE}/debian + @DEBIANBINDIR=$${WORKSPACE}/DEBIAN + @mkdir --parents --verbose $$WORKSPACE $$DEBIANBINDIR + #@mkdir --parents --verbose $$DEBIANDIR + @CONTROLFILE="$${BASEDIR}/scripts/cc-metric-store.deb.control" + @COMMITISH="HEAD" + @VERS=$$(git describe --tags --abbrev=0 $${COMMITISH}) + @VERS=$${VERS#v} + @VERS=$$(echo $$VERS | sed -e s+'-'+'_'+g) + @if [ "$${VERS}" = "" ]; then VERS="0.0.1"; fi + @ARCH=$$(uname -m) + @ARCH=$$(echo $$ARCH | sed -e s+'_'+'-'+g) + @if [ "$${ARCH}" = "x86-64" ]; then ARCH=amd64; fi + @PREFIX="$${NAME}-$${VERSION}_$${ARCH}" + @SIZE_BYTES=$$(du -bcs --exclude=.dpkgbuild "$$WORKSPACE"/ | awk '{print $$1}' | head -1 | sed -e 's/^0\+//') + @SIZE="$$(awk -v size="$$SIZE_BYTES" 'BEGIN {print (size/1024)+1}' | awk '{print int($$0)}')" + #@sed -e s+"{VERSION}"+"$$VERS"+g -e s+"{INSTALLED_SIZE}"+"$$SIZE"+g -e s+"{ARCH}"+"$$ARCH"+g $$CONTROLFILE > $${DEBIANDIR}/control + @sed -e s+"{VERSION}"+"$$VERS"+g -e s+"{INSTALLED_SIZE}"+"$$SIZE"+g -e s+"{ARCH}"+"$$ARCH"+g $$CONTROLFILE > $${DEBIANBINDIR}/control + @make PREFIX=$${WORKSPACE} install + @DEB_FILE="cc-metric-store_$${VERS}_$${ARCH}.deb" + @dpkg-deb -b $${WORKSPACE} "$$DEB_FILE" + @rm -r "$${WORKSPACE}" + @if [ "$${GITHUB_ACTIONS}" = "true" ]; then + @ echo "::set-output name=DEB::$${DEB_FILE}" + @fi +>>>>>>> main diff --git a/cc-metric-store.go b/cc-metric-store.go deleted file mode 100644 index 8e69c95..0000000 --- a/cc-metric-store.go +++ /dev/null @@ -1,349 +0,0 @@ -package main - -import ( - "bufio" - "context" - "encoding/json" - "flag" - "fmt" - "io" - "log" - "os" - "os/signal" - "runtime" - "runtime/debug" - "sync" - "syscall" - "time" - - "github.com/google/gops/agent" -) - -// For aggregation over multiple values at different cpus/sockets/..., not time! -type AggregationStrategy int - -const ( - NoAggregation AggregationStrategy = iota - SumAggregation - AvgAggregation -) - -func (as *AggregationStrategy) UnmarshalJSON(data []byte) error { - var str string - if err := json.Unmarshal(data, &str); err != nil { - return err - } - - switch str { - case "": - *as = NoAggregation - case "sum": - *as = SumAggregation - case "avg": - *as = AvgAggregation - default: - return fmt.Errorf("invalid aggregation strategy: %#v", str) - } - return nil -} - -type MetricConfig struct { - // Interval in seconds at which measurements will arive. - Frequency int64 `json:"frequency"` - - // Can be 'sum', 'avg' or null. Describes how to aggregate metrics from the same timestep over the hierarchy. - Aggregation AggregationStrategy `json:"aggregation"` - - // Private, used internally... - offset int -} - -type HttpConfig struct { - // Address to bind to, for example "0.0.0.0:8081" - Address string `json:"address"` - - // If not the empty string, use https with this as the certificate file - CertFile string `json:"https-cert-file"` - - // If not the empty string, use https with this as the key file - KeyFile string `json:"https-key-file"` -} - -type NatsConfig struct { - // Address of the nats server - Address string `json:"address"` - - // Username/Password, optional - Username string `json:"username"` - Password string `json:"password"` - - Subscriptions []struct { - // Channel name - SubscribeTo string `json:"subscribe-to"` - - // Allow lines without a cluster tag, use this as default, optional - ClusterTag string `json:"cluster-tag"` - } `json:"subscriptions"` -} - -type Config struct { - Metrics map[string]MetricConfig `json:"metrics"` - RetentionInMemory string `json:"retention-in-memory"` - Nats []*NatsConfig `json:"nats"` - JwtPublicKey string `json:"jwt-public-key"` - HttpConfig *HttpConfig `json:"http-api"` - Checkpoints struct { - Interval string `json:"interval"` - RootDir string `json:"directory"` - Restore string `json:"restore"` - } `json:"checkpoints"` - Archive struct { - Interval string `json:"interval"` - RootDir string `json:"directory"` - DeleteInstead bool `json:"delete-instead"` - } `json:"archive"` - Debug struct { - EnableGops bool `json:"gops"` - DumpToFile string `json:"dump-to-file"` - } `json:"debug"` -} - -var conf Config -var memoryStore *MemoryStore = nil -var lastCheckpoint time.Time - -var debugDumpLock sync.Mutex -var debugDump io.Writer = io.Discard - -func loadConfiguration(file string) Config { - var config Config - configFile, err := os.Open(file) - if err != nil { - log.Fatal(err) - } - defer configFile.Close() - dec := json.NewDecoder(configFile) - dec.DisallowUnknownFields() - if err := dec.Decode(&config); err != nil { - log.Fatal(err) - } - return config -} - -func intervals(wg *sync.WaitGroup, ctx context.Context) { - wg.Add(3) - // go func() { - // defer wg.Done() - // ticks := time.Tick(30 * time.Minute) - // for { - // select { - // case <-ctx.Done(): - // return - // case <-ticks: - // runtime.GC() - // } - // } - // }() - - go func() { - defer wg.Done() - d, err := time.ParseDuration(conf.RetentionInMemory) - if err != nil { - log.Fatal(err) - } - if d <= 0 { - return - } - - ticks := time.Tick(d / 2) - for { - select { - case <-ctx.Done(): - return - case <-ticks: - t := time.Now().Add(-d) - log.Printf("start freeing buffers (older than %s)...\n", t.Format(time.RFC3339)) - freed, err := memoryStore.Free(nil, t.Unix()) - if err != nil { - log.Printf("freeing up buffers failed: %s\n", err.Error()) - } else { - log.Printf("done: %d buffers freed\n", freed) - } - } - } - }() - - lastCheckpoint = time.Now() - go func() { - defer wg.Done() - d, err := time.ParseDuration(conf.Checkpoints.Interval) - if err != nil { - log.Fatal(err) - } - if d <= 0 { - return - } - - ticks := time.Tick(d) - for { - select { - case <-ctx.Done(): - return - case <-ticks: - log.Printf("start checkpointing (starting at %s)...\n", lastCheckpoint.Format(time.RFC3339)) - now := time.Now() - n, err := memoryStore.ToCheckpoint(conf.Checkpoints.RootDir, - lastCheckpoint.Unix(), now.Unix()) - if err != nil { - log.Printf("checkpointing failed: %s\n", err.Error()) - } else { - log.Printf("done: %d checkpoint files created\n", n) - lastCheckpoint = now - } - } - } - }() - - go func() { - defer wg.Done() - d, err := time.ParseDuration(conf.Archive.Interval) - if err != nil { - log.Fatal(err) - } - if d <= 0 { - return - } - - ticks := time.Tick(d) - for { - select { - case <-ctx.Done(): - return - case <-ticks: - t := time.Now().Add(-d) - log.Printf("start archiving checkpoints (older than %s)...\n", t.Format(time.RFC3339)) - n, err := ArchiveCheckpoints(conf.Checkpoints.RootDir, conf.Archive.RootDir, t.Unix(), conf.Archive.DeleteInstead) - if err != nil { - log.Printf("archiving failed: %s\n", err.Error()) - } else { - log.Printf("done: %d files zipped and moved to archive\n", n) - } - } - } - }() -} - -func main() { - var configFile string - var enableGopsAgent bool - flag.StringVar(&configFile, "config", "./config.json", "configuration file") - flag.BoolVar(&enableGopsAgent, "gops", false, "Listen via github.com/google/gops/agent") - flag.Parse() - - startupTime := time.Now() - conf = loadConfiguration(configFile) - memoryStore = NewMemoryStore(conf.Metrics) - - if enableGopsAgent || conf.Debug.EnableGops { - if err := agent.Listen(agent.Options{}); err != nil { - log.Fatal(err) - } - } - - if conf.Debug.DumpToFile != "" { - f, err := os.Create(conf.Debug.DumpToFile) - if err != nil { - log.Fatal(err) - } - - debugDump = f - } - - d, err := time.ParseDuration(conf.Checkpoints.Restore) - if err != nil { - log.Fatal(err) - } - - restoreFrom := startupTime.Add(-d) - log.Printf("Loading checkpoints newer than %s\n", restoreFrom.Format(time.RFC3339)) - files, err := memoryStore.FromCheckpoint(conf.Checkpoints.RootDir, restoreFrom.Unix()) - loadedData := memoryStore.SizeInBytes() / 1024 / 1024 // In MB - if err != nil { - log.Fatalf("Loading checkpoints failed: %s\n", err.Error()) - } else { - log.Printf("Checkpoints loaded (%d files, %d MB, that took %fs)\n", files, loadedData, time.Since(startupTime).Seconds()) - } - - // Try to use less memory by forcing a GC run here and then - // lowering the target percentage. The default of 100 means - // that only once the ratio of new allocations execeds the - // previously active heap, a GC is triggered. - // Forcing a GC here will set the "previously active heap" - // to a minumum. - runtime.GC() - if loadedData > 1000 && os.Getenv("GOGC") == "" { - debug.SetGCPercent(10) - } - - ctx, shutdown := context.WithCancel(context.Background()) - - var wg sync.WaitGroup - sigs := make(chan os.Signal, 1) - signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM, syscall.SIGUSR1) - go func() { - for { - sig := <-sigs - if sig == syscall.SIGUSR1 { - memoryStore.DebugDump(bufio.NewWriter(os.Stdout), nil) - continue - } - - log.Println("Shuting down...") - shutdown() - } - }() - - intervals(&wg, ctx) - - wg.Add(1) - - go func() { - err := StartApiServer(ctx, conf.HttpConfig) - if err != nil { - log.Fatal(err) - } - wg.Done() - }() - - if conf.Nats != nil { - for _, natsConf := range conf.Nats { - // TODO: When multiple nats configs share a URL, do a single connect. - wg.Add(1) - nc := natsConf - go func() { - // err := ReceiveNats(conf.Nats, decodeLine, runtime.NumCPU()-1, ctx) - err := ReceiveNats(nc, decodeLine, 1, ctx) - - if err != nil { - log.Fatal(err) - } - wg.Done() - }() - } - } - - wg.Wait() - - log.Printf("Writing to '%s'...\n", conf.Checkpoints.RootDir) - files, err = memoryStore.ToCheckpoint(conf.Checkpoints.RootDir, lastCheckpoint.Unix(), time.Now().Unix()) - if err != nil { - log.Printf("Writing checkpoint failed: %s\n", err.Error()) - } - log.Printf("Done! (%d files written)\n", files) - - if closer, ok := debugDump.(io.Closer); ok { - if err := closer.Close(); err != nil { - log.Printf("error: %s", err.Error()) - } - } -} diff --git a/cmd/cc-metric-store/main.go b/cmd/cc-metric-store/main.go new file mode 100644 index 0000000..523b7c6 --- /dev/null +++ b/cmd/cc-metric-store/main.go @@ -0,0 +1,133 @@ +package main + +import ( + "bufio" + "context" + "flag" + "fmt" + "log" + "os" + "os/signal" + "runtime" + "runtime/debug" + "sync" + "syscall" + "time" + + "github.com/ClusterCockpit/cc-metric-store/internal/api" + "github.com/ClusterCockpit/cc-metric-store/internal/config" + "github.com/ClusterCockpit/cc-metric-store/internal/memorystore" + "github.com/google/gops/agent" +) + +var ( + date string + commit string + version string +) + +func main() { + var configFile string + var enableGopsAgent, flagVersion bool + flag.StringVar(&configFile, "config", "./config.json", "configuration file") + flag.BoolVar(&enableGopsAgent, "gops", false, "Listen via github.com/google/gops/agent") + flag.BoolVar(&flagVersion, "version", false, "Show version information and exit") + flag.Parse() + + if flagVersion { + fmt.Printf("Version:\t%s\n", version) + fmt.Printf("Git hash:\t%s\n", commit) + fmt.Printf("Build time:\t%s\n", date) + os.Exit(0) + } + + startupTime := time.Now() + config.Init(configFile) + memorystore.Init(config.Keys.Metrics) + ms := memorystore.GetMemoryStore() + + if enableGopsAgent || config.Keys.Debug.EnableGops { + if err := agent.Listen(agent.Options{}); err != nil { + log.Fatal(err) + } + } + + d, err := time.ParseDuration(config.Keys.Checkpoints.Restore) + if err != nil { + log.Fatal(err) + } + + restoreFrom := startupTime.Add(-d) + log.Printf("Loading checkpoints newer than %s\n", restoreFrom.Format(time.RFC3339)) + files, err := ms.FromCheckpoint(config.Keys.Checkpoints.RootDir, restoreFrom.Unix()) + loadedData := ms.SizeInBytes() / 1024 / 1024 // In MB + if err != nil { + log.Fatalf("Loading checkpoints failed: %s\n", err.Error()) + } else { + log.Printf("Checkpoints loaded (%d files, %d MB, that took %fs)\n", files, loadedData, time.Since(startupTime).Seconds()) + } + + // Try to use less memory by forcing a GC run here and then + // lowering the target percentage. The default of 100 means + // that only once the ratio of new allocations execeds the + // previously active heap, a GC is triggered. + // Forcing a GC here will set the "previously active heap" + // to a minumum. + runtime.GC() + if loadedData > 1000 && os.Getenv("GOGC") == "" { + debug.SetGCPercent(10) + } + + ctx, shutdown := context.WithCancel(context.Background()) + + var wg sync.WaitGroup + sigs := make(chan os.Signal, 1) + signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM, syscall.SIGUSR1) + go func() { + for { + sig := <-sigs + if sig == syscall.SIGUSR1 { + ms.DebugDump(bufio.NewWriter(os.Stdout), nil) + continue + } + + log.Println("Shutting down...") + shutdown() + } + }() + + wg.Add(3) + + memorystore.Retention(&wg, ctx) + memorystore.Checkpointing(&wg, ctx) + memorystore.Archiving(&wg, ctx) + + wg.Add(1) + + go func() { + err := api.StartApiServer(ctx, config.Keys.HttpConfig) + if err != nil { + log.Fatal(err) + } + wg.Done() + }() + + if config.Keys.Nats != nil { + for _, natsConf := range config.Keys.Nats { + // TODO: When multiple nats configs share a URL, do a single connect. + wg.Add(1) + nc := natsConf + go func() { + // err := ReceiveNats(conf.Nats, decodeLine, runtime.NumCPU()-1, ctx) + err := api.ReceiveNats(nc, ms, 1, ctx) + if err != nil { + log.Fatal(err) + } + wg.Done() + }() + } + } + + wg.Wait() + memorystore.Shutdown() +} diff --git a/go.mod b/go.mod index ddbf11c..18d91be 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/ClusterCockpit/cc-metric-store go 1.19 require ( - github.com/golang-jwt/jwt/v5 v5.2.1 + github.com/golang-jwt/jwt/v4 v4.5.0 github.com/google/gops v0.3.28 github.com/gorilla/mux v1.8.1 github.com/influxdata/line-protocol/v2 v2.2.1 diff --git a/go.mod.orig b/go.mod.orig new file mode 100644 index 0000000..cd78a68 --- /dev/null +++ b/go.mod.orig @@ -0,0 +1,37 @@ +module github.com/ClusterCockpit/cc-metric-store + +go 1.19 + +require ( +<<<<<<< HEAD + github.com/golang-jwt/jwt/v4 v4.0.0 + github.com/google/gops v0.3.22 + github.com/gorilla/mux v1.8.0 + github.com/influxdata/line-protocol/v2 v2.2.0 + github.com/nats-io/nats.go v1.11.0 +======= + github.com/golang-jwt/jwt/v5 v5.2.1 + github.com/google/gops v0.3.28 + github.com/gorilla/mux v1.8.1 + github.com/influxdata/line-protocol/v2 v2.2.1 + github.com/nats-io/nats.go v1.33.1 +) + +require ( + github.com/klauspost/compress v1.17.7 // indirect + github.com/nats-io/nkeys v0.4.7 // indirect + github.com/nats-io/nuid v1.0.1 // indirect + golang.org/x/crypto v0.21.0 // indirect + golang.org/x/sys v0.18.0 // indirect +>>>>>>> main +) + +require ( + github.com/golang/protobuf v1.5.2 // indirect + github.com/nats-io/nats-server/v2 v2.2.6 // indirect + github.com/nats-io/nkeys v0.3.0 // indirect + github.com/nats-io/nuid v1.0.1 // indirect + golang.org/x/crypto v0.0.0-20210314154223-e6e6c4f2bb5b // indirect + golang.org/x/sys v0.0.0-20210902050250-f475640dd07b // indirect + google.golang.org/protobuf v1.26.0 // indirect +) diff --git a/go.sum b/go.sum index b76f81f..e278527 100644 --- a/go.sum +++ b/go.sum @@ -3,8 +3,8 @@ github.com/frankban/quicktest v1.11.0/go.mod h1:K+q6oSqb0W0Ininfk863uOk1lMy69l/P github.com/frankban/quicktest v1.11.2/go.mod h1:K+q6oSqb0W0Ininfk863uOk1lMy69l/P6txr3mVT54s= github.com/frankban/quicktest v1.13.0 h1:yNZif1OkDfNoDfb9zZa9aXIpejNR4F23Wely0c+Qdqk= github.com/frankban/quicktest v1.13.0/go.mod h1:qLE0fzW0VuyUAJgPU19zByoIr0HtCHN/r/VLSOOIySU= -github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk= -github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= +github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= +github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= diff --git a/go.sum.orig b/go.sum.orig new file mode 100644 index 0000000..686ec91 --- /dev/null +++ b/go.sum.orig @@ -0,0 +1,110 @@ +<<<<<<< HEAD +github.com/StackExchange/wmi v1.2.1/go.mod h1:rcmrprowKIVzvc+NUiLncP2uuArMWLCbu9SBzvHz7e8= +======= +>>>>>>> main +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/frankban/quicktest v1.11.0/go.mod h1:K+q6oSqb0W0Ininfk863uOk1lMy69l/P6txr3mVT54s= +github.com/frankban/quicktest v1.11.2/go.mod h1:K+q6oSqb0W0Ininfk863uOk1lMy69l/P6txr3mVT54s= +github.com/frankban/quicktest v1.13.0 h1:yNZif1OkDfNoDfb9zZa9aXIpejNR4F23Wely0c+Qdqk= +github.com/frankban/quicktest v1.13.0/go.mod h1:qLE0fzW0VuyUAJgPU19zByoIr0HtCHN/r/VLSOOIySU= +<<<<<<< HEAD +github.com/go-ole/go-ole v1.2.5/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/go-ole/go-ole v1.2.6-0.20210915003542-8b1f7f90f6b1/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/golang-jwt/jwt/v4 v4.0.0 h1:RAqyYixv1p7uEnocuy8P1nru5wprCh/MH2BIlW5z5/o= +github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +======= +github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk= +github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= +>>>>>>> main +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/gops v0.3.28 h1:2Xr57tqKAmQYRAfG12E+yLcoa2Y42UJo2lOrUFL9ark= +github.com/google/gops v0.3.28/go.mod h1:6f6+Nl8LcHrzJwi8+p0ii+vmBFSlB4f8cOOkTJ7sk4c= +github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= +github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= +github.com/influxdata/line-protocol-corpus v0.0.0-20210519164801-ca6fa5da0184/go.mod h1:03nmhxzZ7Xk2pdG+lmMd7mHDfeVOYFyhOgwO61qWU98= +github.com/influxdata/line-protocol-corpus v0.0.0-20210922080147-aa28ccfb8937 h1:MHJNQ+p99hFATQm6ORoLmpUCF7ovjwEFshs/NHzAbig= +github.com/influxdata/line-protocol-corpus v0.0.0-20210922080147-aa28ccfb8937/go.mod h1:BKR9c0uHSmRgM/se9JhFHtTT7JTO67X23MtKMHtZcpo= +github.com/influxdata/line-protocol/v2 v2.0.0-20210312151457-c52fdecb625a/go.mod h1:6+9Xt5Sq1rWx+glMgxhcg2c0DUaehK+5TDcPZ76GypY= +github.com/influxdata/line-protocol/v2 v2.1.0/go.mod h1:QKw43hdUBg3GTk2iC3iyCxksNj7PX9aUSeYOYE/ceHY= +<<<<<<< HEAD +github.com/influxdata/line-protocol/v2 v2.2.0 h1:UPmAqE15Hw5zu9E10SYhoXVLWnEJkWnuCbaCiRsA3c0= +github.com/influxdata/line-protocol/v2 v2.2.0/go.mod h1:DmB3Cnh+3oxmG6LOBIxce4oaL4CPj3OmMPgvauXh+tM= +github.com/keybase/go-ps v0.0.0-20190827175125-91aafc93ba19/go.mod h1:hY+WOq6m2FpbvyrI93sMaypsttvaIL5nhVR92dTMUcQ= +github.com/klauspost/compress v1.11.12 h1:famVnQVu7QwryBN4jNseQdUKES71ZAOnB6UQQJPZvqk= +github.com/klauspost/compress v1.11.12/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= +======= +github.com/influxdata/line-protocol/v2 v2.2.1 h1:EAPkqJ9Km4uAxtMRgUubJyqAr6zgWM0dznKMLRauQRE= +github.com/influxdata/line-protocol/v2 v2.2.1/go.mod h1:DmB3Cnh+3oxmG6LOBIxce4oaL4CPj3OmMPgvauXh+tM= +github.com/klauspost/compress v1.17.7 h1:ehO88t2UGzQK66LMdE8tibEd1ErmzZjNEqWkjLAKQQg= +github.com/klauspost/compress v1.17.7/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +>>>>>>> main +github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/nats-io/nats.go v1.33.1 h1:8TxLZZ/seeEfR97qV0/Bl939tpDnt2Z2fK3HkPypj70= +github.com/nats-io/nats.go v1.33.1/go.mod h1:Ubdu4Nh9exXdSz0RVWRFBbRfrbSxOYd26oF0wkWclB8= +github.com/nats-io/nkeys v0.4.7 h1:RwNJbbIdYCoClSDNY7QVKZlyb/wfT6ugvFCiKy6vDvI= +github.com/nats-io/nkeys v0.4.7/go.mod h1:kqXRgRDPlGy7nGaEDMuYzmiJCIAAWDK0IMBtDmGD0nc= +github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +<<<<<<< HEAD +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/shirou/gopsutil/v3 v3.21.9/go.mod h1:YWp/H8Qs5fVmf17v7JNZzA0mPJ+mS2e9JdiUF9LlKzQ= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/tklauser/go-sysconf v0.3.9/go.mod h1:11DU/5sG7UexIrp/O6g35hrWzu0JxlwQ3LSFUzyeuhs= +github.com/tklauser/numcpus v0.3.0/go.mod h1:yFGUr7TUHQRAhyqBcEg0Ge34zDBAsIvJJcyE6boqnA8= +github.com/xlab/treeprint v1.1.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210314154223-e6e6c4f2bb5b h1:wSOdpTq0/eI46Ez/LkDwIsAKA71YP2SRKBODiRWM0as= +golang.org/x/crypto v0.0.0-20210314154223-e6e6c4f2bb5b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/sys v0.0.0-20190130150945-aca44879d564/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210816074244-15123e1e1f71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210902050250-f475640dd07b h1:S7hKs0Flbq0bbc9xgYt4stIEG1zNDFqyrPwAX2Wj/sE= +golang.org/x/sys v0.0.0-20210902050250-f475640dd07b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1 h1:NusfzzA6yGQ+ua51ck7E3omNUX/JuqbFSaRGqU8CcLI= +golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +======= +golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= +golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= +golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= +golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +>>>>>>> main +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +<<<<<<< HEAD +rsc.io/goversion v1.2.0/go.mod h1:Eih9y/uIBS3ulggl7KNJ09xGSLcuNaLgmvvqa07sgfo= +======= +>>>>>>> main diff --git a/scripts/cc-metric-store.service b/init/cc-metric-store.service similarity index 100% rename from scripts/cc-metric-store.service rename to init/cc-metric-store.service diff --git a/internal/api/api.go b/internal/api/api.go new file mode 100644 index 0000000..efd5e53 --- /dev/null +++ b/internal/api/api.go @@ -0,0 +1,368 @@ +package api + +import ( + "bufio" + "context" + "crypto/ed25519" + "encoding/base64" + "encoding/json" + "fmt" + "io" + "log" + "math" + "net/http" + "strconv" + "strings" + "time" + + "github.com/ClusterCockpit/cc-metric-store/internal/config" + "github.com/ClusterCockpit/cc-metric-store/internal/memorystore" + "github.com/ClusterCockpit/cc-metric-store/internal/util" + "github.com/gorilla/mux" + "github.com/influxdata/line-protocol/v2/lineprotocol" +) + +type ApiMetricData struct { + Error *string `json:"error,omitempty"` + Data util.FloatArray `json:"data,omitempty"` + From int64 `json:"from"` + To int64 `json:"to"` + Avg util.Float `json:"avg"` + Min util.Float `json:"min"` + Max util.Float `json:"max"` +} + +// TODO: Optimize this, just like the stats endpoint! +func (data *ApiMetricData) AddStats() { + n := 0 + sum, min, max := 0.0, math.MaxFloat64, -math.MaxFloat64 + for _, x := range data.Data { + if x.IsNaN() { + continue + } + + n += 1 + sum += float64(x) + min = math.Min(min, float64(x)) + max = math.Max(max, float64(x)) + } + + if n > 0 { + avg := sum / float64(n) + data.Avg = util.Float(avg) + data.Min = util.Float(min) + data.Max = util.Float(max) + } else { + data.Avg, data.Min, data.Max = util.NaN, util.NaN, util.NaN + } +} + +func (data *ApiMetricData) ScaleBy(f util.Float) { + if f == 0 || f == 1 { + return + } + + data.Avg *= f + data.Min *= f + data.Max *= f + for i := 0; i < len(data.Data); i++ { + data.Data[i] *= f + } +} + +func (data *ApiMetricData) PadDataWithNull(ms *memorystore.MemoryStore, from, to int64, metric string) { + minfo, ok := ms.Metrics[metric] + if !ok { + return + } + + if (data.From / minfo.Frequency) > (from / minfo.Frequency) { + padfront := int((data.From / minfo.Frequency) - (from / minfo.Frequency)) + ndata := make([]util.Float, 0, padfront+len(data.Data)) + for i := 0; i < padfront; i++ { + ndata = append(ndata, util.NaN) + } + for j := 0; j < len(data.Data); j++ { + ndata = append(ndata, data.Data[j]) + } + data.Data = ndata + } +} + +func handleFree(rw http.ResponseWriter, r *http.Request) { + rawTo := r.URL.Query().Get("to") + if rawTo == "" { + http.Error(rw, "'to' is a required query parameter", http.StatusBadRequest) + return + } + + to, err := strconv.ParseInt(rawTo, 10, 64) + if err != nil { + http.Error(rw, err.Error(), http.StatusBadRequest) + return + } + + // // TODO: lastCheckpoint might be modified by different go-routines. + // // Load it using the sync/atomic package? + // freeUpTo := lastCheckpoint.Unix() + // if to < freeUpTo { + // freeUpTo = to + // } + + if r.Method != http.MethodPost { + http.Error(rw, "Method Not Allowed", http.StatusMethodNotAllowed) + return + } + + bodyDec := json.NewDecoder(r.Body) + var selectors [][]string + err = bodyDec.Decode(&selectors) + if err != nil { + http.Error(rw, err.Error(), http.StatusBadRequest) + return + } + + ms := memorystore.GetMemoryStore() + n := 0 + for _, sel := range selectors { + bn, err := ms.Free(sel, to) + if err != nil { + http.Error(rw, err.Error(), http.StatusInternalServerError) + return + } + + n += bn + } + + rw.WriteHeader(http.StatusOK) + fmt.Fprintf(rw, "buffers freed: %d\n", n) +} + +func handleWrite(rw http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + http.Error(rw, "Method Not Allowed", http.StatusMethodNotAllowed) + return + } + + bytes, err := io.ReadAll(r.Body) + if err != nil { + log.Printf("error while reading request body: %s", err.Error()) + http.Error(rw, err.Error(), http.StatusInternalServerError) + return + } + + ms := memorystore.GetMemoryStore() + dec := lineprotocol.NewDecoderWithBytes(bytes) + if err := decodeLine(dec, ms, r.URL.Query().Get("cluster")); err != nil { + log.Printf("/api/write error: %s", err.Error()) + http.Error(rw, err.Error(), http.StatusBadRequest) + return + } + rw.WriteHeader(http.StatusOK) +} + +type ApiQueryRequest struct { + Cluster string `json:"cluster"` + Queries []ApiQuery `json:"queries"` + ForAllNodes []string `json:"for-all-nodes"` + From int64 `json:"from"` + To int64 `json:"to"` + WithStats bool `json:"with-stats"` + WithData bool `json:"with-data"` + WithPadding bool `json:"with-padding"` +} + +type ApiQueryResponse struct { + Queries []ApiQuery `json:"queries,omitempty"` + Results [][]ApiMetricData `json:"results"` +} + +type ApiQuery struct { + Type *string `json:"type,omitempty"` + SubType *string `json:"subtype,omitempty"` + Metric string `json:"metric"` + Hostname string `json:"host"` + TypeIds []string `json:"type-ids,omitempty"` + SubTypeIds []string `json:"subtype-ids,omitempty"` + ScaleFactor util.Float `json:"scale-by,omitempty"` + Aggregate bool `json:"aggreg"` +} + +func handleQuery(rw http.ResponseWriter, r *http.Request) { + var err error + req := ApiQueryRequest{WithStats: true, WithData: true, WithPadding: true} + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + http.Error(rw, err.Error(), http.StatusBadRequest) + return + } + + ms := memorystore.GetMemoryStore() + + response := ApiQueryResponse{ + Results: make([][]ApiMetricData, 0, len(req.Queries)), + } + if req.ForAllNodes != nil { + nodes := ms.ListChildren([]string{req.Cluster}) + for _, node := range nodes { + for _, metric := range req.ForAllNodes { + q := ApiQuery{ + Metric: metric, + Hostname: node, + } + req.Queries = append(req.Queries, q) + response.Queries = append(response.Queries, q) + } + } + } + + for _, query := range req.Queries { + sels := make([]util.Selector, 0, 1) + if query.Aggregate || query.Type == nil { + sel := util.Selector{{String: req.Cluster}, {String: query.Hostname}} + if query.Type != nil { + if len(query.TypeIds) == 1 { + sel = append(sel, util.SelectorElement{String: *query.Type + query.TypeIds[0]}) + } else { + ids := make([]string, len(query.TypeIds)) + for i, id := range query.TypeIds { + ids[i] = *query.Type + id + } + sel = append(sel, util.SelectorElement{Group: ids}) + } + + if query.SubType != nil { + if len(query.SubTypeIds) == 1 { + sel = append(sel, util.SelectorElement{String: *query.SubType + query.SubTypeIds[0]}) + } else { + ids := make([]string, len(query.SubTypeIds)) + for i, id := range query.SubTypeIds { + ids[i] = *query.SubType + id + } + sel = append(sel, util.SelectorElement{Group: ids}) + } + } + } + sels = append(sels, sel) + } else { + for _, typeId := range query.TypeIds { + if query.SubType != nil { + for _, subTypeId := range query.SubTypeIds { + sels = append(sels, util.Selector{ + {String: req.Cluster}, + {String: query.Hostname}, + {String: *query.Type + typeId}, + {String: *query.SubType + subTypeId}, + }) + } + } else { + sels = append(sels, util.Selector{ + {String: req.Cluster}, + {String: query.Hostname}, + {String: *query.Type + typeId}, + }) + } + } + } + + // log.Printf("query: %#v\n", query) + // log.Printf("sels: %#v\n", sels) + + res := make([]ApiMetricData, 0, len(sels)) + for _, sel := range sels { + data := ApiMetricData{} + data.Data, data.From, data.To, err = ms.Read(sel, query.Metric, req.From, req.To) + // log.Printf("data: %#v, %#v, %#v, %#v", data.Data, data.From, data.To, err) + if err != nil { + msg := err.Error() + data.Error = &msg + res = append(res, data) + continue + } + + if req.WithStats { + data.AddStats() + } + if query.ScaleFactor != 0 { + data.ScaleBy(query.ScaleFactor) + } + if req.WithPadding { + data.PadDataWithNull(ms, req.From, req.To, query.Metric) + } + if !req.WithData { + data.Data = nil + } + res = append(res, data) + } + response.Results = append(response.Results, res) + } + + rw.Header().Set("Content-Type", "application/json") + bw := bufio.NewWriter(rw) + defer bw.Flush() + if err := json.NewEncoder(bw).Encode(response); err != nil { + log.Print(err) + return + } +} + +func handleDebug(rw http.ResponseWriter, r *http.Request) { + raw := r.URL.Query().Get("selector") + selector := []string{} + if len(raw) != 0 { + selector = strings.Split(raw, ":") + } + + ms := memorystore.GetMemoryStore() + if err := ms.DebugDump(bufio.NewWriter(rw), selector); err != nil { + rw.WriteHeader(http.StatusBadRequest) + rw.Write([]byte(err.Error())) + } +} + +func StartApiServer(ctx context.Context, httpConfig *config.HttpConfig) error { + r := mux.NewRouter() + + r.HandleFunc("/api/free", handleFree) + r.HandleFunc("/api/write", handleWrite) + r.HandleFunc("/api/query", handleQuery) + r.HandleFunc("/api/debug", handleDebug) + + server := &http.Server{ + Handler: r, + Addr: httpConfig.Address, + WriteTimeout: 30 * time.Second, + ReadTimeout: 30 * time.Second, + } + + if len(config.Keys.JwtPublicKey) > 0 { + buf, err := base64.StdEncoding.DecodeString(config.Keys.JwtPublicKey) + if err != nil { + return err + } + publicKey := ed25519.PublicKey(buf) + server.Handler = authentication(server.Handler, publicKey) + } + + go func() { + if httpConfig.CertFile != "" && httpConfig.KeyFile != "" { + log.Printf("API https endpoint listening on '%s'\n", httpConfig.Address) + err := server.ListenAndServeTLS(httpConfig.CertFile, httpConfig.KeyFile) + if err != nil && err != http.ErrServerClosed { + log.Println(err) + } + } else { + log.Printf("API http endpoint listening on '%s'\n", httpConfig.Address) + err := server.ListenAndServe() + if err != nil && err != http.ErrServerClosed { + log.Println(err) + } + } + }() + + for { + <-ctx.Done() + err := server.Shutdown(context.Background()) + log.Println("API server shut down") + return err + } +} diff --git a/api.go b/internal/api/api.go.orig similarity index 72% rename from api.go rename to internal/api/api.go.orig index 1a12f1c..6226b7b 100644 --- a/api.go +++ b/internal/api/api.go.orig @@ -1,4 +1,4 @@ -package main +package api import ( "bufio" @@ -6,7 +6,6 @@ import ( "crypto/ed25519" "encoding/base64" "encoding/json" - "errors" "fmt" "io" "log" @@ -14,15 +13,29 @@ import ( "net/http" "strconv" "strings" - "sync" "time" +<<<<<<< HEAD:internal/api/api.go + "github.com/ClusterCockpit/cc-metric-store/internal/config" + "github.com/ClusterCockpit/cc-metric-store/internal/memorystore" + "github.com/ClusterCockpit/cc-metric-store/internal/util" +======= "github.com/golang-jwt/jwt/v5" +>>>>>>> main:api.go "github.com/gorilla/mux" "github.com/influxdata/line-protocol/v2/lineprotocol" ) type ApiMetricData struct { +<<<<<<< HEAD:internal/api/api.go + Error *string `json:"error,omitempty"` + Data util.FloatArray `json:"data,omitempty"` + From int64 `json:"from"` + To int64 `json:"to"` + Avg util.Float `json:"avg"` + Min util.Float `json:"min"` + Max util.Float `json:"max"` +======= Error *string `json:"error,omitempty"` Data FloatArray `json:"data,omitempty"` From int64 `json:"from"` @@ -30,6 +43,7 @@ type ApiMetricData struct { Avg Float `json:"avg"` Min Float `json:"min"` Max Float `json:"max"` +>>>>>>> main:api.go } // TODO: Optimize this, just like the stats endpoint! @@ -49,15 +63,15 @@ func (data *ApiMetricData) AddStats() { if n > 0 { avg := sum / float64(n) - data.Avg = Float(avg) - data.Min = Float(min) - data.Max = Float(max) + data.Avg = util.Float(avg) + data.Min = util.Float(min) + data.Max = util.Float(max) } else { - data.Avg, data.Min, data.Max = NaN, NaN, NaN + data.Avg, data.Min, data.Max = util.NaN, util.NaN, util.NaN } } -func (data *ApiMetricData) ScaleBy(f Float) { +func (data *ApiMetricData) ScaleBy(f util.Float) { if f == 0 || f == 1 { return } @@ -70,17 +84,17 @@ func (data *ApiMetricData) ScaleBy(f Float) { } } -func (data *ApiMetricData) PadDataWithNull(from, to int64, metric string) { - minfo, ok := memoryStore.metrics[metric] +func (data *ApiMetricData) PadDataWithNull(ms *memorystore.MemoryStore, from, to int64, metric string) { + minfo, ok := ms.Metrics[metric] if !ok { return } if (data.From / minfo.Frequency) > (from / minfo.Frequency) { padfront := int((data.From / minfo.Frequency) - (from / minfo.Frequency)) - ndata := make([]Float, 0, padfront+len(data.Data)) + ndata := make([]util.Float, 0, padfront+len(data.Data)) for i := 0; i < padfront; i++ { - ndata = append(ndata, NaN) + ndata = append(ndata, util.NaN) } for j := 0; j < len(data.Data); j++ { ndata = append(ndata, data.Data[j]) @@ -102,12 +116,12 @@ func handleFree(rw http.ResponseWriter, r *http.Request) { return } - // TODO: lastCheckpoint might be modified by different go-routines. - // Load it using the sync/atomic package? - freeUpTo := lastCheckpoint.Unix() - if to < freeUpTo { - freeUpTo = to - } + // // TODO: lastCheckpoint might be modified by different go-routines. + // // Load it using the sync/atomic package? + // freeUpTo := lastCheckpoint.Unix() + // if to < freeUpTo { + // freeUpTo = to + // } if r.Method != http.MethodPost { http.Error(rw, "Method Not Allowed", http.StatusMethodNotAllowed) @@ -122,9 +136,10 @@ func handleFree(rw http.ResponseWriter, r *http.Request) { return } + ms := memorystore.GetMemoryStore() n := 0 for _, sel := range selectors { - bn, err := memoryStore.Free(sel, freeUpTo) + bn, err := ms.Free(sel, to) if err != nil { http.Error(rw, err.Error(), http.StatusInternalServerError) return @@ -150,26 +165,9 @@ func handleWrite(rw http.ResponseWriter, r *http.Request) { return } - if debugDump != io.Discard { - now := time.Now() - msg := make([]byte, 0, 512) - msg = append(msg, "\n--- local unix time: "...) - msg = strconv.AppendInt(msg, now.Unix(), 10) - msg = append(msg, " ---\n"...) - - debugDumpLock.Lock() - defer debugDumpLock.Unlock() - if _, err := debugDump.Write(msg); err != nil { - log.Printf("error while writing to debug dump: %s", err.Error()) - } - if _, err := debugDump.Write(bytes); err != nil { - log.Printf("error while writing to debug dump: %s", err.Error()) - } - return - } - + ms := memorystore.GetMemoryStore() dec := lineprotocol.NewDecoderWithBytes(bytes) - if err := decodeLine(dec, r.URL.Query().Get("cluster")); err != nil { + if err := decodeLine(dec, ms, r.URL.Query().Get("cluster")); err != nil { log.Printf("/api/write error: %s", err.Error()) http.Error(rw, err.Error(), http.StatusBadRequest) return @@ -194,6 +192,16 @@ type ApiQueryResponse struct { } type ApiQuery struct { +<<<<<<< HEAD:internal/api/api.go + Type *string `json:"type,omitempty"` + SubType *string `json:"subtype,omitempty"` + Metric string `json:"metric"` + Hostname string `json:"host"` + TypeIds []string `json:"type-ids,omitempty"` + SubTypeIds []string `json:"subtype-ids,omitempty"` + ScaleFactor util.Float `json:"scale-by,omitempty"` + Aggregate bool `json:"aggreg"` +======= Type *string `json:"type,omitempty"` SubType *string `json:"subtype,omitempty"` Metric string `json:"metric"` @@ -202,21 +210,28 @@ type ApiQuery struct { SubTypeIds []string `json:"subtype-ids,omitempty"` ScaleFactor Float `json:"scale-by,omitempty"` Aggregate bool `json:"aggreg"` +>>>>>>> main:api.go } func handleQuery(rw http.ResponseWriter, r *http.Request) { var err error req := ApiQueryRequest{WithStats: true, WithData: true, WithPadding: true} +<<<<<<< HEAD:internal/api/api.go + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { +======= if err = json.NewDecoder(r.Body).Decode(&req); err != nil { +>>>>>>> main:api.go http.Error(rw, err.Error(), http.StatusBadRequest) return } + ms := memorystore.GetMemoryStore() + response := ApiQueryResponse{ Results: make([][]ApiMetricData, 0, len(req.Queries)), } if req.ForAllNodes != nil { - nodes := memoryStore.ListChildren([]string{req.Cluster}) + nodes := ms.ListChildren([]string{req.Cluster}) for _, node := range nodes { for _, metric := range req.ForAllNodes { q := ApiQuery{ @@ -230,29 +245,29 @@ func handleQuery(rw http.ResponseWriter, r *http.Request) { } for _, query := range req.Queries { - sels := make([]Selector, 0, 1) + sels := make([]util.Selector, 0, 1) if query.Aggregate || query.Type == nil { - sel := Selector{{String: req.Cluster}, {String: query.Hostname}} + sel := util.Selector{{String: req.Cluster}, {String: query.Hostname}} if query.Type != nil { if len(query.TypeIds) == 1 { - sel = append(sel, SelectorElement{String: *query.Type + query.TypeIds[0]}) + sel = append(sel, util.SelectorElement{String: *query.Type + query.TypeIds[0]}) } else { ids := make([]string, len(query.TypeIds)) for i, id := range query.TypeIds { ids[i] = *query.Type + id } - sel = append(sel, SelectorElement{Group: ids}) + sel = append(sel, util.SelectorElement{Group: ids}) } if query.SubType != nil { if len(query.SubTypeIds) == 1 { - sel = append(sel, SelectorElement{String: *query.SubType + query.SubTypeIds[0]}) + sel = append(sel, util.SelectorElement{String: *query.SubType + query.SubTypeIds[0]}) } else { ids := make([]string, len(query.SubTypeIds)) for i, id := range query.SubTypeIds { ids[i] = *query.SubType + id } - sel = append(sel, SelectorElement{Group: ids}) + sel = append(sel, util.SelectorElement{Group: ids}) } } } @@ -261,7 +276,11 @@ func handleQuery(rw http.ResponseWriter, r *http.Request) { for _, typeId := range query.TypeIds { if query.SubType != nil { for _, subTypeId := range query.SubTypeIds { +<<<<<<< HEAD:internal/api/api.go + sels = append(sels, util.Selector{ +======= sels = append(sels, Selector{ +>>>>>>> main:api.go {String: req.Cluster}, {String: query.Hostname}, {String: *query.Type + typeId}, @@ -269,7 +288,7 @@ func handleQuery(rw http.ResponseWriter, r *http.Request) { }) } } else { - sels = append(sels, Selector{ + sels = append(sels, util.Selector{ {String: req.Cluster}, {String: query.Hostname}, {String: *query.Type + typeId}, @@ -284,7 +303,7 @@ func handleQuery(rw http.ResponseWriter, r *http.Request) { res := make([]ApiMetricData, 0, len(sels)) for _, sel := range sels { data := ApiMetricData{} - data.Data, data.From, data.To, err = memoryStore.Read(sel, query.Metric, req.From, req.To) + data.Data, data.From, data.To, err = ms.Read(sel, query.Metric, req.From, req.To) // log.Printf("data: %#v, %#v, %#v, %#v", data.Data, data.From, data.To, err) if err != nil { msg := err.Error() @@ -300,7 +319,7 @@ func handleQuery(rw http.ResponseWriter, r *http.Request) { data.ScaleBy(query.ScaleFactor) } if req.WithPadding { - data.PadDataWithNull(req.From, req.To, query.Metric) + data.PadDataWithNull(ms, req.From, req.To, query.Metric) } if !req.WithData { data.Data = nil @@ -319,10 +338,20 @@ func handleQuery(rw http.ResponseWriter, r *http.Request) { } } -func authentication(next http.Handler, publicKey ed25519.PublicKey) http.Handler { - cacheLock := sync.RWMutex{} - cache := map[string]*jwt.Token{} +func handleDebug(rw http.ResponseWriter, r *http.Request) { + raw := r.URL.Query().Get("selector") + selector := []string{} + if len(raw) != 0 { + selector = strings.Split(raw, ":") + } +<<<<<<< HEAD:internal/api/api.go + ms := memorystore.GetMemoryStore() + if err := ms.DebugDump(bufio.NewWriter(rw), selector); err != nil { + rw.WriteHeader(http.StatusBadRequest) + rw.Write([]byte(err.Error())) + } +======= return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) { authheader := r.Header.Get("Authorization") if authheader == "" || !strings.HasPrefix(authheader, "Bearer ") { @@ -373,26 +402,16 @@ func authentication(next http.Handler, publicKey ed25519.PublicKey) http.Handler // Let request through... next.ServeHTTP(rw, r) }) +>>>>>>> main:api.go } -func StartApiServer(ctx context.Context, httpConfig *HttpConfig) error { +func StartApiServer(ctx context.Context, httpConfig *config.HttpConfig) error { r := mux.NewRouter() r.HandleFunc("/api/free", handleFree) r.HandleFunc("/api/write", handleWrite) r.HandleFunc("/api/query", handleQuery) - r.HandleFunc("/api/debug", func(rw http.ResponseWriter, r *http.Request) { - raw := r.URL.Query().Get("selector") - selector := []string{} - if len(raw) != 0 { - selector = strings.Split(raw, ":") - } - - if err := memoryStore.DebugDump(bufio.NewWriter(rw), selector); err != nil { - rw.WriteHeader(http.StatusBadRequest) - rw.Write([]byte(err.Error())) - } - }) + r.HandleFunc("/api/debug", handleDebug) server := &http.Server{ Handler: r, @@ -401,8 +420,8 @@ func StartApiServer(ctx context.Context, httpConfig *HttpConfig) error { ReadTimeout: 30 * time.Second, } - if len(conf.JwtPublicKey) > 0 { - buf, err := base64.StdEncoding.DecodeString(conf.JwtPublicKey) + if len(config.Keys.JwtPublicKey) > 0 { + buf, err := base64.StdEncoding.DecodeString(config.Keys.JwtPublicKey) if err != nil { return err } diff --git a/internal/api/authentication.go b/internal/api/authentication.go new file mode 100644 index 0000000..015810a --- /dev/null +++ b/internal/api/authentication.go @@ -0,0 +1,56 @@ +package api + +import ( + "crypto/ed25519" + "errors" + "net/http" + "strings" + "sync" + + "github.com/golang-jwt/jwt/v4" +) + +func authentication(next http.Handler, publicKey ed25519.PublicKey) http.Handler { + cacheLock := sync.RWMutex{} + cache := map[string]*jwt.Token{} + + return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) { + authheader := r.Header.Get("Authorization") + if authheader == "" || !strings.HasPrefix(authheader, "Bearer ") { + http.Error(rw, "Use JWT Authentication", http.StatusUnauthorized) + return + } + + rawtoken := authheader[len("Bearer "):] + cacheLock.RLock() + token, ok := cache[rawtoken] + cacheLock.RUnlock() + if ok && token.Claims.Valid() == nil { + next.ServeHTTP(rw, r) + return + } + + // The actual token is ignored for now. + // In case expiration and so on are specified, the Parse function + // already returns an error for expired tokens. + var err error + token, err = jwt.Parse(rawtoken, func(t *jwt.Token) (interface{}, error) { + if t.Method != jwt.SigningMethodEdDSA { + return nil, errors.New("only Ed25519/EdDSA supported") + } + + return publicKey, nil + }) + if err != nil { + http.Error(rw, err.Error(), http.StatusUnauthorized) + return + } + + cacheLock.Lock() + cache[rawtoken] = token + cacheLock.Unlock() + + // Let request through... + next.ServeHTTP(rw, r) + }) +} diff --git a/lineprotocol.go b/internal/api/lineprotocol.go similarity index 85% rename from lineprotocol.go rename to internal/api/lineprotocol.go index b2e1692..9081638 100644 --- a/lineprotocol.go +++ b/internal/api/lineprotocol.go @@ -1,4 +1,4 @@ -package main +package api import ( "context" @@ -9,20 +9,18 @@ import ( "sync" "time" + "github.com/ClusterCockpit/cc-metric-store/internal/config" + "github.com/ClusterCockpit/cc-metric-store/internal/memorystore" + "github.com/ClusterCockpit/cc-metric-store/internal/util" "github.com/influxdata/line-protocol/v2/lineprotocol" "github.com/nats-io/nats.go" ) -type Metric struct { - Name string - Value Float - - mc MetricConfig -} - -// Currently unused, could be used to send messages via raw TCP. // Each connection is handled in it's own goroutine. This is a blocking function. -func ReceiveRaw(ctx context.Context, listener net.Listener, handleLine func(*lineprotocol.Decoder, string) error) error { +func ReceiveRaw(ctx context.Context, + listener net.Listener, + handleLine func(*lineprotocol.Decoder, string) error, +) error { var wg sync.WaitGroup wg.Add(1) @@ -84,7 +82,11 @@ func ReceiveRaw(ctx context.Context, listener net.Listener, handleLine func(*lin // Connect to a nats server and subscribe to "updates". This is a blocking // function. handleLine will be called for each line recieved via nats. // Send `true` through the done channel for gracefull termination. -func ReceiveNats(conf *NatsConfig, handleLine func(*lineprotocol.Decoder, string) error, workers int, ctx context.Context) error { +func ReceiveNats(conf *config.NatsConfig, + ms *memorystore.MemoryStore, + workers int, + ctx context.Context, +) error { var opts []nats.Option if conf.Username != "" && conf.Password != "" { opts = append(opts, nats.UserInfo(conf.Username, conf.Password)) @@ -111,7 +113,7 @@ func ReceiveNats(conf *NatsConfig, handleLine func(*lineprotocol.Decoder, string go func() { for m := range msgs { dec := lineprotocol.NewDecoderWithBytes(m.Data) - if err := handleLine(dec, clusterTag); err != nil { + if err := decodeLine(dec, ms, clusterTag); err != nil { log.Printf("error: %s\n", err.Error()) } } @@ -126,7 +128,7 @@ func ReceiveNats(conf *NatsConfig, handleLine func(*lineprotocol.Decoder, string } else { sub, err = nc.Subscribe(sc.SubscribeTo, func(m *nats.Msg) { dec := lineprotocol.NewDecoderWithBytes(m.Data) - if err := handleLine(dec, clusterTag); err != nil { + if err := decodeLine(dec, ms, clusterTag); err != nil { log.Printf("error: %s\n", err.Error()) } }) @@ -175,18 +177,21 @@ func reorder(buf, prefix []byte) []byte { // Decode lines using dec and make write calls to the MemoryStore. // If a line is missing its cluster tag, use clusterDefault as default. -func decodeLine(dec *lineprotocol.Decoder, clusterDefault string) error { +func decodeLine(dec *lineprotocol.Decoder, + ms *memorystore.MemoryStore, + clusterDefault string, +) error { // Reduce allocations in loop: t := time.Now() - metric, metricBuf := Metric{}, make([]byte, 0, 16) + metric, metricBuf := memorystore.Metric{}, make([]byte, 0, 16) selector := make([]string, 0, 4) typeBuf, subTypeBuf := make([]byte, 0, 16), make([]byte, 0) // Optimize for the case where all lines in a "batch" are about the same // cluster and host. By using `WriteToLevel` (level = host), we do not need // to take the root- and cluster-level lock as often. - var lvl *level = nil - var prevCluster, prevHost string = "", "" + var lvl *memorystore.Level = nil + prevCluster, prevHost := "", "" var ok bool for dec.Next() { @@ -200,7 +205,7 @@ func decodeLine(dec *lineprotocol.Decoder, clusterDefault string) error { metricBuf = append(metricBuf[:0], rawmeasurement...) // The go compiler optimizes map[string(byteslice)] lookups: - metric.mc, ok = memoryStore.metrics[string(rawmeasurement)] + metric.MetricConfig, ok = ms.Metrics[string(rawmeasurement)] if !ok { continue } @@ -264,7 +269,7 @@ func decodeLine(dec *lineprotocol.Decoder, clusterDefault string) error { if lvl == nil { selector = selector[:2] selector[0], selector[1] = cluster, host - lvl = memoryStore.GetLevel(selector) + lvl = ms.GetLevel(selector) prevCluster, prevHost = cluster, host } @@ -292,11 +297,11 @@ func decodeLine(dec *lineprotocol.Decoder, clusterDefault string) error { } if val.Kind() == lineprotocol.Float { - metric.Value = Float(val.FloatV()) + metric.Value = util.Float(val.FloatV()) } else if val.Kind() == lineprotocol.Int { - metric.Value = Float(val.IntV()) + metric.Value = util.Float(val.IntV()) } else if val.Kind() == lineprotocol.Uint { - metric.Value = Float(val.UintV()) + metric.Value = util.Float(val.UintV()) } else { return fmt.Errorf("unsupported value type in message: %s", val.Kind().String()) } @@ -306,7 +311,7 @@ func decodeLine(dec *lineprotocol.Decoder, clusterDefault string) error { return err } - if err := memoryStore.WriteToLevel(lvl, selector, t.Unix(), []Metric{metric}); err != nil { + if err := ms.WriteToLevel(lvl, selector, t.Unix(), []memorystore.Metric{metric}); err != nil { return err } } diff --git a/internal/config/config.go b/internal/config/config.go new file mode 100644 index 0000000..b829b5d --- /dev/null +++ b/internal/config/config.go @@ -0,0 +1,112 @@ +package config + +import ( + "encoding/json" + "fmt" + "log" + "os" +) + +// For aggregation over multiple values at different cpus/sockets/..., not time! +type AggregationStrategy int + +const ( + NoAggregation AggregationStrategy = iota + SumAggregation + AvgAggregation +) + +func (as *AggregationStrategy) UnmarshalJSON(data []byte) error { + var str string + if err := json.Unmarshal(data, &str); err != nil { + return err + } + + switch str { + case "": + *as = NoAggregation + case "sum": + *as = SumAggregation + case "avg": + *as = AvgAggregation + default: + return fmt.Errorf("invalid aggregation strategy: %#v", str) + } + return nil +} + +type MetricConfig struct { + // Interval in seconds at which measurements will arive. + Frequency int64 `json:"frequency"` + + // Can be 'sum', 'avg' or null. Describes how to aggregate metrics from the same timestep over the hierarchy. + Aggregation AggregationStrategy `json:"aggregation"` + + // Private, used internally... + Offset int +} + +type HttpConfig struct { + // Address to bind to, for example "0.0.0.0:8081" + Address string `json:"address"` + + // If not the empty string, use https with this as the certificate file + CertFile string `json:"https-cert-file"` + + // If not the empty string, use https with this as the key file + KeyFile string `json:"https-key-file"` +} + +type NatsConfig struct { + // Address of the nats server + Address string `json:"address"` + + // Username/Password, optional + Username string `json:"username"` + Password string `json:"password"` + + Subscriptions []struct { + // Channel name + SubscribeTo string `json:"subscribe-to"` + + // Allow lines without a cluster tag, use this as default, optional + ClusterTag string `json:"cluster-tag"` + } `json:"subscriptions"` +} + +type Config struct { + Metrics map[string]MetricConfig `json:"metrics"` + HttpConfig *HttpConfig `json:"http-api"` + Checkpoints struct { + Interval string `json:"interval"` + RootDir string `json:"directory"` + Restore string `json:"restore"` + } `json:"checkpoints"` + Debug struct { + DumpToFile string `json:"dump-to-file"` + EnableGops bool `json:"gops"` + } `json:"debug"` + RetentionInMemory string `json:"retention-in-memory"` + JwtPublicKey string `json:"jwt-public-key"` + Archive struct { + Interval string `json:"interval"` + RootDir string `json:"directory"` + DeleteInstead bool `json:"delete-instead"` + } `json:"archive"` + Nats []*NatsConfig `json:"nats"` +} + +var Keys Config + +func Init(file string) { + configFile, err := os.Open(file) + if err != nil { + log.Fatal(err) + } + defer configFile.Close() + dec := json.NewDecoder(configFile) + dec.DisallowUnknownFields() + if err := dec.Decode(&Keys); err != nil { + log.Fatal(err) + } +} diff --git a/internal/memorystore/archive.go b/internal/memorystore/archive.go new file mode 100644 index 0000000..fab457c --- /dev/null +++ b/internal/memorystore/archive.go @@ -0,0 +1,184 @@ +package memorystore + +import ( + "archive/zip" + "bufio" + "context" + "errors" + "fmt" + "io" + "log" + "os" + "path/filepath" + "sync" + "sync/atomic" + "time" + + "github.com/ClusterCockpit/cc-metric-store/internal/config" +) + +func Archiving(wg *sync.WaitGroup, ctx context.Context) { + go func() { + defer wg.Done() + d, err := time.ParseDuration(config.Keys.Archive.Interval) + if err != nil { + log.Fatal(err) + } + if d <= 0 { + return + } + + ticks := func() <-chan time.Time { + if d <= 0 { + return nil + } + return time.NewTicker(d).C + }() + for { + select { + case <-ctx.Done(): + return + case <-ticks: + t := time.Now().Add(-d) + log.Printf("start archiving checkpoints (older than %s)...\n", t.Format(time.RFC3339)) + n, err := ArchiveCheckpoints(config.Keys.Checkpoints.RootDir, config.Keys.Archive.RootDir, t.Unix(), config.Keys.Archive.DeleteInstead) + if err != nil { + log.Printf("archiving failed: %s\n", err.Error()) + } else { + log.Printf("done: %d files zipped and moved to archive\n", n) + } + } + } + }() +} + +var ErrNoNewData error = errors.New("all data already archived") + +// ZIP all checkpoint files older than `from` together and write them to the `archiveDir`, +// deleting them from the `checkpointsDir`. +func ArchiveCheckpoints(checkpointsDir, archiveDir string, from int64, deleteInstead bool) (int, error) { + entries1, err := os.ReadDir(checkpointsDir) + if err != nil { + return 0, err + } + + type workItem struct { + cdir, adir string + cluster, host string + } + + var wg sync.WaitGroup + n, errs := int32(0), int32(0) + work := make(chan workItem, NumWorkers) + + wg.Add(NumWorkers) + for worker := 0; worker < NumWorkers; worker++ { + go func() { + defer wg.Done() + for workItem := range work { + m, err := archiveCheckpoints(workItem.cdir, workItem.adir, from, deleteInstead) + if err != nil { + log.Printf("error while archiving %s/%s: %s", workItem.cluster, workItem.host, err.Error()) + atomic.AddInt32(&errs, 1) + } + atomic.AddInt32(&n, int32(m)) + } + }() + } + + for _, de1 := range entries1 { + entries2, e := os.ReadDir(filepath.Join(checkpointsDir, de1.Name())) + if e != nil { + err = e + } + + for _, de2 := range entries2 { + cdir := filepath.Join(checkpointsDir, de1.Name(), de2.Name()) + adir := filepath.Join(archiveDir, de1.Name(), de2.Name()) + work <- workItem{ + adir: adir, cdir: cdir, + cluster: de1.Name(), host: de2.Name(), + } + } + } + + close(work) + wg.Wait() + + if err != nil { + return int(n), err + } + + if errs > 0 { + return int(n), fmt.Errorf("%d errors happend while archiving (%d successes)", errs, n) + } + return int(n), nil +} + +// Helper function for `ArchiveCheckpoints`. +func archiveCheckpoints(dir string, archiveDir string, from int64, deleteInstead bool) (int, error) { + entries, err := os.ReadDir(dir) + if err != nil { + return 0, err + } + + files, err := findFiles(entries, from, false) + if err != nil { + return 0, err + } + + if deleteInstead { + n := 0 + for _, checkpoint := range files { + filename := filepath.Join(dir, checkpoint) + if err = os.Remove(filename); err != nil { + return n, err + } + n += 1 + } + return n, nil + } + + filename := filepath.Join(archiveDir, fmt.Sprintf("%d.zip", from)) + f, err := os.OpenFile(filename, os.O_CREATE|os.O_WRONLY, 0o644) + if err != nil && os.IsNotExist(err) { + err = os.MkdirAll(archiveDir, 0o755) + if err == nil { + f, err = os.OpenFile(filename, os.O_CREATE|os.O_WRONLY, 0o644) + } + } + if err != nil { + return 0, err + } + defer f.Close() + bw := bufio.NewWriter(f) + defer bw.Flush() + zw := zip.NewWriter(bw) + defer zw.Close() + + n := 0 + for _, checkpoint := range files { + filename := filepath.Join(dir, checkpoint) + r, err := os.Open(filename) + if err != nil { + return n, err + } + defer r.Close() + + w, err := zw.Create(checkpoint) + if err != nil { + return n, err + } + + if _, err = io.Copy(w, r); err != nil { + return n, err + } + + if err = os.Remove(filename); err != nil { + return n, err + } + n += 1 + } + + return n, nil +} diff --git a/internal/memorystore/buffer.go b/internal/memorystore/buffer.go new file mode 100644 index 0000000..34fee5d --- /dev/null +++ b/internal/memorystore/buffer.go @@ -0,0 +1,233 @@ +package memorystore + +import ( + "errors" + "sync" + + "github.com/ClusterCockpit/cc-metric-store/internal/util" +) + +// Default buffer capacity. +// `buffer.data` will only ever grow up to it's capacity and a new link +// in the buffer chain will be created if needed so that no copying +// of data or reallocation needs to happen on writes. +const ( + BUFFER_CAP int = 512 +) + +// So that we can reuse allocations +var bufferPool sync.Pool = sync.Pool{ + New: func() interface{} { + return &buffer{ + data: make([]util.Float, 0, BUFFER_CAP), + } + }, +} + +var ( + ErrNoData error = errors.New("no data for this metric/level") + ErrDataDoesNotAlign error = errors.New("data from lower granularities does not align") +) + +// Each metric on each level has it's own buffer. +// This is where the actual values go. +// If `cap(data)` is reached, a new buffer is created and +// becomes the new head of a buffer list. +type buffer struct { + prev *buffer + next *buffer + data []util.Float + frequency int64 + start int64 + archived bool + closed bool +} + +func newBuffer(ts, freq int64) *buffer { + b := bufferPool.Get().(*buffer) + b.frequency = freq + b.start = ts - (freq / 2) + b.prev = nil + b.next = nil + b.archived = false + b.closed = false + b.data = b.data[:0] + return b +} + +// If a new buffer was created, the new head is returnd. +// Otherwise, the existing buffer is returnd. +// Normaly, only "newer" data should be written, but if the value would +// end up in the same buffer anyways it is allowed. +func (b *buffer) write(ts int64, value util.Float) (*buffer, error) { + if ts < b.start { + return nil, errors.New("cannot write value to buffer from past") + } + + // idx := int((ts - b.start + (b.frequency / 3)) / b.frequency) + idx := int((ts - b.start) / b.frequency) + if idx >= cap(b.data) { + newbuf := newBuffer(ts, b.frequency) + newbuf.prev = b + b.next = newbuf + b.close() + b = newbuf + idx = 0 + } + + // Overwriting value or writing value from past + if idx < len(b.data) { + b.data[idx] = value + return b, nil + } + + // Fill up unwritten slots with NaN + for i := len(b.data); i < idx; i++ { + b.data = append(b.data, util.NaN) + } + + b.data = append(b.data, value) + return b, nil +} + +func (b *buffer) end() int64 { + return b.firstWrite() + int64(len(b.data))*b.frequency +} + +func (b *buffer) firstWrite() int64 { + return b.start + (b.frequency / 2) +} + +func (b *buffer) close() {} + +/* +func (b *buffer) close() { + if b.closed { + return + } + + b.closed = true + n, sum, min, max := 0, 0., math.MaxFloat64, -math.MaxFloat64 + for _, x := range b.data { + if x.IsNaN() { + continue + } + + n += 1 + f := float64(x) + sum += f + min = math.Min(min, f) + max = math.Max(max, f) + } + + b.statisticts.samples = n + if n > 0 { + b.statisticts.avg = Float(sum / float64(n)) + b.statisticts.min = Float(min) + b.statisticts.max = Float(max) + } else { + b.statisticts.avg = NaN + b.statisticts.min = NaN + b.statisticts.max = NaN + } +} +*/ + +// func interpolate(idx int, data []Float) Float { +// if idx == 0 || idx+1 == len(data) { +// return NaN +// } +// return (data[idx-1] + data[idx+1]) / 2.0 +// } + +// Return all known values from `from` to `to`. Gaps of information are represented as NaN. +// Simple linear interpolation is done between the two neighboring cells if possible. +// If values at the start or end are missing, instead of NaN values, the second and thrid +// return values contain the actual `from`/`to`. +// This function goes back the buffer chain if `from` is older than the currents buffer start. +// The loaded values are added to `data` and `data` is returned, possibly with a shorter length. +// If `data` is not long enough to hold all values, this function will panic! +func (b *buffer) read(from, to int64, data []util.Float) ([]util.Float, int64, int64, error) { + if from < b.firstWrite() { + if b.prev != nil { + return b.prev.read(from, to, data) + } + from = b.firstWrite() + } + + i := 0 + t := from + for ; t < to; t += b.frequency { + idx := int((t - b.start) / b.frequency) + if idx >= cap(b.data) { + if b.next == nil { + break + } + b = b.next + idx = 0 + } + + if idx >= len(b.data) { + if b.next == nil || to <= b.next.start { + break + } + data[i] += util.NaN + } else if t < b.start { + data[i] += util.NaN + // } else if b.data[idx].IsNaN() { + // data[i] += interpolate(idx, b.data) + } else { + data[i] += b.data[idx] + } + i++ + } + + return data[:i], from, t, nil +} + +// Returns true if this buffer needs to be freed. +func (b *buffer) free(t int64) (delme bool, n int) { + if b.prev != nil { + delme, m := b.prev.free(t) + n += m + if delme { + b.prev.next = nil + if cap(b.prev.data) == BUFFER_CAP { + bufferPool.Put(b.prev) + } + b.prev = nil + } + } + + end := b.end() + if end < t { + return true, n + 1 + } + + return false, n +} + +// Call `callback` on every buffer that contains data in the range from `from` to `to`. +func (b *buffer) iterFromTo(from, to int64, callback func(b *buffer) error) error { + if b == nil { + return nil + } + + if err := b.prev.iterFromTo(from, to, callback); err != nil { + return err + } + + if from <= b.end() && b.start <= to { + return callback(b) + } + + return nil +} + +func (b *buffer) count() int64 { + res := int64(len(b.data)) + if b.prev != nil { + res += b.prev.count() + } + return res +} diff --git a/archive.go b/internal/memorystore/checkpoint.go similarity index 69% rename from archive.go rename to internal/memorystore/checkpoint.go index 302afb3..9b036d5 100644 --- a/archive.go +++ b/internal/memorystore/checkpoint.go @@ -1,12 +1,11 @@ -package main +package memorystore import ( - "archive/zip" "bufio" + "context" "encoding/json" "errors" "fmt" - "io" "io/fs" "log" "os" @@ -18,13 +17,66 @@ import ( "strings" "sync" "sync/atomic" + "time" + + "github.com/ClusterCockpit/cc-metric-store/internal/config" + "github.com/ClusterCockpit/cc-metric-store/internal/util" ) // Whenever changed, update MarshalJSON as well! type CheckpointMetrics struct { - Frequency int64 `json:"frequency"` - Start int64 `json:"start"` - Data []Float `json:"data"` + Data []util.Float `json:"data"` + Frequency int64 `json:"frequency"` + Start int64 `json:"start"` +} + +type CheckpointFile struct { + Metrics map[string]*CheckpointMetrics `json:"metrics"` + Children map[string]*CheckpointFile `json:"children"` + From int64 `json:"from"` + To int64 `json:"to"` +} + +var lastCheckpoint time.Time + +func Checkpointing(wg *sync.WaitGroup, ctx context.Context) { + lastCheckpoint = time.Now() + ms := GetMemoryStore() + + go func() { + defer wg.Done() + d, err := time.ParseDuration(config.Keys.Checkpoints.Interval) + if err != nil { + log.Fatal(err) + } + if d <= 0 { + return + } + + ticks := func() <-chan time.Time { + if d <= 0 { + return nil + } + return time.NewTicker(d).C + }() + for { + select { + case <-ctx.Done(): + return + case <-ticks: + log.Printf("start checkpointing (starting at %s)...\n", lastCheckpoint.Format(time.RFC3339)) + now := time.Now() + n, err := ms.ToCheckpoint(config.Keys.Checkpoints.RootDir, + lastCheckpoint.Unix(), now.Unix()) + if err != nil { + log.Printf("checkpointing failed: %s\n", err.Error()) + } else { + log.Printf("done: %d checkpoint files created\n", n) + lastCheckpoint = now + } + } + } + }() } // As `Float` implements a custom MarshalJSON() function, @@ -51,31 +103,12 @@ func (cm *CheckpointMetrics) MarshalJSON() ([]byte, error) { return buf, nil } -type CheckpointFile struct { - From int64 `json:"from"` - To int64 `json:"to"` - Metrics map[string]*CheckpointMetrics `json:"metrics"` - Children map[string]*CheckpointFile `json:"children"` -} - -var ErrNoNewData error = errors.New("all data already archived") - -var NumWorkers int = 4 - -func init() { - maxWorkers := 10 - NumWorkers = runtime.NumCPU()/2 + 1 - if NumWorkers > maxWorkers { - NumWorkers = maxWorkers - } -} - // Metrics stored at the lowest 2 levels are not stored away (root and cluster)! // On a per-host basis a new JSON file is created. I have no idea if this will scale. // The good thing: Only a host at a time is locked, so this function can run // in parallel to writes/reads. func (m *MemoryStore) ToCheckpoint(dir string, from, to int64) (int, error) { - levels := make([]*level, 0) + levels := make([]*Level, 0) selectors := make([][]string, 0) m.root.lock.RLock() for sel1, l1 := range m.root.children { @@ -89,7 +122,7 @@ func (m *MemoryStore) ToCheckpoint(dir string, from, to int64) (int, error) { m.root.lock.RUnlock() type workItem struct { - level *level + level *Level dir string selector []string } @@ -136,7 +169,7 @@ func (m *MemoryStore) ToCheckpoint(dir string, from, to int64) (int, error) { return int(n), nil } -func (l *level) toCheckpointFile(from, to int64, m *MemoryStore) (*CheckpointFile, error) { +func (l *Level) toCheckpointFile(from, to int64, m *MemoryStore) (*CheckpointFile, error) { l.lock.RLock() defer l.lock.RUnlock() @@ -147,8 +180,8 @@ func (l *level) toCheckpointFile(from, to int64, m *MemoryStore) (*CheckpointFil Children: make(map[string]*CheckpointFile), } - for metric, minfo := range m.metrics { - b := l.metrics[minfo.offset] + for metric, minfo := range m.Metrics { + b := l.metrics[minfo.Offset] if b == nil { continue } @@ -165,14 +198,14 @@ func (l *level) toCheckpointFile(from, to int64, m *MemoryStore) (*CheckpointFil continue } - data := make([]Float, (to-from)/b.frequency+1) + data := make([]util.Float, (to-from)/b.frequency+1) data, start, end, err := b.read(from, to, data) if err != nil { return nil, err } for i := int((end - start) / b.frequency); i < len(data); i++ { - data[i] = NaN + data[i] = util.NaN } retval.Metrics[metric] = &CheckpointMetrics{ @@ -200,7 +233,7 @@ func (l *level) toCheckpointFile(from, to int64, m *MemoryStore) (*CheckpointFil return retval, nil } -func (l *level) toCheckpoint(dir string, from, to int64, m *MemoryStore) error { +func (l *Level) toCheckpoint(dir string, from, to int64, m *MemoryStore) error { cf, err := l.toCheckpointFile(from, to, m) if err != nil { return err @@ -211,11 +244,11 @@ func (l *level) toCheckpoint(dir string, from, to int64, m *MemoryStore) error { } filepath := path.Join(dir, fmt.Sprintf("%d.json", from)) - f, err := os.OpenFile(filepath, os.O_CREATE|os.O_WRONLY, 0644) + f, err := os.OpenFile(filepath, os.O_CREATE|os.O_WRONLY, 0o644) if err != nil && os.IsNotExist(err) { - err = os.MkdirAll(dir, 0755) + err = os.MkdirAll(dir, 0o755) if err == nil { - f, err = os.OpenFile(filepath, os.O_CREATE|os.O_WRONLY, 0644) + f, err = os.OpenFile(filepath, os.O_CREATE|os.O_WRONLY, 0o644) } } if err != nil { @@ -244,7 +277,7 @@ func (m *MemoryStore) FromCheckpoint(dir string, from int64) (int, error) { go func() { defer wg.Done() for host := range work { - lvl := m.root.findLevelOrCreate(host[:], len(m.metrics)) + lvl := m.root.findLevelOrCreate(host[:], len(m.Metrics)) nn, err := lvl.fromCheckpoint(filepath.Join(dir, host[0], host[1]), from, m) if err != nil { log.Fatalf("error while loading checkpoints: %s", err.Error()) @@ -302,7 +335,7 @@ done: return int(n), nil } -func (l *level) loadFile(cf *CheckpointFile, m *MemoryStore) error { +func (l *Level) loadFile(cf *CheckpointFile, m *MemoryStore) error { for name, metric := range cf.Metrics { n := len(metric.Data) b := &buffer{ @@ -315,15 +348,15 @@ func (l *level) loadFile(cf *CheckpointFile, m *MemoryStore) error { } b.close() - minfo, ok := m.metrics[name] + minfo, ok := m.Metrics[name] if !ok { continue // return errors.New("Unkown metric: " + name) } - prev := l.metrics[minfo.offset] + prev := l.metrics[minfo.Offset] if prev == nil { - l.metrics[minfo.offset] = b + l.metrics[minfo.Offset] = b } else { if prev.start > b.start { return errors.New("wooops") @@ -332,18 +365,18 @@ func (l *level) loadFile(cf *CheckpointFile, m *MemoryStore) error { b.prev = prev prev.next = b } - l.metrics[minfo.offset] = b + l.metrics[minfo.Offset] = b } if len(cf.Children) > 0 && l.children == nil { - l.children = make(map[string]*level) + l.children = make(map[string]*Level) } for sel, childCf := range cf.Children { child, ok := l.children[sel] if !ok { - child = &level{ - metrics: make([]*buffer, len(m.metrics)), + child = &Level{ + metrics: make([]*buffer, len(m.Metrics)), children: nil, } l.children[sel] = child @@ -357,7 +390,7 @@ func (l *level) loadFile(cf *CheckpointFile, m *MemoryStore) error { return nil } -func (l *level) fromCheckpoint(dir string, from int64, m *MemoryStore) (int, error) { +func (l *Level) fromCheckpoint(dir string, from int64, m *MemoryStore) (int, error) { direntries, err := os.ReadDir(dir) if err != nil { if os.IsNotExist(err) { @@ -371,9 +404,9 @@ func (l *level) fromCheckpoint(dir string, from int64, m *MemoryStore) (int, err filesLoaded := 0 for _, e := range direntries { if e.IsDir() { - child := &level{ - metrics: make([]*buffer, len(m.metrics)), - children: make(map[string]*level), + child := &Level{ + metrics: make([]*buffer, len(m.Metrics)), + children: make(map[string]*Level), } files, err := child.fromCheckpoint(path.Join(dir, e.Name()), from, m) @@ -466,132 +499,3 @@ func findFiles(direntries []fs.DirEntry, t int64, findMoreRecentFiles bool) ([]s return filenames, nil } - -// ZIP all checkpoint files older than `from` together and write them to the `archiveDir`, -// deleting them from the `checkpointsDir`. -func ArchiveCheckpoints(checkpointsDir, archiveDir string, from int64, deleteInstead bool) (int, error) { - entries1, err := os.ReadDir(checkpointsDir) - if err != nil { - return 0, err - } - - type workItem struct { - cdir, adir string - cluster, host string - } - - var wg sync.WaitGroup - n, errs := int32(0), int32(0) - work := make(chan workItem, NumWorkers) - - wg.Add(NumWorkers) - for worker := 0; worker < NumWorkers; worker++ { - go func() { - defer wg.Done() - for workItem := range work { - m, err := archiveCheckpoints(workItem.cdir, workItem.adir, from, deleteInstead) - if err != nil { - log.Printf("error while archiving %s/%s: %s", workItem.cluster, workItem.host, err.Error()) - atomic.AddInt32(&errs, 1) - } - atomic.AddInt32(&n, int32(m)) - } - }() - } - - for _, de1 := range entries1 { - entries2, e := os.ReadDir(filepath.Join(checkpointsDir, de1.Name())) - if e != nil { - err = e - } - - for _, de2 := range entries2 { - cdir := filepath.Join(checkpointsDir, de1.Name(), de2.Name()) - adir := filepath.Join(archiveDir, de1.Name(), de2.Name()) - work <- workItem{ - adir: adir, cdir: cdir, - cluster: de1.Name(), host: de2.Name(), - } - } - } - - close(work) - wg.Wait() - - if err != nil { - return int(n), err - } - - if errs > 0 { - return int(n), fmt.Errorf("%d errors happend while archiving (%d successes)", errs, n) - } - return int(n), nil -} - -// Helper function for `ArchiveCheckpoints`. -func archiveCheckpoints(dir string, archiveDir string, from int64, deleteInstead bool) (int, error) { - entries, err := os.ReadDir(dir) - if err != nil { - return 0, err - } - - files, err := findFiles(entries, from, false) - if err != nil { - return 0, err - } - - if deleteInstead { - n := 0 - for _, checkpoint := range files { - filename := filepath.Join(dir, checkpoint) - if err = os.Remove(filename); err != nil { - return n, err - } - n += 1 - } - return n, nil - } - - filename := filepath.Join(archiveDir, fmt.Sprintf("%d.zip", from)) - f, err := os.OpenFile(filename, os.O_CREATE|os.O_WRONLY, 0644) - if err != nil && os.IsNotExist(err) { - err = os.MkdirAll(archiveDir, 0755) - if err == nil { - f, err = os.OpenFile(filename, os.O_CREATE|os.O_WRONLY, 0644) - } - } - if err != nil { - return 0, err - } - defer f.Close() - bw := bufio.NewWriter(f) - defer bw.Flush() - zw := zip.NewWriter(bw) - defer zw.Close() - - n := 0 - for _, checkpoint := range files { - filename := filepath.Join(dir, checkpoint) - r, err := os.Open(filename) - if err != nil { - return n, err - } - defer r.Close() - - w, err := zw.Create(checkpoint) - if err != nil { - return n, err - } - - if _, err = io.Copy(w, r); err != nil { - return n, err - } - - if err = os.Remove(filename); err != nil { - return n, err - } - n += 1 - } - - return n, nil -} diff --git a/debug.go b/internal/memorystore/debug.go similarity index 92% rename from debug.go rename to internal/memorystore/debug.go index 88af59f..2743a45 100644 --- a/debug.go +++ b/internal/memorystore/debug.go @@ -1,4 +1,4 @@ -package main +package memorystore import ( "bufio" @@ -29,7 +29,7 @@ func (b *buffer) debugDump(buf []byte) []byte { return buf } -func (l *level) debugDump(m *MemoryStore, w *bufio.Writer, lvlname string, buf []byte, depth int) ([]byte, error) { +func (l *Level) debugDump(m *MemoryStore, w *bufio.Writer, lvlname string, buf []byte, depth int) ([]byte, error) { l.lock.RLock() defer l.lock.RUnlock() for i := 0; i < depth; i++ { @@ -40,8 +40,8 @@ func (l *level) debugDump(m *MemoryStore, w *bufio.Writer, lvlname string, buf [ buf = append(buf, "\":{\n"...) depth += 1 objitems := 0 - for name, mc := range m.metrics { - if b := l.metrics[mc.offset]; b != nil { + for name, mc := range m.Metrics { + if b := l.metrics[mc.Offset]; b != nil { for i := 0; i < depth; i++ { buf = append(buf, '\t') } diff --git a/internal/memorystore/level.go b/internal/memorystore/level.go new file mode 100644 index 0000000..4bbfe7c --- /dev/null +++ b/internal/memorystore/level.go @@ -0,0 +1,183 @@ +package memorystore + +import ( + "sync" + "unsafe" + + "github.com/ClusterCockpit/cc-metric-store/internal/util" +) + +// Could also be called "node" as this forms a node in a tree structure. +// Called Level because "node" might be confusing here. +// Can be both a leaf or a inner node. In this tree structue, inner nodes can +// also hold data (in `metrics`). +type Level struct { + children map[string]*Level + metrics []*buffer + lock sync.RWMutex +} + +// Find the correct level for the given selector, creating it if +// it does not exist. Example selector in the context of the +// ClusterCockpit could be: []string{ "emmy", "host123", "cpu0" }. +// This function would probably benefit a lot from `level.children` beeing a `sync.Map`? +func (l *Level) findLevelOrCreate(selector []string, nMetrics int) *Level { + if len(selector) == 0 { + return l + } + + // Allow concurrent reads: + l.lock.RLock() + var child *Level + var ok bool + if l.children == nil { + // Children map needs to be created... + l.lock.RUnlock() + } else { + child, ok := l.children[selector[0]] + l.lock.RUnlock() + if ok { + return child.findLevelOrCreate(selector[1:], nMetrics) + } + } + + // The level does not exist, take write lock for unqiue access: + l.lock.Lock() + // While this thread waited for the write lock, another thread + // could have created the child node. + if l.children != nil { + child, ok = l.children[selector[0]] + if ok { + l.lock.Unlock() + return child.findLevelOrCreate(selector[1:], nMetrics) + } + } + + child = &Level{ + metrics: make([]*buffer, nMetrics), + children: nil, + } + + if l.children != nil { + l.children[selector[0]] = child + } else { + l.children = map[string]*Level{selector[0]: child} + } + l.lock.Unlock() + return child.findLevelOrCreate(selector[1:], nMetrics) +} + +func (l *Level) free(t int64) (int, error) { + l.lock.Lock() + defer l.lock.Unlock() + + n := 0 + for i, b := range l.metrics { + if b != nil { + delme, m := b.free(t) + n += m + if delme { + if cap(b.data) == BUFFER_CAP { + bufferPool.Put(b) + } + l.metrics[i] = nil + } + } + } + + for _, l := range l.children { + m, err := l.free(t) + n += m + if err != nil { + return n, err + } + } + + return n, nil +} + +func (l *Level) sizeInBytes() int64 { + l.lock.RLock() + defer l.lock.RUnlock() + size := int64(0) + + for _, b := range l.metrics { + if b != nil { + size += b.count() * int64(unsafe.Sizeof(util.Float(0))) + } + } + + return size +} + +func (l *Level) findLevel(selector []string) *Level { + if len(selector) == 0 { + return l + } + + l.lock.RLock() + defer l.lock.RUnlock() + + lvl := l.children[selector[0]] + if lvl == nil { + return nil + } + + return lvl.findLevel(selector[1:]) +} + +func (l *Level) findBuffers(selector util.Selector, offset int, f func(b *buffer) error) error { + l.lock.RLock() + defer l.lock.RUnlock() + + if len(selector) == 0 { + b := l.metrics[offset] + if b != nil { + return f(b) + } + + for _, lvl := range l.children { + err := lvl.findBuffers(nil, offset, f) + if err != nil { + return err + } + } + return nil + } + + sel := selector[0] + if len(sel.String) != 0 && l.children != nil { + lvl, ok := l.children[sel.String] + if ok { + err := lvl.findBuffers(selector[1:], offset, f) + if err != nil { + return err + } + } + return nil + } + + if sel.Group != nil && l.children != nil { + for _, key := range sel.Group { + lvl, ok := l.children[key] + if ok { + err := lvl.findBuffers(selector[1:], offset, f) + if err != nil { + return err + } + } + } + return nil + } + + if sel.Any && l.children != nil { + for _, lvl := range l.children { + if err := lvl.findBuffers(selector[1:], offset, f); err != nil { + return err + } + } + return nil + } + + return nil +} diff --git a/internal/memorystore/memorystore.go b/internal/memorystore/memorystore.go new file mode 100644 index 0000000..4868a85 --- /dev/null +++ b/internal/memorystore/memorystore.go @@ -0,0 +1,283 @@ +package memorystore + +import ( + "context" + "errors" + "log" + "runtime" + "sync" + "time" + + "github.com/ClusterCockpit/cc-metric-store/internal/config" + "github.com/ClusterCockpit/cc-metric-store/internal/util" +) + +var ( + singleton sync.Once + msInstance *MemoryStore +) + +var NumWorkers int = 4 + +func init() { + maxWorkers := 10 + NumWorkers = runtime.NumCPU()/2 + 1 + if NumWorkers > maxWorkers { + NumWorkers = maxWorkers + } +} + +type Metric struct { + Name string + Value util.Float + MetricConfig config.MetricConfig +} + +type MemoryStore struct { + Metrics map[string]config.MetricConfig + root Level +} + +// Create a new, initialized instance of a MemoryStore. +// Will panic if values in the metric configurations are invalid. +func Init(metrics map[string]config.MetricConfig) { + singleton.Do(func() { + offset := 0 + for key, cfg := range metrics { + if cfg.Frequency == 0 { + panic("invalid frequency") + } + + metrics[key] = config.MetricConfig{ + Frequency: cfg.Frequency, + Aggregation: cfg.Aggregation, + Offset: offset, + } + offset += 1 + } + + msInstance = &MemoryStore{ + root: Level{ + metrics: make([]*buffer, len(metrics)), + children: make(map[string]*Level), + }, + Metrics: metrics, + } + }) +} + +func GetMemoryStore() *MemoryStore { + if msInstance == nil { + log.Fatalf("MemoryStore not initialized!") + } + + return msInstance +} + +func Shutdown() { + ms := GetMemoryStore() + log.Printf("Writing to '%s'...\n", config.Keys.Checkpoints.RootDir) + files, err := ms.ToCheckpoint(config.Keys.Checkpoints.RootDir, lastCheckpoint.Unix(), time.Now().Unix()) + if err != nil { + log.Printf("Writing checkpoint failed: %s\n", err.Error()) + } + log.Printf("Done! (%d files written)\n", files) +} + +func Retention(wg *sync.WaitGroup, ctx context.Context) { + ms := GetMemoryStore() + + go func() { + defer wg.Done() + d, err := time.ParseDuration(config.Keys.RetentionInMemory) + if err != nil { + log.Fatal(err) + } + if d <= 0 { + return + } + + ticks := func() <-chan time.Time { + d := d / 2 + if d <= 0 { + return nil + } + return time.NewTicker(d).C + }() + for { + select { + case <-ctx.Done(): + return + case <-ticks: + t := time.Now().Add(-d) + log.Printf("start freeing buffers (older than %s)...\n", t.Format(time.RFC3339)) + freed, err := ms.Free(nil, t.Unix()) + if err != nil { + log.Printf("freeing up buffers failed: %s\n", err.Error()) + } else { + log.Printf("done: %d buffers freed\n", freed) + } + } + } + }() +} + +// Write all values in `metrics` to the level specified by `selector` for time `ts`. +// Look at `findLevelOrCreate` for how selectors work. +func (m *MemoryStore) Write(selector []string, ts int64, metrics []Metric) error { + var ok bool + for i, metric := range metrics { + if metric.MetricConfig.Frequency == 0 { + metric.MetricConfig, ok = m.Metrics[metric.Name] + if !ok { + metric.MetricConfig.Frequency = 0 + } + metrics[i] = metric + } + } + + return m.WriteToLevel(&m.root, selector, ts, metrics) +} + +func (m *MemoryStore) GetLevel(selector []string) *Level { + return m.root.findLevelOrCreate(selector, len(m.Metrics)) +} + +// Assumes that `minfo` in `metrics` is filled in! +func (m *MemoryStore) WriteToLevel(l *Level, selector []string, ts int64, metrics []Metric) error { + l = l.findLevelOrCreate(selector, len(m.Metrics)) + l.lock.Lock() + defer l.lock.Unlock() + + for _, metric := range metrics { + if metric.MetricConfig.Frequency == 0 { + continue + } + + b := l.metrics[metric.MetricConfig.Offset] + if b == nil { + // First write to this metric and level + b = newBuffer(ts, metric.MetricConfig.Frequency) + l.metrics[metric.MetricConfig.Offset] = b + } + + nb, err := b.write(ts, metric.Value) + if err != nil { + return err + } + + // Last write created a new buffer... + if b != nb { + l.metrics[metric.MetricConfig.Offset] = nb + } + } + return nil +} + +// Returns all values for metric `metric` from `from` to `to` for the selected level(s). +// If the level does not hold the metric itself, the data will be aggregated recursively from the children. +// The second and third return value are the actual from/to for the data. Those can be different from +// the range asked for if no data was available. +func (m *MemoryStore) Read(selector util.Selector, metric string, from, to int64) ([]util.Float, int64, int64, error) { + if from > to { + return nil, 0, 0, errors.New("invalid time range") + } + + minfo, ok := m.Metrics[metric] + if !ok { + return nil, 0, 0, errors.New("unkown metric: " + metric) + } + + n, data := 0, make([]util.Float, (to-from)/minfo.Frequency+1) + err := m.root.findBuffers(selector, minfo.Offset, func(b *buffer) error { + cdata, cfrom, cto, err := b.read(from, to, data) + if err != nil { + return err + } + + if n == 0 { + from, to = cfrom, cto + } else if from != cfrom || to != cto || len(data) != len(cdata) { + missingfront, missingback := int((from-cfrom)/minfo.Frequency), int((to-cto)/minfo.Frequency) + if missingfront != 0 { + return ErrDataDoesNotAlign + } + + newlen := len(cdata) - missingback + if newlen < 1 { + return ErrDataDoesNotAlign + } + cdata = cdata[0:newlen] + if len(cdata) != len(data) { + return ErrDataDoesNotAlign + } + + from, to = cfrom, cto + } + + data = cdata + n += 1 + return nil + }) + + if err != nil { + return nil, 0, 0, err + } else if n == 0 { + return nil, 0, 0, errors.New("metric or host not found") + } else if n > 1 { + if minfo.Aggregation == config.AvgAggregation { + normalize := 1. / util.Float(n) + for i := 0; i < len(data); i++ { + data[i] *= normalize + } + } else if minfo.Aggregation != config.SumAggregation { + return nil, 0, 0, errors.New("invalid aggregation") + } + } + + return data, from, to, nil +} + +// Release all buffers for the selected level and all its children that contain only +// values older than `t`. +func (m *MemoryStore) Free(selector []string, t int64) (int, error) { + return m.GetLevel(selector).free(t) +} + +func (m *MemoryStore) FreeAll() error { + for k := range m.root.children { + delete(m.root.children, k) + } + + return nil +} + +func (m *MemoryStore) SizeInBytes() int64 { + return m.root.sizeInBytes() +} + +// Given a selector, return a list of all children of the level selected. +func (m *MemoryStore) ListChildren(selector []string) []string { + lvl := &m.root + for lvl != nil && len(selector) != 0 { + lvl.lock.RLock() + next := lvl.children[selector[0]] + lvl.lock.RUnlock() + lvl = next + selector = selector[1:] + } + + if lvl == nil { + return nil + } + + lvl.lock.RLock() + defer lvl.lock.RUnlock() + + children := make([]string, 0, len(lvl.children)) + for child := range lvl.children { + children = append(children, child) + } + + return children +} diff --git a/stats.go b/internal/memorystore/stats.go similarity index 70% rename from stats.go rename to internal/memorystore/stats.go index 510891b..5ddecfc 100644 --- a/stats.go +++ b/internal/memorystore/stats.go @@ -1,15 +1,18 @@ -package main +package memorystore import ( "errors" "math" + + "github.com/ClusterCockpit/cc-metric-store/internal/config" + "github.com/ClusterCockpit/cc-metric-store/internal/util" ) type Stats struct { Samples int - Avg Float - Min Float - Max Float + Avg util.Float + Min util.Float + Max util.Float } func (b *buffer) stats(from, to int64) (Stats, int64, int64, error) { @@ -54,28 +57,28 @@ func (b *buffer) stats(from, to int64) (Stats, int64, int64, error) { return Stats{ Samples: samples, - Avg: Float(sum) / Float(samples), - Min: Float(min), - Max: Float(max), + Avg: util.Float(sum) / util.Float(samples), + Min: util.Float(min), + Max: util.Float(max), }, from, t, nil } // Returns statistics for the requested metric on the selected node/level. // Data is aggregated to the selected level the same way as in `MemoryStore.Read`. // If `Stats.Samples` is zero, the statistics should not be considered as valid. -func (m *MemoryStore) Stats(selector Selector, metric string, from, to int64) (*Stats, int64, int64, error) { +func (m *MemoryStore) Stats(selector util.Selector, metric string, from, to int64) (*Stats, int64, int64, error) { if from > to { return nil, 0, 0, errors.New("invalid time range") } - minfo, ok := m.metrics[metric] + minfo, ok := m.Metrics[metric] if !ok { return nil, 0, 0, errors.New("unkown metric: " + metric) } n, samples := 0, 0 - avg, min, max := Float(0), math.MaxFloat32, -math.MaxFloat32 - err := m.root.findBuffers(selector, minfo.offset, func(b *buffer) error { + avg, min, max := util.Float(0), math.MaxFloat32, -math.MaxFloat32 + err := m.root.findBuffers(selector, minfo.Offset, func(b *buffer) error { stats, cfrom, cto, err := b.stats(from, to) if err != nil { return err @@ -94,7 +97,6 @@ func (m *MemoryStore) Stats(selector Selector, metric string, from, to int64) (* n += 1 return nil }) - if err != nil { return nil, 0, 0, err } @@ -103,16 +105,16 @@ func (m *MemoryStore) Stats(selector Selector, metric string, from, to int64) (* return nil, 0, 0, ErrNoData } - if minfo.Aggregation == AvgAggregation { - avg /= Float(n) - } else if n > 1 && minfo.Aggregation != SumAggregation { + if minfo.Aggregation == config.AvgAggregation { + avg /= util.Float(n) + } else if n > 1 && minfo.Aggregation != config.SumAggregation { return nil, 0, 0, errors.New("invalid aggregation") } return &Stats{ Samples: samples, Avg: avg, - Min: Float(min), - Max: Float(max), + Min: util.Float(min), + Max: util.Float(max), }, from, to, nil } diff --git a/float.go b/internal/util/float.go similarity index 92% rename from float.go rename to internal/util/float.go index eae2d98..603bc91 100644 --- a/float.go +++ b/internal/util/float.go @@ -1,4 +1,4 @@ -package main +package util import ( "math" @@ -11,8 +11,10 @@ import ( // we have to use our own type which implements encoding/json.Marshaler itself. type Float float64 -var NaN Float = Float(math.NaN()) -var nullAsBytes []byte = []byte("null") +var ( + NaN Float = Float(math.NaN()) + nullAsBytes []byte = []byte("null") +) func (f Float) IsNaN() bool { return math.IsNaN(float64(f)) @@ -55,7 +57,6 @@ func (fa FloatArray) MarshalJSON() ([]byte, error) { buf = append(buf, `null`...) } else { buf = strconv.AppendFloat(buf, float64(fa[i]), 'f', 3, 64) - } } buf = append(buf, ']') diff --git a/internal/util/selector.go b/internal/util/selector.go new file mode 100644 index 0000000..27557ef --- /dev/null +++ b/internal/util/selector.go @@ -0,0 +1,51 @@ +package util + +import ( + "encoding/json" + "errors" +) + +type SelectorElement struct { + String string + Group []string + Any bool +} + +func (se *SelectorElement) UnmarshalJSON(input []byte) error { + if input[0] == '"' { + if err := json.Unmarshal(input, &se.String); err != nil { + return err + } + + if se.String == "*" { + se.Any = true + se.String = "" + } + + return nil + } + + if input[0] == '[' { + return json.Unmarshal(input, &se.Group) + } + + return errors.New("the Go SelectorElement type can only be a string or an array of strings") +} + +func (se *SelectorElement) MarshalJSON() ([]byte, error) { + if se.Any { + return []byte("\"*\""), nil + } + + if se.String != "" { + return json.Marshal(se.String) + } + + if se.Group != nil { + return json.Marshal(se.Group) + } + + return nil, errors.New("a Go Selector must be a non-empty string or a non-empty slice of strings") +} + +type Selector []SelectorElement diff --git a/lineprotocol_test.go b/lineprotocol_test.go deleted file mode 100644 index a6df786..0000000 --- a/lineprotocol_test.go +++ /dev/null @@ -1,144 +0,0 @@ -package main - -import ( - "bytes" - "log" - "strconv" - "testing" - - "github.com/influxdata/line-protocol/v2/lineprotocol" -) - -const TestDataClassicFormat string = ` -m1,cluster=ctest,hostname=htest1,type=node value=1 123456789 -m2,cluster=ctest,hostname=htest1,type=node value=2 123456789 -m3,hostname=htest2,type=node value=3 123456789 -m4,cluster=ctest,hostname=htest2,type=core,type-id=1 value=4 123456789 -m4,cluster=ctest,hostname=htest2,type-id=2,type=core value=5 123456789 -` - -const BenchmarkLineBatch string = ` -nm1,cluster=ctest,hostname=htest1,type=node value=123.0 123456789 -nm2,cluster=ctest,hostname=htest1,type=node value=123.0 123456789 -nm3,cluster=ctest,hostname=htest1,type=node value=123.0 123456789 -nm4,cluster=ctest,hostname=htest1,type=node value=123.0 123456789 -nm5,cluster=ctest,hostname=htest1,type=node value=123.0 123456789 -nm6,cluster=ctest,hostname=htest1,type=node value=123.0 123456789 -nm7,cluster=ctest,hostname=htest1,type=node value=123.0 123456789 -nm8,cluster=ctest,hostname=htest1,type=node value=123.0 123456789 -nm9,cluster=ctest,hostname=htest1,type=node value=123.0 123456789 -cm1,cluster=ctest,hostname=htest1,type=core,type-id=1 value=234.0 123456789 -cm2,cluster=ctest,hostname=htest1,type=core,type-id=1 value=234.0 123456789 -cm3,cluster=ctest,hostname=htest1,type=core,type-id=1 value=234.0 123456789 -cm4,cluster=ctest,hostname=htest1,type=core,type-id=1 value=234.0 123456789 -cm5,cluster=ctest,hostname=htest1,type=core,type-id=1 value=234.0 123456789 -cm6,cluster=ctest,hostname=htest1,type=core,type-id=1 value=234.0 123456789 -cm7,cluster=ctest,hostname=htest1,type=core,type-id=1 value=234.0 123456789 -cm8,cluster=ctest,hostname=htest1,type=core,type-id=1 value=234.0 123456789 -cm9,cluster=ctest,hostname=htest1,type=core,type-id=1 value=234.0 123456789 -cm1,cluster=ctest,hostname=htest1,type=core,type-id=2 value=345.0 123456789 -cm2,cluster=ctest,hostname=htest1,type=core,type-id=2 value=345.0 123456789 -cm3,cluster=ctest,hostname=htest1,type=core,type-id=2 value=345.0 123456789 -cm4,cluster=ctest,hostname=htest1,type=core,type-id=2 value=345.0 123456789 -cm5,cluster=ctest,hostname=htest1,type=core,type-id=2 value=345.0 123456789 -cm6,cluster=ctest,hostname=htest1,type=core,type-id=2 value=345.0 123456789 -cm7,cluster=ctest,hostname=htest1,type=core,type-id=2 value=345.0 123456789 -cm8,cluster=ctest,hostname=htest1,type=core,type-id=2 value=345.0 123456789 -cm9,cluster=ctest,hostname=htest1,type=core,type-id=2 value=345.0 123456789 -cm1,cluster=ctest,hostname=htest1,type=core,type-id=3 value=456.0 123456789 -cm2,cluster=ctest,hostname=htest1,type=core,type-id=3 value=456.0 123456789 -cm3,cluster=ctest,hostname=htest1,type=core,type-id=3 value=456.0 123456789 -cm4,cluster=ctest,hostname=htest1,type=core,type-id=3 value=456.0 123456789 -cm5,cluster=ctest,hostname=htest1,type=core,type-id=3 value=456.0 123456789 -cm6,cluster=ctest,hostname=htest1,type=core,type-id=3 value=456.0 123456789 -cm7,cluster=ctest,hostname=htest1,type=core,type-id=3 value=456.0 123456789 -cm8,cluster=ctest,hostname=htest1,type=core,type-id=3 value=456.0 123456789 -cm9,cluster=ctest,hostname=htest1,type=core,type-id=3 value=456.0 123456789 -cm1,cluster=ctest,hostname=htest1,type=core,type-id=4 value=567.0 123456789 -cm2,cluster=ctest,hostname=htest1,type=core,type-id=4 value=567.0 123456789 -cm3,cluster=ctest,hostname=htest1,type=core,type-id=4 value=567.0 123456789 -cm4,cluster=ctest,hostname=htest1,type=core,type-id=4 value=567.0 123456789 -cm5,cluster=ctest,hostname=htest1,type=core,type-id=4 value=567.0 123456789 -cm6,cluster=ctest,hostname=htest1,type=core,type-id=4 value=567.0 123456789 -cm7,cluster=ctest,hostname=htest1,type=core,type-id=4 value=567.0 123456789 -cm8,cluster=ctest,hostname=htest1,type=core,type-id=4 value=567.0 123456789 -cm9,cluster=ctest,hostname=htest1,type=core,type-id=4 value=567.0 123456789 -` - -func TestLineprotocolDecoder(t *testing.T) { - prevMemoryStore := memoryStore - t.Cleanup(func() { - memoryStore = prevMemoryStore - }) - - memoryStore = NewMemoryStore(map[string]MetricConfig{ - "m1": {Frequency: 1}, - "m2": {Frequency: 1}, - "m3": {Frequency: 1}, - "m4": {Frequency: 1}, - }) - - dec := lineprotocol.NewDecoderWithBytes([]byte(TestDataClassicFormat)) - if err := decodeLine(dec, "ctest"); err != nil { - log.Fatal(err) - } - - // memoryStore.DebugDump(bufio.NewWriter(os.Stderr)) - - h1 := memoryStore.GetLevel([]string{"ctest", "htest1"}) - h1b1 := h1.metrics[memoryStore.metrics["m1"].offset] - h1b2 := h1.metrics[memoryStore.metrics["m2"].offset] - if h1b1.data[0] != 1.0 || h1b2.data[0] != 2.0 { - log.Fatal() - } - - h2 := memoryStore.GetLevel([]string{"ctest", "htest2"}) - h2b3 := h2.metrics[memoryStore.metrics["m3"].offset] - if h2b3.data[0] != 3.0 { - log.Fatal() - } - - h2c1 := memoryStore.GetLevel([]string{"ctest", "htest2", "core1"}) - h2c1b4 := h2c1.metrics[memoryStore.metrics["m4"].offset] - h2c2 := memoryStore.GetLevel([]string{"ctest", "htest2", "core2"}) - h2c2b4 := h2c2.metrics[memoryStore.metrics["m4"].offset] - if h2c1b4.data[0] != 4.0 || h2c2b4.data[0] != 5.0 { - log.Fatal() - } -} - -func BenchmarkLineprotocolDecoder(b *testing.B) { - b.StopTimer() - memoryStore = NewMemoryStore(map[string]MetricConfig{ - "nm1": {Frequency: 1}, - "nm2": {Frequency: 1}, - "nm3": {Frequency: 1}, - "nm4": {Frequency: 1}, - "nm5": {Frequency: 1}, - "nm6": {Frequency: 1}, - "nm7": {Frequency: 1}, - "nm8": {Frequency: 1}, - "nm9": {Frequency: 1}, - "cm1": {Frequency: 1}, - "cm2": {Frequency: 1}, - "cm3": {Frequency: 1}, - "cm4": {Frequency: 1}, - "cm5": {Frequency: 1}, - "cm6": {Frequency: 1}, - "cm7": {Frequency: 1}, - "cm8": {Frequency: 1}, - "cm9": {Frequency: 1}, - }) - - for i := 0; i < b.N; i++ { - data := []byte(BenchmarkLineBatch) - data = bytes.ReplaceAll(data, []byte("123456789"), []byte(strconv.Itoa(i+123456789))) - dec := lineprotocol.NewDecoderWithBytes(data) - - b.StartTimer() - if err := decodeLine(dec, "ctest"); err != nil { - b.Fatal(err) - } - b.StopTimer() - } -} diff --git a/memoryStore_test.go.orig b/memoryStore_test.go.orig deleted file mode 100644 index b64c85b..0000000 --- a/memoryStore_test.go.orig +++ /dev/null @@ -1,504 +0,0 @@ -package main - -import ( - "fmt" - "log" - "math" - "testing" - - "github.com/ClusterCockpit/cc-metric-store/lineprotocol" -) - -var testMetrics [][]lineprotocol.Metric = [][]lineprotocol.Metric{ - {{"flops", 100.5}, {"mem_bw", 2088.67}}, - {{"flops", 180.5}, {"mem_bw", 4078.32}, {"mem_capacity", 1020}}, - {{"flops", 980.5}, {"mem_bw", 9078.32}, {"mem_capacity", 5010}}, - {{"flops", 940.5}, {"mem_bw", 9278.32}, {"mem_capacity", 6010}}, - {{"flops", 930.5}, {"mem_bw", 9378.32}, {"mem_capacity", 7010}}, - {{"flops", 980.5}, {"mem_bw", 9478.32}, {"mem_capacity", 8010}}, - {{"flops", 980.5}, {"mem_bw", 9478.32}, {"mem_capacity", 8010}}, - {{"flops", 980.5}, {"mem_bw", 9478.32}, {"mem_capacity", 8010}}, - {{"flops", 970.5}, {"mem_bw", 9178.32}, {"mem_capacity", 2010}}, - {{"flops", 970.5}, {"mem_bw", 9178.32}, {"mem_capacity", 2010}}} - -var testMetricsAlt [][]lineprotocol.Metric = [][]lineprotocol.Metric{ - {{"flops", 120.5}, {"mem_bw", 2080.67}}, - {{"flops", 130.5}, {"mem_bw", 4071.32}, {"mem_capacity", 1120}}, - {{"flops", 940.5}, {"mem_bw", 9072.32}, {"mem_capacity", 5210}}, - {{"flops", 950.5}, {"mem_bw", 9273.32}, {"mem_capacity", 6310}}, - {{"flops", 960.5}, {"mem_bw", 9374.32}, {"mem_capacity", 7410}}, - {{"flops", 970.5}, {"mem_bw", 9475.32}, {"mem_capacity", 8510}}, - {{"flops", 990.5}, {"mem_bw", 9476.32}, {"mem_capacity", 8610}}, - {{"flops", 910.5}, {"mem_bw", 9477.32}, {"mem_capacity", 8710}}, - {{"flops", 920.5}, {"mem_bw", 9178.32}, {"mem_capacity", 2810}}, - {{"flops", 930.5}, {"mem_bw", 9179.32}, {"mem_capacity", 2910}}} - -func dumpStoreBuffer(s *storeBuffer) { - log.Printf("Start TS %d\n", s.start) - ctr := 0 - - for _, val := range s.store { - fmt.Printf("%f\t", val) - ctr++ - - if ctr == 10 { - fmt.Printf("\n") - ctr = 0 - } - } -} - -func printMemStore(m *MemoryStore) { - log.Println("########################") - log.Printf("Frequency %d, Metrics %d Slots %d\n", - m.frequency, m.numMetrics, m.numSlots) - log.Println("##Offsets") - for key, val := range m.offsets { - log.Printf("\t%s = %d\n", key, val) - } - log.Println("##Containers") - for key, c := range m.containers { - log.Printf("ID %s\n", key) - log.Println("###current") - dumpStoreBuffer(c.current) - log.Println("###next") - dumpStoreBuffer(c.next) - } - log.Println("########################") -} - -//############################ -//#### Whitebox tests ######## -//############################ -func TestAddMetricSimple(t *testing.T) { - key := "m1220" - m := newMemoryStore([]string{"flops", "mem_bw", "mem_capacity"}, 10, 60) - // printMemStore(m) - - m.AddMetrics(key, 1584022800, testMetrics[0]) - m.AddMetrics(key, 1584022890, testMetrics[1]) - - want := testMetrics[0][0].Value - got := m.containers[key].current.store[0] - if got != want { - t.Errorf("Want %f got %f\n", want, got) - } - - want = testMetrics[1][2].Value - got = m.containers[key].current.store[21] - if got != want { - t.Errorf("Want %f got %f\n", want, got) - } - // printMemStore(m) -} - -func TestAddMetricReplace(t *testing.T) { - key := "m1220" - m := newMemoryStore([]string{"flops", "mem_bw", "mem_capacity"}, 10, 60) - // printMemStore(m) - - m.AddMetrics(key, 1584022800, testMetrics[0]) - m.AddMetrics(key, 1584022800, testMetrics[1]) - - want := testMetrics[1][0].Value - got := m.containers[key].current.store[0] - if got != want { - t.Errorf("Want %f got %f\n", want, got) - } - - m.AddMetrics(key, 1584022850, testMetrics[0]) - want = testMetrics[0][0].Value - got = m.containers[key].current.store[0] - if got != want { - t.Errorf("Want %f got %f\n", want, got) - } - - m.AddMetrics(key, 1584022860, testMetrics[1]) - want = testMetrics[0][0].Value - got = m.containers[key].current.store[0] - if got != want { - t.Errorf("Want %f got %f\n", want, got) - } - // printMemStore(m) -} - -func TestAddMetricSwitch(t *testing.T) { - key := "m1220" - m := newMemoryStore([]string{"flops", "mem_bw", "mem_capacity"}, 10, 60) - // printMemStore(m) - - m.AddMetrics(key, 1584023000, testMetrics[0]) - m.AddMetrics(key, 1584023580, testMetrics[1]) - - want := testMetrics[1][2].Value - got := m.containers[key].current.store[29] - if got != want { - t.Errorf("Want %f got %f\n", want, got) - } - - m.AddMetrics(key, 1584023600, testMetrics[2]) - want = testMetrics[2][2].Value - got = m.containers[key].current.store[20] - if got != want { - t.Errorf("Want %f got %f\n", want, got) - } - - // printMemStore(m) -} - -//############################ -//#### Blackbox tests ######## -//############################ - -func TestAddMetricOutOfBounds(t *testing.T) { - key := "m1220" - m := newMemoryStore([]string{"flops", "mem_bw", "mem_capacity"}, 30, 60) - - err := m.AddMetrics(key, 1584023000, testMetrics[0]) - if err != nil { - t.Errorf("Got error 1584023000\n") - } - err = m.AddMetrics(key, 1584026600, testMetrics[0]) - if err == nil { - t.Errorf("Got no error 1584026600\n") - } - err = m.AddMetrics(key, 1584021580, testMetrics[1]) - if err == nil { - t.Errorf("Got no error 1584021580\n") - } - err = m.AddMetrics(key, 1584024580, testMetrics[1]) - if err != nil { - t.Errorf("Got error 1584024580\n") - } - err = m.AddMetrics(key, 1584091580, testMetrics[1]) - if err == nil { - t.Errorf("Got no error 1584091580\n") - } - err = m.AddMetrics(key, 1584024780, testMetrics[0]) - if err != nil { - t.Errorf("Got error 1584024780\n") - } -} - -func TestGetMetricPlainCurrent(t *testing.T) { - key := "m1220" - m := newMemoryStore([]string{"flops", "mem_bw", "mem_capacity"}, 10, 60) - - for i := 0; i < len(testMetrics); i++ { - m.AddMetrics(key, int64(1584023000+i*60), testMetrics[i]) - } - - // printMemStore(m) - val, tsFrom, err := m.GetMetric(key, "flops", 1584023000, 1584023560) - - if err != nil { - t.Errorf("Got error\n") - } - if tsFrom != 1584023000 { - t.Errorf("Start ts differs: %d\n", tsFrom) - } - if len(val) != 9 { - t.Errorf("Want 9. Got %d\n", len(val)) - } - if val[0] != 100.5 { - t.Errorf("Want 100.5 Got %f\n", val[0]) - } - if val[8] != 970.5 { - t.Errorf("Want 970.5 Got %f\n", val[9]) - } -} - -func TestGetMetricPlainNext(t *testing.T) { - key := "m1220" - m := newMemoryStore([]string{"flops", "mem_bw", "mem_capacity"}, 10, 60) - - for i := 0; i < len(testMetrics); i++ { - m.AddMetrics(key, int64(1584023000+i*60), testMetrics[i]) - } - for i := 0; i < len(testMetrics); i++ { - m.AddMetrics(key, int64(1584023600+i*60), testMetricsAlt[i]) - } - - // printMemStore(m) - val, tsFrom, err := m.GetMetric(key, "flops", 1584023000, 1584023560) - - if err != nil { - t.Errorf("Got error\n") - } - if tsFrom != 1584023000 { - t.Errorf("Start ts differs: %d\n", tsFrom) - } - if len(val) != 9 { - t.Errorf("Want 9. Got %d\n", len(val)) - } - if val[0] != 100.5 { - t.Errorf("Want 100.5 Got %f\n", val[0]) - } - if val[8] != 970.5 { - t.Errorf("Want 970.5 Got %f\n", val[9]) - } -} - -func TestGetMetricGap(t *testing.T) { - key := "m1220" - m := newMemoryStore([]string{"flops", "mem_bw", "mem_capacity"}, 10, 60) - - for i := 0; i < len(testMetrics); i++ { - m.AddMetrics(key, int64(1584023000+i*120), testMetrics[i]) - } - - val, tsFrom, err := m.GetMetric(key, "flops", 1584023000, 1584023600) - - if err != nil { - t.Errorf("Got error\n") - } - if tsFrom != 1584023000 { - t.Errorf("Start ts differs: %d\n", tsFrom) - } - if len(val) != 10 { - t.Errorf("Want 10. Got %d\n", len(val)) - } - if val[0] != 100.5 { - t.Errorf("Want 100.5 Got %f\n", val[0]) - } - if !math.IsNaN(float64(val[1])) { - t.Errorf("Want NaN Got %f\n", val[1]) - } - if val[0] != 100.5 { - t.Errorf("Want 100.5 Got %f\n", val[0]) - } - - // fmt.Println(val) -} - -func TestGetMetricSplit(t *testing.T) { - key := "m1220" - m := newMemoryStore([]string{"flops", "mem_bw", "mem_capacity"}, 10, 60) - - for i := 0; i < len(testMetrics); i++ { - m.AddMetrics(key, int64(1584023000+i*60), testMetrics[i]) - } - for i := 0; i < len(testMetrics); i++ { - m.AddMetrics(key, int64(1584023600+i*60), testMetricsAlt[i]) - } - - // printMemStore(m) - - val, tsFrom, err := m.GetMetric(key, "flops", 1584023200, 1584023860) - - if err != nil { - t.Errorf("Got error\n") - } - if tsFrom != 1584023200 { - t.Errorf("Start ts differs: %d\n", tsFrom) - } - if len(val) != 11 { - t.Errorf("Want 11. Got %d\n", len(val)) - } - if val[0] != 940.5 { - t.Errorf("Want 940.5 Got %f\n", val[0]) - } - if val[10] != 950.5 { - t.Errorf("Want 950.5 Got %f\n", val[0]) - } -} - -func TestGetMetricExceedNext(t *testing.T) { - key := "m1220" - m := newMemoryStore([]string{"flops", "mem_bw", "mem_capacity"}, 10, 60) - - for i := 0; i < len(testMetrics); i++ { - m.AddMetrics(key, int64(1584023000+i*60), testMetrics[i]) - } - for i := 0; i < len(testMetrics); i++ { - m.AddMetrics(key, int64(1584023600+i*60), testMetricsAlt[i]) - } - - // printMemStore(m) - - val, tsFrom, err := m.GetMetric(key, "flops", 1584022800, 1584023400) - - if err != nil { - t.Errorf("Got error\n") - } - if tsFrom != 1584023000 { - t.Errorf("Start ts differs: %d\n", tsFrom) - } - if len(val) != 6 { - t.Errorf("Want 6. Got %d\n", len(val)) - } - if val[0] != 100.5 { - t.Errorf("Want 100.5 Got %f\n", val[0]) - } - if val[5] != 980.5 { - t.Errorf("Want 980.5 Got %f\n", val[5]) - } -} - -func TestGetMetricExceedNextSplit(t *testing.T) { - key := "m1220" - m := newMemoryStore([]string{"flops", "mem_bw", "mem_capacity"}, 10, 60) - - for i := 0; i < len(testMetrics); i++ { - m.AddMetrics(key, int64(1584023000+i*60), testMetrics[i]) - } - for i := 0; i < len(testMetrics); i++ { - m.AddMetrics(key, int64(1584023600+i*60), testMetricsAlt[i]) - } - - // printMemStore(m) - - val, tsFrom, err := m.GetMetric(key, "flops", 1584022800, 1584023900) - - if err != nil { - t.Errorf("Got error\n") - } - if tsFrom != 1584023000 { - t.Errorf("Start ts differs: %d\n", tsFrom) - } - if len(val) != 15 { - t.Errorf("Want 14. Got %d\n", len(val)) - } - if val[0] != 100.5 { - t.Errorf("Want 100.5 Got %f\n", val[0]) - } - if val[14] != 960.5 { - t.Errorf("Want 960.5 Got %f\n", val[13]) - } -} - -func TestGetMetricExceedCurrent(t *testing.T) { - key := "m1220" - m := newMemoryStore([]string{"flops", "mem_bw", "mem_capacity"}, 10, 60) - - for i := 0; i < len(testMetrics); i++ { - m.AddMetrics(key, int64(1584023000+i*60), testMetrics[i]) - } - for i := 0; i < len(testMetrics); i++ { - m.AddMetrics(key, int64(1584023600+i*60), testMetricsAlt[i]) - } - - // printMemStore(m) - - val, tsFrom, err := m.GetMetric(key, "flops", 1584023800, 1584027900) - - if err != nil { - t.Errorf("Got error\n") - } - if tsFrom != 1584023800 { - t.Errorf("Start ts differs: %d\n", tsFrom) - } - if len(val) != 7 { - t.Errorf("Want 6. Got %d\n", len(val)) - } - if val[0] != 950.5 { - t.Errorf("Want 950.5 Got %f\n", val[0]) - } - if val[6] != 930.5 { - t.Errorf("Want 930.5 Got %f\n", val[5]) - } -} - -func TestGetMetricExceedCurrentSplit(t *testing.T) { - key := "m1220" - m := newMemoryStore([]string{"flops", "mem_bw", "mem_capacity"}, 10, 60) - - for i := 0; i < len(testMetrics); i++ { - m.AddMetrics(key, int64(1584023000+i*60), testMetrics[i]) - } - for i := 0; i < len(testMetrics); i++ { - m.AddMetrics(key, int64(1584023600+i*60), testMetricsAlt[i]) - } - - // printMemStore(m) - - val, tsFrom, err := m.GetMetric(key, "flops", 1584023120, 1584027900) - - if err != nil { - t.Errorf("Got error\n") - } - if tsFrom != 1584023120 { - t.Errorf("Start ts differs: %d\n", tsFrom) - } - if len(val) != 18 { - t.Errorf("Want 18. Got %d\n", len(val)) - } - if val[0] != 980.5 { - t.Errorf("Want 950.5 Got %f\n", val[0]) - } - if val[17] != 930.5 { - t.Errorf("Want 930.5 Got %f\n", val[17]) - } -} - -func TestGetMetricExceedBoth(t *testing.T) { - key := "m1220" - m := newMemoryStore([]string{"flops", "mem_bw", "mem_capacity"}, 10, 60) - - for i := 0; i < len(testMetrics); i++ { - m.AddMetrics(key, int64(1584023000+i*60), testMetrics[i]) - } - for i := 0; i < len(testMetrics); i++ { - m.AddMetrics(key, int64(1584023600+i*60), testMetricsAlt[i]) - } - - // printMemStore(m) - - val, tsFrom, err := m.GetMetric(key, "flops", 1584022800, 1584027900) - - if err != nil { - t.Errorf("Got error\n") - } - if tsFrom != 1584023000 { - t.Errorf("Start ts differs: %d\n", tsFrom) - } - if len(val) != 20 { - t.Errorf("Want 20. Got %d\n", len(val)) - } - if val[0] != 100.5 { - t.Errorf("Want 950.5 Got %f\n", val[0]) - } - if val[19] != 930.5 { - t.Errorf("Want 930.5 Got %f\n", val[17]) - } -} - -func TestGetMetricOutUpper(t *testing.T) { - key := "m1220" - m := newMemoryStore([]string{"flops", "mem_bw", "mem_capacity"}, 10, 60) - - for i := 0; i < len(testMetrics); i++ { - m.AddMetrics(key, int64(1584023000+i*60), testMetrics[i]) - } - for i := 0; i < len(testMetrics); i++ { - m.AddMetrics(key, int64(1584023600+i*60), testMetricsAlt[i]) - } - - // printMemStore(m) - - _, _, err := m.GetMetric(key, "flops", 1584032800, 1584037900) - - if err == nil { - t.Errorf("Got no error\n") - } -} - -func TestGetMetricOutLower(t *testing.T) { - key := "m1220" - m := newMemoryStore([]string{"flops", "mem_bw", "mem_capacity"}, 10, 60) - - for i := 0; i < len(testMetrics); i++ { - m.AddMetrics(key, int64(1584023000+i*60), testMetrics[i]) - } - for i := 0; i < len(testMetrics); i++ { - m.AddMetrics(key, int64(1584023600+i*60), testMetricsAlt[i]) - } - - // printMemStore(m) - - _, _, err := m.GetMetric(key, "flops", 1584002800, 1584007900) - - if err == nil { - t.Errorf("Got no error\n") - } -} diff --git a/memstore.go b/memstore.go deleted file mode 100644 index 79b3f05..0000000 --- a/memstore.go +++ /dev/null @@ -1,538 +0,0 @@ -package main - -import ( - "errors" - "sync" - "unsafe" -) - -// Default buffer capacity. -// `buffer.data` will only ever grow up to it's capacity and a new link -// in the buffer chain will be created if needed so that no copying -// of data or reallocation needs to happen on writes. -const ( - BUFFER_CAP int = 512 -) - -// So that we can reuse allocations -var bufferPool sync.Pool = sync.Pool{ - New: func() interface{} { - return &buffer{ - data: make([]Float, 0, BUFFER_CAP), - } - }, -} - -var ( - ErrNoData error = errors.New("no data for this metric/level") - ErrDataDoesNotAlign error = errors.New("data from lower granularities does not align") -) - -// Each metric on each level has it's own buffer. -// This is where the actual values go. -// If `cap(data)` is reached, a new buffer is created and -// becomes the new head of a buffer list. -type buffer struct { - frequency int64 // Time between two "slots" - start int64 // Timestamp of when `data[0]` was written. - data []Float // The slice should never reallocacte as `cap(data)` is respected. - prev, next *buffer // `prev` contains older data, `next` newer data. - archived bool // If true, this buffer is already archived - - closed bool - /* - statisticts struct { - samples int - min Float - max Float - avg Float - } - */ -} - -func newBuffer(ts, freq int64) *buffer { - b := bufferPool.Get().(*buffer) - b.frequency = freq - b.start = ts - (freq / 2) - b.prev = nil - b.next = nil - b.archived = false - b.closed = false - b.data = b.data[:0] - return b -} - -// If a new buffer was created, the new head is returnd. -// Otherwise, the existing buffer is returnd. -// Normaly, only "newer" data should be written, but if the value would -// end up in the same buffer anyways it is allowed. -func (b *buffer) write(ts int64, value Float) (*buffer, error) { - if ts < b.start { - return nil, errors.New("cannot write value to buffer from past") - } - - // idx := int((ts - b.start + (b.frequency / 3)) / b.frequency) - idx := int((ts - b.start) / b.frequency) - if idx >= cap(b.data) { - newbuf := newBuffer(ts, b.frequency) - newbuf.prev = b - b.next = newbuf - b.close() - b = newbuf - idx = 0 - } - - // Overwriting value or writing value from past - if idx < len(b.data) { - b.data[idx] = value - return b, nil - } - - // Fill up unwritten slots with NaN - for i := len(b.data); i < idx; i++ { - b.data = append(b.data, NaN) - } - - b.data = append(b.data, value) - return b, nil -} - -func (b *buffer) end() int64 { - return b.firstWrite() + int64(len(b.data))*b.frequency -} - -func (b *buffer) firstWrite() int64 { - return b.start + (b.frequency / 2) -} - -func (b *buffer) close() {} - -/* -func (b *buffer) close() { - if b.closed { - return - } - - b.closed = true - n, sum, min, max := 0, 0., math.MaxFloat64, -math.MaxFloat64 - for _, x := range b.data { - if x.IsNaN() { - continue - } - - n += 1 - f := float64(x) - sum += f - min = math.Min(min, f) - max = math.Max(max, f) - } - - b.statisticts.samples = n - if n > 0 { - b.statisticts.avg = Float(sum / float64(n)) - b.statisticts.min = Float(min) - b.statisticts.max = Float(max) - } else { - b.statisticts.avg = NaN - b.statisticts.min = NaN - b.statisticts.max = NaN - } -} -*/ - -// func interpolate(idx int, data []Float) Float { -// if idx == 0 || idx+1 == len(data) { -// return NaN -// } -// return (data[idx-1] + data[idx+1]) / 2.0 -// } - -// Return all known values from `from` to `to`. Gaps of information are represented as NaN. -// Simple linear interpolation is done between the two neighboring cells if possible. -// If values at the start or end are missing, instead of NaN values, the second and thrid -// return values contain the actual `from`/`to`. -// This function goes back the buffer chain if `from` is older than the currents buffer start. -// The loaded values are added to `data` and `data` is returned, possibly with a shorter length. -// If `data` is not long enough to hold all values, this function will panic! -func (b *buffer) read(from, to int64, data []Float) ([]Float, int64, int64, error) { - if from < b.firstWrite() { - if b.prev != nil { - return b.prev.read(from, to, data) - } - from = b.firstWrite() - } - - var i int = 0 - var t int64 = from - for ; t < to; t += b.frequency { - idx := int((t - b.start) / b.frequency) - if idx >= cap(b.data) { - if b.next == nil { - break - } - b = b.next - idx = 0 - } - - if idx >= len(b.data) { - if b.next == nil || to <= b.next.start { - break - } - data[i] += NaN - } else if t < b.start { - data[i] += NaN - // } else if b.data[idx].IsNaN() { - // data[i] += interpolate(idx, b.data) - } else { - data[i] += b.data[idx] - } - i++ - } - - return data[:i], from, t, nil -} - -// Returns true if this buffer needs to be freed. -func (b *buffer) free(t int64) (delme bool, n int) { - if b.prev != nil { - delme, m := b.prev.free(t) - n += m - if delme { - b.prev.next = nil - if cap(b.prev.data) == BUFFER_CAP { - bufferPool.Put(b.prev) - } - b.prev = nil - } - } - - end := b.end() - if end < t { - return true, n + 1 - } - - return false, n -} - -// Call `callback` on every buffer that contains data in the range from `from` to `to`. -func (b *buffer) iterFromTo(from, to int64, callback func(b *buffer) error) error { - if b == nil { - return nil - } - - if err := b.prev.iterFromTo(from, to, callback); err != nil { - return err - } - - if from <= b.end() && b.start <= to { - return callback(b) - } - - return nil -} - -func (b *buffer) count() int64 { - res := int64(len(b.data)) - if b.prev != nil { - res += b.prev.count() - } - return res -} - -// Could also be called "node" as this forms a node in a tree structure. -// Called level because "node" might be confusing here. -// Can be both a leaf or a inner node. In this tree structue, inner nodes can -// also hold data (in `metrics`). -type level struct { - lock sync.RWMutex - metrics []*buffer // Every level can store metrics. - children map[string]*level // Lower levels. -} - -// Find the correct level for the given selector, creating it if -// it does not exist. Example selector in the context of the -// ClusterCockpit could be: []string{ "emmy", "host123", "cpu0" }. -// This function would probably benefit a lot from `level.children` beeing a `sync.Map`? -func (l *level) findLevelOrCreate(selector []string, nMetrics int) *level { - if len(selector) == 0 { - return l - } - - // Allow concurrent reads: - l.lock.RLock() - var child *level - var ok bool - if l.children == nil { - // Children map needs to be created... - l.lock.RUnlock() - } else { - child, ok := l.children[selector[0]] - l.lock.RUnlock() - if ok { - return child.findLevelOrCreate(selector[1:], nMetrics) - } - } - - // The level does not exist, take write lock for unqiue access: - l.lock.Lock() - // While this thread waited for the write lock, another thread - // could have created the child node. - if l.children != nil { - child, ok = l.children[selector[0]] - if ok { - l.lock.Unlock() - return child.findLevelOrCreate(selector[1:], nMetrics) - } - } - - child = &level{ - metrics: make([]*buffer, nMetrics), - children: nil, - } - - if l.children != nil { - l.children[selector[0]] = child - } else { - l.children = map[string]*level{selector[0]: child} - } - l.lock.Unlock() - return child.findLevelOrCreate(selector[1:], nMetrics) -} - -func (l *level) free(t int64) (int, error) { - l.lock.Lock() - defer l.lock.Unlock() - - n := 0 - for i, b := range l.metrics { - if b != nil { - delme, m := b.free(t) - n += m - if delme { - if cap(b.data) == BUFFER_CAP { - bufferPool.Put(b) - } - l.metrics[i] = nil - } - } - } - - for _, l := range l.children { - m, err := l.free(t) - n += m - if err != nil { - return n, err - } - } - - return n, nil -} - -func (l *level) sizeInBytes() int64 { - l.lock.RLock() - defer l.lock.RUnlock() - size := int64(0) - - for _, b := range l.metrics { - if b != nil { - size += b.count() * int64(unsafe.Sizeof(Float(0))) - } - } - - for _, child := range l.children { - size += child.sizeInBytes() - } - - return size -} - -type MemoryStore struct { - root level // root of the tree structure - metrics map[string]MetricConfig -} - -// Return a new, initialized instance of a MemoryStore. -// Will panic if values in the metric configurations are invalid. -func NewMemoryStore(metrics map[string]MetricConfig) *MemoryStore { - offset := 0 - for key, config := range metrics { - if config.Frequency == 0 { - panic("invalid frequency") - } - - metrics[key] = MetricConfig{ - Frequency: config.Frequency, - Aggregation: config.Aggregation, - offset: offset, - } - offset += 1 - } - - return &MemoryStore{ - root: level{ - metrics: make([]*buffer, len(metrics)), - children: make(map[string]*level), - }, - metrics: metrics, - } -} - -// Write all values in `metrics` to the level specified by `selector` for time `ts`. -// Look at `findLevelOrCreate` for how selectors work. -func (m *MemoryStore) Write(selector []string, ts int64, metrics []Metric) error { - var ok bool - for i, metric := range metrics { - if metric.mc.Frequency == 0 { - metric.mc, ok = m.metrics[metric.Name] - if !ok { - metric.mc.Frequency = 0 - } - metrics[i] = metric - } - } - - return m.WriteToLevel(&m.root, selector, ts, metrics) -} - -func (m *MemoryStore) GetLevel(selector []string) *level { - return m.root.findLevelOrCreate(selector, len(m.metrics)) -} - -// Assumes that `minfo` in `metrics` is filled in! -func (m *MemoryStore) WriteToLevel(l *level, selector []string, ts int64, metrics []Metric) error { - l = l.findLevelOrCreate(selector, len(m.metrics)) - l.lock.Lock() - defer l.lock.Unlock() - - for _, metric := range metrics { - if metric.mc.Frequency == 0 { - continue - } - - b := l.metrics[metric.mc.offset] - if b == nil { - // First write to this metric and level - b = newBuffer(ts, metric.mc.Frequency) - l.metrics[metric.mc.offset] = b - } - - nb, err := b.write(ts, metric.Value) - if err != nil { - return err - } - - // Last write created a new buffer... - if b != nb { - l.metrics[metric.mc.offset] = nb - } - } - return nil -} - -// Returns all values for metric `metric` from `from` to `to` for the selected level(s). -// If the level does not hold the metric itself, the data will be aggregated recursively from the children. -// The second and third return value are the actual from/to for the data. Those can be different from -// the range asked for if no data was available. -func (m *MemoryStore) Read(selector Selector, metric string, from, to int64) ([]Float, int64, int64, error) { - if from > to { - return nil, 0, 0, errors.New("invalid time range") - } - - minfo, ok := m.metrics[metric] - if !ok { - return nil, 0, 0, errors.New("unkown metric: " + metric) - } - - n, data := 0, make([]Float, (to-from)/minfo.Frequency+1) - err := m.root.findBuffers(selector, minfo.offset, func(b *buffer) error { - cdata, cfrom, cto, err := b.read(from, to, data) - if err != nil { - return err - } - - if n == 0 { - from, to = cfrom, cto - } else if from != cfrom || to != cto || len(data) != len(cdata) { - missingfront, missingback := int((from-cfrom)/minfo.Frequency), int((to-cto)/minfo.Frequency) - if missingfront != 0 { - return ErrDataDoesNotAlign - } - - newlen := len(cdata) - missingback - if newlen < 1 { - return ErrDataDoesNotAlign - } - cdata = cdata[0:newlen] - if len(cdata) != len(data) { - return ErrDataDoesNotAlign - } - - from, to = cfrom, cto - } - - data = cdata - n += 1 - return nil - }) - - if err != nil { - return nil, 0, 0, err - } else if n == 0 { - return nil, 0, 0, errors.New("metric or host not found") - } else if n > 1 { - if minfo.Aggregation == AvgAggregation { - normalize := 1. / Float(n) - for i := 0; i < len(data); i++ { - data[i] *= normalize - } - } else if minfo.Aggregation != SumAggregation { - return nil, 0, 0, errors.New("invalid aggregation") - } - } - - return data, from, to, nil -} - -// Release all buffers for the selected level and all its children that contain only -// values older than `t`. -func (m *MemoryStore) Free(selector []string, t int64) (int, error) { - return m.GetLevel(selector).free(t) -} - -func (m *MemoryStore) FreeAll() error { - for k := range m.root.children { - delete(m.root.children, k) - } - - return nil -} - -func (m *MemoryStore) SizeInBytes() int64 { - return m.root.sizeInBytes() -} - -// Given a selector, return a list of all children of the level selected. -func (m *MemoryStore) ListChildren(selector []string) []string { - lvl := &m.root - for lvl != nil && len(selector) != 0 { - lvl.lock.RLock() - next := lvl.children[selector[0]] - lvl.lock.RUnlock() - lvl = next - selector = selector[1:] - } - - if lvl == nil { - return nil - } - - lvl.lock.RLock() - defer lvl.lock.RUnlock() - - children := make([]string, 0, len(lvl.children)) - for child := range lvl.children { - children = append(children, child) - } - - return children -} diff --git a/memstore_test.go b/memstore_test.go deleted file mode 100644 index 8821e7a..0000000 --- a/memstore_test.go +++ /dev/null @@ -1,512 +0,0 @@ -package main - -import ( - "fmt" - "math" - "math/rand" - "sync" - "testing" -) - -func TestMemoryStoreBasics(t *testing.T) { - frequency := int64(10) - start, count := int64(100), int64(5000) - store := NewMemoryStore(map[string]MetricConfig{ - "a": {Frequency: frequency}, - "b": {Frequency: frequency * 2}, - }) - - for i := int64(0); i < count; i++ { - err := store.Write([]string{"testhost"}, start+i*frequency, []Metric{ - {Name: "a", Value: Float(i)}, - {Name: "b", Value: Float(i / 2)}, - }) - if err != nil { - t.Error(err) - return - } - } - - sel := Selector{{String: "testhost"}} - adata, from, to, err := store.Read(sel, "a", start, start+count*frequency) - if err != nil || from != start || to != start+count*frequency { - t.Error(err) - return - } - bdata, _, _, err := store.Read(sel, "b", start, start+count*frequency) - if err != nil { - t.Error(err) - return - } - - if len(adata) != int(count) || len(bdata) != int(count/2) { - t.Error("unexpected count of returned values") - return - } - - for i := 0; i < int(count); i++ { - if adata[i] != Float(i) { - t.Errorf("incorrect value for metric a (%f vs. %f)", adata[i], Float(i)) - return - } - } - - for i := 0; i < int(count/2); i++ { - if bdata[i] != Float(i) && bdata[i] != Float(i-1) { - t.Errorf("incorrect value for metric b (%f) at index %d", bdata[i], i) - return - } - } - -} - -func TestMemoryStoreTooMuchWrites(t *testing.T) { - frequency := int64(10) - count := BUFFER_CAP*3 + 10 - store := NewMemoryStore(map[string]MetricConfig{ - "a": {Frequency: frequency}, - "b": {Frequency: frequency * 2}, - "c": {Frequency: frequency / 2}, - "d": {Frequency: frequency * 3}, - }) - - start := int64(100) - for i := 0; i < count; i++ { - if err := store.Write([]string{"test"}, start+int64(i)*frequency, []Metric{ - {Name: "a", Value: Float(i)}, - {Name: "b", Value: Float(i / 2)}, - {Name: "c", Value: Float(i * 2)}, - {Name: "d", Value: Float(i / 3)}, - }); err != nil { - t.Fatal(err) - } - } - - end := start + int64(count)*frequency - data, from, to, err := store.Read(Selector{{String: "test"}}, "a", start, end) - if len(data) != count || from != start || to != end || err != nil { - t.Fatalf("a: err=%#v, from=%d, to=%d, data=%#v\n", err, from, to, data) - } - - data, from, to, err = store.Read(Selector{{String: "test"}}, "b", start, end) - if len(data) != count/2 || from != start || to != end || err != nil { - t.Fatalf("b: err=%#v, from=%d, to=%d, data=%#v\n", err, from, to, data) - } - - data, from, to, err = store.Read(Selector{{String: "test"}}, "c", start, end) - if len(data) != count*2-1 || from != start || to != end-frequency/2 || err != nil { - t.Fatalf("c: err=%#v, from=%d, to=%d, data=%#v\n", err, from, to, data) - } - - data, from, to, err = store.Read(Selector{{String: "test"}}, "d", start, end) - if len(data) != count/3+1 || from != start || to != end+frequency*2 || err != nil { - t.Errorf("expected: err=nil, from=%d, to=%d, len(data)=%d\n", start, end+frequency*2, count/3) - t.Fatalf("d: err=%#v, from=%d, to=%d, data=%#v\n", err, from, to, data) - } -} - -func TestMemoryStoreOutOfBounds(t *testing.T) { - count := 2000 - toffset := 1000 - store := NewMemoryStore(map[string]MetricConfig{ - "a": {Frequency: 60}, - }) - - for i := 0; i < count; i++ { - if err := store.Write([]string{"cluster", "host", "cpu"}, int64(toffset+i*60), []Metric{ - {Name: "a", Value: Float(i)}, - }); err != nil { - t.Fatal(err) - } - } - - sel := Selector{{String: "cluster"}, {String: "host"}, {String: "cpu"}} - data, from, to, err := store.Read(sel, "a", 500, int64(toffset+count*60+500)) - if err != nil { - t.Fatal(err) - } - - if from/60 != int64(toffset)/60 || to/60 != int64(toffset+count*60)/60 { - t.Fatalf("Got %d-%d, expected %d-%d", from, to, toffset, toffset+count*60) - } - - if len(data) != count || data[0] != 0 || data[len(data)-1] != Float((count-1)) { - t.Fatalf("Wrong data (got: %d, %f, %f, expected: %d, %f, %f)", - len(data), data[0], data[len(data)-1], count, 0., Float(count-1)) - } - - testfrom, testlen := int64(100000000), int64(10000) - data, from, to, err = store.Read(sel, "a", testfrom, testfrom+testlen) - if len(data) != 0 || from != testfrom || to != testfrom || err != nil { - t.Fatal("Unexpected data returned when reading range after valid data") - } - - testfrom, testlen = 0, 10 - data, from, to, err = store.Read(sel, "a", testfrom, testfrom+testlen) - if len(data) != 0 || from/60 != int64(toffset)/60 || to/60 != int64(toffset)/60 || err != nil { - t.Fatal("Unexpected data returned when reading range before valid data") - } -} - -func TestMemoryStoreMissingDatapoints(t *testing.T) { - count := 3000 - store := NewMemoryStore(map[string]MetricConfig{ - "a": {Frequency: 1}, - }) - - for i := 0; i < count; i++ { - if i%3 != 0 { - continue - } - - err := store.Write([]string{"testhost"}, int64(i), []Metric{ - {Name: "a", Value: Float(i)}, - }) - if err != nil { - t.Error(err) - return - } - } - - sel := Selector{{String: "testhost"}} - adata, _, _, err := store.Read(sel, "a", 0, int64(count)) - if err != nil { - t.Error(err) - return - } - - if len(adata) != count-2 { - t.Error("unexpected len") - return - } - - for i := 0; i < count-2; i++ { - if i%3 == 0 { - if adata[i] != Float(i) { - t.Error("unexpected value") - return - } - } else { - if !math.IsNaN(float64(adata[i])) { - t.Errorf("NaN expected (i = %d, value = %f)\n", i, adata[i]) - return - } - } - } -} - -func TestMemoryStoreAggregation(t *testing.T) { - count := 3000 - store := NewMemoryStore(map[string]MetricConfig{ - "a": {Frequency: 1, Aggregation: SumAggregation}, - }) - - for i := 0; i < count; i++ { - err := store.Write([]string{"host0", "cpu0"}, int64(i), []Metric{ - {Name: "a", Value: Float(i) / 2.}, - }) - if err != nil { - t.Error(err) - return - } - - err = store.Write([]string{"host0", "cpu1"}, int64(i), []Metric{ - {Name: "a", Value: Float(i) * 2.}, - }) - if err != nil { - t.Error(err) - return - } - } - - adata, from, to, err := store.Read(Selector{{String: "host0"}}, "a", int64(0), int64(count)) - if err != nil { - t.Error(err) - return - } - - if len(adata) != count || from != 0 || to != int64(count) { - t.Error("unexpected length or time range of returned data") - return - } - - for i := 0; i < count; i++ { - expected := Float(i)/2. + Float(i)*2. - if adata[i] != expected { - t.Errorf("expected: %f, got: %f", expected, adata[i]) - return - } - } -} - -func TestMemoryStoreStats(t *testing.T) { - count := 3000 - store := NewMemoryStore(map[string]MetricConfig{ - "a": {Frequency: 1}, - "b": {Frequency: 1, Aggregation: AvgAggregation}, - }) - - sel1 := []string{"cluster", "host1"} - sel2 := []string{"cluster", "host2", "left"} - sel3 := []string{"cluster", "host2", "right"} - - samples := 0 - asum, amin, amax := 0., math.MaxFloat32, -math.MaxFloat32 - bsum, bmin, bmax := 0., math.MaxFloat32, -math.MaxFloat32 - - for i := 0; i < count; i++ { - if i%5 == 0 { - // Skip some writes, test if samples is calculated correctly - continue - } - - samples += 1 - a := float64(rand.Int()%100 - 50) - asum += a - amin = math.Min(amin, a) - amax = math.Max(amax, a) - b := float64(rand.Int()%100 - 50) - bsum += b * 2 - bmin = math.Min(bmin, b) - bmax = math.Max(bmax, b) - - store.Write(sel1, int64(i), []Metric{ - {Name: "a", Value: Float(a)}, - }) - store.Write(sel2, int64(i), []Metric{ - {Name: "b", Value: Float(b)}, - }) - store.Write(sel3, int64(i), []Metric{ - {Name: "b", Value: Float(b)}, - }) - } - - stats, from, to, err := store.Stats(Selector{{String: "cluster"}, {String: "host1"}}, "a", 0, int64(count)) - if err != nil { - t.Fatal(err) - } - - if from != 1 || to != int64(count) || stats.Samples != samples { - t.Fatalf("unexpected: from=%d, to=%d, stats.Samples=%d (expected samples=%d)\n", from, to, stats.Samples, samples) - } - - if stats.Avg != Float(asum/float64(samples)) || stats.Min != Float(amin) || stats.Max != Float(amax) { - t.Fatalf("wrong stats: %#v\n", stats) - } - - stats, from, to, err = store.Stats(Selector{{String: "cluster"}, {String: "host2"}}, "b", 0, int64(count)) - if err != nil { - t.Fatal(err) - } - - if from != 1 || to != int64(count) || stats.Samples != samples*2 { - t.Fatalf("unexpected: from=%d, to=%d, stats.Samples=%d (expected samples=%d)\n", from, to, stats.Samples, samples*2) - } - - if stats.Avg != Float(bsum/float64(samples*2)) || stats.Min != Float(bmin) || stats.Max != Float(bmax) { - t.Fatalf("wrong stats: %#v (expected: avg=%f, min=%f, max=%f)\n", stats, bsum/float64(samples*2), bmin, bmax) - } -} - -func TestMemoryStoreArchive(t *testing.T) { - store1 := NewMemoryStore(map[string]MetricConfig{ - "a": {Frequency: 1}, - "b": {Frequency: 1}, - }) - - count := 2000 - for i := 0; i < count; i++ { - err := store1.Write([]string{"cluster", "host", "cpu0"}, 100+int64(i), []Metric{ - {Name: "a", Value: Float(i)}, - {Name: "b", Value: Float(i * 2)}, - }) - if err != nil { - t.Error(err) - return - } - } - - // store1.DebugDump(bufio.NewWriter(os.Stdout)) - - archiveRoot := t.TempDir() - _, err := store1.ToCheckpoint(archiveRoot, 100, 100+int64(count/2)) - if err != nil { - t.Error(err) - return - } - - _, err = store1.ToCheckpoint(archiveRoot, 100+int64(count/2), 100+int64(count)) - if err != nil { - t.Error(err) - return - } - - store2 := NewMemoryStore(map[string]MetricConfig{ - "a": {Frequency: 1}, - "b": {Frequency: 1}, - }) - n, err := store2.FromCheckpoint(archiveRoot, 100) - if err != nil { - t.Error(err) - return - } - - sel := Selector{{String: "cluster"}, {String: "host"}, {String: "cpu0"}} - adata, from, to, err := store2.Read(sel, "a", 100, int64(100+count)) - if err != nil { - t.Error(err) - return - } - - if n != 2 || len(adata) != count || from != 100 || to != int64(100+count) { - t.Errorf("unexpected: n=%d, len=%d, from=%d, to=%d\n", n, len(adata), from, to) - return - } - - for i := 0; i < count; i++ { - expected := Float(i) - if adata[i] != expected { - t.Errorf("expected: %f, got: %f", expected, adata[i]) - } - } -} - -func TestMemoryStoreFree(t *testing.T) { - store := NewMemoryStore(map[string]MetricConfig{ - "a": {Frequency: 1}, - "b": {Frequency: 2}, - }) - - count := 3000 - sel := []string{"cluster", "host", "1"} - for i := 0; i < count; i++ { - err := store.Write(sel, int64(i), []Metric{ - {Name: "a", Value: Float(i)}, - {Name: "b", Value: Float(i)}, - }) - if err != nil { - t.Fatal(err) - } - } - - n, err := store.Free([]string{"cluster", "host"}, int64(BUFFER_CAP*2)+100) - if err != nil { - t.Fatal(err) - } - - if n != 3 { - t.Fatal("two buffers expected to be released") - } - - adata, from, to, err := store.Read(Selector{{String: "cluster"}, {String: "host"}, {String: "1"}}, "a", 0, int64(count)) - if err != nil { - t.Fatal(err) - } - - if from != int64(BUFFER_CAP*2) || to != int64(count) || len(adata) != count-2*BUFFER_CAP { - t.Fatalf("unexpected values from call to `Read`: from=%d, to=%d, len=%d", from, to, len(adata)) - } - - // bdata, from, to, err := store.Read(Selector{{String: "cluster"}, {String: "host"}, {String: "1"}}, "b", 0, int64(count)) - // if err != nil { - // t.Fatal(err) - // } - - // if from != int64(BUFFER_CAP*2) || to != int64(count) || len(bdata) != (count-2*BUFFER_CAP)/2 { - // t.Fatalf("unexpected values from call to `Read`: from=%d (expected: %d), to=%d (expected: %d), len=%d (expected: %d)", - // from, BUFFER_CAP*2, to, count, len(bdata), (count-2*BUFFER_CAP)/2) - // } - - if adata[0] != Float(BUFFER_CAP*2) || adata[len(adata)-1] != Float(count-1) { - t.Fatal("wrong values") - } -} - -func BenchmarkMemoryStoreConcurrentWrites(b *testing.B) { - frequency := int64(5) - count := b.N - goroutines := 4 - store := NewMemoryStore(map[string]MetricConfig{ - "a": {Frequency: frequency}, - }) - - var wg sync.WaitGroup - wg.Add(goroutines) - - for g := 0; g < goroutines; g++ { - go func(g int) { - host := fmt.Sprintf("host%d", g) - for i := 0; i < count; i++ { - store.Write([]string{"cluster", host, "cpu0"}, int64(i)*frequency, []Metric{ - {Name: "a", Value: Float(i)}, - }) - } - wg.Done() - }(g) - } - - wg.Wait() - b.StopTimer() - - for g := 0; g < goroutines; g++ { - host := fmt.Sprintf("host%d", g) - sel := Selector{{String: "cluster"}, {String: host}, {String: "cpu0"}} - adata, _, _, err := store.Read(sel, "a", 0, int64(count)*frequency) - if err != nil { - b.Error(err) - return - } - - if len(adata) != count { - b.Error("unexpected count") - return - } - - for i := 0; i < count; i++ { - expected := Float(i) - if adata[i] != expected { - b.Error("incorrect value for metric a") - return - } - } - } -} - -func BenchmarkMemoryStoreAggregation(b *testing.B) { - b.StopTimer() - count := 2000 - store := NewMemoryStore(map[string]MetricConfig{ - "flops_any": {Frequency: 1, Aggregation: AvgAggregation}, - }) - - sel := []string{"testcluster", "host123", "cpu0"} - for i := 0; i < count; i++ { - sel[2] = "cpu0" - err := store.Write(sel, int64(i), []Metric{ - {Name: "flops_any", Value: Float(i)}, - }) - if err != nil { - b.Fatal(err) - } - - sel[2] = "cpu1" - err = store.Write(sel, int64(i), []Metric{ - {Name: "flops_any", Value: Float(i)}, - }) - if err != nil { - b.Fatal(err) - } - } - - b.StartTimer() - for n := 0; n < b.N; n++ { - data, from, to, err := store.Read(Selector{{String: "testcluster"}, {String: "host123"}}, "flops_any", 0, int64(count)) - if err != nil { - b.Fatal(err) - } - - if len(data) != count || from != 0 || to != int64(count) { - b.Fatal() - } - } -} diff --git a/scripts/cc-metric-store.config b/scripts/cc-metric-store.config deleted file mode 100644 index feb0740..0000000 --- a/scripts/cc-metric-store.config +++ /dev/null @@ -1,17 +0,0 @@ -CC_USER=clustercockpit - -CC_GROUP=clustercockpit - -CC_HOME=/tmp - -LOG_DIR=/var/log - -DATA_DIR=/var/run/cc-metric-store - -MAX_OPEN_FILES=10000 - -CONF_DIR=/etc/cc-metric-store - -CONF_FILE=/etc/cc-metric-store/cc-metric-store.json - -RESTART_ON_UPGRADE=true diff --git a/scripts/cc-metric-store.deb.control b/scripts/cc-metric-store.deb.control deleted file mode 100644 index 6978971..0000000 --- a/scripts/cc-metric-store.deb.control +++ /dev/null @@ -1,12 +0,0 @@ -Package: cc-metric-store -Version: {VERSION} -Installed-Size: {INSTALLED_SIZE} -Architecture: {ARCH} -Maintainer: thomas.gruber@fau.de -Depends: libc6 (>= 2.2.1) -Build-Depends: debhelper-compat (= 13), git, golang-go -Description: In-memory metric store daemon from the ClusterCockpit suite -Homepage: https://github.com/ClusterCockpit/cc-metric-store -Source: cc-metric-store -Rules-Requires-Root: no - diff --git a/scripts/cc-metric-store.init b/scripts/cc-metric-store.init deleted file mode 100644 index 80cab6f..0000000 --- a/scripts/cc-metric-store.init +++ /dev/null @@ -1,141 +0,0 @@ -#! /usr/bin/env bash - -# chkconfig: 2345 80 05 -# description: ClusterCockpit metric store -# processname: cc-metric-store -# config: /etc/default/cc-metric-store -# pidfile: /var/run/cc-metric-store.pid - -### BEGIN INIT INFO -# Provides: cc-metric-store -# Required-Start: $all -# Required-Stop: $remote_fs $syslog -# Default-Start: 2 3 4 5 -# Default-Stop: 0 1 6 -# Short-Description: Start ClusterCockpit metric store at boot time -### END INIT INFO - - -PATH=/bin:/usr/bin:/sbin:/usr/sbin -NAME=cc-metric-store -DESC="ClusterCockpit metric store" -DEFAULT=/etc/default/${NAME}.json - -CC_USER=clustercockpit -CC_GROUP=clustercockpit -CONF_DIR=/etc/cc-metric-store -PID_FILE=/var/run/$NAME.pid -DAEMON=/usr/sbin/$NAME -CONF_FILE=${CONF_DIR}/cc-metric-store.json - -umask 0027 - -if [ ! -x $DAEMON ]; then - echo "Program not installed or not executable" - exit 5 -fi - -. /lib/lsb/init-functions - -if [ -r /etc/default/rcS ]; then - . /etc/default/rcS -fi - -# overwrite settings from default file -if [ -f "$DEFAULT" ]; then - . "$DEFAULT" -fi - -CC_OPTS="--config=${CONF_FILE}" - -function checkUser() { - if [ `id -u` -ne 0 ]; then - echo "You need root privileges to run this script" - exit 4 - fi -} - -case "$1" in - start) - checkUser - log_daemon_msg "Starting $DESC" - - pid=`pidofproc -p $PID_FILE $NAME` - if [ -n "$pid" ] ; then - log_begin_msg "Already running." - log_end_msg 0 - exit 0 - fi - - # Prepare environment - touch "$PID_FILE" && chown "$CC_USER":"$CC_GROUP" "$PID_FILE" - - if [ -n "$MAX_OPEN_FILES" ]; then - ulimit -n $MAX_OPEN_FILES - fi - - # Start Daemon - start-stop-daemon --start -b --chdir "$WORK_DIR" --user "$CC_USER" -c "$CC_USER" --pidfile "$PID_FILE" --exec $DAEMON -- $DAEMON_OPTS - return=$? - if [ $return -eq 0 ] - then - sleep 1 - - # check if pid file has been written to - if ! [[ -s $PID_FILE ]]; then - log_end_msg 1 - exit 1 - fi - - i=0 - timeout=10 - # Wait for the process to be properly started before exiting - until { cat "$PID_FILE" | xargs kill -0; } >/dev/null 2>&1 - do - sleep 1 - i=$(($i + 1)) - if [ $i -gt $timeout ]; then - log_end_msg 1 - exit 1 - fi - done - fi - log_end_msg $return - ;; - stop) - checkUser - log_daemon_msg "Stopping $DESC" - - if [ -f "$PID_FILE" ]; then - start-stop-daemon --stop --pidfile "$PID_FILE" \ - --user "$CC_USER" \ - --retry=TERM/20/KILL/5 >/dev/null - if [ $? -eq 1 ]; then - log_progress_msg "$DESC is not running but pid file exists, cleaning up" - elif [ $? -eq 3 ]; then - PID="`cat $PID_FILE`" - log_failure_msg "Failed to stop $DESC (pid $PID)" - exit 1 - fi - rm -f "$PID_FILE" - else - log_progress_msg "(not running)" - fi - log_end_msg 0 - ;; - status) - status_of_proc -p $PID_FILE $NAME $NAME && exit 0 || exit $? - ;; - restart|force-reload) - if [ -f "$PID_FILE" ]; then - $0 stop - sleep 1 - fi - $0 start - ;; - *) - log_success_msg "Usage: $0 {start|stop|restart|force-reload|status}" - exit 3 - ;; -esac - diff --git a/scripts/cc-metric-store.spec b/scripts/cc-metric-store.spec deleted file mode 100644 index ae88883..0000000 --- a/scripts/cc-metric-store.spec +++ /dev/null @@ -1,62 +0,0 @@ -Name: cc-metric-store -Version: %{VERS} -Release: 1%{?dist} -Summary: In-memory metric database from the ClusterCockpit suite - -License: MIT -Source0: %{name}-%{version}.tar.gz - -BuildRequires: go-toolset -BuildRequires: systemd-rpm-macros - -Provides: %{name} = %{version} - -%description -In-memory metric database from the ClusterCockpit suite - -%global debug_package %{nil} - -%prep -%autosetup - - -%build -make - - -%install -# Install cc-metric-store -make PREFIX=%{buildroot} install -# Integrate into system -install -Dpm 0644 scripts/%{name}.service %{buildroot}%{_unitdir}/%{name}.service -install -Dpm 0600 scripts/%{name}.config %{buildroot}%{_sysconfdir}/default/%{name} -install -Dpm 0644 scripts/%{name}.sysusers %{buildroot}%{_sysusersdir}/%{name}.conf - - -%check -# go test should be here... :) - -%pre -%sysusers_create_package scripts/%{name}.sysusers - -%post -%systemd_post %{name}.service - -%preun -%systemd_preun %{name}.service - -%files -# Binary -%attr(-,clustercockpit,clustercockpit) %{_bindir}/%{name} -# Config -%dir %{_sysconfdir}/%{name} -%attr(0600,clustercockpit,clustercockpit) %config(noreplace) %{_sysconfdir}/%{name}/%{name}.json -# Systemd -%{_unitdir}/%{name}.service -%{_sysconfdir}/default/%{name} -%{_sysusersdir}/%{name}.conf - -%changelog -* Mon Mar 07 2022 Thomas Gruber - 0.1 -- Initial metric store implementation - diff --git a/scripts/cc-metric-store.sysusers b/scripts/cc-metric-store.sysusers deleted file mode 100644 index 6ce3700..0000000 --- a/scripts/cc-metric-store.sysusers +++ /dev/null @@ -1,2 +0,0 @@ -#Type Name ID GECOS Home directory Shell -u clustercockpit - "User for ClusterCockpit" /run/cc-metric-collector /sbin/nologin diff --git a/scripts/send-fake-data.go b/scripts/send-fake-data.go deleted file mode 100644 index 2fdc479..0000000 --- a/scripts/send-fake-data.go +++ /dev/null @@ -1,105 +0,0 @@ -package main - -import ( - "bytes" - "fmt" - "io" - "log" - "math" - "math/rand" - "net/http" - "time" -) - -const token = "eyJ0eXAiOiJKV1QiLCJhbGciOiJFZERTQSJ9.eyJ1c2VyIjoiYWRtaW4iLCJyb2xlcyI6WyJST0xFX0FETUlOIiwiUk9MRV9BTkFMWVNUIiwiUk9MRV9VU0VSIl19.d-3_3FZTsadPjDEdsWrrQ7nS0edMAR4zjl-eK7rJU3HziNBfI9PDHDIpJVHTNN5E5SlLGLFXctWyKAkwhXL-Dw" -const ccmsurl = "http://localhost:8081/api/write" -const cluster = "fakedev" -const sockets = 2 -const cpus = 8 -const freq = 15 * time.Second - -var hosts = []string{"fake001", "fake002", "fake003", "fake004", "fake005"} -var metrics = []struct { - Name string - Type string - AvgValue float64 -}{ - {"flops_any", "cpu", 10.0}, - {"mem_bw", "socket", 50.0}, - {"ipc", "cpu", 1.25}, - {"cpu_load", "node", 4}, - {"mem_used", "node", 20}, -} - -var states = make([]float64, 0) - -func send(client *http.Client, t int64) { - msg := &bytes.Buffer{} - - i := 0 - for _, host := range hosts { - for _, metric := range metrics { - n := 1 - if metric.Type == "socket" { - n = sockets - } else if metric.Type == "cpu" { - n = cpus - } - - for j := 0; j < n; j++ { - fmt.Fprintf(msg, "%s,cluster=%s,host=%s,type=%s", metric.Name, cluster, host, metric.Type) - if metric.Type == "socket" { - fmt.Fprintf(msg, ",type-id=%d", j) - } else if metric.Type == "cpu" { - fmt.Fprintf(msg, ",type-id=%d", j) - } - - x := metric.AvgValue + math.Sin(states[i])*(metric.AvgValue/10.) - states[i] += 0.1 - fmt.Fprintf(msg, " value=%f ", x) - - fmt.Fprintf(msg, "%d\n", t) - i++ - } - } - } - - req, _ := http.NewRequest(http.MethodPost, ccmsurl, msg) - req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token)) - res, err := client.Do(req) - if err != nil { - log.Print(err) - return - } - if res.StatusCode != http.StatusOK { - body, _ := io.ReadAll(res.Body) - log.Printf("%s: %s", res.Status, string(body)) - } -} - -func main() { - for range hosts { - for _, m := range metrics { - n := 1 - if m.Type == "socket" { - n = sockets - } else if m.Type == "cpu" { - n = cpus - } - - for i := 0; i < n; i++ { - states = append(states, rand.Float64()*100) - } - } - } - - client := &http.Client{} - - i := 0 - for t := range time.Tick(freq) { - log.Printf("tick... (#%d)", i) - i++ - - send(client, t.Unix()) - } -} diff --git a/selector.go b/selector.go deleted file mode 100644 index 25fe209..0000000 --- a/selector.go +++ /dev/null @@ -1,123 +0,0 @@ -package main - -import ( - "encoding/json" - "errors" -) - -type SelectorElement struct { - Any bool - String string - Group []string -} - -func (se *SelectorElement) UnmarshalJSON(input []byte) error { - if input[0] == '"' { - if err := json.Unmarshal(input, &se.String); err != nil { - return err - } - - if se.String == "*" { - se.Any = true - se.String = "" - } - - return nil - } - - if input[0] == '[' { - return json.Unmarshal(input, &se.Group) - } - - return errors.New("the Go SelectorElement type can only be a string or an array of strings") -} - -func (se *SelectorElement) MarshalJSON() ([]byte, error) { - if se.Any { - return []byte("\"*\""), nil - } - - if se.String != "" { - return json.Marshal(se.String) - } - - if se.Group != nil { - return json.Marshal(se.Group) - } - - return nil, errors.New("a Go Selector must be a non-empty string or a non-empty slice of strings") -} - -type Selector []SelectorElement - -func (l *level) findLevel(selector []string) *level { - if len(selector) == 0 { - return l - } - - l.lock.RLock() - defer l.lock.RUnlock() - - lvl := l.children[selector[0]] - if lvl == nil { - return nil - } - - return lvl.findLevel(selector[1:]) -} - -func (l *level) findBuffers(selector Selector, offset int, f func(b *buffer) error) error { - l.lock.RLock() - defer l.lock.RUnlock() - - if len(selector) == 0 { - b := l.metrics[offset] - if b != nil { - return f(b) - } - - for _, lvl := range l.children { - err := lvl.findBuffers(nil, offset, f) - if err != nil { - return err - } - } - return nil - } - - sel := selector[0] - if len(sel.String) != 0 && l.children != nil { - lvl, ok := l.children[sel.String] - if ok { - err := lvl.findBuffers(selector[1:], offset, f) - if err != nil { - return err - } - } - return nil - } - - if sel.Group != nil && l.children != nil { - for _, key := range sel.Group { - lvl, ok := l.children[key] - if ok { - err := lvl.findBuffers(selector[1:], offset, f) - if err != nil { - return err - } - } - } - return nil - } - - if sel.Any && l.children != nil { - for _, lvl := range l.children { - if err := lvl.findBuffers(selector[1:], offset, f); err != nil { - return err - } - } - return nil - } - - return nil -}