Merge branch 'dev' into add_editmeta_byrequest

- With this commit, all SV4 components are migrated to SV5

.github/workflows/Release.yml

@@ -1,331 +0,0 @@
-# See: https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions
-
-# Workflow name
-name: Release
-
-# Run on tag push
-on:
- push:
-  tags:
-  - '**'
-
-jobs:
-
-  #
-  # Build on AlmaLinux 8.5 using golang-1.18.2
-  #
-  AlmaLinux-RPM-build:
-    runs-on: ubuntu-latest
-    # See: https://hub.docker.com/_/almalinux
-    container: almalinux:8.5
-    # The job outputs link to the outputs of the 'rpmrename' step
-    # Only job outputs can be used in child jobs
-    outputs:
-      rpm : ${{steps.rpmrename.outputs.RPM}}
-      srpm : ${{steps.rpmrename.outputs.SRPM}}
-    steps:
-
-    # Use dnf to install development packages
-    - name: Install development packages
-      run: |
-          dnf --assumeyes group install "Development Tools" "RPM Development Tools"
-          dnf --assumeyes install wget openssl-devel diffutils delve which npm
-          dnf --assumeyes install 'dnf-command(builddep)'
-
-    # Checkout git repository and submodules
-    # fetch-depth must be 0 to use git describe
-    # See: https://github.com/marketplace/actions/checkout
-    - name: Checkout
-      uses: actions/checkout@v2
-      with:
-        submodules: recursive
-        fetch-depth: 0
-
-    # Use dnf to install build dependencies
-    - name: Install build dependencies
-      run: |
-          wget -q http://mirror.centos.org/centos/8-stream/AppStream/x86_64/os/Packages/golang-1.18.2-1.module_el8.7.0+1173+5d37c0fd.x86_64.rpm \
-                  http://mirror.centos.org/centos/8-stream/AppStream/x86_64/os/Packages/golang-bin-1.18.2-1.module_el8.7.0+1173+5d37c0fd.x86_64.rpm \
-                  http://mirror.centos.org/centos/8-stream/AppStream/x86_64/os/Packages/golang-src-1.18.2-1.module_el8.7.0+1173+5d37c0fd.noarch.rpm \
-                  http://mirror.centos.org/centos/8-stream/AppStream/x86_64/os/Packages/go-toolset-1.18.2-1.module_el8.7.0+1173+5d37c0fd.x86_64.rpm
-          rpm -i go*.rpm
-          npm install --global yarn rollup svelte rollup-plugin-svelte
-          #dnf --assumeyes builddep build/package/cc-backend.spec
-
-    - name: RPM build ClusterCockpit
-      id: rpmbuild
-      run: make RPM
-
-    # AlmaLinux 8.5 is a derivate of RedHat Enterprise Linux 8 (UBI8),
-    # so the created RPM both contain the substring 'el8' in the RPM file names
-    # This step replaces the substring 'el8' to 'alma85'. It uses the move operation
-    # because it is unclear whether the default AlmaLinux 8.5 container contains the 
-    # 'rename' command. This way we also get the new names for output.
-    - name: Rename RPMs (s/el8/alma85/)
-      id: rpmrename
-      run: |
-        OLD_RPM="${{steps.rpmbuild.outputs.RPM}}"
-        OLD_SRPM="${{steps.rpmbuild.outputs.SRPM}}"
-        NEW_RPM="${OLD_RPM/el8/alma85}"
-        NEW_SRPM=${OLD_SRPM/el8/alma85}
-        mv "${OLD_RPM}" "${NEW_RPM}"
-        mv "${OLD_SRPM}" "${NEW_SRPM}"
-        echo "::set-output name=SRPM::${NEW_SRPM}"
-        echo "::set-output name=RPM::${NEW_RPM}"
-
-    # See: https://github.com/actions/upload-artifact
-    - name: Save RPM as artifact
-      uses: actions/upload-artifact@v2
-      with:
-        name: cc-backend RPM for AlmaLinux 8.5
-        path: ${{ steps.rpmrename.outputs.RPM }}
-    - name: Save SRPM as artifact
-      uses: actions/upload-artifact@v2
-      with:
-        name: cc-backend SRPM for AlmaLinux 8.5
-        path: ${{ steps.rpmrename.outputs.SRPM }}
-
-  #
-  # Build on UBI 8 using golang-1.18.2
-  #
-  UBI-8-RPM-build:
-    runs-on: ubuntu-latest
-    # See: https://catalog.redhat.com/software/containers/ubi8/ubi/5c359854d70cc534b3a3784e?container-tabs=gti
-    container: registry.access.redhat.com/ubi8/ubi:8.5-226.1645809065
-    # The job outputs link to the outputs of the 'rpmbuild' step
-    outputs:
-      rpm : ${{steps.rpmbuild.outputs.RPM}}
-      srpm : ${{steps.rpmbuild.outputs.SRPM}}
-    steps:
-
-    # Use dnf to install development packages
-    - name: Install development packages
-      run: dnf --assumeyes --disableplugin=subscription-manager install rpm-build go-srpm-macros rpm-build-libs rpm-libs gcc make python38 git wget openssl-devel diffutils delve which
-
-    # Checkout git repository and submodules
-    # fetch-depth must be 0 to use git describe
-    # See: https://github.com/marketplace/actions/checkout
-    - name: Checkout
-      uses: actions/checkout@v2
-      with:
-        submodules: recursive
-        fetch-depth: 0
-
-    # Use dnf to install build dependencies
-    - name: Install build dependencies
-      run: |
-          wget -q http://mirror.centos.org/centos/8-stream/AppStream/x86_64/os/Packages/golang-1.18.2-1.module_el8.7.0+1173+5d37c0fd.x86_64.rpm \
-                  http://mirror.centos.org/centos/8-stream/AppStream/x86_64/os/Packages/golang-bin-1.18.2-1.module_el8.7.0+1173+5d37c0fd.x86_64.rpm \
-                  http://mirror.centos.org/centos/8-stream/AppStream/x86_64/os/Packages/golang-src-1.18.2-1.module_el8.7.0+1173+5d37c0fd.noarch.rpm \
-                  http://mirror.centos.org/centos/8-stream/AppStream/x86_64/os/Packages/go-toolset-1.18.2-1.module_el8.7.0+1173+5d37c0fd.x86_64.rpm
-          rpm -i go*.rpm
-          dnf --assumeyes --disableplugin=subscription-manager install npm
-          npm install --global yarn rollup svelte rollup-plugin-svelte
-          #dnf --assumeyes builddep build/package/cc-backend.spec
-
-    - name: RPM build ClusterCockpit
-      id: rpmbuild
-      run: make RPM
-
-    # See: https://github.com/actions/upload-artifact
-    - name: Save RPM as artifact
-      uses: actions/upload-artifact@v2
-      with:
-        name: cc-backend RPM for UBI 8
-        path: ${{ steps.rpmbuild.outputs.RPM }}
-    - name: Save SRPM as artifact
-      uses: actions/upload-artifact@v2
-      with:
-        name: cc-backend SRPM for UBI 8
-        path: ${{ steps.rpmbuild.outputs.SRPM }}
-
-  #
-  # Build on Ubuntu 20.04 using official go 1.19.1 package
-  #
-  Ubuntu-focal-build:
-    runs-on: ubuntu-latest
-    container: ubuntu:20.04
-    # The job outputs link to the outputs of the 'debrename' step
-    # Only job outputs can be used in child jobs
-    outputs:
-      deb : ${{steps.debrename.outputs.DEB}}
-    steps:
-    # Use apt to install development packages
-    - name: Install development packages
-      run: |
-          apt update && apt --assume-yes upgrade
-          apt --assume-yes install build-essential sed git wget bash
-          apt --assume-yes install npm
-          npm install --global yarn rollup svelte rollup-plugin-svelte
-    # Checkout git repository and submodules
-    # fetch-depth must be 0 to use git describe
-    # See: https://github.com/marketplace/actions/checkout
-    - name: Checkout
-      uses: actions/checkout@v2
-      with:
-        submodules: recursive
-        fetch-depth: 0
-    # Use official golang package
-    - name: Install Golang
-      run: |
-          wget -q https://go.dev/dl/go1.19.1.linux-amd64.tar.gz
-          tar -C /usr/local -xzf go1.19.1.linux-amd64.tar.gz
-          export PATH=/usr/local/go/bin:/usr/local/go/pkg/tool/linux_amd64:$PATH
-          go version
-    - name: DEB build ClusterCockpit
-      id: dpkg-build
-      run: |
-          ls -la
-          pwd
-          env
-          export PATH=/usr/local/go/bin:/usr/local/go/pkg/tool/linux_amd64:$PATH
-          git config --global --add safe.directory $(pwd)
-          make DEB
-    - name: Rename DEB (add '_ubuntu20.04')
-      id: debrename
-      run: |
-        OLD_DEB_NAME=$(echo "${{steps.dpkg-build.outputs.DEB}}" | rev | cut -d '.' -f 2- | rev)
-        NEW_DEB_FILE="${OLD_DEB_NAME}_ubuntu20.04.deb"
-        mv "${{steps.dpkg-build.outputs.DEB}}" "${NEW_DEB_FILE}"
-        echo "::set-output name=DEB::${NEW_DEB_FILE}"
-    # See: https://github.com/actions/upload-artifact
-    - name: Save DEB as artifact
-      uses: actions/upload-artifact@v2
-      with:
-        name: cc-backend DEB for Ubuntu 20.04
-        path: ${{ steps.debrename.outputs.DEB }}
-
-  #
-  # Build on Ubuntu 20.04 using official go 1.19.1 package
-  #
-  Ubuntu-jammy-build:
-    runs-on: ubuntu-latest
-    container: ubuntu:22.04
-    # The job outputs link to the outputs of the 'debrename' step
-    # Only job outputs can be used in child jobs
-    outputs:
-      deb : ${{steps.debrename.outputs.DEB}}
-    steps:
-    # Use apt to install development packages
-    - name: Install development packages
-      run: |
-          apt update && apt --assume-yes upgrade
-          apt --assume-yes install build-essential sed git wget bash npm
-          npm install --global yarn rollup svelte rollup-plugin-svelte
-    # Checkout git repository and submodules
-    # fetch-depth must be 0 to use git describe
-    # See: https://github.com/marketplace/actions/checkout
-    - name: Checkout
-      uses: actions/checkout@v2
-      with:
-        submodules: recursive
-        fetch-depth: 0
-    # Use official golang package
-    - name: Install Golang
-      run: |
-          wget -q https://go.dev/dl/go1.19.1.linux-amd64.tar.gz
-          tar -C /usr/local -xzf go1.19.1.linux-amd64.tar.gz
-          export PATH=/usr/local/go/bin:/usr/local/go/pkg/tool/linux_amd64:$PATH
-          go version
-    - name: DEB build ClusterCockpit
-      id: dpkg-build
-      run: |
-          ls -la
-          pwd
-          env
-          export PATH=/usr/local/go/bin:/usr/local/go/pkg/tool/linux_amd64:$PATH
-          git config --global --add safe.directory $(pwd)
-          make DEB
-    - name: Rename DEB (add '_ubuntu22.04')
-      id: debrename
-      run: |
-        OLD_DEB_NAME=$(echo "${{steps.dpkg-build.outputs.DEB}}" | rev | cut -d '.' -f 2- | rev)
-        NEW_DEB_FILE="${OLD_DEB_NAME}_ubuntu22.04.deb"
-        mv "${{steps.dpkg-build.outputs.DEB}}" "${NEW_DEB_FILE}"
-        echo "::set-output name=DEB::${NEW_DEB_FILE}"
-    # See: https://github.com/actions/upload-artifact
-    - name: Save DEB as artifact
-      uses: actions/upload-artifact@v2
-      with:
-        name: cc-backend DEB for Ubuntu 22.04
-        path: ${{ steps.debrename.outputs.DEB }}
-
-  #
-  # Create release with fresh RPMs
-  #
-  Release:
-    runs-on: ubuntu-latest
-    # We need the RPMs, so add dependency
-    needs: [AlmaLinux-RPM-build, UBI-8-RPM-build, Ubuntu-focal-build, Ubuntu-jammy-build]
-
-    steps:
-    # See: https://github.com/actions/download-artifact
-    - name: Download AlmaLinux 8.5 RPM
-      uses: actions/download-artifact@v2
-      with:
-        name: cc-backend RPM for AlmaLinux 8.5
-    - name: Download AlmaLinux 8.5 SRPM
-      uses: actions/download-artifact@v2
-      with:
-        name: cc-backend SRPM for AlmaLinux 8.5
-    
-    - name: Download UBI 8 RPM
-      uses: actions/download-artifact@v2
-      with:
-        name: cc-backend RPM for UBI 8
-    - name: Download UBI 8 SRPM
-      uses: actions/download-artifact@v2
-      with:
-        name: cc-backend SRPM for UBI 8
-
-    - name: Download Ubuntu 20.04 DEB
-      uses: actions/download-artifact@v2
-      with:
-        name: cc-backend DEB for Ubuntu 20.04
-
-    - name: Download Ubuntu 22.04 DEB
-      uses: actions/download-artifact@v2
-      with:
-        name: cc-backend DEB for Ubuntu 22.04
-
-    # The download actions do not publish the name of the downloaded file,
-    # so we re-use the job outputs of the parent jobs. The files are all
-    # downloaded to the current folder.
-    # The gh-release action afterwards does not accept file lists but all
-    # files have to be listed at 'files'. The step creates one output per
-    # RPM package (2 per distro)
-    - name: Set RPM variables
-      id: files
-      run: |
-        ALMA_85_RPM=$(basename "${{ needs.AlmaLinux-RPM-build.outputs.rpm}}")
-        ALMA_85_SRPM=$(basename "${{ needs.AlmaLinux-RPM-build.outputs.srpm}}")
-        UBI_8_RPM=$(basename "${{ needs.UBI-8-RPM-build.outputs.rpm}}")
-        UBI_8_SRPM=$(basename "${{ needs.UBI-8-RPM-build.outputs.srpm}}")
-        U_2004_DEB=$(basename "${{ needs.Ubuntu-focal-build.outputs.deb}}")
-        U_2204_DEB=$(basename "${{ needs.Ubuntu-jammy-build.outputs.deb}}")
-        echo "ALMA_85_RPM::${ALMA_85_RPM}"
-        echo "ALMA_85_SRPM::${ALMA_85_SRPM}"
-        echo "UBI_8_RPM::${UBI_8_RPM}"
-        echo "UBI_8_SRPM::${UBI_8_SRPM}"
-        echo "U_2004_DEB::${U_2004_DEB}"
-        echo "U_2204_DEB::${U_2204_DEB}"
-        echo "::set-output name=ALMA_85_RPM::${ALMA_85_RPM}"
-        echo "::set-output name=ALMA_85_SRPM::${ALMA_85_SRPM}"
-        echo "::set-output name=UBI_8_RPM::${UBI_8_RPM}"
-        echo "::set-output name=UBI_8_SRPM::${UBI_8_SRPM}"
-        echo "::set-output name=U_2004_DEB::${U_2004_DEB}"
-        echo "::set-output name=U_2204_DEB::${U_2204_DEB}"
-
-    # See: https://github.com/softprops/action-gh-release
-    - name: Release
-      uses: softprops/action-gh-release@v1
-      if: startsWith(github.ref, 'refs/tags/')
-      with:
-        name: cc-backend-${{github.ref_name}}
-        files: |
-         ${{ steps.files.outputs.ALMA_85_RPM }}
-         ${{ steps.files.outputs.ALMA_85_SRPM }}
-         ${{ steps.files.outputs.UBI_8_RPM }}
-         ${{ steps.files.outputs.UBI_8_SRPM }}
-         ${{ steps.files.outputs.U_2004_DEB }}
-         ${{ steps.files.outputs.U_2204_DEB }}

.github/workflows/test.yml

@@ -7,7 +7,7 @@ jobs:
       - name: Install Go
         uses: actions/setup-go@v4
         with:
-          go-version: 1.19.x
+          go-version: 1.24.x
       - name: Checkout code
         uses: actions/checkout@v3
       - name: Build, Vet & Test

.gitignore

@@ -1,19 +1,23 @@
 /cc-backend
-
-/var/job-archive
-/var/*.db
-/var/machine-state
-
 /.env
 /config.json
 
+/var/job-archive
+/var/machine-state
+/var/job.db-shm
+/var/job.db-wal
+/var/*.db
+/var/*.txt
+
 /web/frontend/public/build
 /web/frontend/node_modules
-/.vscode/*
+
 /archive-migration
 /archive-manager
-var/job.db-shm
-var/job.db-wal
 
+/internal/repository/testdata/job.db-shm
+/internal/repository/testdata/job.db-wal
+
+/.vscode/*
 dist/
 *.db

.goreleaser.yaml

@@ -34,19 +34,6 @@ builds:
     main: ./tools/archive-manager
     tags:
       - static_build
-  - env:
-      - CGO_ENABLED=0
-    goos:
-      - linux
-    goarch:
-      - amd64
-    goamd64:
-      - v3
-    id: "archive-migration"
-    binary: archive-migration
-    main: ./tools/archive-migration
-    tags:
-      - static_build
   - env:
       - CGO_ENABLED=0
     goos:
@@ -70,7 +57,7 @@ archives:
       {{- else }}{{ .Arch }}{{ end }}
       {{- if .Arm }}v{{ .Arm }}{{ end }}
 checksum:
-  name_template: 'checksums.txt'
+  name_template: "checksums.txt"
 snapshot:
   name_template: "{{ incpatch .Version }}-next"
 changelog:
@@ -100,7 +87,7 @@ changelog:
 release:
   draft: false
   footer: |
-    Supports job archive version 1 and database version 6.
+    Supports job archive version 2 and database version 8.
     Please check out the [Release Notes](https://github.com/ClusterCockpit/cc-backend/blob/master/ReleaseNotes.md) for further details on breaking changes.
 
 # vim: set ts=2 sw=2 tw=0 fo=cnqoj

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2022 NHR@FAU, University Erlangen-Nuremberg
+Copyright (c) NHR@FAU, University Erlangen-Nuremberg
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

Makefile

@@ -2,7 +2,7 @@ TARGET = ./cc-backend
 VAR = ./var
 CFG = config.json .env
 FRONTEND = ./web/frontend
-VERSION = 1.2.2
+VERSION = 1.4.4
 GIT_HASH := $(shell git rev-parse --short HEAD || echo 'development')
 CURRENT_TIME = $(shell date +"%Y-%m-%d:T%H:%M:%S")
 LD_FLAGS = '-s -X main.date=${CURRENT_TIME} -X main.version=${VERSION} -X main.commit=${GIT_HASH}'
@@ -22,13 +22,23 @@ SVELTE_COMPONENTS = status   \
 					header
 
 SVELTE_TARGETS = $(addprefix $(FRONTEND)/public/build/,$(addsuffix .js, $(SVELTE_COMPONENTS)))
-SVELTE_SRC = $(wildcard $(FRONTEND)/src/*.svelte)         \
-			 $(wildcard $(FRONTEND)/src/*.js)             \
-			 $(wildcard $(FRONTEND)/src/filters/*.svelte) \
-			 $(wildcard $(FRONTEND)/src/plots/*.svelte)   \
-			 $(wildcard $(FRONTEND)/src/joblist/*.svelte)
+SVELTE_SRC = $(wildcard $(FRONTEND)/src/*.svelte)                 \
+			 $(wildcard $(FRONTEND)/src/*.js)                     \
+			 $(wildcard $(FRONTEND)/src/analysis/*.svelte)        \
+			 $(wildcard $(FRONTEND)/src/config/*.svelte)          \
+			 $(wildcard $(FRONTEND)/src/config/admin/*.svelte)    \
+			 $(wildcard $(FRONTEND)/src/config/user/*.svelte)     \
+			 $(wildcard $(FRONTEND)/src/generic/*.js)             \
+			 $(wildcard $(FRONTEND)/src/generic/*.svelte)         \
+			 $(wildcard $(FRONTEND)/src/generic/filters/*.svelte) \
+			 $(wildcard $(FRONTEND)/src/generic/plots/*.svelte)   \
+			 $(wildcard $(FRONTEND)/src/generic/joblist/*.svelte) \
+			 $(wildcard $(FRONTEND)/src/generic/helper/*.svelte)  \
+			 $(wildcard $(FRONTEND)/src/generic/select/*.svelte)  \
+			 $(wildcard $(FRONTEND)/src/header/*.svelte)          \
+			 $(wildcard $(FRONTEND)/src/job/*.svelte)
 
-.PHONY: clean distclean test tags frontend $(TARGET)
+.PHONY: clean distclean test tags frontend swagger graphql $(TARGET)
 
 .NOTPARALLEL:
 
@@ -40,6 +50,15 @@ frontend:
 	$(info ===>  BUILD frontend)
 	cd web/frontend && npm install && npm run build
 
+swagger:
+	$(info ===>  GENERATE swagger)
+	@go run github.com/swaggo/swag/cmd/swag init -d ./internal/api,./pkg/schema -g rest.go -o ./api
+	@mv ./api/docs.go ./internal/api/docs.go
+
+graphql:
+	$(info ===>  GENERATE graphql)
+	@go run github.com/99designs/gqlgen
+
 clean:
 	$(info ===>  CLEAN)
 	@go clean
@@ -63,7 +82,7 @@ tags:
 	@ctags -R
 
 $(VAR):
-	@mkdir $(VAR)
+	@mkdir -p $(VAR)
 
 config.json:
 	$(info ===>  Initialize config.json file)
@@ -76,90 +95,3 @@ config.json:
 $(SVELTE_TARGETS): $(SVELTE_SRC)
 	$(info ===>  BUILD frontend)
 	cd web/frontend && npm install && npm run build
-
-install: $(TARGET)
-	@WORKSPACE=$(PREFIX)
-	@if [ -z "$${WORKSPACE}" ]; then exit 1; fi
-	@mkdir --parents --verbose $${WORKSPACE}/usr/$(BINDIR)
-	@install -Dpm 755 $(TARGET) $${WORKSPACE}/usr/$(BINDIR)/$(TARGET)
-	@install -Dpm 600 configs/config.json $${WORKSPACE}/etc/$(TARGET)/$(TARGET).json
-
-.ONESHELL:
-.PHONY: RPM
-RPM: build/package/cc-backend.spec
-	@WORKSPACE="$${PWD}"
-	@SPECFILE="$${WORKSPACE}/build/package/cc-backend.spec"
-	# Setup RPM build tree
-	@eval $$(rpm --eval "ARCH='%{_arch}' RPMDIR='%{_rpmdir}' SOURCEDIR='%{_sourcedir}' SPECDIR='%{_specdir}' SRPMDIR='%{_srcrpmdir}' BUILDDIR='%{_builddir}'")
-	@mkdir --parents --verbose "$${RPMDIR}" "$${SOURCEDIR}" "$${SPECDIR}" "$${SRPMDIR}" "$${BUILDDIR}"
-	# Create source tarball
-	@COMMITISH="HEAD"
-	@VERS=$$(git describe --tags $${COMMITISH})
-	@VERS=$${VERS#v}
-	@VERS=$$(echo $$VERS | sed -e s+'-'+'_'+g)
-	@if [ "$${VERS}" = "" ]; then VERS="$(VERSION)"; fi
-	@eval $$(rpmspec --query --queryformat "NAME='%{name}' VERSION='%{version}' RELEASE='%{release}' NVR='%{NVR}' NVRA='%{NVRA}'" --define="VERS $${VERS}" "$${SPECFILE}")
-	@PREFIX="$${NAME}-$${VERSION}"
-	@FORMAT="tar.gz"
-	@SRCFILE="$${SOURCEDIR}/$${PREFIX}.$${FORMAT}"
-	@git archive --verbose --format "$${FORMAT}" --prefix="$${PREFIX}/" --output="$${SRCFILE}" $${COMMITISH}
-	# Build RPM and SRPM
-	@rpmbuild -ba --define="VERS $${VERS}" --rmsource --clean "$${SPECFILE}"
-	# Report RPMs and SRPMs when in GitHub Workflow
-	@if [ "$${GITHUB_ACTIONS}" = true ]; then
-	@     RPMFILE="$${RPMDIR}/$${ARCH}/$${NVRA}.rpm"
-	@     SRPMFILE="$${SRPMDIR}/$${NVR}.src.rpm"
-	@     echo "RPM: $${RPMFILE}"
-	@     echo "SRPM: $${SRPMFILE}"
-	@     echo "::set-output name=SRPM::$${SRPMFILE}"
-	@     echo "::set-output name=RPM::$${RPMFILE}"
-	@fi
-
-.ONESHELL:
-.PHONY: DEB
-DEB: build/package/cc-backend.deb.control
-	@BASEDIR=$${PWD}
-	@WORKSPACE=$${PWD}/.dpkgbuild
-	@DEBIANDIR=$${WORKSPACE}/debian
-	@DEBIANBINDIR=$${WORKSPACE}/DEBIAN
-	@mkdir --parents --verbose $$WORKSPACE $$DEBIANBINDIR
-	#@mkdir --parents --verbose $$DEBIANDIR
-	@CONTROLFILE="$${BASEDIR}/build/package/cc-backend.deb.control"
-	@COMMITISH="HEAD"
-	@VERS=$$(git describe --tags --abbrev=0 $${COMMITISH})
-	@VERS=$${VERS#v}
-	@VERS=$$(echo $$VERS | sed -e s+'-'+'_'+g)
-	@if [ "$${VERS}" = "" ]; then VERS="$(VERSION)"; fi
-	@ARCH=$$(uname -m)
-	@ARCH=$$(echo $$ARCH | sed -e s+'_'+'-'+g)
-	@if [ "$${ARCH}" = "x86-64" ]; then ARCH=amd64; fi
-	@PREFIX="$${NAME}-$${VERSION}_$${ARCH}"
-	@SIZE_BYTES=$$(du -bcs --exclude=.dpkgbuild "$${WORKSPACE}"/ | awk '{print $$1}' | head -1 | sed -e 's/^0\+//')
-	@SIZE="$$(awk -v size="$$SIZE_BYTES" 'BEGIN {print (size/1024)+1}' | awk '{print int($$0)}')"
-	#@sed -e s+"{VERSION}"+"$$VERS"+g -e s+"{INSTALLED_SIZE}"+"$$SIZE"+g -e s+"{ARCH}"+"$$ARCH"+g $$CONTROLFILE > $${DEBIANDIR}/control
-	@sed -e s+"{VERSION}"+"$$VERS"+g -e s+"{INSTALLED_SIZE}"+"$$SIZE"+g -e s+"{ARCH}"+"$$ARCH"+g $$CONTROLFILE > $${DEBIANBINDIR}/control
-	@mkdir --parents --verbose "$${WORKSPACE}"/$(VAR)
-	@touch "$${WORKSPACE}"/$(VAR)/job.db
-	@cd web/frontend && yarn install && yarn build && cd -
-	@go build -ldflags=${LD_FLAGS} ./cmd/cc-backend
-	@mkdir --parents --verbose $${WORKSPACE}/usr/$(BINDIR)
-	@cp $(TARGET) $${WORKSPACE}/usr/$(BINDIR)/$(TARGET)
-	@chmod 0755 $${WORKSPACE}/usr/$(BINDIR)/$(TARGET)
-	@mkdir --parents --verbose $${WORKSPACE}/etc/$(TARGET)
-	@cp configs/config.json $${WORKSPACE}/etc/$(TARGET)/$(TARGET).json
-	@chmod 0600 $${WORKSPACE}/etc/$(TARGET)/$(TARGET).json
-	@mkdir --parents --verbose $${WORKSPACE}/usr/lib/systemd/system
-	@cp build/package/$(TARGET).service $${WORKSPACE}/usr/lib/systemd/system/$(TARGET).service
-	@chmod 0644 $${WORKSPACE}/usr/lib/systemd/system/$(TARGET).service
-	@mkdir --parents --verbose $${WORKSPACE}/etc/default
-	@cp build/package/$(TARGET).config $${WORKSPACE}/etc/default/$(TARGET)
-	@chmod 0600 $${WORKSPACE}/etc/default/$(TARGET)
-	@mkdir --parents --verbose $${WORKSPACE}/usr/lib/sysusers.d
-	@cp build/package/$(TARGET).sysusers $${WORKSPACE}/usr/lib/sysusers.d/$(TARGET).conf
-	@chmod 0644 $${WORKSPACE}/usr/lib/sysusers.d/$(TARGET).conf
-	@DEB_FILE="cc-metric-store_$${VERS}_$${ARCH}.deb"
-	@dpkg-deb -b $${WORKSPACE} "$$DEB_FILE"
-	@rm -r "$${WORKSPACE}"
-	@if [ "$${GITHUB_ACTIONS}" = "true" ]; then
-	@     echo "::set-output name=DEB::$${DEB_FILE}"
-	@fi

README.md

@@ -1,12 +1,16 @@
 # NOTE
-Please have a look at the [Release Notes](https://github.com/ClusterCockpit/cc-backend/blob/master/ReleaseNotes.md) for breaking changes!
+
+Please have a look at the [Release
+Notes](https://github.com/ClusterCockpit/cc-backend/blob/master/ReleaseNotes.md)
+for breaking changes!
 
 # ClusterCockpit REST and GraphQL API backend
 
 [![Build](https://github.com/ClusterCockpit/cc-backend/actions/workflows/test.yml/badge.svg)](https://github.com/ClusterCockpit/cc-backend/actions/workflows/test.yml)
 
 This is a Golang backend implementation for a REST and GraphQL API according to
-the [ClusterCockpit specifications](https://github.com/ClusterCockpit/cc-specifications). It also
+the [ClusterCockpit
+specifications](https://github.com/ClusterCockpit/cc-specifications). It also
 includes a web interface for ClusterCockpit. This implementation replaces the
 previous PHP Symfony based ClusterCockpit web interface. The reasons for
 switching from PHP Symfony to a Golang based solution are explained
@@ -14,31 +18,31 @@ switching from PHP Symfony to a Golang based solution are explained
 
 ## Overview
 
+This is a Golang web backend for the ClusterCockpit job-specific performance
+monitoring framework. It provides a REST API for integrating ClusterCockpit with
+an HPC cluster batch system and external analysis scripts. Data exchange between
+the web front-end and the back-end is based on a GraphQL API. The web frontend
+is also served by the backend using [Svelte](https://svelte.dev/) components.
+Layout and styling are based on [Bootstrap 5](https://getbootstrap.com/) using
+[Bootstrap Icons](https://icons.getbootstrap.com/).
 
-This is a Golang web backend for the ClusterCockpit job-specific performance monitoring framework.
-It provides a REST API for integrating ClusterCockpit with an HPC cluster batch system and external analysis scripts.
-Data exchange between the web front-end and the back-end is based on a GraphQL API.
-The web frontend is also served by the backend using [Svelte](https://svelte.dev/) components.
-Layout and styling are based on [Bootstrap 5](https://getbootstrap.com/) using [Bootstrap Icons](https://icons.getbootstrap.com/).
-
-The backend uses [SQLite 3](https://sqlite.org/) as a relational SQL database by default.
-Optionally it can use a MySQL/MariaDB database server.
-While there are metric data  backends for the InfluxDB and Prometheus time series databases, the only tested and supported setup is to use cc-metric-store as the metric data backend.
-Documentation on how to integrate ClusterCockpit with other time series databases will be added in the future. 
+The backend uses [SQLite 3](https://sqlite.org/) as a relational SQL database by
+default. Optionally it can use a MySQL/MariaDB database server. While there are
+metric data  backends for the InfluxDB and Prometheus time series databases, the
+only tested and supported setup is to use cc-metric-store as the metric data
+backend. Documentation on how to integrate ClusterCockpit with other time series
+databases will be added in the future.
 
 Completed batch jobs are stored in a file-based job archive according to
-[this specification] (https://github.com/ClusterCockpit/cc-specifications/tree/master/job-archive).
+[this specification](https://github.com/ClusterCockpit/cc-specifications/tree/master/job-archive).
 The backend supports authentication via local accounts, an external LDAP
 directory, and JWT tokens. Authorization for APIs is implemented with
 [JWT](https://jwt.io/) tokens created with public/private key encryption.
 
-You find more detailed information here:
-* `./configs/README.md`: Infos about configuration and setup of cc-backend.
-* `./init/README.md`: Infos on how to setup cc-backend as systemd service on Linux.
-* `./tools/README.md`: Infos on the JWT authorizatin token workflows in ClusterCockpit.
-* `./docs`: You can find further documentation here. There is also a Hands-on tutorial that is recommended to get familiar with the ClusterCockpit setup.
+You find a detailed documentation on the [ClusterCockpit
+Webpage](https://clustercockpit.org).
 
-**NOTE**
+## Build requirements
 
 ClusterCockpit requires a current version of the golang toolchain and node.js.
 You can check `go.mod` to see what is the current minimal golang version needed.
@@ -49,7 +53,7 @@ on the Go standard library, it is crucial for security and performance to use a
 current version of golang. In addition, an old golang toolchain may limit the supported
 versions of third-party packages.
 
-## How to try ClusterCockpit with a demo setup.
+## How to try ClusterCockpit with a demo setup
 
 We provide a shell script that downloads demo data and automatically starts the
 cc-backend. You will need `wget`, `go`, `node`, `npm` in your path to
@@ -61,31 +65,37 @@ cd ./cc-backend
 ./startDemo.sh
 ```
 
-You can also try the demo using the lates release binary.
+You can also try the demo using the latest release binary.
 Create a folder and put the release binary `cc-backend` into this folder.
 Execute the following steps:
-```
-$ ./cc-backend -init
-$ vim config.json (Add a second cluster entry and name the clusters alex and fritz)
-$ wget https://hpc-mover.rrze.uni-erlangen.de/HPC-Data/0x7b58aefb/eig7ahyo6fo2bais0ephuf2aitohv1ai/job-archive-demo.tar
-$ tar xf job-archive-demo.tar
-$ ./cc-backend -init-db -add-user demo:admin:demo -loglevel info
-$ ./cc-backend -server -dev -loglevel info
+
+``` shell
+./cc-backend -init
+vim config.json (Add a second cluster entry and name the clusters alex and fritz)
+wget https://hpc-mover.rrze.uni-erlangen.de/HPC-Data/0x7b58aefb/eig7ahyo6fo2bais0ephuf2aitohv1ai/job-archive-demo.tar
+tar xf job-archive-demo.tar
+./cc-backend -init-db -add-user demo:admin:demo -loglevel info
+./cc-backend -server -dev -loglevel info
 ```
 
-You can access the web interface at http://localhost:8080.
+You can access the web interface at [http://localhost:8080](http://localhost:8080).
 Credentials for login are `demo:demo`.
 Please note that some views do not work without a metric backend (e.g., the
 Analysis, Systems and Status views).
 
-## Howto build and run
+## How to build and run
 
-There is a Makefile to automate the build of cc-backend. The Makefile supports the following targets:
-* `$ make`: Initialize `var` directory and build svelte frontend and backend binary. Note that there is no proper prerequesite handling. Any change of frontend source files will result in a complete rebuild.
-* `$ make clean`: Clean go build cache and remove binary.
-* `$ make test`: Run the tests that are also run in the GitHub workflow setup.
+There is a Makefile to automate the build of cc-backend. The Makefile supports
+the following targets:
+
+* `make`: Initialize `var` directory and build svelte frontend and backend
+binary. Note that there is no proper prerequisite handling. Any change of
+frontend source files will result in a complete rebuild.
+* `make clean`: Clean go build cache and remove binary.
+* `make test`: Run the tests that are also run in the GitHub workflow setup.
 
 A common workflow for setting up cc-backend from scratch is:
+
 ```sh
 git clone https://github.com/ClusterCockpit/cc-backend.git
 
@@ -116,89 +126,43 @@ ln -s <your-existing-job-archive> ./var/job-archive
 ./cc-backend -help
 ```
 
-### Run as systemd daemon
-
-To run this program as a daemon, cc-backend comes with a [example systemd setup](./init/README.md).
-
-## Configuration and setup
-
-cc-backend can be used as a local web interface for an existing job archive or
-as a server for the ClusterCockpit monitoring framework.
-
-Create your job archive according to [this specification] (https://github.com/ClusterCockpit/cc-specifications/tree/master/job-archive).
-At least one cluster directory with a valid `cluster.json` file is required. If
-you configure the job archive from scratch, you must also create the job
-archive version file that contains the job archive version as an integer.
-You can retrieve the currently supported version by running the following
-command:
-```
-$ ./cc-backend -version
-```
-It is ok to have no jobs in the job archive.
-
-### Configuration
-
-A configuration file in JSON format must be specified with `-config` to override the default settings.
-By default, a `config.json` file located in the current directory of the `cc-backend` process will be loaded even without the `-config` flag.
-Documentation of all supported configuration and command line options can be found [here](./configs/README.md).
-
-## Database initialization and migration
-
-Each `cc-backend` version supports a specific database version.
-At startup, the version of the sqlite database is checked and `cc-backend` terminates if the version does not match.
-`cc-backend` supports the migration of the database schema to the required version with the command line option `-migrate-db`.
-If the database file does not exist yet, it will be created and initialized with the command line option `-migrate-db`.
-If you want to use a newer database version with an older version of cc-backend, you can downgrade a database with the external tool [migrate](https://github.com/golang-migrate/migrate).
-In this case, you must specify the path to the migration files in a current source tree: `./internal/repository/migrations/`.
-
-## Development and testing
-When making changes to the REST or GraphQL API, the appropriate code generators must be used.
-You must always rebuild `cc-backend` after updating the API files.
-
-### Update GraphQL schema
-
-This project uses [gqlgen](https://github.com/99designs/gqlgen) for the GraphQL API.
-The schema can be found in `./api/schema.graphqls`.
-After changing it, you need to run `go run github.com/99designs/gqlgen`, which will update `./internal/graph/model`.
-If new resolvers are needed, they will be added to `./internal/graph/schema.resolvers.go`, where you will then need to implement them.
-If you start `cc-backend` with the `-dev` flag, the GraphQL Playground UI is available at http://localhost:8080/playground.
-
-### Update Swagger UI
-
-This project integrates [swagger ui] (https://swagger.io/tools/swagger-ui/) to document and test its REST API.
-The swagger documentation files can be found in `./api/`.
-You can generate the swagger-ui configuration by running `go run github.com/swaggo/swag/cmd/swag init -d ./internal/api,./pkg/schema -g rest.go -o ./api `.
-You need to move the created `./api/docs.go` to `./internal/api/docs.go`.
-If you start cc-backend with the `-dev` flag, the Swagger interface is available
-at http://localhost:8080/swagger/.
-You must enter a JWT key for a user with the API role.
-
-**NOTE**
-
-The user who owns the JWT key must not be logged into the same browser (have a
-running session), or the Swagger requests will not work. It is recommended to
-create a separate user that has only the API role.
-
-## Development and testing
-In case the REST or GraphQL API is changed the according code generators have to be used.
-
 ## Project file structure
 
-- [`api/`](https://github.com/ClusterCockpit/cc-backend/tree/master/api) contains the API schema files for the REST and GraphQL APIs. The REST API is documented in the OpenAPI 3.0 format in [./api/openapi.yaml](./api/openapi.yaml).
-- [`cmd/cc-backend`](https://github.com/ClusterCockpit/cc-backend/tree/master/cmd/cc-backend) contains `main.go` for the main application.
-- [`configs/`](https://github.com/ClusterCockpit/cc-backend/tree/master/configs) contains documentation about configuration and command line options and required environment variables. A sample configuration file is provided.
-- [`docs/`](https://github.com/ClusterCockpit/cc-backend/tree/master/docs) contains more in-depth documentation.
-- [`init/`](https://github.com/ClusterCockpit/cc-backend/tree/master/init) contains an example of setting up systemd for production use.
-- [`internal/`](https://github.com/ClusterCockpit/cc-backend/tree/master/internal) contains library source code that is not intended for use by others.
-- [`pkg/`](https://github.com/ClusterCockpit/cc-backend/tree/master/pkg) contains Go packages that can be used by other projects.
-- [`tools/`](https://github.com/ClusterCockpit/cc-backend/tree/master/tools) Additional command line helper tools.
-   - [`archive-manager`](https://github.com/ClusterCockpit/cc-backend/tree/master/tools/archive-manager) Commands for getting infos about and existing job archive.
-   - [`archive-migration`](https://github.com/ClusterCockpit/cc-backend/tree/master/tools/archive-migration) Tool to migrate from previous to current job archive version.
-   - [`convert-pem-pubkey`](https://github.com/ClusterCockpit/cc-backend/tree/master/tools/convert-pem-pubkey) Tool to convert external pubkey for use in `cc-backend`.
-   - [`gen-keypair`](https://github.com/ClusterCockpit/cc-backend/tree/master/tools/gen-keypair) contains a small application to generate a compatible JWT keypair. You find documentation on how to use it [here](https://github.com/ClusterCockpit/cc-backend/blob/master/docs/JWT-Handling.md).
-- [`web/`](https://github.com/ClusterCockpit/cc-backend/tree/master/web) Server-side templates and frontend-related files:
-   - [`frontend`](https://github.com/ClusterCockpit/cc-backend/tree/master/web/frontend) Svelte components and static assets for the frontend UI
-   - [`templates`](https://github.com/ClusterCockpit/cc-backend/tree/master/web/templates) Server-side Go templates
-- [`gqlgen.yml`](https://github.com/ClusterCockpit/cc-backend/blob/master/gqlgen.yml) Configures the behaviour and generation of [gqlgen](https://github.com/99designs/gqlgen).
-- [`startDemo.sh`](https://github.com/ClusterCockpit/cc-backend/blob/master/startDemo.sh) is a shell script that sets up demo data, and builds and starts `cc-backend`.
-
+* [`api/`](https://github.com/ClusterCockpit/cc-backend/tree/master/api)
+contains the API schema files for the REST and GraphQL APIs. The REST API is
+documented in the OpenAPI 3.0 format in
+[./api/openapi.yaml](./api/openapi.yaml).
+* [`cmd/cc-backend`](https://github.com/ClusterCockpit/cc-backend/tree/master/cmd/cc-backend)
+contains `main.go` for the main application.
+* [`configs/`](https://github.com/ClusterCockpit/cc-backend/tree/master/configs)
+contains documentation about configuration and command line options and required
+environment variables. A sample configuration file is provided.
+* [`docs/`](https://github.com/ClusterCockpit/cc-backend/tree/master/docs)
+contains more in-depth documentation.
+* [`init/`](https://github.com/ClusterCockpit/cc-backend/tree/master/init)
+contains an example of setting up systemd for production use.
+* [`internal/`](https://github.com/ClusterCockpit/cc-backend/tree/master/internal)
+contains library source code that is not intended for use by others.
+* [`pkg/`](https://github.com/ClusterCockpit/cc-backend/tree/master/pkg)
+contains Go packages that can be used by other projects.
+* [`tools/`](https://github.com/ClusterCockpit/cc-backend/tree/master/tools)
+Additional command line helper tools.
+  * [`archive-manager`](https://github.com/ClusterCockpit/cc-backend/tree/master/tools/archive-manager)
+  Commands for getting infos about and existing job archive.
+  * [`convert-pem-pubkey`](https://github.com/ClusterCockpit/cc-backend/tree/master/tools/convert-pem-pubkey)
+  Tool to convert external pubkey for use in `cc-backend`.
+  * [`gen-keypair`](https://github.com/ClusterCockpit/cc-backend/tree/master/tools/gen-keypair)
+  contains a small application to generate a compatible JWT keypair. You find
+  documentation on how to use it
+  [here](https://github.com/ClusterCockpit/cc-backend/blob/master/docs/JWT-Handling.md).
+* [`web/`](https://github.com/ClusterCockpit/cc-backend/tree/master/web)
+Server-side templates and frontend-related files:
+  * [`frontend`](https://github.com/ClusterCockpit/cc-backend/tree/master/web/frontend)
+  Svelte components and static assets for the frontend UI
+  * [`templates`](https://github.com/ClusterCockpit/cc-backend/tree/master/web/templates)
+  Server-side Go templates
+* [`gqlgen.yml`](https://github.com/ClusterCockpit/cc-backend/blob/master/gqlgen.yml)
+Configures the behaviour and generation of
+[gqlgen](https://github.com/99designs/gqlgen).
+* [`startDemo.sh`](https://github.com/ClusterCockpit/cc-backend/blob/master/startDemo.sh)
+is a shell script that sets up demo data, and builds and starts `cc-backend`.

ReleaseNotes.md

@@ -1,40 +1,47 @@
-# `cc-backend` version 1.2.2
+# `cc-backend` version 1.4.4
 
-Supports job archive version 1 and database version 6.
+Supports job archive version 2 and database version 8.
 
-This is a minor release of `cc-backend`, the API backend and frontend
+This is a bug fix release of `cc-backend`, the API backend and frontend
 implementation of ClusterCockpit.
+For release specific notes visit the [ClusterCockpit Documentation](https://clusterockpit.org/docs/release/).
 
-** Breaking changes **
+## Breaking changes
 
-* The LDAP configuration option `user_filter` was changed and now should not include
-the uid wildcard. Example:
-   - Old: `"user_filter": "(&(objectclass=posixAccount)(uid=*))"`
-   - New: `"user_filter": "(&(objectclass=posixAccount))"`
+The option `apiAllowedIPs` is now a required configuration attribute in
+`config.json`. This option restricts access to the admin API.
 
-* The aggregate job statistic core hours is now computed using the job table
-column `num_hwthreads`. In a future release this column will be renamed to
-`num_cores`. For correct display of core hours `num_hwthreads` must be correctly
-filled on job start. If your existing jobs do not provide the correct value in
-this column then you can set this with one SQL INSERT statement. This only applies
-if you have exclusive jobs, only. Please be aware that we treat this column as
-it is the number of cores. In case you have SMT enabled and `num_hwthreads`
-is not the number of cores the core hours will be too high by a factor!
+To retain the previous behavior that the API is per default accessible from
+everywhere set:
 
-* The jwts key is now mandatory in config.json. It has to set max-age for
-  validity. Some key names have changed, please refer to
-  [config documentation](./configs/README.md) for details.
-
-* The following API endpoints are only accessible from IPs registered using the apiAllowedIPs configuration option:
-   - `/users/` [GET, POST, DELETE]
-   - `/user/{id}` [POST]
-
-** NOTE **
-If you are using the sqlite3 backend the `PRAGMA` option `foreign_keys` must be
-explicitly set to ON. If using the sqlite3 console it is per default set to
-OFF!  On every console session you must set:
+```json
+  "apiAllowedIPs": [
+    "*"
+  ]
 ```
-sqlite> PRAGMA foreign_keys = ON;
 
-```
-Otherwise if you delete jobs the jobtag relation table will not be updated accordingly!
+## Breaking changes for minor release 1.4.x
+
+- You need to perform a database migration. Depending on your database size the
+  migration might require several hours!
+- You need to adapt the `cluster.json` configuration files in the job-archive,
+  add new required attributes to the metric list and after that edit
+  `./job-archive/version.txt` to version 2. Only metrics that have the footprint
+  attribute set can be filtered and show up in the footprint UI and polar plot.
+- Continuous scrolling is default now in all job lists. You can change this back
+  to paging globally, also every user can configure to use paging or continuous
+  scrolling individually.
+- Tags have a scope now. Existing tags will get global scope in the database
+  migration.
+
+## New features
+
+- Enable to delete tags from the web interface
+
+## Known issues
+
+- Currently energy footprint metrics of type energy are ignored for calculating
+  total energy.
+- Resampling for running jobs only works with cc-metric-store
+- With energy footprint metrics of type power the unit is ignored and it is
+  assumed the metric has the unit Watt.

api/schema.graphqls

@@ -4,133 +4,205 @@ scalar Any
 scalar NullableFloat
 scalar MetricScope
 scalar JobState
+scalar NodeState
+scalar MonitoringState
+
+type Node {
+  id: ID!
+  hostname: String!
+  cluster: String!
+  subCluster: String!
+  nodeState: NodeState!
+  HealthState: MonitoringState!
+  metaData: Any
+}
+
+type NodeStats {
+  state: String!
+  count: Int!
+}
 
 type Job {
-  id:               ID!
-  jobId:            Int!
-  user:             String!
-  project:          String!
-  cluster:          String!
-  subCluster:       String!
-  startTime:        Time!
-  duration:         Int!
-  walltime:         Int!
-  numNodes:         Int!
-  numHWThreads:     Int!
-  numAcc:           Int!
-  SMT:              Int!
-  exclusive:        Int!
-  partition:        String!
-  arrayJobId:       Int!
+  id: ID!
+  jobId: Int!
+  user: String!
+  project: String!
+  cluster: String!
+  subCluster: String!
+  startTime: Time!
+  duration: Int!
+  walltime: Int!
+  numNodes: Int!
+  numHWThreads: Int!
+  numAcc: Int!
+  energy: Float!
+  SMT: Int!
+  exclusive: Int!
+  partition: String!
+  arrayJobId: Int!
   monitoringStatus: Int!
-  state:            JobState!
-  tags:             [Tag!]!
-  resources:        [Resource!]!
-  concurrentJobs:   JobLinkResultList
-
-  metaData:         Any
-  userData:         User
+  state: JobState!
+  tags: [Tag!]!
+  resources: [Resource!]!
+  concurrentJobs: JobLinkResultList
+  footprint: [FootprintValue]
+  energyFootprint: [EnergyFootprintValue]
+  metaData: Any
+  userData: User
 }
 
 type JobLink {
-  id:               ID!
-  jobId:            Int!
+  id: ID!
+  jobId: Int!
 }
 
 type Cluster {
-  name:         String!
-  partitions:   [String!]!        # Slurm partitions
-  metricConfig: [MetricConfig!]!
-  subClusters:  [SubCluster!]!    # Hardware partitions/subclusters
+  name: String!
+  partitions: [String!]! # Slurm partitions
+  subClusters: [SubCluster!]! # Hardware partitions/subclusters
 }
 
 type SubCluster {
-  name:            String!
-  nodes:           String!
-  numberOfNodes:   Int!
-  processorType:   String!
-  socketsPerNode:  Int!
-  coresPerSocket:  Int!
-  threadsPerCore:  Int!
-  flopRateScalar:  MetricValue!
-  flopRateSimd:    MetricValue!
+  name: String!
+  nodes: String!
+  numberOfNodes: Int!
+  processorType: String!
+  socketsPerNode: Int!
+  coresPerSocket: Int!
+  threadsPerCore: Int!
+  flopRateScalar: MetricValue!
+  flopRateSimd: MetricValue!
   memoryBandwidth: MetricValue!
-  topology:        Topology!
+  topology: Topology!
+  metricConfig: [MetricConfig!]!
+  footprint: [String!]!
+}
+
+type FootprintValue {
+  name: String!
+  stat: String!
+  value: Float!
+}
+
+type EnergyFootprintValue {
+  hardware: String!
+  metric: String!
+  value: Float!
 }
 
 type MetricValue {
+  name: String
   unit: Unit!
   value: Float!
 }
 
 type Topology {
-  node:         [Int!]
-  socket:       [[Int!]!]
+  node: [Int!]
+  socket: [[Int!]!]
   memoryDomain: [[Int!]!]
-  die:          [[Int!]!]
-  core:         [[Int!]!]
+  die: [[Int!]!]
+  core: [[Int!]!]
   accelerators: [Accelerator!]
 }
 
 type Accelerator {
-  id:    String!
-  type:  String!
+  id: String!
+  type: String!
   model: String!
 }
 
 type SubClusterConfig {
-  name:    String!
-  peak:    Float
-  normal:  Float
+  name: String!
+  peak: Float
+  normal: Float
   caution: Float
-  alert:   Float
-  remove:  Boolean
+  alert: Float
+  remove: Boolean
 }
 
 type MetricConfig {
-  name:        String!
-  unit:        Unit!
-  scope:       MetricScope!
+  name: String!
+  unit: Unit!
+  scope: MetricScope!
   aggregation: String!
-  timestep:    Int!
-  peak:    Float!
-  normal:  Float
+  timestep: Int!
+  peak: Float!
+  normal: Float
   caution: Float!
-  alert:   Float!
+  alert: Float!
+  lowerIsBetter: Boolean
   subClusters: [SubClusterConfig!]!
 }
 
 type Tag {
-  id:   ID!
+  id: ID!
   type: String!
   name: String!
+  scope: String!
 }
 
 type Resource {
-  hostname:      String!
-  hwthreads:     [Int!]
-  accelerators:  [String!]
+  hostname: String!
+  hwthreads: [Int!]
+  accelerators: [String!]
   configuration: String
 }
 
 type JobMetricWithName {
-  name:   String!
-  scope:  MetricScope!
+  name: String!
+  scope: MetricScope!
   metric: JobMetric!
 }
 
 type JobMetric {
-  unit:             Unit
-  timestep:         Int!
-  series:           [Series!]
+  unit: Unit
+  timestep: Int!
+  series: [Series!]
   statisticsSeries: StatsSeries
 }
 
 type Series {
-  hostname:   String!
-  id:         String
+  hostname: String!
+  id: String
   statistics: MetricStatistics
-  data:       [NullableFloat!]!
+  data: [NullableFloat!]!
+}
+
+type StatsSeries {
+  mean: [NullableFloat!]!
+  median: [NullableFloat!]!
+  min: [NullableFloat!]!
+  max: [NullableFloat!]!
+}
+
+type NamedStatsWithScope {
+  name: String!
+  scope: MetricScope!
+  stats: [ScopedStats!]!
+}
+
+type ScopedStats {
+  hostname: String!
+  id: String
+  data: MetricStatistics!
+}
+
+type JobStats {
+  id: Int!
+  jobId: String!
+  startTime: Int!
+  duration: Int!
+  cluster: String!
+  subCluster: String!
+  numNodes: Int!
+  numHWThreads: Int
+  numAccelerators: Int
+  stats: [NamedStats!]!
+}
+
+type NamedStats {
+  name: String!
+  data: MetricStatistics!
 }
 
 type Unit {
@@ -144,20 +216,14 @@ type MetricStatistics {
   max: Float!
 }
 
-type StatsSeries {
-  mean: [NullableFloat!]!
-  min:  [NullableFloat!]!
-  max:  [NullableFloat!]!
-}
-
 type MetricFootprints {
   metric: String!
-  data:   [NullableFloat!]!
+  data: [NullableFloat!]!
 }
 
 type Footprints {
   timeWeights: TimeWeights!
-  metrics:   [MetricFootprints!]!
+  metrics: [MetricFootprints!]!
 }
 
 type TimeWeights {
@@ -166,87 +232,200 @@ type TimeWeights {
   coreHours: [NullableFloat!]!
 }
 
-enum Aggregate { USER, PROJECT, CLUSTER }
-enum SortByAggregate { TOTALWALLTIME, TOTALJOBS, TOTALNODES, TOTALNODEHOURS, TOTALCORES, TOTALCOREHOURS, TOTALACCS, TOTALACCHOURS }
+enum Aggregate {
+  USER
+  PROJECT
+  CLUSTER
+}
+enum SortByAggregate {
+  TOTALWALLTIME
+  TOTALJOBS
+  TOTALNODES
+  TOTALNODEHOURS
+  TOTALCORES
+  TOTALCOREHOURS
+  TOTALACCS
+  TOTALACCHOURS
+}
 
 type NodeMetrics {
-  host:       String!
+  host: String!
   subCluster: String!
-  metrics:    [JobMetricWithName!]!
+  metrics: [JobMetricWithName!]!
+}
+
+type NodesResultList {
+  items: [NodeMetrics!]!
+  offset: Int
+  limit: Int
+  count: Int
+  totalNodes: Int
+  hasNextPage: Boolean
+}
+
+type ClusterSupport {
+  cluster: String!
+  subClusters: [String!]!
+}
+
+type GlobalMetricListItem {
+  name: String!
+  unit: Unit!
+  scope: MetricScope!
+  footprint: String
+  availability: [ClusterSupport!]!
 }
 
 type Count {
-  name:  String!
+  name: String!
   count: Int!
 }
 
 type User {
   username: String!
-  name:     String!
-  email:    String!
+  name: String!
+  email: String!
+}
+
+input MetricStatItem {
+  metricName: String!
+  range: FloatRange!
 }
 
 type Query {
-  clusters:     [Cluster!]!   # List of all clusters
-  tags:         [Tag!]!       # List of all tags
+  clusters: [Cluster!]! # List of all clusters
+  tags: [Tag!]! # List of all tags
+  globalMetrics: [GlobalMetricListItem!]!
 
   user(username: String!): User
   allocatedNodes(cluster: String!): [Count!]!
 
+  node(id: ID!): Node
+  nodes(filter: [NodeFilter!], order: OrderByInput): NodeStateResultList!
+  nodeStats(filter: [NodeFilter!]): [NodeStats!]!
+
   job(id: ID!): Job
-  jobMetrics(id: ID!, metrics: [String!], scopes: [MetricScope!]): [JobMetricWithName!]!
+  jobMetrics(
+    id: ID!
+    metrics: [String!]
+    scopes: [MetricScope!]
+    resolution: Int
+  ): [JobMetricWithName!]!
+
+  jobStats(id: ID!, metrics: [String!]): [NamedStats!]!
+
+  scopedJobStats(
+    id: ID!
+    metrics: [String!]
+    scopes: [MetricScope!]
+  ): [NamedStatsWithScope!]!
+
+  jobs(
+    filter: [JobFilter!]
+    page: PageRequest
+    order: OrderByInput
+  ): JobResultList!
+
+  jobsStatistics(
+    filter: [JobFilter!]
+    metrics: [String!]
+    page: PageRequest
+    sortBy: SortByAggregate
+    groupBy: Aggregate
+    numDurationBins: String
+    numMetricBins: Int
+  ): [JobsStatistics!]!
+
+  jobsMetricStats(filter: [JobFilter!], metrics: [String!]): [JobStats!]!
   jobsFootprints(filter: [JobFilter!], metrics: [String!]!): Footprints
 
-  jobs(filter: [JobFilter!], page: PageRequest, order: OrderByInput): JobResultList!
-  jobsStatistics(filter: [JobFilter!], page: PageRequest, sortBy: SortByAggregate, groupBy: Aggregate): [JobsStatistics!]!
+  rooflineHeatmap(
+    filter: [JobFilter!]!
+    rows: Int!
+    cols: Int!
+    minX: Float!
+    minY: Float!
+    maxX: Float!
+    maxY: Float!
+  ): [[Float!]!]!
 
-  rooflineHeatmap(filter: [JobFilter!]!, rows: Int!, cols: Int!, minX: Float!, minY: Float!, maxX: Float!, maxY: Float!): [[Float!]!]!
-
-  nodeMetrics(cluster: String!, nodes: [String!], scopes: [MetricScope!], metrics: [String!], from: Time!, to: Time!): [NodeMetrics!]!
+  nodeMetrics(
+    cluster: String!
+    nodes: [String!]
+    scopes: [MetricScope!]
+    metrics: [String!]
+    from: Time!
+    to: Time!
+  ): [NodeMetrics!]!
+  nodeMetricsList(
+    cluster: String!
+    subCluster: String!
+    nodeFilter: String!
+    scopes: [MetricScope!]
+    metrics: [String!]
+    from: Time!
+    to: Time!
+    page: PageRequest
+    resolution: Int
+  ): NodesResultList!
 }
 
 type Mutation {
-  createTag(type: String!, name: String!): Tag!
+  createTag(type: String!, name: String!, scope: String!): Tag!
   deleteTag(id: ID!): ID!
   addTagsToJob(job: ID!, tagIds: [ID!]!): [Tag!]!
   removeTagsFromJob(job: ID!, tagIds: [ID!]!): [Tag!]!
+  removeTagFromList(tagIds: [ID!]!): [Int!]!
 
   updateConfiguration(name: String!, value: String!): String
 }
 
-type IntRangeOutput { from: Int!, to: Int! }
-type TimeRangeOutput { from: Time!, to: Time! }
+type IntRangeOutput {
+  from: Int!
+  to: Int!
+}
+type TimeRangeOutput {
+  range: String
+  from: Time!
+  to: Time!
+}
+
+input NodeFilter {
+  hostname: StringInput
+  cluster: StringInput
+  nodeState: NodeState
+  healthState: MonitoringState
+}
 
 input JobFilter {
-  tags:        [ID!]
-  jobId:       StringInput
-  arrayJobId:  Int
-  user:        StringInput
-  project:     StringInput
-  jobName:     StringInput
-  cluster:     StringInput
-  partition:   StringInput
-  duration:    IntRange
+  tags: [ID!]
+  dbId: [ID!]
+  jobId: StringInput
+  arrayJobId: Int
+  user: StringInput
+  project: StringInput
+  jobName: StringInput
+  cluster: StringInput
+  partition: StringInput
+  duration: IntRange
+  energy: FloatRange
 
   minRunningFor: Int
 
-  numNodes:        IntRange
+  numNodes: IntRange
   numAccelerators: IntRange
-  numHWThreads:    IntRange
+  numHWThreads: IntRange
 
-  startTime:   TimeRange
-  state:       [JobState!]
-  flopsAnyAvg: FloatRange
-  memBwAvg:    FloatRange
-  loadAvg:     FloatRange
-  memUsedMax:  FloatRange
-
-  exclusive:     Int
-  node:    StringInput
+  startTime: TimeRange
+  state: [JobState!]
+  metricStats: [MetricStatItem!]
+  exclusive: Int
+  node: StringInput
 }
 
 input OrderByInput {
   field: String!
+  type: String!
   order: SortDirectionEnum! = ASC
 }
 
@@ -256,29 +435,46 @@ enum SortDirectionEnum {
 }
 
 input StringInput {
-  eq:         String
-  neq:        String
-  contains:   String
+  eq: String
+  neq: String
+  contains: String
   startsWith: String
-  endsWith:   String
-  in:         [String!]
+  endsWith: String
+  in: [String!]
 }
 
-input IntRange   { from: Int!,   to: Int! }
-input FloatRange { from: Float!, to: Float! }
-input TimeRange  { from: Time,   to: Time }
+input IntRange {
+  from: Int!
+  to: Int!
+}
+input TimeRange {
+  range: String
+  from: Time
+  to: Time
+}
+
+input FloatRange {
+  from: Float!
+  to: Float!
+}
+
+type NodeStateResultList {
+  items: [Node!]!
+  count: Int
+}
 
 type JobResultList {
-  items:  [Job!]!
+  items: [Job!]!
   offset: Int
-  limit:  Int
-  count:  Int
+  limit: Int
+  count: Int
+  hasNextPage: Boolean
 }
 
 type JobLinkResultList {
   listQuery: String
-  items:  [JobLink!]!
-  count:  Int
+  items: [JobLink!]!
+  count: Int
 }
 
 type HistoPoint {
@@ -286,26 +482,41 @@ type HistoPoint {
   value: Int!
 }
 
-type JobsStatistics  {
-  id:             ID!            # If `groupBy` was used, ID of the user/project/cluster
-  name:           String!        # if User-Statistics: Given Name of Account (ID) Owner
-  totalJobs:      Int!           # Number of jobs
-  runningJobs:    Int!           # Number of running jobs
-  shortJobs:      Int!           # Number of jobs with a duration of less than duration
-  totalWalltime:  Int!           # Sum of the duration of all matched jobs in hours
-  totalNodes:     Int!           # Sum of the nodes of all matched jobs
-  totalNodeHours: Int!           # Sum of the node hours of all matched jobs
-  totalCores:     Int!           # Sum of the cores of all matched jobs
-  totalCoreHours: Int!           # Sum of the core hours of all matched jobs
-  totalAccs:      Int!         # Sum of the accs of all matched jobs
-  totalAccHours:  Int!           # Sum of the gpu hours of all matched jobs
-  histDuration:   [HistoPoint!]! # value: hour, count: number of jobs with a rounded duration of value
-  histNumNodes:   [HistoPoint!]! # value: number of nodes, count: number of jobs with that number of nodes
-  histNumCores:   [HistoPoint!]! # value: number of cores, count: number of jobs with that number of cores
-  histNumAccs:    [HistoPoint!]! # value: number of accs, count: number of jobs with that number of accs
+type MetricHistoPoints {
+  metric: String!
+  unit: String!
+  stat: String
+  data: [MetricHistoPoint!]
+}
+
+type MetricHistoPoint {
+  bin: Int
+  count: Int!
+  min: Int
+  max: Int
+}
+
+type JobsStatistics {
+  id: ID! # If `groupBy` was used, ID of the user/project/cluster
+  name: String! # if User-Statistics: Given Name of Account (ID) Owner
+  totalJobs: Int! # Number of jobs
+  runningJobs: Int! # Number of running jobs
+  shortJobs: Int! # Number of jobs with a duration of less than duration
+  totalWalltime: Int! # Sum of the duration of all matched jobs in hours
+  totalNodes: Int! # Sum of the nodes of all matched jobs
+  totalNodeHours: Int! # Sum of the node hours of all matched jobs
+  totalCores: Int! # Sum of the cores of all matched jobs
+  totalCoreHours: Int! # Sum of the core hours of all matched jobs
+  totalAccs: Int! # Sum of the accs of all matched jobs
+  totalAccHours: Int! # Sum of the gpu hours of all matched jobs
+  histDuration: [HistoPoint!]! # value: hour, count: number of jobs with a rounded duration of value
+  histNumNodes: [HistoPoint!]! # value: number of nodes, count: number of jobs with that number of nodes
+  histNumCores: [HistoPoint!]! # value: number of cores, count: number of jobs with that number of cores
+  histNumAccs: [HistoPoint!]! # value: number of accs, count: number of jobs with that number of accs
+  histMetrics: [MetricHistoPoints!]! # metric: metricname, data array of histopoints: value: metric average bin, count: number of jobs with that metric average
 }
 
 input PageRequest {
   itemsPerPage: Int!
-  page:         Int!
+  page: Int!
 }

build/package/cc-backend.config

@@ -1,17 +0,0 @@
-CC_USER=clustercockpit
-
-CC_GROUP=clustercockpit
-
-CC_HOME=/tmp
-
-LOG_DIR=/var/log
-
-DATA_DIR=/var/run/cc-backend
-
-MAX_OPEN_FILES=10000
-
-CONF_DIR=/etc/cc-backend
-
-CONF_FILE=/etc/cc-backend/cc-backend.json
-
-RESTART_ON_UPGRADE=true

build/package/cc-backend.deb.control

@@ -1,12 +0,0 @@
-Package: cc-backend
-Version: {VERSION}
-Installed-Size: {INSTALLED_SIZE}
-Architecture: {ARCH}
-Maintainer: thomas.gruber@fau.de
-Depends: libc6 (>= 2.2.1)
-Build-Depends: debhelper-compat (= 13), git, golang-go, npm, yarn
-Description: ClusterCockpit backend and web frontend
-Homepage: https://github.com/ClusterCockpit/cc-backend
-Source: cc-backend
-Rules-Requires-Root: no
-

build/package/cc-backend.service

@@ -1,18 +0,0 @@
-[Unit]
-Description=ClusterCockpit backend and web frontend (cc-backend)
-Documentation=https://github.com/ClusterCockpit/cc-backend
-Wants=network-online.target
-After=network-online.target
-
-[Service]
-EnvironmentFile=/etc/default/cc-backend
-Type=simple
-User=clustercockpit
-Group=clustercockpit
-Restart=on-failure
-TimeoutStopSec=100
-LimitNOFILE=infinity
-ExecStart=/usr/bin/cc-backend --config ${CONF_FILE}
-
-[Install]
-WantedBy=multi-user.target

build/package/cc-backend.spec

@@ -1,70 +0,0 @@
-Name:           cc-backend
-Version:        %{VERS}
-Release:        1%{?dist}
-Summary:        ClusterCockpit backend and web frontend
-
-License:        MIT
-Source0:        %{name}-%{version}.tar.gz
-
-#BuildRequires:  go-toolset
-#BuildRequires:  systemd-rpm-macros
-#BuildRequires:  npm
-
-Provides:       %{name} = %{version}
-
-%description
-ClusterCockpit backend and web frontend
-
-%global debug_package %{nil}
-
-%prep
-%autosetup
-
-
-%build
-#CURRENT_TIME=$(date +%Y-%m-%d:T%H:%M:\%S)
-#LD_FLAGS="-s -X main.buildTime=${CURRENT_TIME} -X main.version=%{VERS}"
-mkdir ./var
-touch ./var/job.db
-cd web/frontend && yarn install && yarn build && cd -
-go build -ldflags="-s -X main.version=%{VERS}" ./cmd/cc-backend
-
-
-%install
-# Install cc-backend
-#make PREFIX=%{buildroot} install
-install -Dpm 755 cc-backend %{buildroot}/%{_bindir}/%{name}
-install -Dpm 0600 configs/config.json %{buildroot}%{_sysconfdir}/%{name}/%{name}.json
-# Integrate into system
-install -Dpm 0644 build/package/%{name}.service %{buildroot}%{_unitdir}/%{name}.service
-install -Dpm 0600 build/package/%{name}.config %{buildroot}%{_sysconfdir}/default/%{name}
-install -Dpm 0644 build/package/%{name}.sysusers %{buildroot}%{_sysusersdir}/%{name}.conf
-
-
-%check
-# go test should be here... :)
-
-%pre
-%sysusers_create_package scripts/%{name}.sysusers
-
-%post
-%systemd_post %{name}.service
-
-%preun
-%systemd_preun %{name}.service
-
-%files
-# Binary
-%attr(-,clustercockpit,clustercockpit) %{_bindir}/%{name}
-# Config
-%dir %{_sysconfdir}/%{name}
-%attr(0600,clustercockpit,clustercockpit) %config(noreplace) %{_sysconfdir}/%{name}/%{name}.json
-# Systemd
-%{_unitdir}/%{name}.service
-%{_sysconfdir}/default/%{name}
-%{_sysusersdir}/%{name}.conf
-
-%changelog
-* Mon Mar 07 2022 Thomas Gruber - 0.1
-- Initial metric store implementation
-

build/package/cc-backend.sysusers

@@ -1,2 +0,0 @@
-#Type Name           ID GECOS                     Home directory           Shell
-u     clustercockpit -  "User for ClusterCockpit" /run/cc-backend /sbin/nologin

cmd/cc-backend/cli.go

@@ -0,0 +1,35 @@
+// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
+// All rights reserved. This file is part of cc-backend.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+package main
+
+import "flag"
+
+var (
+	flagReinitDB, flagInit, flagServer, flagSyncLDAP, flagGops, flagMigrateDB, flagRevertDB,
+	flagForceDB, flagDev, flagVersion, flagLogDateTime, flagApplyTags bool
+	flagNewUser, flagDelUser, flagGenJWT, flagConfigFile, flagImportJob, flagLogLevel string
+)
+
+func cliInit() {
+	flag.BoolVar(&flagInit, "init", false, "Setup var directory, initialize sqlite database file, config.json and .env")
+	flag.BoolVar(&flagReinitDB, "init-db", false, "Go through job-archive and re-initialize the 'job', 'tag', and 'jobtag' tables (all running jobs will be lost!)")
+	flag.BoolVar(&flagSyncLDAP, "sync-ldap", false, "Sync the 'hpc_user' table with ldap")
+	flag.BoolVar(&flagServer, "server", false, "Start a server, continues listening on port after initialization and argument handling")
+	flag.BoolVar(&flagGops, "gops", false, "Listen via github.com/google/gops/agent (for debugging)")
+	flag.BoolVar(&flagDev, "dev", false, "Enable development components: GraphQL Playground and Swagger UI")
+	flag.BoolVar(&flagVersion, "version", false, "Show version information and exit")
+	flag.BoolVar(&flagMigrateDB, "migrate-db", false, "Migrate database to supported version and exit")
+	flag.BoolVar(&flagRevertDB, "revert-db", false, "Migrate database to previous version and exit")
+	flag.BoolVar(&flagApplyTags, "apply-tags", false, "Run taggers on all completed jobs and exit")
+	flag.BoolVar(&flagForceDB, "force-db", false, "Force database version, clear dirty flag and exit")
+	flag.BoolVar(&flagLogDateTime, "logdate", false, "Set this flag to add date and time to log messages")
+	flag.StringVar(&flagConfigFile, "config", "./config.json", "Specify alternative path to `config.json`")
+	flag.StringVar(&flagNewUser, "add-user", "", "Add a new user. Argument format: <username>:[admin,support,manager,api,user]:<password>")
+	flag.StringVar(&flagDelUser, "del-user", "", "Remove a existing user. Argument format: <username>")
+	flag.StringVar(&flagGenJWT, "jwt", "", "Generate and print a JWT for the user specified by its `username`")
+	flag.StringVar(&flagImportJob, "import-job", "", "Import a job. Argument format: `<path-to-meta.json>:<path-to-data.json>,...`")
+	flag.StringVar(&flagLogLevel, "loglevel", "warn", "Sets the logging level: `[debug, info (default), warn, err, crit]`")
+	flag.Parse()
+}

cmd/cc-backend/init.go

@@ -0,0 +1,95 @@
+// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
+// All rights reserved. This file is part of cc-backend.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+package main
+
+import (
+	"os"
+
+	"github.com/ClusterCockpit/cc-backend/internal/repository"
+	cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
+	"github.com/ClusterCockpit/cc-lib/util"
+)
+
+const envString = `
+# Base64 encoded Ed25519 keys (DO NOT USE THESE TWO IN PRODUCTION!)
+# You can generate your own keypair using the gen-keypair tool
+JWT_PUBLIC_KEY="kzfYrYy+TzpanWZHJ5qSdMj5uKUWgq74BWhQG6copP0="
+JWT_PRIVATE_KEY="dtPC/6dWJFKZK7KZ78CvWuynylOmjBFyMsUWArwmodOTN9itjL5POlqdZkcnmpJ0yPm4pRaCrvgFaFAbpyik/Q=="
+
+# Some random bytes used as secret for cookie-based sessions (DO NOT USE THIS ONE IN PRODUCTION)
+SESSION_KEY="67d829bf61dc5f87a73fd814e2c9f629"
+`
+
+const configString = `
+{
+    "addr": "127.0.0.1:8080",
+    "archive": {
+        "kind": "file",
+        "path": "./var/job-archive"
+    },
+    "jwts": {
+        "max-age": "2000h"
+    },
+    "apiAllowedIPs": [
+      "*"
+    ],
+    "enable-resampling": {
+      "trigger": 30,
+      "resolutions": [
+        600,
+        300,
+        120,
+        60
+      ]
+    },
+    "clusters": [
+        {
+            "name": "name",
+            "metricDataRepository": {
+                "kind": "cc-metric-store",
+                "url": "http://localhost:8082",
+                "token": ""
+            },
+            "filterRanges": {
+                "numNodes": {
+                    "from": 1,
+                    "to": 64
+                },
+                "duration": {
+                    "from": 0,
+                    "to": 86400
+                },
+                "startTime": {
+                    "from": "2023-01-01T00:00:00Z",
+                    "to": null
+                }
+            }
+        }
+    ]
+}
+`
+
+func initEnv() {
+	if util.CheckFileExists("var") {
+		cclog.Exit("Directory ./var already exists. Cautiously exiting application initialization.")
+	}
+
+	if err := os.WriteFile("config.json", []byte(configString), 0o666); err != nil {
+		cclog.Abortf("Could not write default ./config.json with permissions '0o666'. Application initialization failed, exited.\nError: %s\n", err.Error())
+	}
+
+	if err := os.WriteFile(".env", []byte(envString), 0o666); err != nil {
+		cclog.Abortf("Could not write default ./.env file with permissions '0o666'. Application initialization failed, exited.\nError: %s\n", err.Error())
+	}
+
+	if err := os.Mkdir("var", 0o777); err != nil {
+		cclog.Abortf("Could not create default ./var folder with permissions '0o777'. Application initialization failed, exited.\nError: %s\n", err.Error())
+	}
+
+	err := repository.MigrateDB("sqlite3", "./var/job.db")
+	if err != nil {
+		cclog.Abortf("Could not initialize default sqlite3 database as './var/job.db'. Application initialization failed, exited.\nError: %s\n", err.Error())
+	}
+}

cmd/cc-backend/main.go

@@ -1,157 +1,55 @@
-// Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved.
+// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
+// All rights reserved. This file is part of cc-backend.
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 package main
 
 import (
-	"context"
-	"crypto/tls"
-	"encoding/json"
-	"errors"
-	"flag"
 	"fmt"
-	"io"
-	"net"
-	"net/http"
 	"os"
 	"os/signal"
-	"runtime"
 	"runtime/debug"
 	"strings"
 	"sync"
 	"syscall"
-	"time"
 
-	"github.com/99designs/gqlgen/graphql/handler"
-	"github.com/99designs/gqlgen/graphql/playground"
-	"github.com/ClusterCockpit/cc-backend/internal/api"
+	"github.com/ClusterCockpit/cc-backend/internal/archiver"
 	"github.com/ClusterCockpit/cc-backend/internal/auth"
 	"github.com/ClusterCockpit/cc-backend/internal/config"
-	"github.com/ClusterCockpit/cc-backend/internal/graph"
-	"github.com/ClusterCockpit/cc-backend/internal/graph/generated"
 	"github.com/ClusterCockpit/cc-backend/internal/importer"
 	"github.com/ClusterCockpit/cc-backend/internal/metricdata"
 	"github.com/ClusterCockpit/cc-backend/internal/repository"
-	"github.com/ClusterCockpit/cc-backend/internal/routerConfig"
-	"github.com/ClusterCockpit/cc-backend/internal/runtimeEnv"
-	"github.com/ClusterCockpit/cc-backend/internal/util"
+	"github.com/ClusterCockpit/cc-backend/internal/tagger"
+	"github.com/ClusterCockpit/cc-backend/internal/taskManager"
 	"github.com/ClusterCockpit/cc-backend/pkg/archive"
-	"github.com/ClusterCockpit/cc-backend/pkg/log"
-	"github.com/ClusterCockpit/cc-backend/pkg/schema"
-	"github.com/ClusterCockpit/cc-backend/web"
-	"github.com/go-co-op/gocron"
+	"github.com/ClusterCockpit/cc-backend/pkg/runtimeEnv"
+	cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
+	"github.com/ClusterCockpit/cc-lib/schema"
+	"github.com/ClusterCockpit/cc-lib/util"
 	"github.com/google/gops/agent"
-	"github.com/gorilla/handlers"
-	"github.com/gorilla/mux"
-	httpSwagger "github.com/swaggo/http-swagger"
+	"github.com/joho/godotenv"
 
 	_ "github.com/go-sql-driver/mysql"
 	_ "github.com/mattn/go-sqlite3"
 )
 
 const logoString = `
- ____ _           _             ____           _          _ _
-/ ___| |_   _ ___| |_ ___ _ __ / ___|___   ___| | ___ __ (_) |_
+ _____ _           _             ____           _          _ _
+/  ___| |_   _ ___| |_ ___ _ __ / ___|___   ___| | ___ __ (_) |_
 | |   | | | | / __| __/ _ \ '__| |   / _ \ / __| |/ / '_ \| | __|
 | |___| | |_| \__ \ ||  __/ |  | |__| (_) | (__|   <| |_) | | |_
-\____|_|\__,_|___/\__\___|_|   \____\___/ \___|_|\_\ .__/|_|\__|
+\_____|_|\__,_|___/\__\___|_|   \____\___/ \___|_|\_\ .__/|_|\__|
                                                     |_|
 `
 
-const envString = `
-# Base64 encoded Ed25519 keys (DO NOT USE THESE TWO IN PRODUCTION!)
-# You can generate your own keypair using the gen-keypair tool
-JWT_PUBLIC_KEY="kzfYrYy+TzpanWZHJ5qSdMj5uKUWgq74BWhQG6copP0="
-JWT_PRIVATE_KEY="dtPC/6dWJFKZK7KZ78CvWuynylOmjBFyMsUWArwmodOTN9itjL5POlqdZkcnmpJ0yPm4pRaCrvgFaFAbpyik/Q=="
-
-# Some random bytes used as secret for cookie-based sessions (DO NOT USE THIS ONE IN PRODUCTION)
-SESSION_KEY="67d829bf61dc5f87a73fd814e2c9f629"
-`
-
-const configString = `
-{
-    "addr": "127.0.0.1:8080",
-    "archive": {
-        "kind": "file",
-        "path": "./var/job-archive"
-    },
-    "clusters": [
-        {
-            "name": "name",
-            "metricDataRepository": {
-                "kind": "cc-metric-store",
-                "url": "http://localhost:8082",
-                "token": ""
-            },
-            "filterRanges": {
-                "numNodes": {
-                    "from": 1,
-                    "to": 64
-                },
-                "duration": {
-                    "from": 0,
-                    "to": 86400
-                },
-                "startTime": {
-                    "from": "2023-01-01T00:00:00Z",
-                    "to": null
-                }
-            }
-        }
-    ]
-}
-`
-
 var (
 	date    string
 	commit  string
 	version string
 )
 
-func initEnv() {
-	if util.CheckFileExists("var") {
-		fmt.Print("Directory ./var already exists. Exiting!\n")
-		os.Exit(0)
-	}
-
-	if err := os.WriteFile("config.json", []byte(configString), 0666); err != nil {
-		log.Fatalf("Writing config.json failed: %s", err.Error())
-	}
-
-	if err := os.WriteFile(".env", []byte(envString), 0666); err != nil {
-		log.Fatalf("Writing .env failed: %s", err.Error())
-	}
-
-	if err := os.Mkdir("var", 0777); err != nil {
-		log.Fatalf("Mkdir var failed: %s", err.Error())
-	}
-
-	err := repository.MigrateDB("sqlite3", "./var/job.db")
-	if err != nil {
-		log.Fatalf("Initialize job.db failed: %s", err.Error())
-	}
-}
-
 func main() {
-	var flagReinitDB, flagInit, flagServer, flagSyncLDAP, flagGops, flagMigrateDB, flagDev, flagVersion, flagLogDateTime bool
-	var flagNewUser, flagDelUser, flagGenJWT, flagConfigFile, flagImportJob, flagLogLevel string
-	flag.BoolVar(&flagInit, "init", false, "Setup var directory, initialize swlite database file, config.json and .env")
-	flag.BoolVar(&flagReinitDB, "init-db", false, "Go through job-archive and re-initialize the 'job', 'tag', and 'jobtag' tables (all running jobs will be lost!)")
-	flag.BoolVar(&flagSyncLDAP, "sync-ldap", false, "Sync the 'user' table with ldap")
-	flag.BoolVar(&flagServer, "server", false, "Start a server, continues listening on port after initialization and argument handling")
-	flag.BoolVar(&flagGops, "gops", false, "Listen via github.com/google/gops/agent (for debugging)")
-	flag.BoolVar(&flagDev, "dev", false, "Enable development components: GraphQL Playground and Swagger UI")
-	flag.BoolVar(&flagVersion, "version", false, "Show version information and exit")
-	flag.BoolVar(&flagMigrateDB, "migrate-db", false, "Migrate database to supported version and exit")
-	flag.BoolVar(&flagLogDateTime, "logdate", false, "Set this flag to add date and time to log messages")
-	flag.StringVar(&flagConfigFile, "config", "./config.json", "Specify alternative path to `config.json`")
-	flag.StringVar(&flagNewUser, "add-user", "", "Add a new user. Argument format: `<username>:[admin,support,manager,api,user]:<password>`")
-	flag.StringVar(&flagDelUser, "del-user", "", "Remove user by `username`")
-	flag.StringVar(&flagGenJWT, "jwt", "", "Generate and print a JWT for the user specified by its `username`")
-	flag.StringVar(&flagImportJob, "import-job", "", "Import a job. Argument format: `<path-to-meta.json>:<path-to-data.json>,...`")
-	flag.StringVar(&flagLogLevel, "loglevel", "warn", "Sets the logging level: `[debug,info,warn (default),err,fatal,crit]`")
-	flag.Parse()
+	cliInit()
 
 	if flagVersion {
 		fmt.Print(logoString)
@@ -163,26 +61,26 @@ func main() {
 		os.Exit(0)
 	}
 
-	// Apply config flags for pkg/log
-	log.Init(flagLogLevel, flagLogDateTime)
+	cclog.Init(flagLogLevel, flagLogDateTime)
 
+	// If init flag set, run tasks here before any file dependencies cause errors
 	if flagInit {
 		initEnv()
-		fmt.Print("Succesfully setup environment!\n")
-		fmt.Print("Please review config.json and .env and adjust it to your needs.\n")
-		fmt.Print("Add your job-archive at ./var/job-archive.\n")
-		os.Exit(0)
+		cclog.Exit("Successfully setup environment!\n" +
+			"Please review config.json and .env and adjust it to your needs.\n" +
+			"Add your job-archive at ./var/job-archive.")
 	}
 
 	// See https://github.com/google/gops (Runtime overhead is almost zero)
 	if flagGops {
 		if err := agent.Listen(agent.Options{}); err != nil {
-			log.Fatalf("gops/agent.Listen failed: %s", err.Error())
+			cclog.Abortf("Could not start gops agent with 'gops/agent.Listen(agent.Options{})'. Application startup failed, exited.\nError: %s\n", err.Error())
 		}
 	}
 
-	if err := runtimeEnv.LoadEnv("./.env"); err != nil && !os.IsNotExist(err) {
-		log.Fatalf("parsing './.env' file failed: %s", err.Error())
+	err := godotenv.Load()
+	if err != nil {
+		cclog.Abortf("Could not parse existing .env file at location './.env'. Application startup failed, exited.\nError: %s\n", err.Error())
 	}
 
 	// Initialize sub-modules and handle command line flags.
@@ -200,310 +98,145 @@ func main() {
 	if flagMigrateDB {
 		err := repository.MigrateDB(config.Keys.DBDriver, config.Keys.DB)
 		if err != nil {
-			log.Fatal(err)
+			cclog.Abortf("MigrateDB Failed: Could not migrate '%s' database at location '%s' to version %d.\nError: %s\n", config.Keys.DBDriver, config.Keys.DB, repository.Version, err.Error())
 		}
-		os.Exit(0)
+		cclog.Exitf("MigrateDB Success: Migrated '%s' database at location '%s' to version %d.\n", config.Keys.DBDriver, config.Keys.DB, repository.Version)
+	}
+
+	if flagRevertDB {
+		err := repository.RevertDB(config.Keys.DBDriver, config.Keys.DB)
+		if err != nil {
+			cclog.Abortf("RevertDB Failed: Could not revert '%s' database at location '%s' to version %d.\nError: %s\n", config.Keys.DBDriver, config.Keys.DB, (repository.Version - 1), err.Error())
+		}
+		cclog.Exitf("RevertDB Success: Reverted '%s' database at location '%s' to version %d.\n", config.Keys.DBDriver, config.Keys.DB, (repository.Version - 1))
+	}
+
+	if flagForceDB {
+		err := repository.ForceDB(config.Keys.DBDriver, config.Keys.DB)
+		if err != nil {
+			cclog.Abortf("ForceDB Failed: Could not force '%s' database at location '%s' to version %d.\nError: %s\n", config.Keys.DBDriver, config.Keys.DB, repository.Version, err.Error())
+		}
+		cclog.Exitf("ForceDB Success: Forced '%s' database at location '%s' to version %d.\n", config.Keys.DBDriver, config.Keys.DB, repository.Version)
 	}
 
 	repository.Connect(config.Keys.DBDriver, config.Keys.DB)
-	db := repository.GetConnection()
 
-	var authentication *auth.Authentication
 	if !config.Keys.DisableAuthentication {
-		var err error
-		if authentication, err = auth.Init(); err != nil {
-			log.Fatalf("auth initialization failed: %v", err)
-		}
 
-		if d, err := time.ParseDuration(config.Keys.SessionMaxAge); err != nil {
-			authentication.SessionMaxAge = d
-		}
+		auth.Init()
 
 		if flagNewUser != "" {
 			parts := strings.SplitN(flagNewUser, ":", 3)
 			if len(parts) != 3 || len(parts[0]) == 0 {
-				log.Fatal("invalid argument format for user creation")
+				cclog.Abortf("Add User: Could not parse supplied argument format: No changes.\n"+
+					"Want: <username>:[admin,support,manager,api,user]:<password>\n"+
+					"Have: %s\n", flagNewUser)
 			}
 
 			ur := repository.GetUserRepository()
 			if err := ur.AddUser(&schema.User{
 				Username: parts[0], Projects: make([]string, 0), Password: parts[2], Roles: strings.Split(parts[1], ","),
 			}); err != nil {
-				log.Fatalf("adding '%s' user authentication failed: %v", parts[0], err)
+				cclog.Abortf("Add User: Could not add new user authentication for '%s' and roles '%s'.\nError: %s\n", parts[0], parts[1], err.Error())
+			} else {
+				cclog.Printf("Add User: Added new user '%s' with roles '%s'.\n", parts[0], parts[1])
 			}
 		}
+
 		if flagDelUser != "" {
 			ur := repository.GetUserRepository()
 			if err := ur.DelUser(flagDelUser); err != nil {
-				log.Fatalf("deleting user failed: %v", err)
+				cclog.Abortf("Delete User: Could not delete user '%s' from DB.\nError: %s\n", flagDelUser, err.Error())
+			} else {
+				cclog.Printf("Delete User: Deleted user '%s' from DB.\n", flagDelUser)
 			}
 		}
 
+		authHandle := auth.GetAuthInstance()
+
 		if flagSyncLDAP {
-			if authentication.LdapAuth == nil {
-				log.Fatal("cannot sync: LDAP authentication is not configured")
+			if authHandle.LdapAuth == nil {
+				cclog.Abort("Sync LDAP: LDAP authentication is not configured, could not synchronize. No changes, exited.")
 			}
 
-			if err := authentication.LdapAuth.Sync(); err != nil {
-				log.Fatalf("LDAP sync failed: %v", err)
+			if err := authHandle.LdapAuth.Sync(); err != nil {
+				cclog.Abortf("Sync LDAP: Could not synchronize, failed with error.\nError: %s\n", err.Error())
 			}
-			log.Info("LDAP sync successfull")
+			cclog.Print("Sync LDAP: LDAP synchronization successfull.")
 		}
 
 		if flagGenJWT != "" {
 			ur := repository.GetUserRepository()
 			user, err := ur.GetUser(flagGenJWT)
 			if err != nil {
-				log.Fatalf("could not get user from JWT: %v", err)
+				cclog.Abortf("JWT: Could not get supplied user '%s' from DB. No changes, exited.\nError: %s\n", flagGenJWT, err.Error())
 			}
 
 			if !user.HasRole(schema.RoleApi) {
-				log.Warnf("user '%s' does not have the API role", user.Username)
+				cclog.Warnf("JWT: User '%s' does not have the role 'api'. REST API endpoints will return error!\n", user.Username)
 			}
 
-			jwt, err := authentication.JwtAuth.ProvideJWT(user)
+			jwt, err := authHandle.JwtAuth.ProvideJWT(user)
 			if err != nil {
-				log.Fatalf("failed to provide JWT to user '%s': %v", user.Username, err)
+				cclog.Abortf("JWT: User '%s' found in DB, but failed to provide JWT.\nError: %s\n", user.Username, err.Error())
 			}
 
-			fmt.Printf("MAIN > JWT for '%s': %s\n", user.Username, jwt)
+			cclog.Printf("JWT: Successfully generated JWT for user '%s': %s\n", user.Username, jwt)
 		}
+
 	} else if flagNewUser != "" || flagDelUser != "" {
-		log.Fatal("arguments --add-user and --del-user can only be used if authentication is enabled")
+		cclog.Abort("Error: Arguments '--add-user' and '--del-user' can only be used if authentication is enabled. No changes, exited.")
 	}
 
 	if err := archive.Init(config.Keys.Archive, config.Keys.DisableArchive); err != nil {
-		log.Fatalf("failed to initialize archive: %s", err.Error())
+		cclog.Abortf("Init: Failed to initialize archive.\nError: %s\n", err.Error())
 	}
 
-	if err := metricdata.Init(config.Keys.DisableArchive); err != nil {
-		log.Fatalf("failed to initialize metricdata repository: %s", err.Error())
+	if err := metricdata.Init(); err != nil {
+		cclog.Abortf("Init: Failed to initialize metricdata repository.\nError %s\n", err.Error())
 	}
 
 	if flagReinitDB {
 		if err := importer.InitDB(); err != nil {
-			log.Fatalf("failed to re-initialize repository DB: %s", err.Error())
+			cclog.Abortf("Init DB: Failed to re-initialize repository DB.\nError: %s\n", err.Error())
+		} else {
+			cclog.Print("Init DB: Sucessfully re-initialized repository DB.")
 		}
 	}
 
 	if flagImportJob != "" {
 		if err := importer.HandleImportFlag(flagImportJob); err != nil {
-			log.Fatalf("job import failed: %s", err.Error())
+			cclog.Abortf("Import Job: Job import failed.\nError: %s\n", err.Error())
+		} else {
+			cclog.Printf("Import Job: Imported Job '%s' into DB.\n", flagImportJob)
+		}
+	}
+
+	if config.Keys.EnableJobTaggers {
+		tagger.Init()
+	}
+
+	if flagApplyTags {
+		if err := tagger.RunTaggers(); err != nil {
+			cclog.Abortf("Running job taggers.\nError: %s\n", err.Error())
 		}
 	}
 
 	if !flagServer {
-		return
+		cclog.Exit("No errors, server flag not set. Exiting cc-backend.")
 	}
 
-	// Setup the http.Handler/Router used by the server
-	jobRepo := repository.GetJobRepository()
-	resolver := &graph.Resolver{DB: db.DB, Repo: jobRepo}
-	graphQLEndpoint := handler.NewDefaultServer(generated.NewExecutableSchema(generated.Config{Resolvers: resolver}))
-	if os.Getenv("DEBUG") != "1" {
-		// Having this handler means that a error message is returned via GraphQL instead of the connection simply beeing closed.
-		// The problem with this is that then, no more stacktrace is printed to stderr.
-		graphQLEndpoint.SetRecoverFunc(func(ctx context.Context, err interface{}) error {
-			switch e := err.(type) {
-			case string:
-				return fmt.Errorf("MAIN > Panic: %s", e)
-			case error:
-				return fmt.Errorf("MAIN > Panic caused by: %w", e)
-			}
+	archiver.Start(repository.GetJobRepository())
 
-			return errors.New("MAIN > Internal server error (panic)")
-		})
-	}
-
-	api := &api.RestApi{
-		JobRepository:   jobRepo,
-		Resolver:        resolver,
-		MachineStateDir: config.Keys.MachineStateDir,
-		Authentication:  authentication,
-	}
-
-	r := mux.NewRouter()
-	buildInfo := web.Build{Version: version, Hash: commit, Buildtime: date}
-
-	r.HandleFunc("/login", func(rw http.ResponseWriter, r *http.Request) {
-		rw.Header().Add("Content-Type", "text/html; charset=utf-8")
-		web.RenderTemplate(rw, "login.tmpl", &web.Page{Title: "Login", Build: buildInfo})
-	}).Methods(http.MethodGet)
-	r.HandleFunc("/imprint", func(rw http.ResponseWriter, r *http.Request) {
-		rw.Header().Add("Content-Type", "text/html; charset=utf-8")
-		web.RenderTemplate(rw, "imprint.tmpl", &web.Page{Title: "Imprint", Build: buildInfo})
-	})
-	r.HandleFunc("/privacy", func(rw http.ResponseWriter, r *http.Request) {
-		rw.Header().Add("Content-Type", "text/html; charset=utf-8")
-		web.RenderTemplate(rw, "privacy.tmpl", &web.Page{Title: "Privacy", Build: buildInfo})
-	})
-
-	secured := r.PathPrefix("/").Subrouter()
-
-	if !config.Keys.DisableAuthentication {
-		r.Handle("/login", authentication.Login(
-			// On success:
-			http.RedirectHandler("/", http.StatusTemporaryRedirect),
-
-			// On failure:
-			func(rw http.ResponseWriter, r *http.Request, err error) {
-				rw.Header().Add("Content-Type", "text/html; charset=utf-8")
-				rw.WriteHeader(http.StatusUnauthorized)
-				web.RenderTemplate(rw, "login.tmpl", &web.Page{
-					Title:   "Login failed - ClusterCockpit",
-					MsgType: "alert-warning",
-					Message: err.Error(),
-					Build:   buildInfo,
-				})
-			})).Methods(http.MethodPost)
-
-		r.Handle("/jwt-login", authentication.Login(
-			// On success:
-			http.RedirectHandler("/", http.StatusTemporaryRedirect),
-
-			// On failure:
-			func(rw http.ResponseWriter, r *http.Request, err error) {
-				rw.Header().Add("Content-Type", "text/html; charset=utf-8")
-				rw.WriteHeader(http.StatusUnauthorized)
-				web.RenderTemplate(rw, "login.tmpl", &web.Page{
-					Title:   "Login failed - ClusterCockpit",
-					MsgType: "alert-warning",
-					Message: err.Error(),
-					Build:   buildInfo,
-				})
-			}))
-
-		r.Handle("/logout", authentication.Logout(
-			http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
-				rw.Header().Add("Content-Type", "text/html; charset=utf-8")
-				rw.WriteHeader(http.StatusOK)
-				web.RenderTemplate(rw, "login.tmpl", &web.Page{
-					Title:   "Bye - ClusterCockpit",
-					MsgType: "alert-info",
-					Message: "Logout successful",
-					Build:   buildInfo,
-				})
-			}))).Methods(http.MethodPost)
-
-		secured.Use(func(next http.Handler) http.Handler {
-			return authentication.Auth(
-				// On success;
-				next,
-
-				// On failure:
-				func(rw http.ResponseWriter, r *http.Request, err error) {
-					rw.WriteHeader(http.StatusUnauthorized)
-					web.RenderTemplate(rw, "login.tmpl", &web.Page{
-						Title:   "Authentication failed - ClusterCockpit",
-						MsgType: "alert-danger",
-						Message: err.Error(),
-						Build:   buildInfo,
-					})
-				})
-		})
-	}
-
-	if flagDev {
-		r.Handle("/playground", playground.Handler("GraphQL playground", "/query"))
-		r.PathPrefix("/swagger/").Handler(httpSwagger.Handler(
-			httpSwagger.URL("http://" + config.Keys.Addr + "/swagger/doc.json"))).Methods(http.MethodGet)
-	}
-	secured.Handle("/query", graphQLEndpoint)
-
-	// Send a searchId and then reply with a redirect to a user, or directly send query to job table for jobid and project.
-	secured.HandleFunc("/search", func(rw http.ResponseWriter, r *http.Request) {
-		routerConfig.HandleSearchBar(rw, r, buildInfo)
-	})
-
-	// Mount all /monitoring/... and /api/... routes.
-	routerConfig.SetupRoutes(secured, buildInfo)
-	api.MountRoutes(secured)
-
-	if config.Keys.EmbedStaticFiles {
-		if i, err := os.Stat("./var/img"); err == nil {
-			if i.IsDir() {
-				log.Info("Use local directory for static images")
-				r.PathPrefix("/img/").Handler(http.StripPrefix("/img/", http.FileServer(http.Dir("./var/img"))))
-			}
-		}
-		r.PathPrefix("/").Handler(web.ServeFiles())
-	} else {
-		r.PathPrefix("/").Handler(http.FileServer(http.Dir(config.Keys.StaticFiles)))
-	}
-
-	r.Use(handlers.CompressHandler)
-	r.Use(handlers.RecoveryHandler(handlers.PrintRecoveryStack(true)))
-	r.Use(handlers.CORS(
-		handlers.AllowCredentials(),
-		handlers.AllowedHeaders([]string{"X-Requested-With", "Content-Type", "Authorization", "Origin"}),
-		handlers.AllowedMethods([]string{"GET", "POST", "HEAD", "OPTIONS"}),
-		handlers.AllowedOrigins([]string{"*"})))
-	handler := handlers.CustomLoggingHandler(io.Discard, r, func(_ io.Writer, params handlers.LogFormatterParams) {
-		if strings.HasPrefix(params.Request.RequestURI, "/api/") {
-			log.Debugf("%s %s (%d, %.02fkb, %dms)",
-				params.Request.Method, params.URL.RequestURI(),
-				params.StatusCode, float32(params.Size)/1024,
-				time.Since(params.TimeStamp).Milliseconds())
-		} else {
-			log.Debugf("%s %s (%d, %.02fkb, %dms)",
-				params.Request.Method, params.URL.RequestURI(),
-				params.StatusCode, float32(params.Size)/1024,
-				time.Since(params.TimeStamp).Milliseconds())
-		}
-	})
+	taskManager.Start()
+	serverInit()
 
 	var wg sync.WaitGroup
-	server := http.Server{
-		ReadTimeout:  10 * time.Second,
-		WriteTimeout: 10 * time.Second,
-		Handler:      handler,
-		Addr:         config.Keys.Addr,
-	}
-
-	// Start http or https server
-	listener, err := net.Listen("tcp", config.Keys.Addr)
-	if err != nil {
-		log.Fatalf("starting http listener failed: %v", err)
-	}
-
-	if !strings.HasSuffix(config.Keys.Addr, ":80") && config.Keys.RedirectHttpTo != "" {
-		go func() {
-			http.ListenAndServe(":80", http.RedirectHandler(config.Keys.RedirectHttpTo, http.StatusMovedPermanently))
-		}()
-	}
-
-	if config.Keys.HttpsCertFile != "" && config.Keys.HttpsKeyFile != "" {
-		cert, err := tls.LoadX509KeyPair(config.Keys.HttpsCertFile, config.Keys.HttpsKeyFile)
-		if err != nil {
-			log.Fatalf("loading X509 keypair failed: %v", err)
-		}
-		listener = tls.NewListener(listener, &tls.Config{
-			Certificates: []tls.Certificate{cert},
-			CipherSuites: []uint16{
-				tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
-				tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
-			},
-			MinVersion:               tls.VersionTLS12,
-			PreferServerCipherSuites: true,
-		})
-		fmt.Printf("HTTPS server listening at %s...", config.Keys.Addr)
-	} else {
-		fmt.Printf("HTTP server listening at %s...", config.Keys.Addr)
-	}
-
-	// Because this program will want to bind to a privileged port (like 80), the listener must
-	// be established first, then the user can be changed, and after that,
-	// the actual http server can be started.
-	if err = runtimeEnv.DropPrivileges(config.Keys.Group, config.Keys.User); err != nil {
-		log.Fatalf("error while preparing server start: %s", err.Error())
-	}
 
 	wg.Add(1)
 	go func() {
 		defer wg.Done()
-		if err = server.Serve(listener); err != nil && err != http.ErrServerClosed {
-			log.Fatalf("starting server failed: %v", err)
-		}
+		serverStart()
 	}()
 
 	wg.Add(1)
@@ -514,117 +247,17 @@ func main() {
 		<-sigs
 		runtimeEnv.SystemdNotifiy(false, "Shutting down ...")
 
-		// First shut down the server gracefully (waiting for all ongoing requests)
-		server.Shutdown(context.Background())
+		serverShutdown()
 
-		// Then, wait for any async archivings still pending...
-		api.JobRepository.WaitForArchiving()
+		util.FsWatcherShutdown()
+
+		taskManager.Shutdown()
 	}()
 
-	s := gocron.NewScheduler(time.Local)
-
-	if config.Keys.StopJobsExceedingWalltime > 0 {
-		log.Info("Register undead jobs service")
-
-		s.Every(1).Day().At("3:00").Do(func() {
-			err = jobRepo.StopJobsExceedingWalltimeBy(config.Keys.StopJobsExceedingWalltime)
-			if err != nil {
-				log.Warnf("Error while looking for jobs exceeding their walltime: %s", err.Error())
-			}
-			runtime.GC()
-		})
-	}
-
-	var cfg struct {
-		Compression int              `json:"compression"`
-		Retention   schema.Retention `json:"retention"`
-	}
-
-	cfg.Retention.IncludeDB = true
-
-	if err = json.Unmarshal(config.Keys.Archive, &cfg); err != nil {
-		log.Warn("Error while unmarshaling raw config json")
-	}
-
-	switch cfg.Retention.Policy {
-	case "delete":
-		log.Info("Register retention delete service")
-
-		s.Every(1).Day().At("4:00").Do(func() {
-			startTime := time.Now().Unix() - int64(cfg.Retention.Age*24*3600)
-			jobs, err := jobRepo.FindJobsBetween(0, startTime)
-			if err != nil {
-				log.Warnf("Error while looking for retention jobs: %s", err.Error())
-			}
-			archive.GetHandle().CleanUp(jobs)
-
-			if cfg.Retention.IncludeDB {
-				cnt, err := jobRepo.DeleteJobsBefore(startTime)
-				if err != nil {
-					log.Errorf("Error while deleting retention jobs from db: %s", err.Error())
-				} else {
-					log.Infof("Retention: Removed %d jobs from db", cnt)
-				}
-				if err = jobRepo.Optimize(); err != nil {
-					log.Errorf("Error occured in db optimization: %s", err.Error())
-				}
-			}
-		})
-	case "move":
-		log.Info("Register retention move service")
-
-		s.Every(1).Day().At("4:00").Do(func() {
-			startTime := time.Now().Unix() - int64(cfg.Retention.Age*24*3600)
-			jobs, err := jobRepo.FindJobsBetween(0, startTime)
-			if err != nil {
-				log.Warnf("Error while looking for retention jobs: %s", err.Error())
-			}
-			archive.GetHandle().Move(jobs, cfg.Retention.Location)
-
-			if cfg.Retention.IncludeDB {
-				cnt, err := jobRepo.DeleteJobsBefore(startTime)
-				if err != nil {
-					log.Errorf("Error while deleting retention jobs from db: %v", err)
-				} else {
-					log.Infof("Retention: Removed %d jobs from db", cnt)
-				}
-				if err = jobRepo.Optimize(); err != nil {
-					log.Errorf("Error occured in db optimization: %v", err)
-				}
-			}
-		})
-	}
-
-	if cfg.Compression > 0 {
-		log.Info("Register compression service")
-
-		s.Every(1).Day().At("5:00").Do(func() {
-			var jobs []*schema.Job
-
-			ar := archive.GetHandle()
-			startTime := time.Now().Unix() - int64(cfg.Compression*24*3600)
-			lastTime := ar.CompressLast(startTime)
-			if startTime == lastTime {
-				log.Info("Compression Service - Complete archive run")
-				jobs, err = jobRepo.FindJobsBetween(0, startTime)
-
-			} else {
-				jobs, err = jobRepo.FindJobsBetween(lastTime, startTime)
-			}
-
-			if err != nil {
-				log.Warnf("Error while looking for compression jobs: %v", err)
-			}
-			ar.Compress(jobs)
-		})
-	}
-
-	s.StartAsync()
-
 	if os.Getenv("GOGC") == "" {
 		debug.SetGCPercent(25)
 	}
 	runtimeEnv.SystemdNotifiy(true, "running")
 	wg.Wait()
-	log.Print("Gracefull shutdown completed!")
+	cclog.Print("Graceful shutdown completed!")
 }

cmd/cc-backend/server.go

@@ -0,0 +1,330 @@
+// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
+// All rights reserved. This file is part of cc-backend.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+package main
+
+import (
+	"context"
+	"crypto/tls"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/99designs/gqlgen/graphql/handler"
+	"github.com/99designs/gqlgen/graphql/handler/transport"
+	"github.com/99designs/gqlgen/graphql/playground"
+	"github.com/ClusterCockpit/cc-backend/internal/api"
+	"github.com/ClusterCockpit/cc-backend/internal/archiver"
+	"github.com/ClusterCockpit/cc-backend/internal/auth"
+	"github.com/ClusterCockpit/cc-backend/internal/config"
+	"github.com/ClusterCockpit/cc-backend/internal/graph"
+	"github.com/ClusterCockpit/cc-backend/internal/graph/generated"
+	"github.com/ClusterCockpit/cc-backend/internal/routerConfig"
+	"github.com/ClusterCockpit/cc-backend/pkg/runtimeEnv"
+	"github.com/ClusterCockpit/cc-backend/web"
+	cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
+	"github.com/gorilla/handlers"
+	"github.com/gorilla/mux"
+	httpSwagger "github.com/swaggo/http-swagger"
+)
+
+var (
+	router    *mux.Router
+	server    *http.Server
+	apiHandle *api.RestApi
+)
+
+func onFailureResponse(rw http.ResponseWriter, r *http.Request, err error) {
+	rw.Header().Add("Content-Type", "application/json")
+	rw.WriteHeader(http.StatusUnauthorized)
+	json.NewEncoder(rw).Encode(map[string]string{
+		"status": http.StatusText(http.StatusUnauthorized),
+		"error":  err.Error(),
+	})
+}
+
+func serverInit() {
+	// Setup the http.Handler/Router used by the server
+	graph.Init()
+	resolver := graph.GetResolverInstance()
+	graphQLServer := handler.New(
+		generated.NewExecutableSchema(generated.Config{Resolvers: resolver}))
+
+	// graphQLServer.AddTransport(transport.SSE{})
+	graphQLServer.AddTransport(transport.POST{})
+	// graphQLServer.AddTransport(transport.Websocket{
+	// 	KeepAlivePingInterval: 10 * time.Second,
+	// 	Upgrader: websocket.Upgrader{
+	// 		CheckOrigin: func(r *http.Request) bool {
+	// 			return true
+	// 		},
+	// 	},
+	// })
+
+	if os.Getenv("DEBUG") != "1" {
+		// Having this handler means that a error message is returned via GraphQL instead of the connection simply beeing closed.
+		// The problem with this is that then, no more stacktrace is printed to stderr.
+		graphQLServer.SetRecoverFunc(func(ctx context.Context, err any) error {
+			switch e := err.(type) {
+			case string:
+				return fmt.Errorf("MAIN > Panic: %s", e)
+			case error:
+				return fmt.Errorf("MAIN > Panic caused by: %s", e.Error())
+			}
+
+			return errors.New("MAIN > Internal server error (panic)")
+		})
+	}
+
+	authHandle := auth.GetAuthInstance()
+
+	apiHandle = api.New()
+
+	router = mux.NewRouter()
+	buildInfo := web.Build{Version: version, Hash: commit, Buildtime: date}
+
+	info := map[string]any{}
+	info["hasOpenIDConnect"] = false
+
+	if config.Keys.OpenIDConfig != nil {
+		openIDConnect := auth.NewOIDC(authHandle)
+		openIDConnect.RegisterEndpoints(router)
+		info["hasOpenIDConnect"] = true
+	}
+
+	router.HandleFunc("/login", func(rw http.ResponseWriter, r *http.Request) {
+		rw.Header().Add("Content-Type", "text/html; charset=utf-8")
+		cclog.Debugf("##%v##", info)
+		web.RenderTemplate(rw, "login.tmpl", &web.Page{Title: "Login", Build: buildInfo, Infos: info})
+	}).Methods(http.MethodGet)
+	router.HandleFunc("/imprint", func(rw http.ResponseWriter, r *http.Request) {
+		rw.Header().Add("Content-Type", "text/html; charset=utf-8")
+		web.RenderTemplate(rw, "imprint.tmpl", &web.Page{Title: "Imprint", Build: buildInfo})
+	})
+	router.HandleFunc("/privacy", func(rw http.ResponseWriter, r *http.Request) {
+		rw.Header().Add("Content-Type", "text/html; charset=utf-8")
+		web.RenderTemplate(rw, "privacy.tmpl", &web.Page{Title: "Privacy", Build: buildInfo})
+	})
+
+	secured := router.PathPrefix("/").Subrouter()
+	securedapi := router.PathPrefix("/api").Subrouter()
+	userapi := router.PathPrefix("/userapi").Subrouter()
+	configapi := router.PathPrefix("/config").Subrouter()
+	frontendapi := router.PathPrefix("/frontend").Subrouter()
+
+	if !config.Keys.DisableAuthentication {
+		router.Handle("/login", authHandle.Login(
+			// On success: Handled within Login()
+			// On failure:
+			func(rw http.ResponseWriter, r *http.Request, err error) {
+				rw.Header().Add("Content-Type", "text/html; charset=utf-8")
+				rw.WriteHeader(http.StatusUnauthorized)
+				web.RenderTemplate(rw, "login.tmpl", &web.Page{
+					Title:   "Login failed - ClusterCockpit",
+					MsgType: "alert-warning",
+					Message: err.Error(),
+					Build:   buildInfo,
+					Infos:   info,
+				})
+			})).Methods(http.MethodPost)
+
+		router.Handle("/jwt-login", authHandle.Login(
+			// On success: Handled within Login()
+			// On failure:
+			func(rw http.ResponseWriter, r *http.Request, err error) {
+				rw.Header().Add("Content-Type", "text/html; charset=utf-8")
+				rw.WriteHeader(http.StatusUnauthorized)
+				web.RenderTemplate(rw, "login.tmpl", &web.Page{
+					Title:   "Login failed - ClusterCockpit",
+					MsgType: "alert-warning",
+					Message: err.Error(),
+					Build:   buildInfo,
+					Infos:   info,
+				})
+			}))
+
+		router.Handle("/logout", authHandle.Logout(
+			http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
+				rw.Header().Add("Content-Type", "text/html; charset=utf-8")
+				rw.WriteHeader(http.StatusOK)
+				web.RenderTemplate(rw, "login.tmpl", &web.Page{
+					Title:   "Bye - ClusterCockpit",
+					MsgType: "alert-info",
+					Message: "Logout successful",
+					Build:   buildInfo,
+					Infos:   info,
+				})
+			}))).Methods(http.MethodPost)
+
+		secured.Use(func(next http.Handler) http.Handler {
+			return authHandle.Auth(
+				// On success;
+				next,
+
+				// On failure:
+				func(rw http.ResponseWriter, r *http.Request, err error) {
+					rw.WriteHeader(http.StatusUnauthorized)
+					web.RenderTemplate(rw, "login.tmpl", &web.Page{
+						Title:    "Authentication failed - ClusterCockpit",
+						MsgType:  "alert-danger",
+						Message:  err.Error(),
+						Build:    buildInfo,
+						Infos:    info,
+						Redirect: r.RequestURI,
+					})
+				})
+		})
+
+		securedapi.Use(func(next http.Handler) http.Handler {
+			return authHandle.AuthApi(
+				// On success;
+				next,
+				// On failure: JSON Response
+				onFailureResponse)
+		})
+
+		userapi.Use(func(next http.Handler) http.Handler {
+			return authHandle.AuthUserApi(
+				// On success;
+				next,
+				// On failure: JSON Response
+				onFailureResponse)
+		})
+
+		configapi.Use(func(next http.Handler) http.Handler {
+			return authHandle.AuthConfigApi(
+				// On success;
+				next,
+				// On failure: JSON Response
+				onFailureResponse)
+		})
+
+		frontendapi.Use(func(next http.Handler) http.Handler {
+			return authHandle.AuthFrontendApi(
+				// On success;
+				next,
+				// On failure: JSON Response
+				onFailureResponse)
+		})
+	}
+
+	if flagDev {
+		router.Handle("/playground", playground.Handler("GraphQL playground", "/query"))
+		router.PathPrefix("/swagger/").Handler(httpSwagger.Handler(
+			httpSwagger.URL("http://" + config.Keys.Addr + "/swagger/doc.json"))).Methods(http.MethodGet)
+	}
+	secured.Handle("/query", graphQLServer)
+
+	// Send a searchId and then reply with a redirect to a user, or directly send query to job table for jobid and project.
+	secured.HandleFunc("/search", func(rw http.ResponseWriter, r *http.Request) {
+		routerConfig.HandleSearchBar(rw, r, buildInfo)
+	})
+
+	// Mount all /monitoring/... and /api/... routes.
+	routerConfig.SetupRoutes(secured, buildInfo)
+	apiHandle.MountApiRoutes(securedapi)
+	apiHandle.MountUserApiRoutes(userapi)
+	apiHandle.MountConfigApiRoutes(configapi)
+	apiHandle.MountFrontendApiRoutes(frontendapi)
+
+	if config.Keys.EmbedStaticFiles {
+		if i, err := os.Stat("./var/img"); err == nil {
+			if i.IsDir() {
+				cclog.Info("Use local directory for static images")
+				router.PathPrefix("/img/").Handler(http.StripPrefix("/img/", http.FileServer(http.Dir("./var/img"))))
+			}
+		}
+		router.PathPrefix("/").Handler(web.ServeFiles())
+	} else {
+		router.PathPrefix("/").Handler(http.FileServer(http.Dir(config.Keys.StaticFiles)))
+	}
+
+	router.Use(handlers.CompressHandler)
+	router.Use(handlers.RecoveryHandler(handlers.PrintRecoveryStack(true)))
+	router.Use(handlers.CORS(
+		handlers.AllowCredentials(),
+		handlers.AllowedHeaders([]string{"X-Requested-With", "Content-Type", "Authorization", "Origin"}),
+		handlers.AllowedMethods([]string{"GET", "POST", "HEAD", "OPTIONS"}),
+		handlers.AllowedOrigins([]string{"*"})))
+}
+
+func serverStart() {
+	handler := handlers.CustomLoggingHandler(io.Discard, router, func(_ io.Writer, params handlers.LogFormatterParams) {
+		if strings.HasPrefix(params.Request.RequestURI, "/api/") {
+			cclog.Debugf("%s %s (%d, %.02fkb, %dms)",
+				params.Request.Method, params.URL.RequestURI(),
+				params.StatusCode, float32(params.Size)/1024,
+				time.Since(params.TimeStamp).Milliseconds())
+		} else {
+			cclog.Debugf("%s %s (%d, %.02fkb, %dms)",
+				params.Request.Method, params.URL.RequestURI(),
+				params.StatusCode, float32(params.Size)/1024,
+				time.Since(params.TimeStamp).Milliseconds())
+		}
+	})
+
+	server = &http.Server{
+		ReadTimeout:  20 * time.Second,
+		WriteTimeout: 20 * time.Second,
+		Handler:      handler,
+		Addr:         config.Keys.Addr,
+	}
+
+	// Start http or https server
+	listener, err := net.Listen("tcp", config.Keys.Addr)
+	if err != nil {
+		cclog.Abortf("Server Start: Starting http listener on '%s' failed.\nError: %s\n", config.Keys.Addr, err.Error())
+	}
+
+	if !strings.HasSuffix(config.Keys.Addr, ":80") && config.Keys.RedirectHttpTo != "" {
+		go func() {
+			http.ListenAndServe(":80", http.RedirectHandler(config.Keys.RedirectHttpTo, http.StatusMovedPermanently))
+		}()
+	}
+
+	if config.Keys.HttpsCertFile != "" && config.Keys.HttpsKeyFile != "" {
+		cert, err := tls.LoadX509KeyPair(
+			config.Keys.HttpsCertFile, config.Keys.HttpsKeyFile)
+		if err != nil {
+			cclog.Abortf("Server Start: Loading X509 keypair failed. Check options 'https-cert-file' and 'https-key-file' in 'config.json'.\nError: %s\n", err.Error())
+		}
+		listener = tls.NewListener(listener, &tls.Config{
+			Certificates: []tls.Certificate{cert},
+			CipherSuites: []uint16{
+				tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
+				tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
+			},
+			MinVersion:               tls.VersionTLS12,
+			PreferServerCipherSuites: true,
+		})
+		cclog.Printf("HTTPS server listening at %s...\n", config.Keys.Addr)
+	} else {
+		cclog.Printf("HTTP server listening at %s...\n", config.Keys.Addr)
+	}
+	//
+	// Because this program will want to bind to a privileged port (like 80), the listener must
+	// be established first, then the user can be changed, and after that,
+	// the actual http server can be started.
+	if err := runtimeEnv.DropPrivileges(config.Keys.Group, config.Keys.User); err != nil {
+		cclog.Abortf("Server Start: Error while preparing server start.\nError: %s\n", err.Error())
+	}
+
+	if err = server.Serve(listener); err != nil && err != http.ErrServerClosed {
+		cclog.Abortf("Server Start: Starting server failed.\nError: %s\n", err.Error())
+	}
+}
+
+func serverShutdown() {
+	// First shut down the server gracefully (waiting for all ongoing requests)
+	server.Shutdown(context.Background())
+
+	// Then, wait for any async archivings still pending...
+	archiver.WaitForArchiving()
+}

configs/README.md

@@ -1,93 +0,0 @@
-## Intro
-
-cc-backend requires a configuration file that specifies the cluster systems to be used.
-To override the default, specify the location of a json configuration file with the `-config <file path>` command line option.
-All security-related configurations, e.g. keys and passwords, are set using
-environment variables.
-It is supported to set these by means of a `.env` file in the project root.
-
-## Configuration Options
-
-* `addr`: Type string.  Address where the http (or https) server will listen on (for example: 'localhost:80'). Default `:8080`.
-* `apiAllowedIPs`: Type string array.  Addresses from which the secured API endpoints (/users and other auth related endpoints)  can be reached
-* `user`: Type string. Drop root permissions once .env was read and the port was taken. Only applicable if using privileged port.
-* `group`: Type string.  Drop root permissions once .env was read and the port was taken. Only applicable if using privileged port.
-* `disable-authentication`: Type bool.  Disable authentication (for everything: API, Web-UI, ...). Default `false`.
-* `embed-static-files`: Type bool. If all files in `web/frontend/public` should be served from within the binary itself (they are embedded) or not. Default `true`.
-* `static-files`: Type string. Folder where static assets can be found, if `embed-static-files` is `false`. No default.
-* `db-driver`: Type string. 'sqlite3' or 'mysql' (mysql will work for mariadb as well). Default `sqlite3`.
-* `db`: Type string. For sqlite3 a filename, for mysql a DSN in this format: https://github.com/go-sql-driver/mysql#dsn-data-source-name (Without query parameters!). Default: `./var/job.db`.
-* `job-archive`: Type object.
-    - `kind`: Type string. At them moment only file is supported as value.
-    - `path`: Type string. Path to the job-archive. Default: `./var/job-archive`.
-    - `compression`: Type integer. Setup automatic compression for jobs older than number of days.
-    - `retention`: Type object.
-        - `policy`: Type string (required). Retention policy. Possible values none, delete,
-          move.
-        - `includeDB`: Type boolean. Also remove jobs from database.
-        - `age`: Type integer. Act on jobs with startTime older than age (in days).
-        - `location`: Type string. The target directory for retention. Only applicable for retention policy move.
-* `disable-archive`: Type bool. Keep all metric data in the metric data repositories, do not write to the job-archive. Default `false`.
-* `validate`: Type bool. Validate all input json documents against json schema.
-* `session-max-age`: Type string. Specifies for how long a session shall be valid  as a string parsable by time.ParseDuration(). If 0 or empty, the session/token does not expire! Default `168h`.
-* `https-cert-file` and `https-key-file`: Type string. If both those options are not empty, use HTTPS using those certificates.
-* `redirect-http-to`: Type string. If not the empty string and `addr` does not end in ":80", redirect every request incoming at port 80 to that url.
-* `machine-state-dir`: Type string. Where to store MachineState files. TODO: Explain in more detail!
-* `stop-jobs-exceeding-walltime`: Type int. If not zero, automatically mark jobs as stopped running X seconds longer than their walltime. Only applies if walltime is set for job. Default `0`.
-* `short-running-jobs-duration`: Type int. Do not show running jobs shorter than X seconds. Default `300`.
-* `jwts`: Type object (required). For JWT Authentication.
-   - `max-age`: Type string (required). Configure how long a token is valid. As string parsable by time.ParseDuration().
-   - `cookieName`: Type string. Cookie that should be checked for a JWT token.
-   - `vaidateUser`: Type boolean. Deny login for users not in database (but defined in JWT). Overwrite roles in JWT with database roles.
-   - `trustedIssuer`: Type string. Issuer that should be accepted when validating external JWTs. 
-   - `syncUserOnLogin`: Type boolean. Add non-existent user to DB at login attempt with values provided in JWT.
-* `ldap`: Type object. For LDAP Authentication and user synchronisation. Default `nil`.
-   - `url`: Type string (required). URL of LDAP directory server.
-   - `user_base`: Type string (required). Base DN of user tree root.
-   - `search_dn`: Type string (required). DN for authenticating LDAP admin account with general read rights.
-   - `user_bind`: Type string (required). Expression used to authenticate users via LDAP bind. Must contain `uid={username}`.
-   - `user_filter`: Type string (required). Filter to extract users for syncing.
-   - `username_attr`: Type string. Attribute with full user name. Defaults to `gecos` if not provided.
-   - `sync_interval`: Type string. Interval used for syncing local user table with LDAP directory. Parsed using time.ParseDuration.
-   - `sync_del_old_users`: Type boolean. Delete obsolete users in database.
-   - `syncUserOnLogin`: Type boolean. Add non-existent user to DB at login attempt if user exists in Ldap directory.
-* `clusters`: Type array of objects (required)
-   - `name`: Type string. The name of the cluster.
-   - `metricDataRepository`: Type object with properties: `kind` (Type string, can be one of `cc-metric-store`, `influxdb` ), `url` (Type string), `token` (Type string)
-   - `filterRanges` Type object. This option controls the slider ranges for the UI controls of numNodes, duration, and startTime.  Example:
-   ```
-   "filterRanges": {
-                "numNodes": { "from": 1, "to": 64 },
-                "duration": { "from": 0, "to": 86400 },
-                "startTime": { "from": "2022-01-01T00:00:00Z", "to": null }
-            }
-   ```
-* `ui-defaults`: Type object. Default configuration for ui views. If overwritten, all options  must be provided! Most options can be overwritten by the user via the web interface.
-   - `analysis_view_histogramMetrics`: Type string array. Metrics to show as job count histograms in analysis view. Default `["flops_any", "mem_bw", "mem_used"]`.
-   - `analysis_view_scatterPlotMetrics`: Type array of string array. Initial
-   scatter plot configuration in analysis view. Default `[["flops_any", "mem_bw"], ["flops_any", "cpu_load"], ["cpu_load", "mem_bw"]]`.
-   - `job_view_nodestats_selectedMetrics`: Type string array. Initial metrics shown in node statistics table of single job view. Default `["flops_any", "mem_bw", "mem_used"]`.
-   - `job_view_polarPlotMetrics`: Type string array. Metrics shown in polar plot of single job view. Default `["flops_any", "mem_bw", "mem_used", "net_bw", "file_bw"]`.
-   - `job_view_selectedMetrics`: Type string array.  Default `["flops_any", "mem_bw", "mem_used"]`.
-   - `plot_general_colorBackground`: Type bool. Color plot background according to job average threshold limits. Default `true`.
-   - `plot_general_colorscheme`: Type string array. Initial color scheme. Default `"#00bfff", "#0000ff", "#ff00ff", "#ff0000", "#ff8000", "#ffff00", "#80ff00"`.
-   - `plot_general_lineWidth`: Type int. Initial linewidth. Default `3`.
-   - `plot_list_jobsPerPage`: Type int. Jobs shown per page in job lists. Default `50`.
-   - `plot_list_selectedMetrics`: Type string array. Initial metric plots shown in jobs lists. Default `"cpu_load", "ipc", "mem_used", "flops_any", "mem_bw"`.
-   - `plot_view_plotsPerRow`: Type int. Number of plots per row in single job view. Default `3`.
-   - `plot_view_showPolarplot`: Type bool. Option to toggle polar plot in single job view. Default `true`.
-   - `plot_view_showRoofline`: Type bool. Option to toggle roofline plot in single job view. Default `true`.
-   - `plot_view_showStatTable`: Type bool. Option to toggle the node statistic table in single job view. Default `true`.
-   - `system_view_selectedMetric`: Type string. Initial metric shown in system view. Default `cpu_load`.
-
-Some of the `ui-defaults` values can be appended by `:<clustername>` in order to have different settings depending on the current cluster. Those are notably `job_view_nodestats_selectedMetrics`, `job_view_polarPlotMetrics`, `job_view_selectedMetrics` and `plot_list_selectedMetrics`.
-
-## Environment Variables
-
-An example env file is found in this directory. Copy it to `.env` in the project root and adapt it for your needs.
-
-* `JWT_PUBLIC_KEY` and `JWT_PRIVATE_KEY`: Base64 encoded Ed25519 keys used for JSON Web Token (JWT) authentication. You can generate your own keypair using `go run ./cmd/gen-keypair/gen-keypair.go`. More information in [README_TOKENS.md](./README_TOKENS.md).
-* `SESSION_KEY`: Some random bytes used as secret for cookie-based sessions.
-* `LDAP_ADMIN_PASSWORD`: The LDAP admin user password (optional).
-* `CROSS_LOGIN_JWT_HS512_KEY`: Used for token based logins via another authentication service.
-* `LOGLEVEL`: Can be `err`, `warn`, `info` or `debug` (optional, `warn` by default). Can be used to reduce logging.

configs/README_TOKENS.md

@@ -1,51 +0,0 @@
-## Introduction
-
-ClusterCockpit uses JSON Web Tokens (JWT) for authorization of its APIs.
-JSON Web Token (JWT) is an open standard (RFC 7519) that defines a compact and self-contained way for securely transmitting information between parties as a JSON object.
-This information can be verified and trusted because it is digitally signed.
-In ClusterCockpit JWTs are signed using a public/private key pair using ECDSA.
-Because tokens are signed using public/private key pairs, the signature also certifies that only the party holding the private key is the one that signed it.
-Expiration of the generated tokens as well as the max. length of a browser session can be configured in the `config.json` file described [here](./README.md).
-
-The [Ed25519](https://ed25519.cr.yp.to/) algorithm for signatures was used because it is compatible with other tools that require authentication, such as NATS.io, and because these elliptic-curve methods provide simillar security with smaller keys compared to something like RSA. They are sligthly more expensive to validate, but that effect is negligible.
-
-## JWT Payload
-
-You may view the payload of a JWT token at [https://jwt.io/#debugger-io](https://jwt.io/#debugger-io).
-Currently ClusterCockpit sets the following claims:
-* `iat`: Issued at claim. The “iat” claim is used to identify the the time at which the JWT was issued. This claim can be used to determine the age of the JWT.
-* `sub`: Subject claim. Identifies the subject of the JWT, in our case this is the username.
-* `roles`: An array of strings specifying the roles set for the subject.
-* `exp`: Expiration date of the token (only if explicitly configured)
-
-It is important to know that JWTs are not encrypted, only signed. This means that outsiders cannot create new JWTs or modify existing ones, but they are able to read out the username.
-
-## Workflow
-
-1. Create a new ECDSA Public/private keypair:
-```
-$ go build ./cmd/gen-keypair/
-$ ./gen-keypair
-```
-2. Add keypair in your `.env` file. A template can be found in `./configs`.
-
-When a user logs in via the `/login` page using a browser, a session cookie (secured using the random bytes in the `SESSION_KEY` env. variable you shoud change as well) is used for all requests after the successfull login. The JWTs make it easier to use the APIs of ClusterCockpit using scripts or other external programs. The token is specified n the `Authorization` HTTP header using the [Bearer schema](https://datatracker.ietf.org/doc/html/rfc6750) (there is an example below). Tokens can be issued to users from the configuration view in the Web-UI or the command line. In order to use the token for API endpoints such as `/api/jobs/start_job/`, the user that executes it needs to have the `api` role. Regular users can only perform read-only queries and only look at data connected to jobs they started themselves.
-
-## cc-metric-store
-
-The [cc-metric-store](https://github.com/ClusterCockpit/cc-metric-store) also uses JWTs for authentication. As it does not issue new tokens, it does not need to kown the private key. The public key of the keypair that is used to generate the JWTs that grant access to the `cc-metric-store` can be specified in its `config.json`. When configuring the `metricDataRepository` object in the `cluster.json` file, you can put a token issued by ClusterCockpit itself.
-
-## Setup user and JWT token for REST API authorization
-
-1. Create user:
-```
-$ ./cc-backend --add-user <username>:api:<password> --no-server
-```
-2. Issue token for user:
-```
-$ ./cc-backend --jwt <username> --no-server
-```
-3. Use issued token token on client side:
-```
-$ curl -X GET "<API ENDPOINT>" -H "accept: application/json" -H "Content-Type: application/json" -H "Authorization: Bearer <JWT TOKEN>"
-```

configs/config-demo.json

@@ -1,56 +1,70 @@
 {
-    "addr": "127.0.0.1:8080",
-    "archive": {
-        "kind": "file",
-        "path": "./var/job-archive"
-    },
-    "jwts": {
-        "max-age": "2000h"
-    },
-    "clusters": [
-        {
-            "name": "fritz",
-            "metricDataRepository": {
-                "kind": "cc-metric-store",
-                "url": "http://localhost:8082",
-                "token": ""
-            },
-            "filterRanges": {
-                "numNodes": {
-                    "from": 1,
-                    "to": 64
-                },
-                "duration": {
-                    "from": 0,
-                    "to": 86400
-                },
-                "startTime": {
-                    "from": "2022-01-01T00:00:00Z",
-                    "to": null
-                }
-            }
-        },
-        {
-            "name": "alex",
-            "metricDataRepository": {
-                "kind": "cc-metric-store",
-                "url": "http://localhost:8082",
-                "token": ""
-            },
-            "filterRanges": {
-                "numNodes": {
-                    "from": 1,
-                    "to": 64
-                },
-                "duration": {
-                    "from": 0,
-                    "to": 86400
-                },
-                "startTime": {
-                    "from": "2022-01-01T00:00:00Z",
-                    "to": null
-                }
-            }
-        }
+  "addr": "127.0.0.1:8080",
+  "short-running-jobs-duration": 300,
+  "archive": {
+    "kind": "file",
+    "path": "./var/job-archive"
+  },
+  "jwts": {
+    "max-age": "2000h"
+  },
+  "enable-resampling": {
+    "trigger": 30,
+    "resolutions": [
+      600,
+      300,
+      120,
+      60
     ]
+  },
+  "apiAllowedIPs": [
+    "*"
+  ],
+  "emission-constant": 317,
+  "clusters": [
+    {
+      "name": "fritz",
+      "metricDataRepository": {
+        "kind": "cc-metric-store",
+        "url": "http://localhost:8082",
+        "token": ""
+      },
+      "filterRanges": {
+        "numNodes": {
+          "from": 1,
+          "to": 64
+        },
+        "duration": {
+          "from": 0,
+          "to": 86400
+        },
+        "startTime": {
+          "from": "2022-01-01T00:00:00Z",
+          "to": null
+        }
+      }
+    },
+    {
+      "name": "alex",
+      "metricDataRepository": {
+        "kind": "cc-metric-store",
+        "url": "http://localhost:8082",
+        "token": ""
+      },
+      "filterRanges": {
+        "numNodes": {
+          "from": 1,
+          "to": 64
+        },
+        "duration": {
+          "from": 0,
+          "to": 86400
+        },
+        "startTime": {
+          "from": "2022-01-01T00:00:00Z",
+          "to": null
+        }
+      }
+    }
+  ]
 }

configs/config-mariadb.json

@@ -0,0 +1,69 @@
+{
+  "addr": "127.0.0.1:8080",
+  "short-running-jobs-duration": 300,
+  "archive": {
+    "kind": "file",
+    "path": "./var/job-archive"
+  },
+  "jwts": {
+    "max-age": "2000h"
+  },
+  "db-driver": "mysql",
+  "db": "clustercockpit:demo@tcp(127.0.0.1:3306)/clustercockpit",
+  "enable-resampling": {
+    "trigger": 30,
+    "resolutions": [
+      600,
+      300,
+      120,
+      60
+    ]
+  },
+  "emission-constant": 317,
+  "clusters": [
+    {
+      "name": "fritz",
+      "metricDataRepository": {
+        "kind": "cc-metric-store",
+        "url": "http://localhost:8082",
+        "token": ""
+      },
+      "filterRanges": {
+        "numNodes": {
+          "from": 1,
+          "to": 64
+        },
+        "duration": {
+          "from": 0,
+          "to": 86400
+        },
+        "startTime": {
+          "from": "2022-01-01T00:00:00Z",
+          "to": null
+        }
+      }
+    },
+    {
+      "name": "alex",
+      "metricDataRepository": {
+        "kind": "cc-metric-store",
+        "url": "http://localhost:8082",
+        "token": ""
+      },
+      "filterRanges": {
+        "numNodes": {
+          "from": 1,
+          "to": 64
+        },
+        "duration": {
+          "from": 0,
+          "to": 86400
+        },
+        "startTime": {
+          "from": "2022-01-01T00:00:00Z",
+          "to": null
+        }
+      }
+    }
+  ]
+}

configs/config.json

@@ -1,50 +1,62 @@
 {
-    "addr": "0.0.0.0:443",
-    "ldap": {
-        "url": "ldaps://test",
-        "user_base": "ou=people,ou=hpc,dc=test,dc=de",
-        "search_dn": "cn=hpcmonitoring,ou=roadm,ou=profile,ou=hpc,dc=test,dc=de",
-        "user_bind": "uid={username},ou=people,ou=hpc,dc=test,dc=de",
-        "user_filter": "(&(objectclass=posixAccount))"
-    },
-    "https-cert-file": "/etc/letsencrypt/live/url/fullchain.pem",
-    "https-key-file": "/etc/letsencrypt/live/url/privkey.pem",
-    "user": "clustercockpit",
-    "group": "clustercockpit",
-    "archive": {
-        "kind": "file",
-        "path": "./var/job-archive"
-    },
-    "validate": true,
-    "clusters": [
-        {
-            "name": "test",
-            "metricDataRepository": {
-                "kind": "cc-metric-store",
-                "url": "http://localhost:8082",
-                "token": "eyJhbGciOiJF-E-pQBQ"
-            },
-            "filterRanges": {
-                "numNodes": {
-                    "from": 1,
-                    "to": 64
-                },
-                "duration": {
-                    "from": 0,
-                    "to": 86400
-                },
-                "startTime": {
-                    "from": "2022-01-01T00:00:00Z",
-                    "to": null
-                }
-            }
+  "addr": "0.0.0.0:443",
+  "ldap": {
+    "url": "ldaps://test",
+    "user_base": "ou=people,ou=hpc,dc=test,dc=de",
+    "search_dn": "cn=hpcmonitoring,ou=roadm,ou=profile,ou=hpc,dc=test,dc=de",
+    "user_bind": "uid={username},ou=people,ou=hpc,dc=test,dc=de",
+    "user_filter": "(&(objectclass=posixAccount))"
+  },
+  "https-cert-file": "/etc/letsencrypt/live/url/fullchain.pem",
+  "https-key-file": "/etc/letsencrypt/live/url/privkey.pem",
+  "user": "clustercockpit",
+  "group": "clustercockpit",
+  "archive": {
+    "kind": "file",
+    "path": "./var/job-archive"
+  },
+  "validate": false,
+  "apiAllowedIPs": [
+    "*"
+  ],
+  "clusters": [
+    {
+      "name": "test",
+      "metricDataRepository": {
+        "kind": "cc-metric-store",
+        "url": "http://localhost:8082",
+        "token": "eyJhbGciOiJF-E-pQBQ"
+      },
+      "filterRanges": {
+        "numNodes": {
+          "from": 1,
+          "to": 64
+        },
+        "duration": {
+          "from": 0,
+          "to": 86400
+        },
+        "startTime": {
+          "from": "2022-01-01T00:00:00Z",
+          "to": null
         }
-    ],
-    "jwts": {
-        "cookieName": "",
-        "validateUser": false,
-        "max-age": "2000h",
-        "trustedIssuer": ""
-    },
-    "short-running-jobs-duration": 300
+      }
+    }
+  ],
+  "jwts": {
+    "cookieName": "",
+    "validateUser": false,
+    "max-age": "2000h",
+    "trustedIssuer": ""
+  },
+  "enable-resampling": {
+    "trigger": 30,
+    "resolutions": [
+      600,
+      300,
+      120,
+      60
+    ]
+  },
+  "short-running-jobs-duration": 300
 }

configs/default_metrics.json

@@ -0,0 +1,12 @@
+{
+  "clusters": [
+    {
+      "name": "fritz",
+      "default_metrics": "cpu_load, flops_any, core_power, lustre_open, mem_used, mem_bw, net_bytes_in"
+    },
+    {
+      "name": "alex",
+      "default_metrics": "flops_any, mem_bw, mem_used, vectorization_ratio"
+    }
+  ]
+}

configs/generate-subcluster.pl

@@ -117,10 +117,12 @@ foreach my $ln (split("\n", $topo)) {
 
 my $node;
 my @sockets;
+my @nodeCores;
 foreach my $socket ( @{$DOMAINS{socket}} ) {
     push @sockets, "[".join(",", @{$socket})."]";
-    $node .=  join(",", @{$socket})
+    push @nodeCores, join(",", @{$socket});
 }
+$node =  join(",", @nodeCores);
 $INFO{sockets} = join(",\n", @sockets);
 
 my @memDomains;
@@ -212,9 +214,27 @@ print <<"END";
       "socketsPerNode": $INFO{socketsPerNode},
       "coresPerSocket": $INFO{coresPerSocket},
       "threadsPerCore": $INFO{threadsPerCore},
-      "flopRateScalar": $flopsScalar,
-      "flopRateSimd": $flopsSimd,
-      "memoryBandwidth": $memBw,
+      "flopRateScalar": {
+           "unit": {
+               "base": "F/s",
+               "prefix": "G"
+           },
+           "value": $flopsScalar
+      },
+      "flopRateSimd": {
+           "unit": {
+               "base": "F/s",
+               "prefix": "G"
+           },
+           "value": $flopsSimd
+      },
+      "memoryBandwidth": {
+           "unit": {
+               "base": "B/s",
+               "prefix": "G"
+           },
+           "value": $memBw
+      },
       "nodes": "<FILL IN NODE RANGES>",
       "topology": {
           "node": [$node],

docs/ConfigurationManagement.md

@@ -1,38 +0,0 @@
-# Release versions
-
-Versions are marked according to [semantic versioning] (https://semver.org).
-Each version embeds the following static assets in the binary:
-* Web frontend with javascript files and all static assets.
-* Golang template files for server-side rendering.
-* JSON schema files for validation.
-* Database migration files.
-
-The remaining external assets are:
-* The SQL database used.
-* The job archive
-* The configuration files `config.json` and `.env`.
-
-The external assets are versioned with integer IDs.
-This means that each release binary is bound to specific versions of the SQL
-database and the job archive.
-The configuration file is checked against the current schema at startup.
-The `-migrate-db` command line switch can be used to upgrade the SQL database
-to migrate from a previous version to the latest one.
-We offer a separate tool `archive-migration` to migrate an existing job archive
-archive from the previous to the latest version.
-
-# Versioning of APIs
-
-cc-backend provides two API backends:
-* A REST API for querying jobs.
-* A GraphQL API for data exchange between web frontend and cc-backend.
-
-The REST API will also be versioned. We still have to decide whether we will also
-support older REST API versions by versioning the endpoint URLs.
-The GraphQL API is for internal use and will not be versioned.
-
-# How to build
-
-In general it is recommended to use the provided release binary.
-In case you want to build build `cc-backend` please always use the provided makefile. This will ensure
-that the frontend is also built correctly and that the version in the binary is encoded in the binary.

docs/Hands-on.md

@@ -1,234 +0,0 @@
-# Hands-on setup ClusterCockpit from scratch (w/o docker)
-
-## Prerequisites
-* perl
-* go
-* npm
-* Optional: curl
-* Script migrateTimestamp.pl
-
-## Documentation
-You find READMEs or api docs in
-* ./cc-backend/configs
-* ./cc-backend/init
-* ./cc-backend/api
-
-## ClusterCockpit configuration files
-### cc-backend
-* `./.env` Passwords and Tokens set in the environment
-* `./config.json` Configuration options for cc-backend
-
-### cc-metric-store
-* `./config.json` Optional to overwrite configuration options
-
-### cc-metric-collector
-Not yet included in the hands-on setup.
-
-## Setup Components
-Start by creating a base folder for all of the following steps.
-* `mkdir clustercockpit`
-* `cd clustercockpit`
-
-### Setup cc-backend
-* Clone Repository
-    - `git clone https://github.com/ClusterCockpit/cc-backend.git`
-    - `cd cc-backend`
-* Build
-    - `make`
-* Activate & configure environment for cc-backend
-    - `cp configs/env-template.txt  .env`
-    - Optional: Have a look via `vim .env`
-    - Copy the `config.json` file included in this tarball into the root directory of cc-backend: `cp ../../config.json  ./`
-* Back to toplevel `clustercockpit`
-    - `cd ..`
-* Prepare Datafolder and Database file
-    - `mkdir var`
-    - `./cc-backend -migrate-db`
-
-### Setup cc-metric-store
-* Clone Repository
-    - `git clone https://github.com/ClusterCockpit/cc-metric-store.git`
-    - `cd cc-metric-store`
-* Build Go Executable
-    - `go get`
-    - `go build`
-* Prepare Datafolders
-    - `mkdir -p var/checkpoints`
-    - `mkdir -p var/archive`
-* Update Config
-    - `vim config.json`
-    - Exchange existing setting in `metrics` with the following:
-```
-"clock":      { "frequency": 60, "aggregation": null },
-"cpi":        { "frequency": 60, "aggregation": null },
-"cpu_load":   { "frequency": 60, "aggregation": null },
-"flops_any":  { "frequency": 60, "aggregation": null },
-"flops_dp":   { "frequency": 60, "aggregation": null },
-"flops_sp":   { "frequency": 60, "aggregation": null },
-"ib_bw":      { "frequency": 60, "aggregation": null },
-"lustre_bw":  { "frequency": 60, "aggregation": null },
-"mem_bw":     { "frequency": 60, "aggregation": null },
-"mem_used":   { "frequency": 60, "aggregation": null },
-"rapl_power": { "frequency": 60, "aggregation": null }
-```
-* Back to toplevel `clustercockpit`
-    - `cd ..`
-
-### Setup Demo Data
-* `mkdir source-data`
-* `cd source-data`
-* Download JobArchive-Source:
-    - `wget https://hpc-mover.rrze.uni-erlangen.de/HPC-Data/0x7b58aefb/eig7ahyo6fo2bais0ephuf2aitohv1ai/job-archive-dev.tar.xz`
-    - `tar xJf job-archive-dev.tar.xz`
-    - `mv ./job-archive ./job-archive-source`
-    - `rm ./job-archive-dev.tar.xz`
-* Download CC-Metric-Store Checkpoints:
-    - `mkdir -p cc-metric-store-source/checkpoints`
-    - `cd cc-metric-store-source/checkpoints`
-    - `wget https://hpc-mover.rrze.uni-erlangen.de/HPC-Data/0x7b58aefb/eig7ahyo6fo2bais0ephuf2aitohv1ai/cc-metric-store-checkpoints.tar.xz`
-    - `tar xf cc-metric-store-checkpoints.tar.xz`
-    - `rm cc-metric-store-checkpoints.tar.xz`
-* Back to `source-data`
-    - `cd ../..`
-* Run timestamp migration script. This may take tens of minutes!
-    - `cp ../migrateTimestamps.pl .`
-    - `./migrateTimestamps.pl`
-    - Expected output:
-```
-Starting to update start- and stoptimes in job-archive for emmy
-Starting to update start- and stoptimes in job-archive for woody
-Done for job-archive
-Starting to update checkpoint filenames and data starttimes for emmy
-Starting to update checkpoint filenames and data starttimes for woody
-Done for checkpoints
-```
-* Copy `cluster.json` files from source to migrated folders
-    - `cp source-data/job-archive-source/emmy/cluster.json cc-backend/var/job-archive/emmy/`
-    - `cp source-data/job-archive-source/woody/cluster.json cc-backend/var/job-archive/woody/`
-* Initialize Job-Archive in SQLite3 job.db and add demo user
-    - `cd cc-backend`
-    - `./cc-backend -init-db -add-user demo:admin:demo`
-    - Expected output:
-```
-<6>[INFO]    new user "demo" created (roles: ["admin"], auth-source: 0)
-<6>[INFO]    Building job table...
-<6>[INFO]    A total of 3936 jobs have been registered in 1.791 seconds.
-```
-* Back to toplevel `clustercockpit`
-    - `cd ..`
-
-### Startup both Apps
-* In cc-backend root:  `$./cc-backend -server -dev`
-    - Starts Clustercockpit at `http:localhost:8080`
-        - Log: `<6>[INFO]    HTTP server listening at :8080...`
-    - Use local internet browser to access interface
-        - You should see and be able to browse finished Jobs
-        - Metadata is read from SQLite3 database
-        - Metricdata is read from job-archive/JSON-Files
-    - Create User in settings (top-right corner)
-        - Name `apiuser`
-        - Username `apiuser`
-        - Role `API`
-        - Submit & Refresh Page
-    - Create JTW for `apiuser`
-        - In Userlist, press `Gen. JTW` for `apiuser`
-        - Save JWT for later use
-* In cc-metric-store root:  `$./cc-metric-store`
-    - Start the cc-metric-store on `http:localhost:8081`, Log:
-```
-2022/07/15 17:17:42 Loading checkpoints newer than 2022-07-13T17:17:42+02:00
-2022/07/15 17:17:45 Checkpoints loaded (5621 files, 319 MB, that took 3.034652s)
-2022/07/15 17:17:45 API http endpoint listening on '0.0.0.0:8081'
-```
-    - Does *not* have a graphical interface
-    - Otpional: Test function by executing:
-```
-$ curl -H "Authorization: Bearer eyJ0eXAiOiJKV1QiLCJhbGciOiJFZERTQSJ9.eyJ1c2VyIjoiYWRtaW4iLCJyb2xlcyI6WyJST0xFX0FETUlOIiwiUk9MRV9BTkFMWVNUIiwiUk9MRV9VU0VSIl19.d-3_3FZTsadPjDEdsWrrQ7nS0edMAR4zjl-eK7rJU3HziNBfI9PDHDIpJVHTNN5E5SlLGLFXctWyKAkwhXL-Dw" -D - "http://localhost:8081/api/query" -d "{ \"cluster\": \"emmy\", \"from\": $(expr $(date +%s) - 60), \"to\": $(date +%s), \"queries\": [{
-  \"metric\": \"flops_any\",
-  \"host\": \"e1111\"
-}] }"
-
-HTTP/1.1 200 OK
-Content-Type: application/json
-Date: Fri, 15 Jul 2022 13:57:22 GMT
-Content-Length: 119
-{"results":[[JSON-DATA-ARRAY]]}
-```
-
-### Development API web interfaces
-The `-dev` flag enables web interfaces to document and test the apis:
-* http://localhost:8080/playground - A GraphQL playground. To use it you must have a authenticated session in the same browser.
-* http://localhost:8080/swagger - A Swagger UI. To use it you have to be logged out, so no user session in the same browser. Use the JWT token with role Api generate previously to authenticate via http header.
-
-### Use cc-backend API to start job
-* Enter the URL `http://localhost:8080/swagger/index.html` in your browser.
-* Enter your JWT token you generated for the API user by clicking the green Authorize button in the upper right part of the window.
-* Click the `/job/start_job` endpoint and click the Try it out button.
-* Enter the following json into the request body text area and fill in a recent start timestamp by executing `date +%s`.:
-```
-{
-    "jobId":         100000,
-    "arrayJobId":    0,
-    "user":          "ccdemouser",
-    "subCluster":    "main",
-    "cluster":       "emmy",
-    "startTime":    <date +%s>,
-    "project":       "ccdemoproject",
-    "resources":  [
-        {"hostname":  "e0601"},
-        {"hostname":  "e0823"},
-        {"hostname":  "e0337"},
-        {"hostname": "e1111"}],
-    "numNodes":      4,
-    "numHwthreads":  80,
-    "walltime":      86400
-}
-```
-* The response body should be the database id of the started job, for example:
-```
-{
-  "id": 3937
-}
-```
-* Check in ClusterCockpit
-    - User `ccdemouser` should appear in Users-Tab with one running job
-    - It could take up to 5 Minutes until the Job is displayed with some current data (5 Min Short-Job Filter)
-    - Job then is marked with a green `running` tag
-    - Metricdata displayed is read from cc-metric-store!
-
-
-### Use cc-backend API to stop job
-* Enter the URL `http://localhost:8080/swagger/index.html` in your browser.
-* Enter your JWT token you generated for the API user by clicking the green Authorize button in the upper right part of the window.
-* Click the `/job/stop_job/{id}` endpoint and click the Try it out button.
-* Enter the database id at id that was returned by `start_job` and copy the following into the request body. Replace the timestamp with a recent one:
-```
-{
-  "cluster": "emmy",
-  "jobState": "completed",
-  "stopTime": <RECENT TS>
-}
-```
-* On success a json document with the job meta data is returned.
-
-* Check in ClusterCockpit
-    - User `ccdemouser` should appear in Users-Tab with one completed job
-    - Job is no longer marked with a green `running` tag -> Completed!
-    - Metricdata displayed is now read from job-archive!
-* Check in job-archive
-    - `cd ./cc-backend/var/job-archive/emmy/100/000`
-    - `cd $STARTTIME`
-    - Inspect `meta.json` and `data.json`
-
-## Helper scripts
-* In this tarball you can find the perl script `generate_subcluster.pl` that helps to generate the subcluster section for your system.
-Usage:
-* Log into an exclusive cluster node.
-* The LIKWID tools likwid-topology and likwid-bench must be in the PATH!
-* `$./generate_subcluster.pl` outputs the subcluster section on `stdout`
-
-Please be aware that
-* You have to enter the name and node list for the subCluster manually.
-* GPU detection only works if LIKWID was build with Cuda avalable and you run likwid-topology also with Cuda loaded.
-* Do not blindly trust the measured peakflops values.
-* Because the script blindly relies on the CSV format output by likwid-topology this is a fragile undertaking!

docs/JWT-Handling.md

@@ -1,99 +0,0 @@
-## Introduction
-
-ClusterCockpit uses JSON Web Tokens (JWT) for authorization of its APIs. JSON
-Web Token (JWT) is an open standard (RFC 7519) that defines a compact and
-self-contained way for securely transmitting information between parties as a
-JSON object. This information can be verified and trusted because it is
-digitally signed. In ClusterCockpit JWTs are signed using a public/private key
-pair using ECDSA. Because tokens are signed using public/private key pairs, the
-signature also certifies that only the party holding the private key is the one
-that signed it. Token expiration is set to the configuration option MaxAge.
-
-## JWT Payload
-
-You may view the payload of a JWT token at [https://jwt.io/#debugger-io](https://jwt.io/#debugger-io).
-Currently ClusterCockpit sets the following claims:
-* `iat`: Issued at claim. The “iat” claim is used to identify the the time at which the JWT was issued. This claim can be used to determine the age of the JWT.
-* `sub`: Subject claim. Identifies the subject of the JWT, in our case this is the username.
-* `roles`: An array of strings specifying the roles set for the subject.
-
-## Workflow
-
-1. Create a new ECDSA Public/private keypair:
-```
-$ go build ./tools/gen-keypair.go
-$ ./gen-keypair
-```
-2. Add keypair in your `.env` file. A template can be found in `./configs`.
-
-There are two usage scenarios:
-* The APIs are used during a browser session. API accesses are authorized with
-  the active session.
-* The REST API is used outside a browser session, e.g. by scripts. In this case
-  you have to issue a token manually. This possible from within the
-  configuration view or on the command line. It is recommended to issue a JWT
-  token in this case for a special user that only has the `api` role. By using
-  different users for different purposes a fine grained access control and
-  access revocation management is possible.
-
-The token is commonly specified in the Authorization HTTP header using the Bearer schema.
-
-## Setup user and JWT token for REST API authorization
-
-1. Create user:
-```
-$ ./cc-backend --add-user <username>:api:<Password> --no-server
-```
-2. Issue token for user:
-```
-$ ./cc-backend -jwt <username>  -no-server
-```
-3. Use issued token token on client side:
-```
-$ curl -X GET "<API ENDPOINT>" -H "accept: application/json"  -H "Content-Type: application/json"  -H "Authorization: Bearer <JWT TOKEN>"
-```
-
-## Accept externally generated JWTs provided via cookie
-If there is an external service like an AuthAPI that can generate JWTs and hand
-them over to ClusterCockpit via cookies, CC can be configured to accept them:
-
-1. `.env`: CC needs a public ed25519 key to verify foreign JWT signatures.
-   Public keys in PEM format can be converted with the instructions in
-   [/tools/convert-pem-pubkey-for-cc](../tools/convert-pem-pubkey-for-cc/Readme.md)
-   .
-
-```
-CROSS_LOGIN_JWT_PUBLIC_KEY="+51iXX8BdLFocrppRxIw52xCOf8xFSH/eNilN5IHVGc="
-```
-
-2. `config.json`: Insert a name for the cookie (set by the external service)
-   containing the JWT so that CC knows where to look at. Define a trusted issuer
-   (JWT claim 'iss'), otherwise it will be rejected. If you want usernames and
-   user roles from JWTs ('sub' and 'roles' claim) to be validated against CC's
-   internal database, you need to enable it here. Unknown users will then be
-   rejected and roles set via JWT will be ignored.
-
-```json
-"jwts": {
-    "cookieName": "access_cc",
-    "forceJWTValidationViaDatabase": true,
-    "trustedExternalIssuer": "auth.example.com"
-}
-```
-
-3. Make sure your external service includes the same issuer (`iss`) in its JWTs.
-   Example JWT payload:
-
-```json
-{
-  "iat": 1668161471,
-  "nbf": 1668161471,
-  "exp": 1668161531,
-  "sub": "alice",
-  "roles": [
-    "user"
-  ],
-  "jti": "a1b2c3d4-1234-5678-abcd-a1b2c3d4e5f6",
-  "iss": "auth.example.com"
-}
-```

docs/Job-Archive.md

@@ -1,78 +0,0 @@
-The job archive specifies an exchange format for job meta and performance metric
-data. It consists of two parts:
-* a [SQLite database schema](https://github.com/ClusterCockpit/cc-backend/wiki/Job-Archive#sqlite-database-schema)  for job meta data and performance statistics
-* a [Json file format](https://github.com/ClusterCockpit/cc-backend/wiki/Job-Archive#json-file-format) together with a [Directory hierarchy specification](https://github.com/ClusterCockpit/cc-backend/wiki/Job-Archive#directory-hierarchy-specification)
-
-By using an open, portable and simple specification based on files it is
-possible to exchange job performance data for research and analysis purposes as
-well as use it as a robust way for archiving job performance data to disk.
-
-# SQLite database schema
-## Introduction
-
-A SQLite 3 database schema is provided to standardize the job meta data
-information in a portable way. The schema also includes optional columns for job
-performance statistics (called a job performance footprint). The database acts
-as a front end to filter and select subsets of job IDs, that are the keys to get
-the full job performance data in the job performance tree hierarchy.
-
-## Database schema
-
-The schema includes 3 tables: the job table, a tag table and a jobtag table
-representing the MANY-TO-MANY relation between jobs and tags. The SQL schema is
-specified
-[here](https://github.com/ClusterCockpit/cc-specifications/blob/master/schemas/jobs-sqlite.sql).
-Explanation of the various columns including the JSON datatypes is documented
-[here](https://github.com/ClusterCockpit/cc-specifications/blob/master/datastructures/job-meta.schema.json).
-
-# Directory hierarchy specification
-
-## Specification
-
-To manage the number of directories within a single directory a tree approach is
-used splitting the integer job ID. The job id is split in junks of 1000 each.
-Usually 2 layers of directories is sufficient but the concept can be used for an
-arbitrary number of layers.
-
-For a 2 layer schema this can be achieved with (code example in Perl):
-``` perl
-$level1 = $jobID/1000;
-$level2 = $jobID%1000;
-$dstPath = sprintf("%s/%s/%d/%03d", $trunk, $destdir, $level1, $level2);
-```
-
-## Example
-
-For the job ID 1034871 the directory path is `./1034/871/`.
-
-# Json file format
-## Overview
-
-Every cluster must be configured in a `cluster.json` file.
-
-The job data consists of two files:
-* `meta.json`: Contains job meta information and job statistics.
-* `data.json`: Contains complete job data with time series
-
-The description of the json format specification is available as [[json
-schema|https://json-schema.org/]] format file. The latest version of the json
-schema is part of the `cc-backend` source tree. For external reference it is
-also available in a separate repository.
-
-## Specification `cluster.json`
-
-The json schema specification is available
-[here](https://github.com/ClusterCockpit/cc-specifications/blob/master/datastructures/cluster.schema.json).
-
-## Specification `meta.json`
-
-The json schema specification is available
-[here](https://github.com/RRZE-HPC/HPCJobDatabase/blob/master/json-schema/job-meta.schema.json).
-
-## Specification `data.json`
-
-The json schema specification is available
-[here](https://github.com/RRZE-HPC/HPCJobDatabase/blob/master/json-schema/job-data.schema.json).
-Metric time series data is stored for a fixed time step. The time step is set
-per metric. If no value is available for a metric time series data timestamp
-`null` is entered. 

docs/adm-customization.md

@@ -1,29 +0,0 @@
-# Overview
-
-Customizing `cc-backend` means changing the logo, legal texts, and the login
-template instead of the placeholders. You can also place a text file in `./var`
-to add dynamic status or notification messages to the clusterCockpit homepage.
-
-# Replace legal texts
-
-To replace the `imprint.tmpl` and `privacy.tmpl` legal texts, you can place your
-version in `./var/`. At startup `cc-backend` will check if `./var/imprint.tmpl` and/or
-`./var/privacy.tmpl` exist and use them instead of the built-in placeholders.
-You can use the placeholders in `web/templates` as a blueprint.
-
-# Replace login template
-To replace the default login layout and styling, you can place your version in
-`./var/`. At startup `cc-backend` will check if `./var/login.tmpl` exist and use
-it instead of the built-in placeholder. You can use the default temaplte
-`web/templates/login.tmpl` as a blueprint.
-
-# Replace logo
-To change the logo displayed in the navigation bar, you can provide the file
-`logo.png` in the folder `./var/img/`. On startup `cc-backend` will check if the
-folder exists and use the images provided there instead of the built-in images.
-You may also place additional images there you use in a custom login template.
-
-# Add notification banner on homepage
-To add a notification banner you can add a file `notice.txt` to `./var`. As long
-as this file is present all text in this file is shown in an info banner on the
-homepage.

docs/adm-upgrade.md

@@ -1,78 +0,0 @@
-In general, an upgrade is nothing more than a replacement of the binary file.
-All the necessary files, except the database file, the configuration file and
-the job archive, are embedded in the binary file. It is recommended to use a
-directory where the file names of the binary files are named with a version
-indicator. This can be, for example, the date or the Unix epoch time. A symbolic
-link points to the version to be used. This makes it easier to switch to earlier
-versions.
-
-The database and the job archive are versioned. Each release binary supports
-specific versions of the database and job archive. If a version mismatch is
-detected, the application is terminated and migration is required.
-
-**IMPORTANT NOTE**
-
-It is recommended to make a backup copy of the database before each update. This
-is mandatory in case the database needs to be migrated. In the case of sqlite,
-this means to stopping `cc-backend` and copying the sqlite database file
-somewhere.
-
-#  Migrating the database
-
-After you have backed up the database, run the following command to migrate the
-database to the latest version:
-```
-$ ./cc-backend -migrate-db
-```
-
-The migration files are embedded in the binary and can also be viewed in the cc
-backend [source tree](https://github.com/ClusterCockpit/cc-backend/tree/master/internal/repository/migrations).
-There are separate migration files for both supported
-database backends.
-We use the [migrate library](https://github.com/golang-migrate/migrate).
-
-If something goes wrong, you can check the status and get the current schema
-(here for sqlite):
-```
-$ sqlite3 var/job.db
-```
-In the sqlite console execute:
-```
-.schema
-```
-to get the current databse schema.
-You can query the current version and whether the migration failed with:
-```
-SELECT * FROM schema_migrations;
-```
-The first column indicates the current database version and the second column is
-a dirty flag indicating whether the migration was successful.
-
-# Migrating the job archive
-
-Job archive migration requires a separate tool (`archive-migration`), which is
-part of the cc-backend source tree (build with `go build ./tools/archive-migration`)
-and is also provided as part of the releases.
-
-Migration is supported only between two successive releases. The migration tool
-migrates the existing job archive to a new job archive. This means that there
-must be enough disk space for two complete job archives. If the tool is called
-without options:
-```
-$ ./archive-migration
-```
-
-it is assumed that a job archive exists in `./var/job-archive`. The new job
-archive is written to `./var/job-archive-new`. Since execution is threaded in case
-of a fatal error, it is impossible to determine in which job the error occurred.
-In this case, you can run the tool in debug mode (with the `-debug` flag). In
-debug mode, threading is disabled and the job ID of each migrated job is output.
-Jobs with empty files will be skipped. Between multiple runs of the tools, the
-`job-archive-new` directory must be moved or deleted.
-
-The `cluster.json` files in `job-archive-new` must be checked for errors, especially
-whether the aggregation attribute is set correctly for all metrics.
-
-Migration takes several hours for relatively large job archives (several hundred
-GB). A versioned job archive contains a version.txt file in the root directory
-of the job archive. This file contains the version as an unsigned integer.

docs/dev-authentication.md

@@ -1,180 +0,0 @@
-# Overview
-
-The authentication is implemented in `internal/auth/`. In `auth.go`
-an interface is defined that any authentication provider must fulfill. It also
-acts as a dispatcher to delegate the calls to the available authentication
-providers.
-
-Two authentication types are available:
-* JWT authentication for the REST API that does not create a session cookie
-* Session based authentication using a session cookie
-
-The most important routines in auth are:
-* `Login()` Handle POST request to login user and start a new session
-* `Auth()`  Authenticate user and put User Object in context of the request
-
-The http router calls auth in the following cases:
-* `r.Handle("/login", authentication.Login( ... )).Methods(http.MethodPost)`:
-  The POST request on the `/login` route will call the Login callback.
-* `r.Handle("/jwt-login", authentication.Login( ... ))`:
-  Any request on the `/jwt-login` route will call the Login callback. Intended
-  for use for the JWT token based authenticators.
-* Any route in the secured subrouter will always call Auth(), on success it will
-  call the next handler in the chain, on failure it will render the login
-  template.
-```
-secured.Use(func(next http.Handler) http.Handler {
-	return authentication.Auth(
-		// On success;
-		next,
-
-		// On failure:
-		func(rw http.ResponseWriter, r *http.Request, err error) {
-               // Render login form
-		})
-})
-```
-
-A JWT token can be used to initiate an authenticated user
-session. This can either happen by calling the login route with a token
-provided in a header or via a special cookie containing the JWT token.
-For API routes the access is authenticated on every request using the JWT token
-and no session is initiated.
-
-# Login
-
-The Login function (located in `auth.go`):
-* Extracts the user name and gets the user from the user database table. In case the
-  user is not found the user object is set to nil.
-* Iterates over all authenticators and:
-  - Calls its `CanLogin` function which checks if the authentication method is
-    supported for this user.
-  - Calls its `Login` function to authenticate the user. On success a valid user
-    object is returned.
-  - Creates a new session object, stores the user attributes in the session and
-    saves the session.
-  - Starts the `onSuccess` http handler
-
-## Local authenticator
-
-This authenticator is applied if 
-```
-return user != nil && user.AuthSource == AuthViaLocalPassword
-```
-
-Compares the password provided by the login form to the password hash stored in
-the user database table:
-```
-if e := bcrypt.CompareHashAndPassword([]byte(user.Password), []byte(r.FormValue("password"))); e != nil {
-	log.Errorf("AUTH/LOCAL > Authentication for user %s failed!", user.Username)
-	return nil, fmt.Errorf("Authentication failed")
-}
-```
-
-## LDAP authenticator
-
-This authenticator is applied if the user was found in the database and its
-AuthSource is LDAP:
-```
-if user != nil {
-	if user.AuthSource == schema.AuthViaLDAP {
-		return user, true
-	}
-} 
-```
-
-If the option `SyncUserOnLogin` is set it tried to sync the user from the LDAP
-directory. In case this succeeds the user is persisted to the database and can
-login.
-
-Gets the LDAP connection and tries a bind with the provided credentials:
-```
-if err := l.Bind(userDn, r.FormValue("password")); err != nil {
-	log.Errorf("AUTH/LDAP > Authentication for user %s failed: %v", user.Username, err)
-	return nil, fmt.Errorf("Authentication failed")
-}
-```
-
-## JWT Session authenticator
-
-Login via JWT token will create a session without password.
-For login the `X-Auth-Token` header is not supported. This authenticator is
-applied if the Authorization header or query parameter login-token is present:
-```
-	return user, r.Header.Get("Authorization") != "" ||
-		r.URL.Query().Get("login-token") != ""
-```
-
-The Login function:
-* Parses the token and checks if it is expired
-* Check if the signing method is EdDSA or HS256 or HS512
-* Check if claims are valid and extracts the claims
-* The following claims have to be present:
-   - `sub`: The subject, in this case this is the username
-   - `exp`: Expiration in Unix epoch time
-   - `roles`: String array with roles of user
-* In case user does not exist in the database and the option `SyncUserOnLogin`
-  is set add user to user database table with `AuthViaToken` AuthSource.
-* Return valid user object
-
-## JWT Cookie Session authenticator
-
-Login via JWT cookie token will create a session without password.
-It is first checked if the required configuration options are set:
-* `trustedIssuer`
-* `CookieName`
-
-and optionally the environment variable `CROSS_LOGIN_JWT_PUBLIC_KEY` is set.
-
-This authenticator is applied if the configured cookie is present:
-```
-	jwtCookie, err := r.Cookie(cookieName)
-
-	if err == nil && jwtCookie.Value != "" {
-		return true
-	}
-```
-
-The Login function:
-* Extracts and parses the token
-* Checks if signing method is Ed25519/EdDSA 
-* In case publicKeyCrossLogin is configured:
-   - Check if `iss` issuer claim matched trusted issuer from configuration
-   - Return public cross login key
-   - Otherwise return standard public key
-* Check if claims are valid
-* Depending on the option `validateUser` the roles are
-  extracted from JWT token or taken from user object fetched from database
-* Ask browser to delete the JWT cookie
-* In case user does not exist in the database and the option `SyncUserOnLogin`
-  is set add user to user database table with `AuthViaToken` AuthSource.
-* Return valid user object
-
-# Auth
-
-The Auth function (located in `auth.go`):
-* Returns a new http handler function that is defined right away
-* This handler tries two methods to authenticate a user:
-   - Via a JWT API token in `AuthViaJWT()`
-   - Via a valid session in `AuthViaSession()`
-* If err is not nil and the user object is valid it puts the user object in the
-  request context and starts the onSuccess http handler
-* Otherwise it calls the onFailure handler
-
-## AuthViaJWT
-
-Implemented in JWTAuthenticator:
-* Extract token either from header `X-Auth-Token` or `Authorization` with Bearer
-  prefix
-* Parse token and check if it is valid. The Parse routine will also check if the
-  token is expired.
-* If the option `validateUser` is set it will ensure the
-  user object exists in the database and takes the roles from the database user
-* Otherwise the roles are extracted from the roles claim
-* Returns a valid user object with AuthType set to AuthToken
-
-## AuthViaSession
-
-* Extracts session
-* Get values username, projects, and roles from session
-* Returns a valid user object with AuthType set to AuthSession

docs/dev-frontend.md

@@ -1,33 +0,0 @@
-## Tips for frontend development
-
-The frontend assets including the Svelte js files are per default embedded in
-the bgo binary. To enable a quick turnaround cycle for web development of the
-frontend disable embedding of static assets in `config.json`:
-```
-"embed-static-files": false,
-"static-files": "./web/frontend/public/",
-
-```
-
-Start the node build process (in directory `./web/frontend`) in development mode:
-```
-$ npm run dev
-```
-
-This will start the build process in listen mode. Whenever you change a source
-files the depending javascript targets will be automatically rebuild.
-In case the javascript files are minified you may need to set the production
-flag by hand to false in `./web/frontend/rollup.config.mjs`:
-```
-const production = false
-```
-
-Usually this should work automatically.
-
-Because the files are still served by ./cc-backend you have to reload the view
-explicitly in your browser.
-
-A common setup is to have three terminals open:
-* One running cc-backend (working directory repository root): `./cc-backend -server -dev`
-* Another running npm in developer mode (working directory `./web/frontend`): `npm run dev`
-* And the last one editing the frontend source files

docs/dev-release.md

@@ -1,13 +0,0 @@
-# Steps to prepare a release
-
-1. On `hotfix` branch:
-   * Update ReleaseNotes.md
-   * Update version in Makefile
-   * Commit, push, and pull request
-   * Merge in master
-
-2. On Linux host:
-   * Pull master
-   * Ensure that GitHub Token environment variable `GITHUB_TOKEN` is set
-   * Create release tag: `git tag v1.1.0 -m release`
-   * Execute `goreleaser release`

docs/dev-testing.md

@@ -1,34 +0,0 @@
-## Overview
-
-We use the standard golang testing environment.
-
-The following conventions are used:
-
-* *White box unit tests*: Tests for internal functionality are placed in files
-* *Black box unit tests*: Tests for public interfaces are placed in files
-with `<package name>_test.go` and belong to the package `<package_name>_test`.
-There only exists one package test file per package.
-* *Integration tests*: Tests that use multiple componenents are placed in a
-package test file. These are named `<package name>_test.go` and belong to the
-package `<package_name>_test`.
-* *Test assets*: Any required files are placed in a directory `./testdata`
-within each package directory.
-
-## Executing tests
-
-Visual Studio Code has a very good golang test integration.
-For debugging a test this is the recommended solution.
-
-The Makefile provided by us has a `test` target that executes:
-```
-$ go clean -testcache
-$ go build ./...
-$ go vet ./...
-$ go test ./...
-```
-
-Of course the commands can also be used on the command line.
-For details about golang testing refer to the standard documentation:
-
-* [Testing package](https://pkg.go.dev/testing)
-* [go test command](https://pkg.go.dev/cmd/go#hdr-Test_packages)

docs/migrateTimestamps.pl

@@ -1,229 +0,0 @@
-#!/usr/bin/env perl
-use strict;
-use warnings;
-use utf8;
-
-use JSON::PP; # from Perl default install
-use Time::Local qw( timelocal ); # from Perl default install
-use Time::Piece; # from Perl default install
-
-### JSON
-my $json = JSON::PP->new->allow_nonref;
-
-### TIME AND DATE
-# now
-my $localtime = localtime;
-my $epochtime = $localtime->epoch;
-# 5 days ago: Via epoch due to possible reverse month borders
-my $epochlessfive = $epochtime - (86400 * 5);
-my $locallessfive = localtime($epochlessfive);
-# Calc like `date --date 'TZ="Europe/Berlin" 0:00 5 days ago' +%s`)
-my ($day, $month, $year) = ($locallessfive->mday, $locallessfive->_mon, $locallessfive->year);
-my $checkpointStart = timelocal(0, 0, 0, $day, $month, $year);
-# for checkpoints
-my $halfday = 43200;
-
-### JOB-ARCHIVE
-my $archiveTarget = './cc-backend/var/job-archive';
-my $archiveSrc = './source-data/job-archive-source';
-my @ArchiveClusters;
-
-# Gen folder
-if ( not -d $archiveTarget ){
-    mkdir( $archiveTarget ) or die "Couldn't create $archiveTarget directory, $!";
-}
-
-# Get clusters by job-archive/$subfolder
-opendir my $dh, $archiveSrc  or die "can't open directory: $!";
-while ( readdir $dh ) {
-    chomp; next if $_ eq '.' or $_ eq '..'  or $_ eq 'job-archive';
-    my $cluster = $_;
-    push @ArchiveClusters, $cluster;
-}
-
-# start for jobarchive
-foreach my $cluster ( @ArchiveClusters ) {
-    print "Starting to update start- and stoptimes in job-archive for $cluster\n";
-
-    my $clusterTarget = "$archiveTarget/$cluster";
-
-    if ( not -d $clusterTarget ){
-        mkdir( $clusterTarget ) or die "Couldn't create $clusterTarget directory, $!";
-    }
-
-    opendir my $dhLevel1, "$archiveSrc/$cluster" or die "can't open directory: $!";
-    while ( readdir $dhLevel1 ) {
-        chomp; next if $_ eq '.' or $_ eq '..';
-        my $level1 = $_;
-
-        if ( -d "$archiveSrc/$cluster/$level1" ) {
-            opendir my $dhLevel2, "$archiveSrc/$cluster/$level1" or die "can't open directory: $!";
-            while ( readdir $dhLevel2 ) {
-                chomp; next if $_ eq '.' or $_ eq '..';
-                my $level2 = $_;
-                my $jobSource = "$archiveSrc/$cluster/$level1/$level2";
-                my $jobOrigin = "$jobSource";
-                my $jobTargetL1 = "$clusterTarget/$level1";
-                my $jobTargetL2 = "$jobTargetL1/$level2";
-
-                # check if files are directly accessible (old format) else get subfolders as file and update path
-                if ( ! -e "$jobSource/meta.json") {
-                    opendir(D, "$jobSource") || die "Can't open directory $jobSource: $!\n";
-                    my @folders = readdir(D);
-                    closedir(D);
-                    if (!@folders) {
-                        next;
-                    }
-
-                    foreach my $folder ( @folders ) {
-                        next if $folder eq '.' or $folder eq '..';
-                        $jobSource = "$jobSource/".$folder;
-                    }
-                }
-                # check if subfolder contains file, else skip
-                if ( ! -e "$jobSource/meta.json") {
-                    print "$jobSource skipped\n";
-                    next;
-                }
-
-                open my $metafh, '<', "$jobSource/meta.json" or die "Can't open file $!";
-                my $rawstr = do { local $/; <$metafh> };
-                close($metafh);
-                my $metadata = $json->decode($rawstr);
-
-                # NOTE Start meta.json iteration here
-                # my $random_number = int(rand(UPPERLIMIT)) + LOWERLIMIT;
-                # Set new startTime: Between 5 days and 1 day before now
-
-                #  Remove id from attributes
-                $metadata->{startTime} = $epochtime - (int(rand(432000)) + 86400);
-                $metadata->{stopTime} = $metadata->{startTime} + $metadata->{duration};
-
-                # Add starttime subfolder to target path
-                my $jobTargetL3 = "$jobTargetL2/".$metadata->{startTime};
-
-                if ( not -d $jobTargetL1 ){
-                    mkdir( $jobTargetL1 ) or die "Couldn't create $jobTargetL1 directory, $!";
-                }
-
-                if ( not -d $jobTargetL2 ){
-                    mkdir( $jobTargetL2 ) or die "Couldn't create $jobTargetL2 directory, $!";
-                }
-
-                # target is not directory
-                if ( not -d $jobTargetL3 ){
-                    mkdir( $jobTargetL3 ) or die "Couldn't create $jobTargetL3 directory, $!";
-
-                    my $outstr = $json->encode($metadata);
-                    open my $metaout, '>', "$jobTargetL3/meta.json" or die "Can't write to file $!";
-                    print $metaout $outstr;
-                    close($metaout);
-
-                    open my $datafh, '<', "$jobSource/data.json" or die "Can't open file $!";
-                    my $datastr = do { local $/; <$datafh> };
-                    close($datafh);
-
-                    open my $dataout, '>', "$jobTargetL3/data.json" or die "Can't write to file $!";
-                    print $dataout $datastr;
-                    close($dataout);
-                }
-            }
-        }
-    }
-}
-print "Done for job-archive\n";
-sleep(1);
-exit;
-
-## CHECKPOINTS
-my $checkpTarget = './cc-metric-store/var/checkpoints';
-my $checkpSource = './source-data/cc-metric-store-source/checkpoints';
-my @CheckpClusters;
-
-# Gen folder
-if ( not -d $checkpTarget ){
-    mkdir( $checkpTarget ) or die "Couldn't create $checkpTarget directory, $!";
-}
-
-# Get clusters by cc-metric-store/$subfolder
-opendir my $dhc, $checkpSource  or die "can't open directory: $!";
-while ( readdir $dhc ) {
-    chomp; next if $_ eq '.' or $_ eq '..'  or $_ eq 'job-archive';
-    my $cluster = $_;
-    push @CheckpClusters, $cluster;
-}
-closedir($dhc);
-
-# start for checkpoints
-foreach my $cluster ( @CheckpClusters ) {
-    print "Starting to update checkpoint filenames and data starttimes for $cluster\n";
-
-    my $clusterTarget = "$checkpTarget/$cluster";
-
-    if ( not -d $clusterTarget ){
-        mkdir( $clusterTarget ) or die "Couldn't create $clusterTarget directory, $!";
-    }
-
-    opendir my $dhLevel1, "$checkpSource/$cluster" or die "can't open directory: $!";
-    while ( readdir $dhLevel1 ) {
-        chomp; next if $_ eq '.' or $_ eq '..';
-        # Nodename as level1-folder
-        my $level1 = $_;
-
-        if ( -d "$checkpSource/$cluster/$level1" ) {
-
-            my $nodeSource = "$checkpSource/$cluster/$level1/";
-            my $nodeOrigin = "$nodeSource";
-            my $nodeTarget = "$clusterTarget/$level1";
-            my @files;
-
-            if ( -e "$nodeSource/1609459200.json") { # 1609459200 == First Checkpoint time in latest dump
-                opendir(D, "$nodeSource") || die "Can't open directory $nodeSource: $!\n";
-                while ( readdir D ) {
-                    chomp; next if $_ eq '.' or $_ eq '..';
-                    my $nodeFile = $_;
-                    push @files, $nodeFile;
-                }
-                closedir(D);
-                my $length = @files;
-                if (!@files || $length != 14) { # needs 14 files == 7 days worth of data
-                    next;
-                }
-            } else {
-                next;
-            }
-
-            # sort for integer timestamp-filename-part (moduleless): Guarantees start with index == 0 == 1609459200.json
-            my @sortedFiles = sort { ($a =~ /^([0-9]{10}).json$/)[0] <=> ($b =~ /^([0-9]{10}).json$/)[0] } @files;
-
-            if ( not -d $nodeTarget ){
-                mkdir( $nodeTarget ) or die "Couldn't create $nodeTarget directory, $!";
-
-                while (my ($index, $file) = each(@sortedFiles)) {
-                    open my $checkfh, '<', "$nodeSource/$file" or die "Can't open file $!";
-                    my $rawstr = do { local $/; <$checkfh> };
-                    close($checkfh);
-                    my $checkpdata = $json->decode($rawstr);
-
-                    my $newTimestamp = $checkpointStart + ($index * $halfday);
-                    # Get Diff from old Timestamp
-                    my $timeDiff = $newTimestamp - $checkpdata->{from};
-                    # Set new timestamp
-                    $checkpdata->{from} = $newTimestamp;
-
-                    foreach my $metric (keys %{$checkpdata->{metrics}}) {
-                        $checkpdata->{metrics}->{$metric}->{start} += $timeDiff;
-                    }
-
-                    my $outstr = $json->encode($checkpdata);
-
-                    open my $checkout, '>', "$nodeTarget/$newTimestamp.json" or die "Can't write to file $!";
-                    print $checkout $outstr;
-                    close($checkout);
-                }
-            }
-        }
-    }
-    closedir($dhLevel1);
-}
-print "Done for checkpoints\n";

docs/searchbar.md

@@ -1,36 +0,0 @@
-# Docs for ClusterCockpit Searchbar
-
-## Usage
-
-* Searchtags are implemented as `type:<query>` search-string
-  * Types `jobId, jobName, projectId, username, name, arrayJobId` for roles `admin` and `support`
-    * `jobName` is jobName as persisted in `job.meta_data` table-column
-    * `username` is actual account identifier as persisted in `job.user` table-column
-    * `name` is account owners name as persisted in `user.name` table-column
-  * Types `jobId, jobName, projectId, arrayJobId` for role `user`
-  * Examples:
-    * `jobName:myJob12`
-    * `jobId:123456`
-    * `username:abcd100`
-    * `name:Paul`
-* If no searchTag used: Best guess search with the following hierarchy
-  * `jobId -> username -> name -> projectId -> jobName`
-* Destinations:
-  * JobId: Job-Table (Allows multiple identical matches, e.g. JobIds from different clusters)
-  * JobName: Job-Table (Allows multiple identical matches, e.g. JobNames from different clusters)
-  * ProjectId: Job-Table
-  * Username: Users-Table
-    * **Please Note**: Only users with jobs will be shown in table! I.e., Users without jobs will be missing in table. Also, a `Last 30 Days` is active by default and might filter out expected users.
-  * Name: Users-Table
-    * **Please Note**: Only users with jobs will be shown in table! I.e., Users without jobs will be missing in table. Also, a `Last 30 Days` is active by default and might filter out expected users.
-  * ArrayJobId: Job-Table (Lists all Jobs of Queried ArrayJobId)
-  * Best guess search always redirects to Job-Table or `/monitoring/user/$USER` (first username match)
-  * Unprocessable queries will display messages detailing the cause (Info, Warning, Error)
-* Spaces trimmed (both for searchTag and queryString)
-  * `  job12` == `job12`
-  * `projectID : abcd ` == `projectId:abcd`
-* `jobName`- and `name-`queries work with a part of the target-string
-  * `jobName:myjob` for jobName "myjob_cluster1"
-  * `name:Paul` for name "Paul Atreides"
-
-* JobName GQL Query is resolved as matching the query as a part of the whole metaData-JSON in the SQL DB.

go.mod

@@ -1,88 +1,91 @@
 module github.com/ClusterCockpit/cc-backend
 
-go 1.18
+go 1.23.5
+
+toolchain go1.24.1
 
 require (
-	github.com/99designs/gqlgen v0.17.36
-	github.com/ClusterCockpit/cc-units v0.4.0
-	github.com/Masterminds/squirrel v1.5.3
-	github.com/go-co-op/gocron v1.25.0
-	github.com/go-ldap/ldap/v3 v3.4.4
-	github.com/go-sql-driver/mysql v1.7.0
-	github.com/golang-jwt/jwt/v4 v4.5.0
-	github.com/golang-migrate/migrate/v4 v4.15.2
-	github.com/google/gops v0.3.27
-	github.com/gorilla/handlers v1.5.1
-	github.com/gorilla/mux v1.8.0
-	github.com/gorilla/sessions v1.2.1
-	github.com/influxdata/influxdb-client-go/v2 v2.12.2
-	github.com/jmoiron/sqlx v1.3.5
-	github.com/mattn/go-sqlite3 v1.14.16
-	github.com/prometheus/client_golang v1.14.0
-	github.com/prometheus/common v0.40.0
+	github.com/99designs/gqlgen v0.17.66
+	github.com/ClusterCockpit/cc-lib v0.3.0
+	github.com/Masterminds/squirrel v1.5.4
+	github.com/coreos/go-oidc/v3 v3.12.0
+	github.com/expr-lang/expr v1.17.3
+	github.com/go-co-op/gocron/v2 v2.16.0
+	github.com/go-ldap/ldap/v3 v3.4.10
+	github.com/go-sql-driver/mysql v1.9.0
+	github.com/golang-jwt/jwt/v5 v5.2.2
+	github.com/golang-migrate/migrate/v4 v4.18.2
+	github.com/google/gops v0.3.28
+	github.com/gorilla/handlers v1.5.2
+	github.com/gorilla/mux v1.8.1
+	github.com/gorilla/sessions v1.4.0
+	github.com/jmoiron/sqlx v1.4.0
+	github.com/joho/godotenv v1.5.1
+	github.com/mattn/go-sqlite3 v1.14.24
+	github.com/prometheus/client_golang v1.22.0
+	github.com/prometheus/common v0.63.0
 	github.com/qustavo/sqlhooks/v2 v2.1.0
-	github.com/santhosh-tekuri/jsonschema/v5 v5.2.0
-	github.com/swaggo/http-swagger v1.3.3
-	github.com/swaggo/swag v1.16.1
-	github.com/vektah/gqlparser/v2 v2.5.8
-	golang.org/x/crypto v0.12.0
-	golang.org/x/exp v0.0.0-20230510235704-dd950f8aeaea
+	github.com/santhosh-tekuri/jsonschema/v5 v5.3.1
+	github.com/swaggo/http-swagger v1.3.4
+	github.com/swaggo/swag v1.16.4
+	github.com/vektah/gqlparser/v2 v2.5.22
+	golang.org/x/crypto v0.37.0
+	golang.org/x/oauth2 v0.27.0
+	golang.org/x/time v0.5.0
 )
 
 require (
+	filippo.io/edwards25519 v1.1.0 // indirect
 	github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect
 	github.com/KyleBanks/depth v1.2.1 // indirect
-	github.com/agnivade/levenshtein v1.1.1 // indirect
-	github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect
+	github.com/agnivade/levenshtein v1.2.1 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
-	github.com/cespare/xxhash/v2 v2.2.0 // indirect
-	github.com/containerd/containerd v1.6.18 // indirect
-	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
-	github.com/deepmap/oapi-codegen v1.12.4 // indirect
-	github.com/felixge/httpsnoop v1.0.3 // indirect
-	github.com/go-asn1-ber/asn1-ber v1.5.4 // indirect
-	github.com/go-openapi/jsonpointer v0.20.0 // indirect
-	github.com/go-openapi/jsonreference v0.20.2 // indirect
-	github.com/go-openapi/spec v0.20.9 // indirect
-	github.com/go-openapi/swag v0.22.4 // indirect
-	github.com/golang/protobuf v1.5.2 // indirect
-	github.com/google/uuid v1.3.0 // indirect
-	github.com/gorilla/securecookie v1.1.1 // indirect
-	github.com/gorilla/websocket v1.5.0 // indirect
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
+	github.com/cpuguy83/go-md2man/v2 v2.0.6 // indirect
+	github.com/felixge/httpsnoop v1.0.4 // indirect
+	github.com/fsnotify/fsnotify v1.9.0 // indirect
+	github.com/go-asn1-ber/asn1-ber v1.5.7 // indirect
+	github.com/go-jose/go-jose/v4 v4.0.5 // indirect
+	github.com/go-openapi/jsonpointer v0.21.0 // indirect
+	github.com/go-openapi/jsonreference v0.21.0 // indirect
+	github.com/go-openapi/spec v0.21.0 // indirect
+	github.com/go-openapi/swag v0.23.0 // indirect
+	github.com/go-viper/mapstructure/v2 v2.2.1 // indirect
+	github.com/google/uuid v1.6.0 // indirect
+	github.com/gorilla/securecookie v1.1.2 // indirect
+	github.com/gorilla/websocket v1.5.3 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
-	github.com/hashicorp/golang-lru/v2 v2.0.3 // indirect
-	github.com/influxdata/line-protocol v0.0.0-20210922203350-b1ad95c89adf // indirect
+	github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
+	github.com/jonboulle/clockwork v0.5.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/jpillora/backoff v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect
 	github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect
-	github.com/mailru/easyjson v0.7.7 // indirect
-	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
-	github.com/mitchellh/mapstructure v1.5.0 // indirect
+	github.com/mailru/easyjson v0.9.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect
-	github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 // indirect
-	github.com/pkg/errors v0.9.1 // indirect
-	github.com/prometheus/client_model v0.3.0 // indirect
-	github.com/prometheus/procfs v0.9.0 // indirect
+	github.com/prometheus/client_model v0.6.2 // indirect
+	github.com/prometheus/procfs v0.16.1 // indirect
 	github.com/robfig/cron/v3 v3.0.1 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
-	github.com/swaggo/files v1.0.0 // indirect
-	github.com/urfave/cli/v2 v2.25.7 // indirect
-	github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
-	go.uber.org/atomic v1.10.0 // indirect
-	golang.org/x/mod v0.12.0 // indirect
-	golang.org/x/net v0.14.0 // indirect
-	golang.org/x/oauth2 v0.5.0 // indirect
-	golang.org/x/sys v0.11.0 // indirect
-	golang.org/x/text v0.12.0 // indirect
-	golang.org/x/tools v0.12.0 // indirect
-	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/protobuf v1.30.0 // indirect
+	github.com/sosodev/duration v1.3.1 // indirect
+	github.com/swaggo/files v1.0.1 // indirect
+	github.com/urfave/cli/v2 v2.27.5 // indirect
+	github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect
+	go.uber.org/atomic v1.11.0 // indirect
+	golang.org/x/exp v0.0.0-20250408133849-7e4ce0ab07d0 // indirect
+	golang.org/x/mod v0.24.0 // indirect
+	golang.org/x/net v0.39.0 // indirect
+	golang.org/x/sync v0.13.0 // indirect
+	golang.org/x/sys v0.32.0 // indirect
+	golang.org/x/text v0.24.0 // indirect
+	golang.org/x/tools v0.32.0 // indirect
+	google.golang.org/protobuf v1.36.6 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
-	sigs.k8s.io/yaml v1.3.0 // indirect
+	sigs.k8s.io/yaml v1.4.0 // indirect
 )

gqlgen.yml

@@ -30,6 +30,7 @@ resolver:
 # gqlgen will search for any type names in the schema in these go packages
 # if they match it will use them, otherwise it will generate them.
 autobind:
+  - "github.com/99designs/gqlgen/graphql/introspection"
   - "github.com/ClusterCockpit/cc-backend/internal/graph/model"
 
 # This section declares type mapping between the GraphQL and go type systems
@@ -50,34 +51,52 @@ models:
       - github.com/99designs/gqlgen/graphql.Int64
       - github.com/99designs/gqlgen/graphql.Int32
   Job:
-    model: "github.com/ClusterCockpit/cc-backend/pkg/schema.Job"
+    model: "github.com/ClusterCockpit/cc-lib/schema.Job"
     fields:
       tags:
         resolver: true
       metaData:
         resolver: true
   Cluster:
-    model: "github.com/ClusterCockpit/cc-backend/pkg/schema.Cluster"
+    model: "github.com/ClusterCockpit/cc-lib/schema.Cluster"
     fields:
       partitions:
         resolver: true
-  NullableFloat: { model: "github.com/ClusterCockpit/cc-backend/pkg/schema.Float" }
-  MetricScope: { model: "github.com/ClusterCockpit/cc-backend/pkg/schema.MetricScope" }
-  MetricValue: { model: "github.com/ClusterCockpit/cc-backend/pkg/schema.MetricValue" }
-  JobStatistics: { model: "github.com/ClusterCockpit/cc-backend/pkg/schema.JobStatistics" }
-  Tag: { model: "github.com/ClusterCockpit/cc-backend/pkg/schema.Tag" }
-  Resource: { model: "github.com/ClusterCockpit/cc-backend/pkg/schema.Resource" }
-  JobState: { model: "github.com/ClusterCockpit/cc-backend/pkg/schema.JobState" }
-  TimeRange: { model: "github.com/ClusterCockpit/cc-backend/pkg/schema.TimeRange" }
-  IntRange: { model: "github.com/ClusterCockpit/cc-backend/pkg/schema.IntRange" }
-  JobMetric: { model: "github.com/ClusterCockpit/cc-backend/pkg/schema.JobMetric" }
-  Series: { model: "github.com/ClusterCockpit/cc-backend/pkg/schema.Series" }
-  MetricStatistics: { model: "github.com/ClusterCockpit/cc-backend/pkg/schema.MetricStatistics" }
-  MetricConfig: { model: "github.com/ClusterCockpit/cc-backend/pkg/schema.MetricConfig" }
-  SubClusterConfig: { model: "github.com/ClusterCockpit/cc-backend/pkg/schema.SubClusterConfig" }
-  Accelerator: { model: "github.com/ClusterCockpit/cc-backend/pkg/schema.Accelerator" }
-  Topology: { model: "github.com/ClusterCockpit/cc-backend/pkg/schema.Topology" }
-  FilterRanges: { model: "github.com/ClusterCockpit/cc-backend/pkg/schema.FilterRanges" }
-  SubCluster: { model: "github.com/ClusterCockpit/cc-backend/pkg/schema.SubCluster" }
-  StatsSeries: { model: "github.com/ClusterCockpit/cc-backend/pkg/schema.StatsSeries" }
-  Unit: { model: "github.com/ClusterCockpit/cc-backend/pkg/schema.Unit" }
+  Node:
+    model: "github.com/ClusterCockpit/cc-lib/schema.Node"
+    fields:
+      metaData:
+        resolver: true
+  NullableFloat: { model: "github.com/ClusterCockpit/cc-lib/schema.Float" }
+  MetricScope: { model: "github.com/ClusterCockpit/cc-lib/schema.MetricScope" }
+  MetricValue: { model: "github.com/ClusterCockpit/cc-lib/schema.MetricValue" }
+  JobStatistics:
+    { model: "github.com/ClusterCockpit/cc-lib/schema.JobStatistics" }
+  GlobalMetricListItem:
+    { model: "github.com/ClusterCockpit/cc-lib/schema.GlobalMetricListItem" }
+  ClusterSupport:
+    { model: "github.com/ClusterCockpit/cc-lib/schema.ClusterSupport" }
+  Tag: { model: "github.com/ClusterCockpit/cc-lib/schema.Tag" }
+  Resource: { model: "github.com/ClusterCockpit/cc-lib/schema.Resource" }
+  JobState: { model: "github.com/ClusterCockpit/cc-lib/schema.JobState" }
+  MonitoringState:
+    { model: "github.com/ClusterCockpit/cc-lib/schema.NodeState" }
+  HealthState:
+    { model: "github.com/ClusterCockpit/cc-lib/schema.MonitoringState" }
+  TimeRange: { model: "github.com/ClusterCockpit/cc-lib/schema.TimeRange" }
+  IntRange: { model: "github.com/ClusterCockpit/cc-lib/schema.IntRange" }
+  JobMetric: { model: "github.com/ClusterCockpit/cc-lib/schema.JobMetric" }
+  Series: { model: "github.com/ClusterCockpit/cc-lib/schema.Series" }
+  MetricStatistics:
+    { model: "github.com/ClusterCockpit/cc-lib/schema.MetricStatistics" }
+  MetricConfig:
+    { model: "github.com/ClusterCockpit/cc-lib/schema.MetricConfig" }
+  SubClusterConfig:
+    { model: "github.com/ClusterCockpit/cc-lib/schema.SubClusterConfig" }
+  Accelerator: { model: "github.com/ClusterCockpit/cc-lib/schema.Accelerator" }
+  Topology: { model: "github.com/ClusterCockpit/cc-lib/schema.Topology" }
+  FilterRanges:
+    { model: "github.com/ClusterCockpit/cc-lib/schema.FilterRanges" }
+  SubCluster: { model: "github.com/ClusterCockpit/cc-lib/schema.SubCluster" }
+  StatsSeries: { model: "github.com/ClusterCockpit/cc-lib/schema.StatsSeries" }
+  Unit: { model: "github.com/ClusterCockpit/cc-lib/schema.Unit" }

init/clustercockpit.service

@@ -1,5 +1,5 @@
 [Unit]
-Description=ClusterCockpit Web Server (Go edition)
+Description=ClusterCockpit Web Server
 Documentation=https://github.com/ClusterCockpit/cc-backend
 Wants=network-online.target
 After=network-online.target

internal/api/api_test.go

@@ -1,5 +1,5 @@
-// Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved.
+// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
+// All rights reserved. This file is part of cc-backend.
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 package api_test
@@ -14,18 +14,21 @@ import (
 	"os"
 	"path/filepath"
 	"reflect"
-	"strconv"
 	"strings"
 	"testing"
+	"time"
 
 	"github.com/ClusterCockpit/cc-backend/internal/api"
+	"github.com/ClusterCockpit/cc-backend/internal/archiver"
+	"github.com/ClusterCockpit/cc-backend/internal/auth"
 	"github.com/ClusterCockpit/cc-backend/internal/config"
 	"github.com/ClusterCockpit/cc-backend/internal/graph"
+	"github.com/ClusterCockpit/cc-backend/internal/metricDataDispatcher"
 	"github.com/ClusterCockpit/cc-backend/internal/metricdata"
 	"github.com/ClusterCockpit/cc-backend/internal/repository"
 	"github.com/ClusterCockpit/cc-backend/pkg/archive"
-	"github.com/ClusterCockpit/cc-backend/pkg/log"
-	"github.com/ClusterCockpit/cc-backend/pkg/schema"
+	cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
+	"github.com/ClusterCockpit/cc-lib/schema"
 	"github.com/gorilla/mux"
 
 	_ "github.com/mattn/go-sqlite3"
@@ -42,6 +45,9 @@ func setup(t *testing.T) *api.RestApi {
     "jwts": {
         "max-age": "2m"
     },
+  "apiAllowedIPs": [
+    "*"
+  ],
 	"clusters": [
 	{
 	   "name": "testcluster",
@@ -110,14 +116,14 @@ func setup(t *testing.T) *api.RestApi {
 		]
 	}`
 
-	log.Init("info", true)
+	cclog.Init("info", true)
 	tmpdir := t.TempDir()
 	jobarchive := filepath.Join(tmpdir, "job-archive")
 	if err := os.Mkdir(jobarchive, 0777); err != nil {
 		t.Fatal(err)
 	}
 
-	if err := os.WriteFile(filepath.Join(jobarchive, "version.txt"), []byte(fmt.Sprintf("%d", 1)), 0666); err != nil {
+	if err := os.WriteFile(filepath.Join(jobarchive, "version.txt"), fmt.Appendf(nil, "%d", 2), 0666); err != nil {
 		t.Fatal(err)
 	}
 
@@ -144,23 +150,20 @@ func setup(t *testing.T) *api.RestApi {
 	archiveCfg := fmt.Sprintf("{\"kind\": \"file\",\"path\": \"%s\"}", jobarchive)
 
 	repository.Connect("sqlite3", dbfilepath)
-	db := repository.GetConnection()
 
 	if err := archive.Init(json.RawMessage(archiveCfg), config.Keys.DisableArchive); err != nil {
 		t.Fatal(err)
 	}
 
-	if err := metricdata.Init(config.Keys.DisableArchive); err != nil {
+	if err := metricdata.Init(); err != nil {
 		t.Fatal(err)
 	}
 
-	jobRepo := repository.GetJobRepository()
-	resolver := &graph.Resolver{DB: db.DB, Repo: jobRepo}
+	archiver.Start(repository.GetJobRepository())
+	auth.Init()
+	graph.Init()
 
-	return &api.RestApi{
-		JobRepository: resolver.Repo,
-		Resolver:      resolver,
-	}
+	return api.New()
 }
 
 func cleanup() {
@@ -175,7 +178,6 @@ func cleanup() {
 func TestRestApi(t *testing.T) {
 	restapi := setup(t)
 	t.Cleanup(cleanup)
-
 	testData := schema.JobData{
 		"load_one": map[schema.MetricScope]*schema.JobMetric{
 			schema.MetricScopeNode: {
@@ -192,15 +194,21 @@ func TestRestApi(t *testing.T) {
 		},
 	}
 
-	metricdata.TestLoadDataCallback = func(job *schema.Job, metrics []string, scopes []schema.MetricScope, ctx context.Context) (schema.JobData, error) {
+	metricdata.TestLoadDataCallback = func(job *schema.Job, metrics []string, scopes []schema.MetricScope, ctx context.Context, resolution int) (schema.JobData, error) {
 		return testData, nil
 	}
 
 	r := mux.NewRouter()
-	restapi.MountRoutes(r)
+	r.PathPrefix("/api").Subrouter()
+	r.StrictSlash(true)
+	restapi.MountApiRoutes(r)
+
+	var TestJobId int64 = 123
+	TestClusterName := "testcluster"
+	var TestStartTime int64 = 123456789
 
 	const startJobBody string = `{
-        "jobId":            123,
+    "jobId":            123,
 		"user":             "testuser",
 		"project":          "testproj",
 		"cluster":          "testcluster",
@@ -213,7 +221,6 @@ func TestRestApi(t *testing.T) {
 		"exclusive":        1,
 		"monitoringStatus": 1,
 		"smt":              1,
-		"tags":             [{ "type": "testTagType", "name": "testTagName" }],
 		"resources": [
 			{
 				"hostname": "host123",
@@ -224,31 +231,37 @@ func TestRestApi(t *testing.T) {
 		"startTime": 123456789
 	}`
 
-	var dbid int64
+	const contextUserKey repository.ContextKey = "user"
+	contextUserValue := &schema.User{
+		Username:   "testuser",
+		Projects:   make([]string, 0),
+		Roles:      []string{"user"},
+		AuthType:   0,
+		AuthSource: 2,
+	}
+
 	if ok := t.Run("StartJob", func(t *testing.T) {
-		req := httptest.NewRequest(http.MethodPost, "/api/jobs/start_job/", bytes.NewBuffer([]byte(startJobBody)))
+		req := httptest.NewRequest(http.MethodPost, "/jobs/start_job/", bytes.NewBuffer([]byte(startJobBody)))
 		recorder := httptest.NewRecorder()
 
-		r.ServeHTTP(recorder, req)
+		ctx := context.WithValue(req.Context(), contextUserKey, contextUserValue)
+
+		r.ServeHTTP(recorder, req.WithContext(ctx))
 		response := recorder.Result()
 		if response.StatusCode != http.StatusCreated {
 			t.Fatal(response.Status, recorder.Body.String())
 		}
-
-		var res api.StartJobApiResponse
-		if err := json.Unmarshal(recorder.Body.Bytes(), &res); err != nil {
-			t.Fatal(err)
-		}
-
-		job, err := restapi.Resolver.Query().Job(context.Background(), strconv.Itoa(int(res.DBID)))
+		// resolver := graph.GetResolverInstance()
+		restapi.JobRepository.SyncJobs()
+		job, err := restapi.JobRepository.Find(&TestJobId, &TestClusterName, &TestStartTime)
 		if err != nil {
 			t.Fatal(err)
 		}
 
-		job.Tags, err = restapi.Resolver.Job().Tags(context.Background(), job)
-		if err != nil {
-			t.Fatal(err)
-		}
+		// job.Tags, err = resolver.Job().Tags(ctx, job)
+		// if err != nil {
+		// 	t.Fatal(err)
+		// }
 
 		if job.JobID != 123 ||
 			job.User != "testuser" ||
@@ -265,15 +278,13 @@ func TestRestApi(t *testing.T) {
 			job.MonitoringStatus != 1 ||
 			job.SMT != 1 ||
 			!reflect.DeepEqual(job.Resources, []*schema.Resource{{Hostname: "host123", HWThreads: []int{0, 1, 2, 3, 4, 5, 6, 7}}}) ||
-			job.StartTime.Unix() != 123456789 {
+			job.StartTime != 123456789 {
 			t.Fatalf("unexpected job properties: %#v", job)
 		}
 
-		if len(job.Tags) != 1 || job.Tags[0].Type != "testTagType" || job.Tags[0].Name != "testTagName" {
-			t.Fatalf("unexpected tags: %#v", job.Tags)
-		}
-
-		dbid = res.DBID
+		// if len(job.Tags) != 1 || job.Tags[0].Type != "testTagType" || job.Tags[0].Name != "testTagName" || job.Tags[0].Scope != "testuser" {
+		// 	t.Fatalf("unexpected tags: %#v", job.Tags)
+		// }
 	}); !ok {
 		return
 	}
@@ -289,17 +300,19 @@ func TestRestApi(t *testing.T) {
 
 	var stoppedJob *schema.Job
 	if ok := t.Run("StopJob", func(t *testing.T) {
-		req := httptest.NewRequest(http.MethodPost, "/api/jobs/stop_job/", bytes.NewBuffer([]byte(stopJobBody)))
+		req := httptest.NewRequest(http.MethodPost, "/jobs/stop_job/", bytes.NewBuffer([]byte(stopJobBody)))
 		recorder := httptest.NewRecorder()
 
-		r.ServeHTTP(recorder, req)
+		ctx := context.WithValue(req.Context(), contextUserKey, contextUserValue)
+
+		r.ServeHTTP(recorder, req.WithContext(ctx))
 		response := recorder.Result()
 		if response.StatusCode != http.StatusOK {
 			t.Fatal(response.Status, recorder.Body.String())
 		}
 
-		restapi.JobRepository.WaitForArchiving()
-		job, err := restapi.Resolver.Query().Job(context.Background(), strconv.Itoa(int(dbid)))
+		archiver.WaitForArchiving()
+		job, err := restapi.JobRepository.Find(&TestJobId, &TestClusterName, &TestStartTime)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -327,7 +340,7 @@ func TestRestApi(t *testing.T) {
 	}
 
 	t.Run("CheckArchive", func(t *testing.T) {
-		data, err := metricdata.LoadData(stoppedJob, []string{"load_one"}, []schema.MetricScope{schema.MetricScopeNode}, context.Background())
+		data, err := metricDataDispatcher.LoadData(stoppedJob, []string{"load_one"}, []schema.MetricScope{schema.MetricScopeNode}, context.Background(), 60)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -339,12 +352,14 @@ func TestRestApi(t *testing.T) {
 
 	t.Run("CheckDoubleStart", func(t *testing.T) {
 		// Starting a job with the same jobId and cluster should only be allowed if the startTime is far appart!
-		body := strings.Replace(startJobBody, `"startTime": 123456789`, `"startTime": 123456790`, -1)
+		body := strings.ReplaceAll(startJobBody, `"startTime": 123456789`, `"startTime": 123456790`)
 
-		req := httptest.NewRequest(http.MethodPost, "/api/jobs/start_job/", bytes.NewBuffer([]byte(body)))
+		req := httptest.NewRequest(http.MethodPost, "/jobs/start_job/", bytes.NewBuffer([]byte(body)))
 		recorder := httptest.NewRecorder()
 
-		r.ServeHTTP(recorder, req)
+		ctx := context.WithValue(req.Context(), contextUserKey, contextUserValue)
+
+		r.ServeHTTP(recorder, req.WithContext(ctx))
 		response := recorder.Result()
 		if response.StatusCode != http.StatusUnprocessableEntity {
 			t.Fatal(response.Status, recorder.Body.String())
@@ -371,10 +386,12 @@ func TestRestApi(t *testing.T) {
 	}`
 
 	ok := t.Run("StartJobFailed", func(t *testing.T) {
-		req := httptest.NewRequest(http.MethodPost, "/api/jobs/start_job/", bytes.NewBuffer([]byte(startJobBodyFailed)))
+		req := httptest.NewRequest(http.MethodPost, "/jobs/start_job/", bytes.NewBuffer([]byte(startJobBodyFailed)))
 		recorder := httptest.NewRecorder()
 
-		r.ServeHTTP(recorder, req)
+		ctx := context.WithValue(req.Context(), contextUserKey, contextUserValue)
+
+		r.ServeHTTP(recorder, req.WithContext(ctx))
 		response := recorder.Result()
 		if response.StatusCode != http.StatusCreated {
 			t.Fatal(response.Status, recorder.Body.String())
@@ -384,8 +401,11 @@ func TestRestApi(t *testing.T) {
 		t.Fatal("subtest failed")
 	}
 
+	time.Sleep(1 * time.Second)
+	restapi.JobRepository.SyncJobs()
+
 	const stopJobBodyFailed string = `{
-        "jobId":     12345,
+    "jobId":     12345,
 		"cluster":   "testcluster",
 
 		"jobState": "failed",
@@ -393,16 +413,18 @@ func TestRestApi(t *testing.T) {
 	}`
 
 	ok = t.Run("StopJobFailed", func(t *testing.T) {
-		req := httptest.NewRequest(http.MethodPost, "/api/jobs/stop_job/", bytes.NewBuffer([]byte(stopJobBodyFailed)))
+		req := httptest.NewRequest(http.MethodPost, "/jobs/stop_job/", bytes.NewBuffer([]byte(stopJobBodyFailed)))
 		recorder := httptest.NewRecorder()
 
-		r.ServeHTTP(recorder, req)
+		ctx := context.WithValue(req.Context(), contextUserKey, contextUserValue)
+
+		r.ServeHTTP(recorder, req.WithContext(ctx))
 		response := recorder.Result()
 		if response.StatusCode != http.StatusOK {
 			t.Fatal(response.Status, recorder.Body.String())
 		}
 
-		restapi.JobRepository.WaitForArchiving()
+		archiver.WaitForArchiving()
 		jobid, cluster := int64(12345), "testcluster"
 		job, err := restapi.JobRepository.Find(&jobid, &cluster, nil)
 		if err != nil {

internal/api/cluster.go

@@ -0,0 +1,70 @@
+// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
+// All rights reserved. This file is part of cc-backend.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+package api
+
+import (
+	"bufio"
+	"encoding/json"
+	"fmt"
+	"net/http"
+
+	"github.com/ClusterCockpit/cc-backend/internal/repository"
+	"github.com/ClusterCockpit/cc-backend/pkg/archive"
+	"github.com/ClusterCockpit/cc-lib/schema"
+)
+
+// GetClustersApiResponse model
+type GetClustersApiResponse struct {
+	Clusters []*schema.Cluster `json:"clusters"` // Array of clusters
+}
+
+// getClusters godoc
+// @summary     Lists all cluster configs
+// @tags Cluster query
+// @description Get a list of all cluster configs. Specific cluster can be requested using query parameter.
+// @produce     json
+// @param       cluster        query    string            false "Job Cluster"
+// @success     200            {object} api.GetClustersApiResponse  "Array of clusters"
+// @failure     400            {object} api.ErrorResponse       "Bad Request"
+// @failure     401            {object} api.ErrorResponse       "Unauthorized"
+// @failure     403            {object} api.ErrorResponse       "Forbidden"
+// @failure     500            {object} api.ErrorResponse       "Internal Server Error"
+// @security    ApiKeyAuth
+// @router      /api/clusters/ [get]
+func (api *RestApi) getClusters(rw http.ResponseWriter, r *http.Request) {
+	if user := repository.GetUserFromContext(r.Context()); user != nil &&
+		!user.HasRole(schema.RoleApi) {
+
+		handleError(fmt.Errorf("missing role: %v", schema.GetRoleString(schema.RoleApi)), http.StatusForbidden, rw)
+		return
+	}
+
+	rw.Header().Add("Content-Type", "application/json")
+	bw := bufio.NewWriter(rw)
+	defer bw.Flush()
+
+	var clusters []*schema.Cluster
+
+	if r.URL.Query().Has("cluster") {
+		name := r.URL.Query().Get("cluster")
+		cluster := archive.GetCluster(name)
+		if cluster == nil {
+			handleError(fmt.Errorf("unknown cluster: %s", name), http.StatusBadRequest, rw)
+			return
+		}
+		clusters = append(clusters, cluster)
+	} else {
+		clusters = archive.Clusters
+	}
+
+	payload := GetClustersApiResponse{
+		Clusters: clusters,
+	}
+
+	if err := json.NewEncoder(bw).Encode(payload); err != nil {
+		handleError(err, http.StatusInternalServerError, rw)
+		return
+	}
+}

internal/api/node.go

@@ -0,0 +1,80 @@
+// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
+// All rights reserved. This file is part of cc-backend.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+package api
+
+import (
+	"fmt"
+	"net/http"
+	"strings"
+
+	"github.com/ClusterCockpit/cc-backend/internal/repository"
+	"github.com/ClusterCockpit/cc-lib/schema"
+)
+
+type Node struct {
+	Name            string   `json:"hostname"`
+	States          []string `json:"states"`
+	CpusAllocated   int      `json:"cpusAllocated"`
+	CpusTotal       int      `json:"cpusTotal"`
+	MemoryAllocated int      `json:"memoryAllocated"`
+	MemoryTotal     int      `json:"memoryTotal"`
+	GpusAllocated   int      `json:"gpusAllocated"`
+	GpusTotal       int      `json:"gpusTotal"`
+}
+
+// updateNodeStatesRequest model
+type UpdateNodeStatesRequest struct {
+	Nodes   []Node `json:"nodes"`
+	Cluster string `json:"cluster" example:"fritz"`
+}
+
+// this routine assumes that only one of them exists per node
+func determineState(states []string) schema.NodeState {
+	for _, state := range states {
+		switch strings.ToLower(state) {
+		case "allocated":
+			return schema.NodeStateAllocated
+		case "reserved":
+			return schema.NodeStateReserved
+		case "idle":
+			return schema.NodeStateIdle
+		case "down":
+			return schema.NodeStateDown
+		case "mixed":
+			return schema.NodeStateMixed
+		}
+	}
+
+	return schema.NodeStateUnknown
+}
+
+// updateNodeStates godoc
+// @summary     Deliver updated Slurm node states
+// @tags Nodestates
+// @description Returns a JSON-encoded list of users.
+// @description Required query-parameter defines if all users or only users with additional special roles are returned.
+// @produce     json
+// @param       request body UpdateNodeStatesRequest true "Request body containing nodes and their states"
+// @success     200     {object} api.DefaultApiResponse "Success message"
+// @failure     400     {object} api.ErrorResponse      "Bad Request"
+// @failure     401     {object} api.ErrorResponse      "Unauthorized"
+// @failure     403     {object} api.ErrorResponse      "Forbidden"
+// @failure     500     {object} api.ErrorResponse      "Internal Server Error"
+// @security    ApiKeyAuth
+// @router      /api/nodestats/ [post]
+func (api *RestApi) updateNodeStates(rw http.ResponseWriter, r *http.Request) {
+	// Parse request body
+	req := UpdateNodeStatesRequest{}
+	if err := decode(r.Body, &req); err != nil {
+		handleError(fmt.Errorf("parsing request body failed: %w", err), http.StatusBadRequest, rw)
+		return
+	}
+	repo := repository.GetNodeRepository()
+
+	for _, node := range req.Nodes {
+		state := determineState(node.States)
+		repo.UpdateNodeState(node.Name, req.Cluster, &state)
+	}
+}

internal/api/user.go

@@ -0,0 +1,159 @@
+// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
+// All rights reserved. This file is part of cc-backend.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+package api
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+
+	"github.com/ClusterCockpit/cc-backend/internal/repository"
+	"github.com/ClusterCockpit/cc-lib/schema"
+	"github.com/gorilla/mux"
+)
+
+type ApiReturnedUser struct {
+	Username string   `json:"username"`
+	Name     string   `json:"name"`
+	Roles    []string `json:"roles"`
+	Email    string   `json:"email"`
+	Projects []string `json:"projects"`
+}
+
+// getUsers godoc
+// @summary     Returns a list of users
+// @tags User
+// @description Returns a JSON-encoded list of users.
+// @description Required query-parameter defines if all users or only users with additional special roles are returned.
+// @produce     json
+// @param       not-just-user query bool true "If returned list should contain all users or only users with additional special roles"
+// @success     200     {array} api.ApiReturnedUser "List of users returned successfully"
+// @failure     400     {string} string             "Bad Request"
+// @failure     401     {string} string             "Unauthorized"
+// @failure     403     {string} string             "Forbidden"
+// @failure     500     {string} string             "Internal Server Error"
+// @security    ApiKeyAuth
+// @router      /api/users/ [get]
+func (api *RestApi) getUsers(rw http.ResponseWriter, r *http.Request) {
+	// SecuredCheck() only worked with TokenAuth: Removed
+
+	if user := repository.GetUserFromContext(r.Context()); !user.HasRole(schema.RoleAdmin) {
+		http.Error(rw, "Only admins are allowed to fetch a list of users", http.StatusForbidden)
+		return
+	}
+
+	users, err := repository.GetUserRepository().ListUsers(r.URL.Query().Get("not-just-user") == "true")
+	if err != nil {
+		http.Error(rw, err.Error(), http.StatusInternalServerError)
+		return
+	}
+
+	json.NewEncoder(rw).Encode(users)
+}
+
+func (api *RestApi) updateUser(rw http.ResponseWriter, r *http.Request) {
+	// SecuredCheck() only worked with TokenAuth: Removed
+
+	if user := repository.GetUserFromContext(r.Context()); !user.HasRole(schema.RoleAdmin) {
+		http.Error(rw, "Only admins are allowed to update a user", http.StatusForbidden)
+		return
+	}
+
+	// Get Values
+	newrole := r.FormValue("add-role")
+	delrole := r.FormValue("remove-role")
+	newproj := r.FormValue("add-project")
+	delproj := r.FormValue("remove-project")
+
+	// TODO: Handle anything but roles...
+	if newrole != "" {
+		if err := repository.GetUserRepository().AddRole(r.Context(), mux.Vars(r)["id"], newrole); err != nil {
+			http.Error(rw, err.Error(), http.StatusUnprocessableEntity)
+			return
+		}
+		rw.Write([]byte("Add Role Success"))
+	} else if delrole != "" {
+		if err := repository.GetUserRepository().RemoveRole(r.Context(), mux.Vars(r)["id"], delrole); err != nil {
+			http.Error(rw, err.Error(), http.StatusUnprocessableEntity)
+			return
+		}
+		rw.Write([]byte("Remove Role Success"))
+	} else if newproj != "" {
+		if err := repository.GetUserRepository().AddProject(r.Context(), mux.Vars(r)["id"], newproj); err != nil {
+			http.Error(rw, err.Error(), http.StatusUnprocessableEntity)
+			return
+		}
+		rw.Write([]byte("Add Project Success"))
+	} else if delproj != "" {
+		if err := repository.GetUserRepository().RemoveProject(r.Context(), mux.Vars(r)["id"], delproj); err != nil {
+			http.Error(rw, err.Error(), http.StatusUnprocessableEntity)
+			return
+		}
+		rw.Write([]byte("Remove Project Success"))
+	} else {
+		http.Error(rw, "Not Add or Del [role|project]?", http.StatusInternalServerError)
+	}
+}
+
+func (api *RestApi) createUser(rw http.ResponseWriter, r *http.Request) {
+	// SecuredCheck() only worked with TokenAuth: Removed
+
+	rw.Header().Set("Content-Type", "text/plain")
+	me := repository.GetUserFromContext(r.Context())
+	if !me.HasRole(schema.RoleAdmin) {
+		http.Error(rw, "Only admins are allowed to create new users", http.StatusForbidden)
+		return
+	}
+
+	username, password, role, name, email, project := r.FormValue("username"),
+		r.FormValue("password"), r.FormValue("role"), r.FormValue("name"),
+		r.FormValue("email"), r.FormValue("project")
+
+	if len(password) == 0 && role != schema.GetRoleString(schema.RoleApi) {
+		http.Error(rw, "Only API users are allowed to have a blank password (login will be impossible)", http.StatusBadRequest)
+		return
+	}
+
+	if len(project) != 0 && role != schema.GetRoleString(schema.RoleManager) {
+		http.Error(rw, "only managers require a project (can be changed later)",
+			http.StatusBadRequest)
+		return
+	} else if len(project) == 0 && role == schema.GetRoleString(schema.RoleManager) {
+		http.Error(rw, "managers require a project to manage (can be changed later)",
+			http.StatusBadRequest)
+		return
+	}
+
+	if err := repository.GetUserRepository().AddUser(&schema.User{
+		Username: username,
+		Name:     name,
+		Password: password,
+		Email:    email,
+		Projects: []string{project},
+		Roles:    []string{role},
+	}); err != nil {
+		http.Error(rw, err.Error(), http.StatusUnprocessableEntity)
+		return
+	}
+
+	fmt.Fprintf(rw, "User %v successfully created!\n", username)
+}
+
+func (api *RestApi) deleteUser(rw http.ResponseWriter, r *http.Request) {
+	// SecuredCheck() only worked with TokenAuth: Removed
+
+	if user := repository.GetUserFromContext(r.Context()); !user.HasRole(schema.RoleAdmin) {
+		http.Error(rw, "Only admins are allowed to delete a user", http.StatusForbidden)
+		return
+	}
+
+	username := r.FormValue("username")
+	if err := repository.GetUserRepository().DelUser(username); err != nil {
+		http.Error(rw, err.Error(), http.StatusUnprocessableEntity)
+		return
+	}
+
+	rw.WriteHeader(http.StatusOK)
+}

internal/archiver/archiveWorker.go

@@ -0,0 +1,98 @@
+// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
+// All rights reserved. This file is part of cc-backend.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+package archiver
+
+import (
+	"context"
+	"sync"
+	"time"
+
+	"github.com/ClusterCockpit/cc-backend/internal/repository"
+	cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
+	"github.com/ClusterCockpit/cc-lib/schema"
+	sq "github.com/Masterminds/squirrel"
+)
+
+var (
+	archivePending sync.WaitGroup
+	archiveChannel chan *schema.Job
+	jobRepo        *repository.JobRepository
+)
+
+func Start(r *repository.JobRepository) {
+	archiveChannel = make(chan *schema.Job, 128)
+	jobRepo = r
+
+	go archivingWorker()
+}
+
+// Archiving worker thread
+func archivingWorker() {
+	for {
+		select {
+		case job, ok := <-archiveChannel:
+			if !ok {
+				break
+			}
+			start := time.Now()
+			// not using meta data, called to load JobMeta into Cache?
+			// will fail if job meta not in repository
+			if _, err := jobRepo.FetchMetadata(job); err != nil {
+				cclog.Errorf("archiving job (dbid: %d) failed at check metadata step: %s", job.ID, err.Error())
+				jobRepo.UpdateMonitoringStatus(*job.ID, schema.MonitoringStatusArchivingFailed)
+				continue
+			}
+
+			// ArchiveJob will fetch all the data from a MetricDataRepository and push into configured archive backend
+			// TODO: Maybe use context with cancel/timeout here
+			jobMeta, err := ArchiveJob(job, context.Background())
+			if err != nil {
+				cclog.Errorf("archiving job (dbid: %d) failed at archiving job step: %s", job.ID, err.Error())
+				jobRepo.UpdateMonitoringStatus(*job.ID, schema.MonitoringStatusArchivingFailed)
+				continue
+			}
+
+			stmt := sq.Update("job").Where("job.id = ?", job.ID)
+
+			if stmt, err = jobRepo.UpdateFootprint(stmt, jobMeta); err != nil {
+				cclog.Errorf("archiving job (dbid: %d) failed at update Footprint step: %s", job.ID, err.Error())
+				continue
+			}
+			if stmt, err = jobRepo.UpdateEnergy(stmt, jobMeta); err != nil {
+				cclog.Errorf("archiving job (dbid: %d) failed at update Energy step: %s", job.ID, err.Error())
+				continue
+			}
+			// Update the jobs database entry one last time:
+			stmt = jobRepo.MarkArchived(stmt, schema.MonitoringStatusArchivingSuccessful)
+			if err := jobRepo.Execute(stmt); err != nil {
+				cclog.Errorf("archiving job (dbid: %d) failed at db execute: %s", job.ID, err.Error())
+				continue
+			}
+			cclog.Debugf("archiving job %d took %s", job.JobID, time.Since(start))
+			cclog.Printf("archiving job (dbid: %d) successful", job.ID)
+
+			repository.CallJobStopHooks(job)
+			archivePending.Done()
+		default:
+			continue
+		}
+	}
+}
+
+// Trigger async archiving
+func TriggerArchiving(job *schema.Job) {
+	if archiveChannel == nil {
+		cclog.Fatal("Cannot archive without archiving channel. Did you Start the archiver?")
+	}
+
+	archivePending.Add(1)
+	archiveChannel <- job
+}
+
+// Wait for background thread to finish pending archiving operations
+func WaitForArchiving() {
+	// close channel and wait for worker to process remaining jobs
+	archivePending.Wait()
+}

internal/archiver/archiver.go

@@ -0,0 +1,79 @@
+// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
+// All rights reserved. This file is part of cc-backend.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+package archiver
+
+import (
+	"context"
+	"math"
+
+	"github.com/ClusterCockpit/cc-backend/internal/config"
+	"github.com/ClusterCockpit/cc-backend/internal/metricDataDispatcher"
+	"github.com/ClusterCockpit/cc-backend/pkg/archive"
+	cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
+	"github.com/ClusterCockpit/cc-lib/schema"
+)
+
+// Writes a running job to the job-archive
+func ArchiveJob(job *schema.Job, ctx context.Context) (*schema.Job, error) {
+	allMetrics := make([]string, 0)
+	metricConfigs := archive.GetCluster(job.Cluster).MetricConfig
+	for _, mc := range metricConfigs {
+		allMetrics = append(allMetrics, mc.Name)
+	}
+
+	scopes := []schema.MetricScope{schema.MetricScopeNode}
+	// FIXME: Add a config option for this
+	if job.NumNodes <= 8 {
+		// This will add the native scope if core scope is not available
+		scopes = append(scopes, schema.MetricScopeCore)
+	}
+
+	if job.NumAcc > 0 {
+		scopes = append(scopes, schema.MetricScopeAccelerator)
+	}
+
+	jobData, err := metricDataDispatcher.LoadData(job, allMetrics, scopes, ctx, 0) // 0 Resulotion-Value retrieves highest res (60s)
+	if err != nil {
+		cclog.Error("Error wile loading job data for archiving")
+		return nil, err
+	}
+
+	job.Statistics = make(map[string]schema.JobStatistics)
+
+	for metric, data := range jobData {
+		avg, min, max := 0.0, math.MaxFloat32, -math.MaxFloat32
+		nodeData, ok := data["node"]
+		if !ok {
+			// This should never happen ?
+			continue
+		}
+
+		for _, series := range nodeData.Series {
+			avg += series.Statistics.Avg
+			min = math.Min(min, series.Statistics.Min)
+			max = math.Max(max, series.Statistics.Max)
+		}
+
+		// Round AVG Result to 2 Digits
+		job.Statistics[metric] = schema.JobStatistics{
+			Unit: schema.Unit{
+				Prefix: archive.GetMetricConfig(job.Cluster, metric).Unit.Prefix,
+				Base:   archive.GetMetricConfig(job.Cluster, metric).Unit.Base,
+			},
+			Avg: (math.Round((avg/float64(job.NumNodes))*100) / 100),
+			Min: min,
+			Max: max,
+		}
+	}
+
+	// If the file based archive is disabled,
+	// only return the JobMeta structure as the
+	// statistics in there are needed.
+	if config.Keys.DisableArchive {
+		return job, nil
+	}
+
+	return job, archive.GetHandle().ImportJob(job, &jobData)
+}

internal/auth/auth.go

@@ -1,5 +1,5 @@
-// Copyright (C) 2023 NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved.
+// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
+// All rights reserved. This file is part of cc-backend.
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 package auth
@@ -10,14 +10,21 @@ import (
 	"database/sql"
 	"encoding/base64"
 	"errors"
+	"fmt"
+	"net"
 	"net/http"
 	"os"
+	"strings"
+	"sync"
 	"time"
 
+	"golang.org/x/time/rate"
+
 	"github.com/ClusterCockpit/cc-backend/internal/config"
 	"github.com/ClusterCockpit/cc-backend/internal/repository"
-	"github.com/ClusterCockpit/cc-backend/pkg/log"
-	"github.com/ClusterCockpit/cc-backend/pkg/schema"
+	cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
+	"github.com/ClusterCockpit/cc-lib/schema"
+	"github.com/ClusterCockpit/cc-lib/util"
 	"github.com/gorilla/sessions"
 )
 
@@ -26,22 +33,40 @@ type Authenticator interface {
 	Login(user *schema.User, rw http.ResponseWriter, r *http.Request) (*schema.User, error)
 }
 
-type Authentication struct {
-	sessionStore  *sessions.CookieStore
-	SessionMaxAge time.Duration
+var (
+	initOnce     sync.Once
+	authInstance *Authentication
+)
 
-	authenticators []Authenticator
+var ipUserLimiters sync.Map
+
+func getIPUserLimiter(ip, username string) *rate.Limiter {
+	key := ip + ":" + username
+	limiter, ok := ipUserLimiters.Load(key)
+	if !ok {
+		newLimiter := rate.NewLimiter(rate.Every(time.Hour/10), 10)
+		ipUserLimiters.Store(key, newLimiter)
+		return newLimiter
+	}
+	return limiter.(*rate.Limiter)
+}
+
+type Authentication struct {
+	sessionStore   *sessions.CookieStore
 	LdapAuth       *LdapAuthenticator
 	JwtAuth        *JWTAuthenticator
 	LocalAuth      *LocalAuthenticator
+	authenticators []Authenticator
+	SessionMaxAge  time.Duration
 }
 
 func (auth *Authentication) AuthViaSession(
 	rw http.ResponseWriter,
-	r *http.Request) (*schema.User, error) {
+	r *http.Request,
+) (*schema.User, error) {
 	session, err := auth.sessionStore.Get(r, "session")
 	if err != nil {
-		log.Error("Error while getting session store")
+		cclog.Error("Error while getting session store")
 		return nil, err
 	}
 
@@ -62,86 +87,166 @@ func (auth *Authentication) AuthViaSession(
 	}, nil
 }
 
-func Init() (*Authentication, error) {
-	auth := &Authentication{}
+func Init() {
+	initOnce.Do(func() {
+		authInstance = &Authentication{}
 
-	sessKey := os.Getenv("SESSION_KEY")
-	if sessKey == "" {
-		log.Warn("environment variable 'SESSION_KEY' not set (will use non-persistent random key)")
-		bytes := make([]byte, 32)
-		if _, err := rand.Read(bytes); err != nil {
-			log.Error("Error while initializing authentication -> failed to generate random bytes for session key")
-			return nil, err
-		}
-		auth.sessionStore = sessions.NewCookieStore(bytes)
-	} else {
-		bytes, err := base64.StdEncoding.DecodeString(sessKey)
-		if err != nil {
-			log.Error("Error while initializing authentication -> decoding session key failed")
-			return nil, err
-		}
-		auth.sessionStore = sessions.NewCookieStore(bytes)
-	}
-
-	if config.Keys.LdapConfig != nil {
-		ldapAuth := &LdapAuthenticator{}
-		if err := ldapAuth.Init(); err != nil {
-			log.Warn("Error while initializing authentication -> ldapAuth init failed")
+		sessKey := os.Getenv("SESSION_KEY")
+		if sessKey == "" {
+			cclog.Warn("environment variable 'SESSION_KEY' not set (will use non-persistent random key)")
+			bytes := make([]byte, 32)
+			if _, err := rand.Read(bytes); err != nil {
+				cclog.Fatal("Error while initializing authentication -> failed to generate random bytes for session key")
+			}
+			authInstance.sessionStore = sessions.NewCookieStore(bytes)
 		} else {
-			auth.LdapAuth = ldapAuth
-			auth.authenticators = append(auth.authenticators, auth.LdapAuth)
-		}
-	} else {
-		log.Info("Missing LDAP configuration: No LDAP support!")
-	}
-
-	if config.Keys.JwtConfig != nil {
-		auth.JwtAuth = &JWTAuthenticator{}
-		if err := auth.JwtAuth.Init(); err != nil {
-			log.Error("Error while initializing authentication -> jwtAuth init failed")
-			return nil, err
+			bytes, err := base64.StdEncoding.DecodeString(sessKey)
+			if err != nil {
+				cclog.Fatal("Error while initializing authentication -> decoding session key failed")
+			}
+			authInstance.sessionStore = sessions.NewCookieStore(bytes)
 		}
 
-		jwtSessionAuth := &JWTSessionAuthenticator{}
-		if err := jwtSessionAuth.Init(); err != nil {
-			log.Info("jwtSessionAuth init failed: No JWT login support!")
+		if d, err := time.ParseDuration(config.Keys.SessionMaxAge); err == nil {
+			authInstance.SessionMaxAge = d
+		}
+
+		if config.Keys.LdapConfig != nil {
+			ldapAuth := &LdapAuthenticator{}
+			if err := ldapAuth.Init(); err != nil {
+				cclog.Warn("Error while initializing authentication -> ldapAuth init failed")
+			} else {
+				authInstance.LdapAuth = ldapAuth
+				authInstance.authenticators = append(authInstance.authenticators, authInstance.LdapAuth)
+			}
 		} else {
-			auth.authenticators = append(auth.authenticators, jwtSessionAuth)
+			cclog.Info("Missing LDAP configuration: No LDAP support!")
 		}
 
-		jwtCookieSessionAuth := &JWTCookieSessionAuthenticator{}
-		if err := jwtCookieSessionAuth.Init(); err != nil {
-			log.Info("jwtCookieSessionAuth init failed: No JWT cookie login support!")
+		if config.Keys.JwtConfig != nil {
+			authInstance.JwtAuth = &JWTAuthenticator{}
+			if err := authInstance.JwtAuth.Init(); err != nil {
+				cclog.Fatal("Error while initializing authentication -> jwtAuth init failed")
+			}
+
+			jwtSessionAuth := &JWTSessionAuthenticator{}
+			if err := jwtSessionAuth.Init(); err != nil {
+				cclog.Info("jwtSessionAuth init failed: No JWT login support!")
+			} else {
+				authInstance.authenticators = append(authInstance.authenticators, jwtSessionAuth)
+			}
+
+			jwtCookieSessionAuth := &JWTCookieSessionAuthenticator{}
+			if err := jwtCookieSessionAuth.Init(); err != nil {
+				cclog.Info("jwtCookieSessionAuth init failed: No JWT cookie login support!")
+			} else {
+				authInstance.authenticators = append(authInstance.authenticators, jwtCookieSessionAuth)
+			}
 		} else {
-			auth.authenticators = append(auth.authenticators, jwtCookieSessionAuth)
+			cclog.Info("Missing JWT configuration: No JWT token support!")
 		}
-	} else {
-		log.Info("Missing JWT configuration: No JWT token support!")
+
+		authInstance.LocalAuth = &LocalAuthenticator{}
+		if err := authInstance.LocalAuth.Init(); err != nil {
+			cclog.Fatal("Error while initializing authentication -> localAuth init failed")
+		}
+		authInstance.authenticators = append(authInstance.authenticators, authInstance.LocalAuth)
+	})
+}
+
+func GetAuthInstance() *Authentication {
+	if authInstance == nil {
+		cclog.Fatal("Authentication module not initialized!")
 	}
 
-	auth.LocalAuth = &LocalAuthenticator{}
-	if err := auth.LocalAuth.Init(); err != nil {
-		log.Error("Error while initializing authentication -> localAuth init failed")
-		return nil, err
-	}
-	auth.authenticators = append(auth.authenticators, auth.LocalAuth)
+	return authInstance
+}
 
-	return auth, nil
+func handleTokenUser(tokenUser *schema.User) {
+	r := repository.GetUserRepository()
+	dbUser, err := r.GetUser(tokenUser.Username)
+
+	if err != nil && err != sql.ErrNoRows {
+		cclog.Errorf("Error while loading user '%s': %v", tokenUser.Username, err)
+	} else if err == sql.ErrNoRows && config.Keys.JwtConfig.SyncUserOnLogin { // Adds New User
+		if err := r.AddUser(tokenUser); err != nil {
+			cclog.Errorf("Error while adding user '%s' to DB: %v", tokenUser.Username, err)
+		}
+	} else if err == nil && config.Keys.JwtConfig.UpdateUserOnLogin { // Update Existing User
+		if err := r.UpdateUser(dbUser, tokenUser); err != nil {
+			cclog.Errorf("Error while updating user '%s' to DB: %v", dbUser.Username, err)
+		}
+	}
+}
+
+func handleOIDCUser(OIDCUser *schema.User) {
+	r := repository.GetUserRepository()
+	dbUser, err := r.GetUser(OIDCUser.Username)
+
+	if err != nil && err != sql.ErrNoRows {
+		cclog.Errorf("Error while loading user '%s': %v", OIDCUser.Username, err)
+	} else if err == sql.ErrNoRows && config.Keys.OpenIDConfig.SyncUserOnLogin { // Adds New User
+		if err := r.AddUser(OIDCUser); err != nil {
+			cclog.Errorf("Error while adding user '%s' to DB: %v", OIDCUser.Username, err)
+		}
+	} else if err == nil && config.Keys.OpenIDConfig.UpdateUserOnLogin { // Update Existing User
+		if err := r.UpdateUser(dbUser, OIDCUser); err != nil {
+			cclog.Errorf("Error while updating user '%s' to DB: %v", dbUser.Username, err)
+		}
+	}
+}
+
+func (auth *Authentication) SaveSession(rw http.ResponseWriter, r *http.Request, user *schema.User) error {
+	session, err := auth.sessionStore.New(r, "session")
+	if err != nil {
+		cclog.Errorf("session creation failed: %s", err.Error())
+		http.Error(rw, err.Error(), http.StatusInternalServerError)
+		return err
+	}
+
+	if auth.SessionMaxAge != 0 {
+		session.Options.MaxAge = int(auth.SessionMaxAge.Seconds())
+	}
+	if config.Keys.HttpsCertFile == "" && config.Keys.HttpsKeyFile == "" {
+		session.Options.Secure = false
+	}
+	session.Options.SameSite = http.SameSiteStrictMode
+	session.Values["username"] = user.Username
+	session.Values["projects"] = user.Projects
+	session.Values["roles"] = user.Roles
+	if err := auth.sessionStore.Save(r, rw, session); err != nil {
+		cclog.Warnf("session save failed: %s", err.Error())
+		http.Error(rw, err.Error(), http.StatusInternalServerError)
+		return err
+	}
+
+	return nil
 }
 
 func (auth *Authentication) Login(
-	onsuccess http.Handler,
-	onfailure func(rw http.ResponseWriter, r *http.Request, loginErr error)) http.Handler {
-
+	onfailure func(rw http.ResponseWriter, r *http.Request, loginErr error),
+) http.Handler {
 	return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
-		username := r.FormValue("username")
-		var dbUser *schema.User
+		ip, _, err := net.SplitHostPort(r.RemoteAddr)
+		if err != nil {
+			ip = r.RemoteAddr
+		}
 
+		username := r.FormValue("username")
+
+		limiter := getIPUserLimiter(ip, username)
+		if !limiter.Allow() {
+			cclog.Warnf("AUTH/RATE > Too many login attempts for combination IP: %s, Username: %s", ip, username)
+			onfailure(rw, r, errors.New("too many login attempts, try again in a few minutes"))
+			return
+		}
+
+		var dbUser *schema.User
 		if username != "" {
 			var err error
 			dbUser, err = repository.GetUserRepository().GetUser(username)
 			if err != nil && err != sql.ErrNoRows {
-				log.Errorf("Error while loading user '%v'", username)
+				cclog.Errorf("Error while loading user '%v'", username)
 			}
 		}
 
@@ -151,79 +256,188 @@ func (auth *Authentication) Login(
 			if user, ok = authenticator.CanLogin(dbUser, username, rw, r); !ok {
 				continue
 			} else {
-				log.Debugf("Can login with user %v", user)
+				cclog.Debugf("Can login with user %v", user)
 			}
 
 			user, err := authenticator.Login(user, rw, r)
 			if err != nil {
-				log.Warnf("user login failed: %s", err.Error())
+				cclog.Warnf("user login failed: %s", err.Error())
 				onfailure(rw, r, err)
 				return
 			}
 
-			session, err := auth.sessionStore.New(r, "session")
-			if err != nil {
-				log.Errorf("session creation failed: %s", err.Error())
-				http.Error(rw, err.Error(), http.StatusInternalServerError)
+			if err := auth.SaveSession(rw, r, user); err != nil {
 				return
 			}
 
-			if auth.SessionMaxAge != 0 {
-				session.Options.MaxAge = int(auth.SessionMaxAge.Seconds())
-			}
-			session.Values["username"] = user.Username
-			session.Values["projects"] = user.Projects
-			session.Values["roles"] = user.Roles
-			if err := auth.sessionStore.Save(r, rw, session); err != nil {
-				log.Warnf("session save failed: %s", err.Error())
-				http.Error(rw, err.Error(), http.StatusInternalServerError)
-				return
-			}
-
-			log.Infof("login successfull: user: %#v (roles: %v, projects: %v)", user.Username, user.Roles, user.Projects)
+			cclog.Infof("login successfull: user: %#v (roles: %v, projects: %v)", user.Username, user.Roles, user.Projects)
 			ctx := context.WithValue(r.Context(), repository.ContextUserKey, user)
-			onsuccess.ServeHTTP(rw, r.WithContext(ctx))
+
+			if r.FormValue("redirect") != "" {
+				http.RedirectHandler(r.FormValue("redirect"), http.StatusFound).ServeHTTP(rw, r.WithContext(ctx))
+				return
+			}
+
+			http.RedirectHandler("/", http.StatusFound).ServeHTTP(rw, r.WithContext(ctx))
 			return
 		}
 
-		log.Debugf("login failed: no authenticator applied")
+		cclog.Debugf("login failed: no authenticator applied")
 		onfailure(rw, r, errors.New("no authenticator applied"))
 	})
 }
 
 func (auth *Authentication) Auth(
 	onsuccess http.Handler,
-	onfailure func(rw http.ResponseWriter, r *http.Request, authErr error)) http.Handler {
-
+	onfailure func(rw http.ResponseWriter, r *http.Request, authErr error),
+) http.Handler {
 	return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
-
 		user, err := auth.JwtAuth.AuthViaJWT(rw, r)
 		if err != nil {
-			log.Infof("authentication failed: %s", err.Error())
+			cclog.Infof("auth -> authentication failed: %s", err.Error())
 			http.Error(rw, err.Error(), http.StatusUnauthorized)
 			return
 		}
-
 		if user == nil {
 			user, err = auth.AuthViaSession(rw, r)
 			if err != nil {
-				log.Infof("authentication failed: %s", err.Error())
+				cclog.Infof("auth -> authentication failed: %s", err.Error())
 				http.Error(rw, err.Error(), http.StatusUnauthorized)
 				return
 			}
 		}
-
 		if user != nil {
 			ctx := context.WithValue(r.Context(), repository.ContextUserKey, user)
 			onsuccess.ServeHTTP(rw, r.WithContext(ctx))
 			return
 		}
 
-		log.Debug("authentication failed")
+		cclog.Info("auth -> authentication failed")
 		onfailure(rw, r, errors.New("unauthorized (please login first)"))
 	})
 }
 
+func (auth *Authentication) AuthApi(
+	onsuccess http.Handler,
+	onfailure func(rw http.ResponseWriter, r *http.Request, authErr error),
+) http.Handler {
+	return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
+		user, err := auth.JwtAuth.AuthViaJWT(rw, r)
+		if err != nil {
+			cclog.Infof("auth api -> authentication failed: %s", err.Error())
+			onfailure(rw, r, err)
+			return
+		}
+
+		ipErr := securedCheck(user, r)
+		if ipErr != nil {
+			cclog.Infof("auth api -> secured check failed: %s", ipErr.Error())
+			onfailure(rw, r, ipErr)
+			return
+		}
+
+		if user != nil {
+			switch {
+			case len(user.Roles) == 1:
+				if user.HasRole(schema.RoleApi) {
+					ctx := context.WithValue(r.Context(), repository.ContextUserKey, user)
+					onsuccess.ServeHTTP(rw, r.WithContext(ctx))
+					return
+				}
+			case len(user.Roles) >= 2:
+				if user.HasAllRoles([]schema.Role{schema.RoleAdmin, schema.RoleApi}) {
+					ctx := context.WithValue(r.Context(), repository.ContextUserKey, user)
+					onsuccess.ServeHTTP(rw, r.WithContext(ctx))
+					return
+				}
+			default:
+				cclog.Info("auth api -> authentication failed: missing role")
+				onfailure(rw, r, errors.New("unauthorized"))
+			}
+		}
+		cclog.Info("auth api -> authentication failed: no auth")
+		onfailure(rw, r, errors.New("unauthorized"))
+	})
+}
+
+func (auth *Authentication) AuthUserApi(
+	onsuccess http.Handler,
+	onfailure func(rw http.ResponseWriter, r *http.Request, authErr error),
+) http.Handler {
+	return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
+		user, err := auth.JwtAuth.AuthViaJWT(rw, r)
+		if err != nil {
+			cclog.Infof("auth user api -> authentication failed: %s", err.Error())
+			onfailure(rw, r, err)
+			return
+		}
+
+		if user != nil {
+			switch {
+			case len(user.Roles) == 1:
+				if user.HasRole(schema.RoleApi) {
+					ctx := context.WithValue(r.Context(), repository.ContextUserKey, user)
+					onsuccess.ServeHTTP(rw, r.WithContext(ctx))
+					return
+				}
+			case len(user.Roles) >= 2:
+				if user.HasRole(schema.RoleApi) && user.HasAnyRole([]schema.Role{schema.RoleUser, schema.RoleManager, schema.RoleAdmin}) {
+					ctx := context.WithValue(r.Context(), repository.ContextUserKey, user)
+					onsuccess.ServeHTTP(rw, r.WithContext(ctx))
+					return
+				}
+			default:
+				cclog.Info("auth user api -> authentication failed: missing role")
+				onfailure(rw, r, errors.New("unauthorized"))
+			}
+		}
+		cclog.Info("auth user api -> authentication failed: no auth")
+		onfailure(rw, r, errors.New("unauthorized"))
+	})
+}
+
+func (auth *Authentication) AuthConfigApi(
+	onsuccess http.Handler,
+	onfailure func(rw http.ResponseWriter, r *http.Request, authErr error),
+) http.Handler {
+	return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
+		user, err := auth.AuthViaSession(rw, r)
+		if err != nil {
+			cclog.Infof("auth config api -> authentication failed: %s", err.Error())
+			onfailure(rw, r, err)
+			return
+		}
+		if user != nil && user.HasRole(schema.RoleAdmin) {
+			ctx := context.WithValue(r.Context(), repository.ContextUserKey, user)
+			onsuccess.ServeHTTP(rw, r.WithContext(ctx))
+			return
+		}
+		cclog.Info("auth config api -> authentication failed: no auth")
+		onfailure(rw, r, errors.New("unauthorized"))
+	})
+}
+
+func (auth *Authentication) AuthFrontendApi(
+	onsuccess http.Handler,
+	onfailure func(rw http.ResponseWriter, r *http.Request, authErr error),
+) http.Handler {
+	return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
+		user, err := auth.AuthViaSession(rw, r)
+		if err != nil {
+			cclog.Infof("auth frontend api -> authentication failed: %s", err.Error())
+			onfailure(rw, r, err)
+			return
+		}
+		if user != nil {
+			ctx := context.WithValue(r.Context(), repository.ContextUserKey, user)
+			onsuccess.ServeHTTP(rw, r.WithContext(ctx))
+			return
+		}
+		cclog.Info("auth frontend api -> authentication failed: no auth")
+		onfailure(rw, r, errors.New("unauthorized"))
+	})
+}
+
 func (auth *Authentication) Logout(onsuccess http.Handler) http.Handler {
 	return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
 		session, err := auth.sessionStore.Get(r, "session")
@@ -243,3 +457,39 @@ func (auth *Authentication) Logout(onsuccess http.Handler) http.Handler {
 		onsuccess.ServeHTTP(rw, r)
 	})
 }
+
+// Helper Moved To MiddleWare Auth Handlers
+func securedCheck(user *schema.User, r *http.Request) error {
+	if user == nil {
+		return fmt.Errorf("no user for secured check")
+	}
+
+	// extract IP address for checking
+	IPAddress := r.Header.Get("X-Real-Ip")
+	if IPAddress == "" {
+		IPAddress = r.Header.Get("X-Forwarded-For")
+	}
+	if IPAddress == "" {
+		IPAddress = r.RemoteAddr
+	}
+
+	// Cannot Handle ipv6! (e.g. localhost -> [::1])
+	if strings.Contains(IPAddress, ":") {
+		IPAddress = strings.Split(IPAddress, ":")[0]
+	}
+
+	// If nothing declared in config: deny all request to this api endpoint
+	if len(config.Keys.ApiAllowedIPs) == 0 {
+		return fmt.Errorf("missing configuration key ApiAllowedIPs")
+	}
+	// If wildcard declared in config: Continue
+	if config.Keys.ApiAllowedIPs[0] == "*" {
+		return nil
+	}
+	// check if IP is allowed
+	if !util.Contains(config.Keys.ApiAllowedIPs, IPAddress) {
+		return fmt.Errorf("unknown ip: %v", IPAddress)
+	}
+
+	return nil
+}

internal/auth/jwt.go

@@ -1,5 +1,5 @@
-// Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved.
+// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
+// All rights reserved. This file is part of cc-backend.
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 package auth
@@ -15,9 +15,9 @@ import (
 
 	"github.com/ClusterCockpit/cc-backend/internal/config"
 	"github.com/ClusterCockpit/cc-backend/internal/repository"
-	"github.com/ClusterCockpit/cc-backend/pkg/log"
-	"github.com/ClusterCockpit/cc-backend/pkg/schema"
-	"github.com/golang-jwt/jwt/v4"
+	cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
+	"github.com/ClusterCockpit/cc-lib/schema"
+	"github.com/golang-jwt/jwt/v5"
 )
 
 type JWTAuthenticator struct {
@@ -28,17 +28,17 @@ type JWTAuthenticator struct {
 func (ja *JWTAuthenticator) Init() error {
 	pubKey, privKey := os.Getenv("JWT_PUBLIC_KEY"), os.Getenv("JWT_PRIVATE_KEY")
 	if pubKey == "" || privKey == "" {
-		log.Warn("environment variables 'JWT_PUBLIC_KEY' or 'JWT_PRIVATE_KEY' not set (token based authentication will not work)")
+		cclog.Warn("environment variables 'JWT_PUBLIC_KEY' or 'JWT_PRIVATE_KEY' not set (token based authentication will not work)")
 	} else {
 		bytes, err := base64.StdEncoding.DecodeString(pubKey)
 		if err != nil {
-			log.Warn("Could not decode JWT public key")
+			cclog.Warn("Could not decode JWT public key")
 			return err
 		}
 		ja.publicKey = ed25519.PublicKey(bytes)
 		bytes, err = base64.StdEncoding.DecodeString(privKey)
 		if err != nil {
-			log.Warn("Could not decode JWT private key")
+			cclog.Warn("Could not decode JWT private key")
 			return err
 		}
 		ja.privateKey = ed25519.PrivateKey(bytes)
@@ -49,8 +49,8 @@ func (ja *JWTAuthenticator) Init() error {
 
 func (ja *JWTAuthenticator) AuthViaJWT(
 	rw http.ResponseWriter,
-	r *http.Request) (*schema.User, error) {
-
+	r *http.Request,
+) (*schema.User, error) {
 	rawtoken := r.Header.Get("X-Auth-Token")
 	if rawtoken == "" {
 		rawtoken = r.Header.Get("Authorization")
@@ -70,12 +70,12 @@ func (ja *JWTAuthenticator) AuthViaJWT(
 		return ja.publicKey, nil
 	})
 	if err != nil {
-		log.Warn("Error while parsing JWT token")
+		cclog.Warn("Error while parsing JWT token")
 		return nil, err
 	}
-	if err := token.Claims.Valid(); err != nil {
-		log.Warn("jwt token claims are not valid")
-		return nil, err
+	if !token.Valid {
+		cclog.Warn("jwt token claims are not valid")
+		return nil, errors.New("jwt token claims are not valid")
 	}
 
 	// Token is valid, extract payload
@@ -88,10 +88,9 @@ func (ja *JWTAuthenticator) AuthViaJWT(
 	if config.Keys.JwtConfig.ValidateUser {
 		ur := repository.GetUserRepository()
 		user, err := ur.GetUser(sub)
-
 		// Deny any logins for unknown usernames
 		if err != nil {
-			log.Warn("Could not find user from JWT in internal database.")
+			cclog.Warn("Could not find user from JWT in internal database.")
 			return nil, errors.New("unknown user")
 		}
 		// Take user roles from database instead of trusting the JWT
@@ -117,7 +116,6 @@ func (ja *JWTAuthenticator) AuthViaJWT(
 
 // Generate a new JWT that can be used for authentication
 func (ja *JWTAuthenticator) ProvideJWT(user *schema.User) (string, error) {
-
 	if ja.privateKey == nil {
 		return "", errors.New("environment variable 'JWT_PRIVATE_KEY' not set")
 	}

internal/auth/jwtCookieSession.go

@@ -1,5 +1,5 @@
-// Copyright (C) 2023 NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved.
+// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
+// All rights reserved. This file is part of cc-backend.
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 package auth
@@ -15,9 +15,9 @@ import (
 
 	"github.com/ClusterCockpit/cc-backend/internal/config"
 	"github.com/ClusterCockpit/cc-backend/internal/repository"
-	"github.com/ClusterCockpit/cc-backend/pkg/log"
-	"github.com/ClusterCockpit/cc-backend/pkg/schema"
-	"github.com/golang-jwt/jwt/v4"
+	cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
+	"github.com/ClusterCockpit/cc-lib/schema"
+	"github.com/golang-jwt/jwt/v5"
 )
 
 type JWTCookieSessionAuthenticator struct {
@@ -31,18 +31,18 @@ var _ Authenticator = (*JWTCookieSessionAuthenticator)(nil)
 func (ja *JWTCookieSessionAuthenticator) Init() error {
 	pubKey, privKey := os.Getenv("JWT_PUBLIC_KEY"), os.Getenv("JWT_PRIVATE_KEY")
 	if pubKey == "" || privKey == "" {
-		log.Warn("environment variables 'JWT_PUBLIC_KEY' or 'JWT_PRIVATE_KEY' not set (token based authentication will not work)")
+		cclog.Warn("environment variables 'JWT_PUBLIC_KEY' or 'JWT_PRIVATE_KEY' not set (token based authentication will not work)")
 		return errors.New("environment variables 'JWT_PUBLIC_KEY' or 'JWT_PRIVATE_KEY' not set (token based authentication will not work)")
 	} else {
 		bytes, err := base64.StdEncoding.DecodeString(pubKey)
 		if err != nil {
-			log.Warn("Could not decode JWT public key")
+			cclog.Warn("Could not decode JWT public key")
 			return err
 		}
 		ja.publicKey = ed25519.PublicKey(bytes)
 		bytes, err = base64.StdEncoding.DecodeString(privKey)
 		if err != nil {
-			log.Warn("Could not decode JWT private key")
+			cclog.Warn("Could not decode JWT private key")
 			return err
 		}
 		ja.privateKey = ed25519.PrivateKey(bytes)
@@ -53,13 +53,13 @@ func (ja *JWTCookieSessionAuthenticator) Init() error {
 	if keyFound && pubKeyCrossLogin != "" {
 		bytes, err := base64.StdEncoding.DecodeString(pubKeyCrossLogin)
 		if err != nil {
-			log.Warn("Could not decode cross login JWT public key")
+			cclog.Warn("Could not decode cross login JWT public key")
 			return err
 		}
 		ja.publicKeyCrossLogin = ed25519.PublicKey(bytes)
 	} else {
 		ja.publicKeyCrossLogin = nil
-		log.Debug("environment variable 'CROSS_LOGIN_JWT_PUBLIC_KEY' not set (cross login token based authentication will not work)")
+		cclog.Debug("environment variable 'CROSS_LOGIN_JWT_PUBLIC_KEY' not set (cross login token based authentication will not work)")
 		return errors.New("environment variable 'CROSS_LOGIN_JWT_PUBLIC_KEY' not set (cross login token based authentication will not work)")
 	}
 
@@ -67,22 +67,22 @@ func (ja *JWTCookieSessionAuthenticator) Init() error {
 	// Warn if other necessary settings are not configured
 	if jc != nil {
 		if jc.CookieName == "" {
-			log.Info("cookieName for JWTs not configured (cross login via JWT cookie will fail)")
+			cclog.Info("cookieName for JWTs not configured (cross login via JWT cookie will fail)")
 			return errors.New("cookieName for JWTs not configured (cross login via JWT cookie will fail)")
 		}
 		if !jc.ValidateUser {
-			log.Info("forceJWTValidationViaDatabase not set to true: CC will accept users and roles defined in JWTs regardless of its own database!")
+			cclog.Info("forceJWTValidationViaDatabase not set to true: CC will accept users and roles defined in JWTs regardless of its own database!")
 		}
 		if jc.TrustedIssuer == "" {
-			log.Info("trustedExternalIssuer for JWTs not configured (cross login via JWT cookie will fail)")
+			cclog.Info("trustedExternalIssuer for JWTs not configured (cross login via JWT cookie will fail)")
 			return errors.New("trustedExternalIssuer for JWTs not configured (cross login via JWT cookie will fail)")
 		}
 	} else {
-		log.Warn("config for JWTs not configured (cross login via JWT cookie will fail)")
+		cclog.Warn("config for JWTs not configured (cross login via JWT cookie will fail)")
 		return errors.New("config for JWTs not configured (cross login via JWT cookie will fail)")
 	}
 
-	log.Info("JWT Cookie Session authenticator successfully registered")
+	cclog.Info("JWT Cookie Session authenticator successfully registered")
 	return nil
 }
 
@@ -90,8 +90,8 @@ func (ja *JWTCookieSessionAuthenticator) CanLogin(
 	user *schema.User,
 	username string,
 	rw http.ResponseWriter,
-	r *http.Request) (*schema.User, bool) {
-
+	r *http.Request,
+) (*schema.User, bool) {
 	jc := config.Keys.JwtConfig
 	cookieName := ""
 	if jc.CookieName != "" {
@@ -113,8 +113,8 @@ func (ja *JWTCookieSessionAuthenticator) CanLogin(
 func (ja *JWTCookieSessionAuthenticator) Login(
 	user *schema.User,
 	rw http.ResponseWriter,
-	r *http.Request) (*schema.User, error) {
-
+	r *http.Request,
+) (*schema.User, error) {
 	jc := config.Keys.JwtConfig
 	jwtCookie, err := r.Cookie(jc.CookieName)
 	var rawtoken string
@@ -140,14 +140,13 @@ func (ja *JWTCookieSessionAuthenticator) Login(
 		return ja.publicKey, nil
 	})
 	if err != nil {
-		log.Warn("JWT cookie session: error while parsing token")
+		cclog.Warn("JWT cookie session: error while parsing token")
 		return nil, err
 	}
 
-	// Check token validity and extract paypload
-	if err := token.Claims.Valid(); err != nil {
-		log.Warn("jwt token claims are not valid")
-		return nil, err
+	if !token.Valid {
+		cclog.Warn("jwt token claims are not valid")
+		return nil, errors.New("jwt token claims are not valid")
 	}
 
 	claims := token.Claims.(jwt.MapClaims)
@@ -160,12 +159,12 @@ func (ja *JWTCookieSessionAuthenticator) Login(
 		var err error
 		user, err = repository.GetUserRepository().GetUser(sub)
 		if err != nil && err != sql.ErrNoRows {
-			log.Errorf("Error while loading user '%v'", sub)
+			cclog.Errorf("Error while loading user '%v'", sub)
 		}
 
 		// Deny any logins for unknown usernames
 		if user == nil {
-			log.Warn("Could not find user from JWT in internal database.")
+			cclog.Warn("Could not find user from JWT in internal database.")
 			return nil, errors.New("unknown user")
 		}
 	} else {
@@ -199,10 +198,8 @@ func (ja *JWTCookieSessionAuthenticator) Login(
 			AuthSource: schema.AuthViaToken,
 		}
 
-		if jc.SyncUserOnLogin {
-			if err := repository.GetUserRepository().AddUser(user); err != nil {
-				log.Errorf("Error while adding user '%s' to DB", user.Username)
-			}
+		if jc.SyncUserOnLogin || jc.UpdateUserOnLogin {
+			handleTokenUser(user)
 		}
 	}
 

internal/auth/jwtSession.go

@@ -1,5 +1,5 @@
-// Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved.
+// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
+// All rights reserved. This file is part of cc-backend.
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 package auth
@@ -15,9 +15,9 @@ import (
 
 	"github.com/ClusterCockpit/cc-backend/internal/config"
 	"github.com/ClusterCockpit/cc-backend/internal/repository"
-	"github.com/ClusterCockpit/cc-backend/pkg/log"
-	"github.com/ClusterCockpit/cc-backend/pkg/schema"
-	"github.com/golang-jwt/jwt/v4"
+	cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
+	"github.com/ClusterCockpit/cc-lib/schema"
+	"github.com/golang-jwt/jwt/v5"
 )
 
 type JWTSessionAuthenticator struct {
@@ -30,13 +30,13 @@ func (ja *JWTSessionAuthenticator) Init() error {
 	if pubKey := os.Getenv("CROSS_LOGIN_JWT_HS512_KEY"); pubKey != "" {
 		bytes, err := base64.StdEncoding.DecodeString(pubKey)
 		if err != nil {
-			log.Warn("Could not decode cross login JWT HS512 key")
+			cclog.Warn("Could not decode cross login JWT HS512 key")
 			return err
 		}
 		ja.loginTokenKey = bytes
 	}
 
-	log.Info("JWT Session authenticator successfully registered")
+	cclog.Info("JWT Session authenticator successfully registered")
 	return nil
 }
 
@@ -44,8 +44,8 @@ func (ja *JWTSessionAuthenticator) CanLogin(
 	user *schema.User,
 	username string,
 	rw http.ResponseWriter,
-	r *http.Request) (*schema.User, bool) {
-
+	r *http.Request,
+) (*schema.User, bool) {
 	return user, r.Header.Get("Authorization") != "" ||
 		r.URL.Query().Get("login-token") != ""
 }
@@ -53,8 +53,8 @@ func (ja *JWTSessionAuthenticator) CanLogin(
 func (ja *JWTSessionAuthenticator) Login(
 	user *schema.User,
 	rw http.ResponseWriter,
-	r *http.Request) (*schema.User, error) {
-
+	r *http.Request,
+) (*schema.User, error) {
 	rawtoken := strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer ")
 	if rawtoken == "" {
 		rawtoken = r.URL.Query().Get("login-token")
@@ -67,13 +67,13 @@ func (ja *JWTSessionAuthenticator) Login(
 		return nil, fmt.Errorf("unkown signing method for login token: %s (known: HS256, HS512, EdDSA)", t.Method.Alg())
 	})
 	if err != nil {
-		log.Warn("Error while parsing jwt token")
+		cclog.Warn("Error while parsing jwt token")
 		return nil, err
 	}
 
-	if err = token.Claims.Valid(); err != nil {
-		log.Warn("jwt token claims are not valid")
-		return nil, err
+	if !token.Valid {
+		cclog.Warn("jwt token claims are not valid")
+		return nil, errors.New("jwt token claims are not valid")
 	}
 
 	claims := token.Claims.(jwt.MapClaims)
@@ -86,12 +86,12 @@ func (ja *JWTSessionAuthenticator) Login(
 		var err error
 		user, err = repository.GetUserRepository().GetUser(sub)
 		if err != nil && err != sql.ErrNoRows {
-			log.Errorf("Error while loading user '%v'", sub)
+			cclog.Errorf("Error while loading user '%v'", sub)
 		}
 
 		// Deny any logins for unknown usernames
 		if user == nil {
-			log.Warn("Could not find user from JWT in internal database.")
+			cclog.Warn("Could not find user from JWT in internal database.")
 			return nil, errors.New("unknown user")
 		}
 	} else {
@@ -138,10 +138,8 @@ func (ja *JWTSessionAuthenticator) Login(
 			AuthSource: schema.AuthViaToken,
 		}
 
-		if config.Keys.JwtConfig.SyncUserOnLogin {
-			if err := repository.GetUserRepository().AddUser(user); err != nil {
-				log.Errorf("Error while adding user '%s' to DB", user.Username)
-			}
+		if config.Keys.JwtConfig.SyncUserOnLogin || config.Keys.JwtConfig.UpdateUserOnLogin {
+			handleTokenUser(user)
 		}
 	}
 

internal/auth/ldap.go

@@ -1,5 +1,5 @@
-// Copyright (C) 2023 NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved.
+// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
+// All rights reserved. This file is part of cc-backend.
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 package auth
@@ -10,18 +10,17 @@ import (
 	"net/http"
 	"os"
 	"strings"
-	"time"
 
 	"github.com/ClusterCockpit/cc-backend/internal/config"
 	"github.com/ClusterCockpit/cc-backend/internal/repository"
-	"github.com/ClusterCockpit/cc-backend/pkg/log"
-	"github.com/ClusterCockpit/cc-backend/pkg/schema"
+	cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
+	"github.com/ClusterCockpit/cc-lib/schema"
 	"github.com/go-ldap/ldap/v3"
 )
 
 type LdapAuthenticator struct {
 	syncPassword string
-	UserAttr string
+	UserAttr     string
 }
 
 var _ Authenticator = (*LdapAuthenticator)(nil)
@@ -29,38 +28,11 @@ var _ Authenticator = (*LdapAuthenticator)(nil)
 func (la *LdapAuthenticator) Init() error {
 	la.syncPassword = os.Getenv("LDAP_ADMIN_PASSWORD")
 	if la.syncPassword == "" {
-		log.Warn("environment variable 'LDAP_ADMIN_PASSWORD' not set (ldap sync will not work)")
+		cclog.Warn("environment variable 'LDAP_ADMIN_PASSWORD' not set (ldap sync will not work)")
 	}
 
 	lc := config.Keys.LdapConfig
 
-	if lc.SyncInterval != "" {
-		interval, err := time.ParseDuration(lc.SyncInterval)
-		if err != nil {
-			log.Warnf("Could not parse duration for sync interval: %v",
-				lc.SyncInterval)
-			return err
-		}
-
-		if interval == 0 {
-			log.Info("Sync interval is zero")
-			return nil
-		}
-
-		go func() {
-			ticker := time.NewTicker(interval)
-			for t := range ticker.C {
-				log.Printf("sync started at %s", t.Format(time.RFC3339))
-				if err := la.Sync(); err != nil {
-					log.Errorf("sync failed: %s", err.Error())
-				}
-				log.Print("sync done")
-			}
-		}()
-	} else {
-		log.Info("LDAP configuration key sync_interval invalid")
-	}
-
 	if lc.UserAttr != "" {
 		la.UserAttr = lc.UserAttr
 	} else {
@@ -74,8 +46,8 @@ func (la *LdapAuthenticator) CanLogin(
 	user *schema.User,
 	username string,
 	rw http.ResponseWriter,
-	r *http.Request) (*schema.User, bool) {
-
+	r *http.Request,
+) (*schema.User, bool) {
 	lc := config.Keys.LdapConfig
 
 	if user != nil {
@@ -86,7 +58,7 @@ func (la *LdapAuthenticator) CanLogin(
 		if lc.SyncUserOnLogin {
 			l, err := la.getLdapConnection(true)
 			if err != nil {
-				log.Error("LDAP connection error")
+				cclog.Error("LDAP connection error")
 			}
 			defer l.Close()
 
@@ -99,12 +71,12 @@ func (la *LdapAuthenticator) CanLogin(
 
 			sr, err := l.Search(searchRequest)
 			if err != nil {
-				log.Warn(err)
+				cclog.Warn(err)
 				return nil, false
 			}
 
 			if len(sr.Entries) != 1 {
-				log.Warn("LDAP: User does not exist or too many entries returned")
+				cclog.Warn("LDAP: User does not exist or too many entries returned")
 				return nil, false
 			}
 
@@ -124,7 +96,7 @@ func (la *LdapAuthenticator) CanLogin(
 			}
 
 			if err := repository.GetUserRepository().AddUser(user); err != nil {
-				log.Errorf("User '%s' LDAP: Insert into DB failed", username)
+				cclog.Errorf("User '%s' LDAP: Insert into DB failed", username)
 				return nil, false
 			}
 
@@ -138,18 +110,18 @@ func (la *LdapAuthenticator) CanLogin(
 func (la *LdapAuthenticator) Login(
 	user *schema.User,
 	rw http.ResponseWriter,
-	r *http.Request) (*schema.User, error) {
-
+	r *http.Request,
+) (*schema.User, error) {
 	l, err := la.getLdapConnection(false)
 	if err != nil {
-		log.Warn("Error while getting ldap connection")
+		cclog.Warn("Error while getting ldap connection")
 		return nil, err
 	}
 	defer l.Close()
 
 	userDn := strings.Replace(config.Keys.LdapConfig.UserBind, "{username}", user.Username, -1)
 	if err := l.Bind(userDn, r.FormValue("password")); err != nil {
-		log.Errorf("AUTH/LDAP > Authentication for user %s failed: %v",
+		cclog.Errorf("AUTH/LDAP > Authentication for user %s failed: %v",
 			user.Username, err)
 		return nil, fmt.Errorf("Authentication failed")
 	}
@@ -176,7 +148,7 @@ func (la *LdapAuthenticator) Sync() error {
 
 	l, err := la.getLdapConnection(true)
 	if err != nil {
-		log.Error("LDAP connection error")
+		cclog.Error("LDAP connection error")
 		return err
 	}
 	defer l.Close()
@@ -187,7 +159,7 @@ func (la *LdapAuthenticator) Sync() error {
 		lc.UserFilter,
 		[]string{"dn", "uid", la.UserAttr}, nil))
 	if err != nil {
-		log.Warn("LDAP search error")
+		cclog.Warn("LDAP search error")
 		return err
 	}
 
@@ -210,7 +182,7 @@ func (la *LdapAuthenticator) Sync() error {
 	for username, where := range users {
 		if where == IN_DB && lc.SyncDelOldUsers {
 			ur.DelUser(username)
-			log.Debugf("sync: remove %v (does not show up in LDAP anymore)", username)
+			cclog.Debugf("sync: remove %v (does not show up in LDAP anymore)", username)
 		} else if where == IN_LDAP {
 			name := newnames[username]
 
@@ -226,9 +198,9 @@ func (la *LdapAuthenticator) Sync() error {
 				AuthSource: schema.AuthViaLDAP,
 			}
 
-			log.Debugf("sync: add %v (name: %v, roles: [user], ldap: true)", username, name)
+			cclog.Debugf("sync: add %v (name: %v, roles: [user], ldap: true)", username, name)
 			if err := ur.AddUser(user); err != nil {
-				log.Errorf("User '%s' LDAP: Insert into DB failed", username)
+				cclog.Errorf("User '%s' LDAP: Insert into DB failed", username)
 				return err
 			}
 		}
@@ -238,18 +210,17 @@ func (la *LdapAuthenticator) Sync() error {
 }
 
 func (la *LdapAuthenticator) getLdapConnection(admin bool) (*ldap.Conn, error) {
-
 	lc := config.Keys.LdapConfig
 	conn, err := ldap.DialURL(lc.Url)
 	if err != nil {
-		log.Warn("LDAP URL dial failed")
+		cclog.Warn("LDAP URL dial failed")
 		return nil, err
 	}
 
 	if admin {
 		if err := conn.Bind(lc.SearchDN, la.syncPassword); err != nil {
 			conn.Close()
-			log.Warn("LDAP connection bind failed")
+			cclog.Warn("LDAP connection bind failed")
 			return nil, err
 		}
 	}

internal/auth/local.go

@@ -1,5 +1,5 @@
-// Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved.
+// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
+// All rights reserved. This file is part of cc-backend.
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 package auth
@@ -8,8 +8,8 @@ import (
 	"fmt"
 	"net/http"
 
-	"github.com/ClusterCockpit/cc-backend/pkg/log"
-	"github.com/ClusterCockpit/cc-backend/pkg/schema"
+	cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
+	"github.com/ClusterCockpit/cc-lib/schema"
 	"golang.org/x/crypto/bcrypt"
 )
 
@@ -27,19 +27,19 @@ func (la *LocalAuthenticator) CanLogin(
 	user *schema.User,
 	username string,
 	rw http.ResponseWriter,
-	r *http.Request) (*schema.User, bool) {
-
+	r *http.Request,
+) (*schema.User, bool) {
 	return user, user != nil && user.AuthSource == schema.AuthViaLocalPassword
 }
 
 func (la *LocalAuthenticator) Login(
 	user *schema.User,
 	rw http.ResponseWriter,
-	r *http.Request) (*schema.User, error) {
-
+	r *http.Request,
+) (*schema.User, error) {
 	if e := bcrypt.CompareHashAndPassword([]byte(user.Password),
 		[]byte(r.FormValue("password"))); e != nil {
-		log.Errorf("AUTH/LOCAL > Authentication for user %s failed!", user.Username)
+		cclog.Errorf("AUTH/LOCAL > Authentication for user %s failed!", user.Username)
 		return nil, fmt.Errorf("Authentication failed")
 	}
 

internal/auth/oidc.go

@@ -0,0 +1,196 @@
+// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
+// All rights reserved. This file is part of cc-backend.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+package auth
+
+import (
+	"context"
+	"crypto/rand"
+	"encoding/base64"
+	"io"
+	"net/http"
+	"os"
+	"time"
+
+	"github.com/ClusterCockpit/cc-backend/internal/config"
+	"github.com/ClusterCockpit/cc-backend/internal/repository"
+	cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
+	"github.com/ClusterCockpit/cc-lib/schema"
+	"github.com/coreos/go-oidc/v3/oidc"
+	"github.com/gorilla/mux"
+	"golang.org/x/oauth2"
+)
+
+type OIDC struct {
+	client         *oauth2.Config
+	provider       *oidc.Provider
+	authentication *Authentication
+	clientID       string
+}
+
+func randString(nByte int) (string, error) {
+	b := make([]byte, nByte)
+	if _, err := io.ReadFull(rand.Reader, b); err != nil {
+		return "", err
+	}
+	return base64.RawURLEncoding.EncodeToString(b), nil
+}
+
+func setCallbackCookie(w http.ResponseWriter, r *http.Request, name, value string) {
+	c := &http.Cookie{
+		Name:     name,
+		Value:    value,
+		MaxAge:   int(time.Hour.Seconds()),
+		Secure:   r.TLS != nil,
+		HttpOnly: true,
+	}
+	http.SetCookie(w, c)
+}
+
+func NewOIDC(a *Authentication) *OIDC {
+	provider, err := oidc.NewProvider(context.Background(), config.Keys.OpenIDConfig.Provider)
+	if err != nil {
+		cclog.Fatal(err)
+	}
+	clientID := os.Getenv("OID_CLIENT_ID")
+	if clientID == "" {
+		cclog.Warn("environment variable 'OID_CLIENT_ID' not set (Open ID connect auth will not work)")
+	}
+	clientSecret := os.Getenv("OID_CLIENT_SECRET")
+	if clientSecret == "" {
+		cclog.Warn("environment variable 'OID_CLIENT_SECRET' not set (Open ID connect auth will not work)")
+	}
+
+	client := &oauth2.Config{
+		ClientID:     clientID,
+		ClientSecret: clientSecret,
+		Endpoint:     provider.Endpoint(),
+		RedirectURL:  "oidc-callback",
+		Scopes:       []string{oidc.ScopeOpenID, "profile", "email"},
+	}
+
+	oa := &OIDC{provider: provider, client: client, clientID: clientID, authentication: a}
+
+	return oa
+}
+
+func (oa *OIDC) RegisterEndpoints(r *mux.Router) {
+	r.HandleFunc("/oidc-login", oa.OAuth2Login)
+	r.HandleFunc("/oidc-callback", oa.OAuth2Callback)
+}
+
+func (oa *OIDC) OAuth2Callback(rw http.ResponseWriter, r *http.Request) {
+	c, err := r.Cookie("state")
+	if err != nil {
+		http.Error(rw, "state cookie not found", http.StatusBadRequest)
+		return
+	}
+	state := c.Value
+
+	c, err = r.Cookie("verifier")
+	if err != nil {
+		http.Error(rw, "verifier cookie not found", http.StatusBadRequest)
+		return
+	}
+	codeVerifier := c.Value
+
+	_ = r.ParseForm()
+	if r.Form.Get("state") != state {
+		http.Error(rw, "State invalid", http.StatusBadRequest)
+		return
+	}
+	code := r.Form.Get("code")
+	if code == "" {
+		http.Error(rw, "Code not found", http.StatusBadRequest)
+		return
+	}
+	token, err := oa.client.Exchange(context.Background(), code, oauth2.VerifierOption(codeVerifier))
+	if err != nil {
+		http.Error(rw, "Failed to exchange token: "+err.Error(), http.StatusInternalServerError)
+		return
+	}
+
+	userInfo, err := oa.provider.UserInfo(context.Background(), oauth2.StaticTokenSource(token))
+	if err != nil {
+		http.Error(rw, "Failed to get userinfo: "+err.Error(), http.StatusInternalServerError)
+		return
+	}
+
+	// // Extract the ID Token from OAuth2 token.
+	// rawIDToken, ok := token.Extra("id_token").(string)
+	// if !ok {
+	// 	http.Error(rw, "Cannot access idToken", http.StatusInternalServerError)
+	// }
+	//
+	// verifier := oa.provider.Verifier(&oidc.Config{ClientID: oa.clientID})
+	// // Parse and verify ID Token payload.
+	// idToken, err := verifier.Verify(context.Background(), rawIDToken)
+	// if err != nil {
+	// 	http.Error(rw, "Failed to extract idToken: "+err.Error(), http.StatusInternalServerError)
+	// }
+
+	projects := make([]string, 0)
+
+	// Extract custom claims
+	var claims struct {
+		Username string `json:"preferred_username"`
+		Name     string `json:"name"`
+		Profile  struct {
+			Client struct {
+				Roles []string `json:"roles"`
+			} `json:"clustercockpit"`
+		} `json:"resource_access"`
+	}
+	if err := userInfo.Claims(&claims); err != nil {
+		http.Error(rw, "Failed to extract Claims: "+err.Error(), http.StatusInternalServerError)
+	}
+
+	var roles []string
+	for _, r := range claims.Profile.Client.Roles {
+		switch r {
+		case "user":
+			roles = append(roles, schema.GetRoleString(schema.RoleUser))
+		case "admin":
+			roles = append(roles, schema.GetRoleString(schema.RoleAdmin))
+		}
+	}
+
+	if len(roles) == 0 {
+		roles = append(roles, schema.GetRoleString(schema.RoleUser))
+	}
+
+	user := &schema.User{
+		Username:   claims.Username,
+		Name:       claims.Name,
+		Roles:      roles,
+		Projects:   projects,
+		AuthSource: schema.AuthViaOIDC,
+	}
+
+	if config.Keys.OpenIDConfig.SyncUserOnLogin || config.Keys.OpenIDConfig.UpdateUserOnLogin {
+		handleOIDCUser(user)
+	}
+
+	oa.authentication.SaveSession(rw, r, user)
+	cclog.Infof("login successfull: user: %#v (roles: %v, projects: %v)", user.Username, user.Roles, user.Projects)
+	ctx := context.WithValue(r.Context(), repository.ContextUserKey, user)
+	http.RedirectHandler("/", http.StatusTemporaryRedirect).ServeHTTP(rw, r.WithContext(ctx))
+}
+
+func (oa *OIDC) OAuth2Login(rw http.ResponseWriter, r *http.Request) {
+	state, err := randString(16)
+	if err != nil {
+		http.Error(rw, "Internal error", http.StatusInternalServerError)
+		return
+	}
+	setCallbackCookie(rw, r, "state", state)
+
+	// use PKCE to protect against CSRF attacks
+	codeVerifier := oauth2.GenerateVerifier()
+	setCallbackCookie(rw, r, "verifier", codeVerifier)
+
+	// Redirect user to consent page to ask for permission
+	url := oa.client.AuthCodeURL(state, oauth2.AccessTypeOffline, oauth2.S256ChallengeOption(codeVerifier))
+	http.Redirect(rw, r, url, http.StatusFound)
+}

internal/config/config.go

@@ -1,5 +1,5 @@
-// Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved.
+// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
+// All rights reserved. This file is part of cc-backend.
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 package config
@@ -7,10 +7,10 @@ package config
 import (
 	"bytes"
 	"encoding/json"
-	"log"
 	"os"
 
-	"github.com/ClusterCockpit/cc-backend/pkg/schema"
+	cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
+	"github.com/ClusterCockpit/cc-lib/schema"
 )
 
 var Keys schema.ProgramConfig = schema.ProgramConfig{
@@ -29,8 +29,9 @@ var Keys schema.ProgramConfig = schema.ProgramConfig{
 		"analysis_view_histogramMetrics":         []string{"flops_any", "mem_bw", "mem_used"},
 		"analysis_view_scatterPlotMetrics":       [][]string{{"flops_any", "mem_bw"}, {"flops_any", "cpu_load"}, {"cpu_load", "mem_bw"}},
 		"job_view_nodestats_selectedMetrics":     []string{"flops_any", "mem_bw", "mem_used"},
-		"job_view_polarPlotMetrics":              []string{"flops_any", "mem_bw", "mem_used"},
 		"job_view_selectedMetrics":               []string{"flops_any", "mem_bw", "mem_used"},
+		"job_view_showFootprint":                 true,
+		"job_list_usePaging":                     false,
 		"plot_general_colorBackground":           true,
 		"plot_general_colorscheme":               []string{"#00bfff", "#0000ff", "#ff00ff", "#ff0000", "#ff8000", "#ffff00", "#80ff00"},
 		"plot_general_lineWidth":                 3,
@@ -52,20 +53,20 @@ func Init(flagConfigFile string) {
 	raw, err := os.ReadFile(flagConfigFile)
 	if err != nil {
 		if !os.IsNotExist(err) {
-			log.Fatalf("CONFIG ERROR: %v", err)
+			cclog.Abortf("Config Init: Could not read config file '%s'.\nError: %s\n", flagConfigFile, err.Error())
 		}
 	} else {
 		if err := schema.Validate(schema.Config, bytes.NewReader(raw)); err != nil {
-			log.Fatalf("Validate config: %v\n", err)
+			cclog.Abortf("Config Init: Could not validate config file '%s'.\nError: %s\n", flagConfigFile, err.Error())
 		}
 		dec := json.NewDecoder(bytes.NewReader(raw))
 		dec.DisallowUnknownFields()
 		if err := dec.Decode(&Keys); err != nil {
-			log.Fatalf("could not decode: %v", err)
+			cclog.Abortf("Config Init: Could not decode config file '%s'.\nError: %s\n", flagConfigFile, err.Error())
 		}
 
 		if Keys.Clusters == nil || len(Keys.Clusters) < 1 {
-			log.Fatal("At least one cluster required in config!")
+			cclog.Abort("Config Init: At least one cluster required in config. Exited with error.")
 		}
 	}
 }

internal/config/config_test.go

@@ -1,5 +1,5 @@
-// Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved.
+// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
+// All rights reserved. This file is part of cc-backend.
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 package config

internal/config/default_metrics.go

@@ -0,0 +1,48 @@
+// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
+// All rights reserved. This file is part of cc-backend.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+package config
+
+import (
+	"encoding/json"
+	"os"
+	"strings"
+)
+
+type DefaultMetricsCluster struct {
+	Name           string `json:"name"`
+	DefaultMetrics string `json:"default_metrics"`
+}
+
+type DefaultMetricsConfig struct {
+	Clusters []DefaultMetricsCluster `json:"clusters"`
+}
+
+func LoadDefaultMetricsConfig() (*DefaultMetricsConfig, error) {
+	filePath := "default_metrics.json"
+	if _, err := os.Stat(filePath); os.IsNotExist(err) {
+		return nil, nil
+	}
+	data, err := os.ReadFile(filePath)
+	if err != nil {
+		return nil, err
+	}
+	var cfg DefaultMetricsConfig
+	if err := json.Unmarshal(data, &cfg); err != nil {
+		return nil, err
+	}
+	return &cfg, nil
+}
+
+func ParseMetricsString(s string) []string {
+	parts := strings.Split(s, ",")
+	var metrics []string
+	for _, p := range parts {
+		trimmed := strings.TrimSpace(p)
+		if trimmed != "" {
+			metrics = append(metrics, trimmed)
+		}
+	}
+	return metrics
+}

internal/graph/model/models.go

@@ -1,5 +1,5 @@
-// Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved.
+// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
+// All rights reserved. This file is part of cc-backend.
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 package model

internal/graph/model/models_gen.go

@@ -8,7 +8,7 @@ import (
 	"strconv"
 	"time"
 
-	"github.com/ClusterCockpit/cc-backend/pkg/schema"
+	"github.com/ClusterCockpit/cc-lib/schema"
 )
 
 type Count struct {
@@ -16,11 +16,23 @@ type Count struct {
 	Count int    `json:"count"`
 }
 
+type EnergyFootprintValue struct {
+	Hardware string  `json:"hardware"`
+	Metric   string  `json:"metric"`
+	Value    float64 `json:"value"`
+}
+
 type FloatRange struct {
 	From float64 `json:"from"`
 	To   float64 `json:"to"`
 }
 
+type FootprintValue struct {
+	Name  string  `json:"name"`
+	Stat  string  `json:"stat"`
+	Value float64 `json:"value"`
+}
+
 type Footprints struct {
 	TimeWeights *TimeWeights        `json:"timeWeights"`
 	Metrics     []*MetricFootprints `json:"metrics"`
@@ -38,6 +50,7 @@ type IntRangeOutput struct {
 
 type JobFilter struct {
 	Tags            []string          `json:"tags,omitempty"`
+	DbID            []string          `json:"dbId,omitempty"`
 	JobID           *StringInput      `json:"jobId,omitempty"`
 	ArrayJobID      *int              `json:"arrayJobId,omitempty"`
 	User            *StringInput      `json:"user,omitempty"`
@@ -46,16 +59,14 @@ type JobFilter struct {
 	Cluster         *StringInput      `json:"cluster,omitempty"`
 	Partition       *StringInput      `json:"partition,omitempty"`
 	Duration        *schema.IntRange  `json:"duration,omitempty"`
+	Energy          *FloatRange       `json:"energy,omitempty"`
 	MinRunningFor   *int              `json:"minRunningFor,omitempty"`
 	NumNodes        *schema.IntRange  `json:"numNodes,omitempty"`
 	NumAccelerators *schema.IntRange  `json:"numAccelerators,omitempty"`
 	NumHWThreads    *schema.IntRange  `json:"numHWThreads,omitempty"`
 	StartTime       *schema.TimeRange `json:"startTime,omitempty"`
 	State           []schema.JobState `json:"state,omitempty"`
-	FlopsAnyAvg     *FloatRange       `json:"flopsAnyAvg,omitempty"`
-	MemBwAvg        *FloatRange       `json:"memBwAvg,omitempty"`
-	LoadAvg         *FloatRange       `json:"loadAvg,omitempty"`
-	MemUsedMax      *FloatRange       `json:"memUsedMax,omitempty"`
+	MetricStats     []*MetricStatItem `json:"metricStats,omitempty"`
 	Exclusive       *int              `json:"exclusive,omitempty"`
 	Node            *StringInput      `json:"node,omitempty"`
 }
@@ -78,29 +89,44 @@ type JobMetricWithName struct {
 }
 
 type JobResultList struct {
-	Items  []*schema.Job `json:"items"`
-	Offset *int          `json:"offset,omitempty"`
-	Limit  *int          `json:"limit,omitempty"`
-	Count  *int          `json:"count,omitempty"`
+	Items       []*schema.Job `json:"items"`
+	Offset      *int          `json:"offset,omitempty"`
+	Limit       *int          `json:"limit,omitempty"`
+	Count       *int          `json:"count,omitempty"`
+	HasNextPage *bool         `json:"hasNextPage,omitempty"`
+}
+
+type JobStats struct {
+	ID              int           `json:"id"`
+	JobID           string        `json:"jobId"`
+	StartTime       int           `json:"startTime"`
+	Duration        int           `json:"duration"`
+	Cluster         string        `json:"cluster"`
+	SubCluster      string        `json:"subCluster"`
+	NumNodes        int           `json:"numNodes"`
+	NumHWThreads    *int          `json:"numHWThreads,omitempty"`
+	NumAccelerators *int          `json:"numAccelerators,omitempty"`
+	Stats           []*NamedStats `json:"stats"`
 }
 
 type JobsStatistics struct {
-	ID             string        `json:"id"`
-	Name           string        `json:"name"`
-	TotalJobs      int           `json:"totalJobs"`
-	RunningJobs    int           `json:"runningJobs"`
-	ShortJobs      int           `json:"shortJobs"`
-	TotalWalltime  int           `json:"totalWalltime"`
-	TotalNodes     int           `json:"totalNodes"`
-	TotalNodeHours int           `json:"totalNodeHours"`
-	TotalCores     int           `json:"totalCores"`
-	TotalCoreHours int           `json:"totalCoreHours"`
-	TotalAccs      int           `json:"totalAccs"`
-	TotalAccHours  int           `json:"totalAccHours"`
-	HistDuration   []*HistoPoint `json:"histDuration"`
-	HistNumNodes   []*HistoPoint `json:"histNumNodes"`
-	HistNumCores   []*HistoPoint `json:"histNumCores"`
-	HistNumAccs    []*HistoPoint `json:"histNumAccs"`
+	ID             string               `json:"id"`
+	Name           string               `json:"name"`
+	TotalJobs      int                  `json:"totalJobs"`
+	RunningJobs    int                  `json:"runningJobs"`
+	ShortJobs      int                  `json:"shortJobs"`
+	TotalWalltime  int                  `json:"totalWalltime"`
+	TotalNodes     int                  `json:"totalNodes"`
+	TotalNodeHours int                  `json:"totalNodeHours"`
+	TotalCores     int                  `json:"totalCores"`
+	TotalCoreHours int                  `json:"totalCoreHours"`
+	TotalAccs      int                  `json:"totalAccs"`
+	TotalAccHours  int                  `json:"totalAccHours"`
+	HistDuration   []*HistoPoint        `json:"histDuration"`
+	HistNumNodes   []*HistoPoint        `json:"histNumNodes"`
+	HistNumCores   []*HistoPoint        `json:"histNumCores"`
+	HistNumAccs    []*HistoPoint        `json:"histNumAccs"`
+	HistMetrics    []*MetricHistoPoints `json:"histMetrics"`
 }
 
 type MetricFootprints struct {
@@ -108,14 +134,74 @@ type MetricFootprints struct {
 	Data   []schema.Float `json:"data"`
 }
 
+type MetricHistoPoint struct {
+	Bin   *int `json:"bin,omitempty"`
+	Count int  `json:"count"`
+	Min   *int `json:"min,omitempty"`
+	Max   *int `json:"max,omitempty"`
+}
+
+type MetricHistoPoints struct {
+	Metric string              `json:"metric"`
+	Unit   string              `json:"unit"`
+	Stat   *string             `json:"stat,omitempty"`
+	Data   []*MetricHistoPoint `json:"data,omitempty"`
+}
+
+type MetricStatItem struct {
+	MetricName string      `json:"metricName"`
+	Range      *FloatRange `json:"range"`
+}
+
+type Mutation struct {
+}
+
+type NamedStats struct {
+	Name string                   `json:"name"`
+	Data *schema.MetricStatistics `json:"data"`
+}
+
+type NamedStatsWithScope struct {
+	Name  string             `json:"name"`
+	Scope schema.MetricScope `json:"scope"`
+	Stats []*ScopedStats     `json:"stats"`
+}
+
+type NodeFilter struct {
+	Hostname    *StringInput      `json:"hostname,omitempty"`
+	Cluster     *StringInput      `json:"cluster,omitempty"`
+	NodeState   *string           `json:"nodeState,omitempty"`
+	HealthState *schema.NodeState `json:"healthState,omitempty"`
+}
+
 type NodeMetrics struct {
 	Host       string               `json:"host"`
 	SubCluster string               `json:"subCluster"`
 	Metrics    []*JobMetricWithName `json:"metrics"`
 }
 
+type NodeStateResultList struct {
+	Items []*schema.Node `json:"items"`
+	Count *int           `json:"count,omitempty"`
+}
+
+type NodeStats struct {
+	State string `json:"state"`
+	Count int    `json:"count"`
+}
+
+type NodesResultList struct {
+	Items       []*NodeMetrics `json:"items"`
+	Offset      *int           `json:"offset,omitempty"`
+	Limit       *int           `json:"limit,omitempty"`
+	Count       *int           `json:"count,omitempty"`
+	TotalNodes  *int           `json:"totalNodes,omitempty"`
+	HasNextPage *bool          `json:"hasNextPage,omitempty"`
+}
+
 type OrderByInput struct {
 	Field string            `json:"field"`
+	Type  string            `json:"type"`
 	Order SortDirectionEnum `json:"order"`
 }
 
@@ -124,6 +210,12 @@ type PageRequest struct {
 	Page         int `json:"page"`
 }
 
+type ScopedStats struct {
+	Hostname string                   `json:"hostname"`
+	ID       *string                  `json:"id,omitempty"`
+	Data     *schema.MetricStatistics `json:"data"`
+}
+
 type StringInput struct {
 	Eq         *string  `json:"eq,omitempty"`
 	Neq        *string  `json:"neq,omitempty"`
@@ -134,8 +226,9 @@ type StringInput struct {
 }
 
 type TimeRangeOutput struct {
-	From time.Time `json:"from"`
-	To   time.Time `json:"to"`
+	Range *string   `json:"range,omitempty"`
+	From  time.Time `json:"from"`
+	To    time.Time `json:"to"`
 }
 
 type TimeWeights struct {
@@ -176,7 +269,7 @@ func (e Aggregate) String() string {
 	return string(e)
 }
 
-func (e *Aggregate) UnmarshalGQL(v interface{}) error {
+func (e *Aggregate) UnmarshalGQL(v any) error {
 	str, ok := v.(string)
 	if !ok {
 		return fmt.Errorf("enums must be strings")
@@ -229,7 +322,7 @@ func (e SortByAggregate) String() string {
 	return string(e)
 }
 
-func (e *SortByAggregate) UnmarshalGQL(v interface{}) error {
+func (e *SortByAggregate) UnmarshalGQL(v any) error {
 	str, ok := v.(string)
 	if !ok {
 		return fmt.Errorf("enums must be strings")
@@ -270,7 +363,7 @@ func (e SortDirectionEnum) String() string {
 	return string(e)
 }
 
-func (e *SortDirectionEnum) UnmarshalGQL(v interface{}) error {
+func (e *SortDirectionEnum) UnmarshalGQL(v any) error {
 	str, ok := v.(string)
 	if !ok {
 		return fmt.Errorf("enums must be strings")

internal/graph/resolver.go

@@ -1,15 +1,39 @@
 package graph
 
 import (
+	"sync"
+
 	"github.com/ClusterCockpit/cc-backend/internal/repository"
+	cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
 	"github.com/jmoiron/sqlx"
 )
 
 // This file will not be regenerated automatically.
 //
 // It serves as dependency injection for your app, add any dependencies you require here.
+var (
+	initOnce         sync.Once
+	resolverInstance *Resolver
+)
 
 type Resolver struct {
 	DB   *sqlx.DB
 	Repo *repository.JobRepository
 }
+
+func Init() {
+	initOnce.Do(func() {
+		db := repository.GetConnection()
+		resolverInstance = &Resolver{
+			DB: db.DB, Repo: repository.GetJobRepository(),
+		}
+	})
+}
+
+func GetResolverInstance() *Resolver {
+	if resolverInstance == nil {
+		cclog.Fatal("Authentication module not initialized!")
+	}
+
+	return resolverInstance
+}

internal/graph/schema.resolvers.go

@@ -2,22 +2,26 @@ package graph
 
 // This file will be automatically regenerated based on the schema, any resolver implementations
 // will be copied through when generating and any unknown code will be moved to the end.
-// Code generated by github.com/99designs/gqlgen version v0.17.36
+// Code generated by github.com/99designs/gqlgen version v0.17.66
 
 import (
 	"context"
 	"errors"
 	"fmt"
+	"regexp"
+	"slices"
 	"strconv"
+	"strings"
 	"time"
 
+	"github.com/ClusterCockpit/cc-backend/internal/config"
 	"github.com/ClusterCockpit/cc-backend/internal/graph/generated"
 	"github.com/ClusterCockpit/cc-backend/internal/graph/model"
-	"github.com/ClusterCockpit/cc-backend/internal/metricdata"
+	"github.com/ClusterCockpit/cc-backend/internal/metricDataDispatcher"
 	"github.com/ClusterCockpit/cc-backend/internal/repository"
 	"github.com/ClusterCockpit/cc-backend/pkg/archive"
-	"github.com/ClusterCockpit/cc-backend/pkg/log"
-	"github.com/ClusterCockpit/cc-backend/pkg/schema"
+	cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
+	"github.com/ClusterCockpit/cc-lib/schema"
 )
 
 // Partitions is the resolver for the partitions field.
@@ -25,17 +29,20 @@ func (r *clusterResolver) Partitions(ctx context.Context, obj *schema.Cluster) (
 	return r.Repo.Partitions(obj.Name)
 }
 
+// StartTime is the resolver for the startTime field.
+func (r *jobResolver) StartTime(ctx context.Context, obj *schema.Job) (*time.Time, error) {
+	timestamp := time.Unix(obj.StartTime, 0)
+	return &timestamp, nil
+}
+
 // Tags is the resolver for the tags field.
 func (r *jobResolver) Tags(ctx context.Context, obj *schema.Job) ([]*schema.Tag, error) {
-	return r.Repo.GetTags(&obj.ID)
+	return r.Repo.GetTags(repository.GetUserFromContext(ctx), obj.ID)
 }
 
 // ConcurrentJobs is the resolver for the concurrentJobs field.
 func (r *jobResolver) ConcurrentJobs(ctx context.Context, obj *schema.Job) (*model.JobLinkResultList, error) {
-	if obj.State == schema.JobStateRunning {
-		obj.Duration = int32(time.Now().Unix() - obj.StartTimeUnix)
-	}
-
+	// FIXME: Make the hardcoded duration configurable
 	if obj.Exclusive != 1 && obj.Duration > 600 {
 		return r.Repo.FindConcurrentJobs(ctx, obj)
 	}
@@ -43,8 +50,72 @@ func (r *jobResolver) ConcurrentJobs(ctx context.Context, obj *schema.Job) (*mod
 	return nil, nil
 }
 
+// Footprint is the resolver for the footprint field.
+func (r *jobResolver) Footprint(ctx context.Context, obj *schema.Job) ([]*model.FootprintValue, error) {
+	rawFootprint, err := r.Repo.FetchFootprint(obj)
+	if err != nil {
+		cclog.Warn("Error while fetching job footprint data")
+		return nil, err
+	}
+
+	res := []*model.FootprintValue{}
+	for name, value := range rawFootprint {
+
+		parts := strings.Split(name, "_")
+		statPart := parts[len(parts)-1]
+		nameParts := parts[:len(parts)-1]
+
+		res = append(res, &model.FootprintValue{
+			Name:  strings.Join(nameParts, "_"),
+			Stat:  statPart,
+			Value: value,
+		})
+	}
+
+	return res, err
+}
+
+// EnergyFootprint is the resolver for the energyFootprint field.
+func (r *jobResolver) EnergyFootprint(ctx context.Context, obj *schema.Job) ([]*model.EnergyFootprintValue, error) {
+	rawEnergyFootprint, err := r.Repo.FetchEnergyFootprint(obj)
+	if err != nil {
+		cclog.Warn("Error while fetching job energy footprint data")
+		return nil, err
+	}
+
+	res := []*model.EnergyFootprintValue{}
+	for name, value := range rawEnergyFootprint {
+		// Suboptimal: Nearly hardcoded metric name expectations
+		matchCpu := regexp.MustCompile(`cpu|Cpu|CPU`)
+		matchAcc := regexp.MustCompile(`acc|Acc|ACC`)
+		matchMem := regexp.MustCompile(`mem|Mem|MEM`)
+		matchCore := regexp.MustCompile(`core|Core|CORE`)
+
+		hwType := ""
+		switch test := name; { // NOtice ';' for var declaration
+		case matchCpu.MatchString(test):
+			hwType = "CPU"
+		case matchAcc.MatchString(test):
+			hwType = "Accelerator"
+		case matchMem.MatchString(test):
+			hwType = "Memory"
+		case matchCore.MatchString(test):
+			hwType = "Core"
+		default:
+			hwType = "Other"
+		}
+
+		res = append(res, &model.EnergyFootprintValue{
+			Hardware: hwType,
+			Metric:   name,
+			Value:    value,
+		})
+	}
+	return res, err
+}
+
 // MetaData is the resolver for the metaData field.
-func (r *jobResolver) MetaData(ctx context.Context, obj *schema.Job) (interface{}, error) {
+func (r *jobResolver) MetaData(ctx context.Context, obj *schema.Job) (any, error) {
 	return r.Repo.FetchMetadata(obj)
 }
 
@@ -53,41 +124,82 @@ func (r *jobResolver) UserData(ctx context.Context, obj *schema.Job) (*model.Use
 	return repository.GetUserRepository().FetchUserInCtx(ctx, obj.User)
 }
 
+// Name is the resolver for the name field.
+func (r *metricValueResolver) Name(ctx context.Context, obj *schema.MetricValue) (*string, error) {
+	panic(fmt.Errorf("not implemented: Name - name"))
+}
+
 // CreateTag is the resolver for the createTag field.
-func (r *mutationResolver) CreateTag(ctx context.Context, typeArg string, name string) (*schema.Tag, error) {
-	id, err := r.Repo.CreateTag(typeArg, name)
-	if err != nil {
-		log.Warn("Error while creating tag")
-		return nil, err
+func (r *mutationResolver) CreateTag(ctx context.Context, typeArg string, name string, scope string) (*schema.Tag, error) {
+	user := repository.GetUserFromContext(ctx)
+	if user == nil {
+		return nil, fmt.Errorf("no user in context")
 	}
 
-	return &schema.Tag{ID: id, Type: typeArg, Name: name}, nil
+	// Test Access: Admins && Admin Tag OR Support/Admin and Global Tag OR Everyone && Private Tag
+	if user.HasRole(schema.RoleAdmin) && scope == "admin" ||
+		user.HasAnyRole([]schema.Role{schema.RoleAdmin, schema.RoleSupport}) && scope == "global" ||
+		user.Username == scope {
+		// Create in DB
+		id, err := r.Repo.CreateTag(typeArg, name, scope)
+		if err != nil {
+			cclog.Warn("Error while creating tag")
+			return nil, err
+		}
+		return &schema.Tag{ID: id, Type: typeArg, Name: name, Scope: scope}, nil
+	} else {
+		cclog.Warnf("Not authorized to create tag with scope: %s", scope)
+		return nil, fmt.Errorf("not authorized to create tag with scope: %s", scope)
+	}
 }
 
 // DeleteTag is the resolver for the deleteTag field.
 func (r *mutationResolver) DeleteTag(ctx context.Context, id string) (string, error) {
+	// This Uses ID string <-> ID string, removeTagFromList uses []string <-> []int
 	panic(fmt.Errorf("not implemented: DeleteTag - deleteTag"))
 }
 
 // AddTagsToJob is the resolver for the addTagsToJob field.
 func (r *mutationResolver) AddTagsToJob(ctx context.Context, job string, tagIds []string) ([]*schema.Tag, error) {
+	user := repository.GetUserFromContext(ctx)
+	if user == nil {
+		return nil, fmt.Errorf("no user in context")
+	}
+
 	jid, err := strconv.ParseInt(job, 10, 64)
 	if err != nil {
-		log.Warn("Error while adding tag to job")
+		cclog.Warn("Error while adding tag to job")
 		return nil, err
 	}
 
 	tags := []*schema.Tag{}
 	for _, tagId := range tagIds {
+		// Get ID
 		tid, err := strconv.ParseInt(tagId, 10, 64)
 		if err != nil {
-			log.Warn("Error while parsing tag id")
+			cclog.Warn("Error while parsing tag id")
 			return nil, err
 		}
 
-		if tags, err = r.Repo.AddTag(jid, tid); err != nil {
-			log.Warn("Error while adding tag")
-			return nil, err
+		// Test Exists
+		_, _, tscope, exists := r.Repo.TagInfo(tid)
+		if !exists {
+			cclog.Warnf("Tag does not exist (ID): %d", tid)
+			return nil, fmt.Errorf("tag does not exist (ID): %d", tid)
+		}
+
+		// Test Access: Admins && Admin Tag OR Support/Admin and Global Tag OR Everyone && Private Tag
+		if user.HasRole(schema.RoleAdmin) && tscope == "admin" ||
+			user.HasAnyRole([]schema.Role{schema.RoleAdmin, schema.RoleSupport}) && tscope == "global" ||
+			user.Username == tscope {
+			// Add to Job
+			if tags, err = r.Repo.AddTag(user, jid, tid); err != nil {
+				cclog.Warn("Error while adding tag")
+				return nil, err
+			}
+		} else {
+			cclog.Warnf("Not authorized to add tag: %d", tid)
+			return nil, fmt.Errorf("not authorized to add tag: %d", tid)
 		}
 	}
 
@@ -96,39 +208,118 @@ func (r *mutationResolver) AddTagsToJob(ctx context.Context, job string, tagIds
 
 // RemoveTagsFromJob is the resolver for the removeTagsFromJob field.
 func (r *mutationResolver) RemoveTagsFromJob(ctx context.Context, job string, tagIds []string) ([]*schema.Tag, error) {
+	user := repository.GetUserFromContext(ctx)
+	if user == nil {
+		return nil, fmt.Errorf("no user in context")
+	}
+
 	jid, err := strconv.ParseInt(job, 10, 64)
 	if err != nil {
-		log.Warn("Error while parsing job id")
+		cclog.Warn("Error while parsing job id")
 		return nil, err
 	}
 
 	tags := []*schema.Tag{}
 	for _, tagId := range tagIds {
+		// Get ID
 		tid, err := strconv.ParseInt(tagId, 10, 64)
 		if err != nil {
-			log.Warn("Error while parsing tag id")
+			cclog.Warn("Error while parsing tag id")
 			return nil, err
 		}
 
-		if tags, err = r.Repo.RemoveTag(jid, tid); err != nil {
-			log.Warn("Error while removing tag")
-			return nil, err
+		// Test Exists
+		_, _, tscope, exists := r.Repo.TagInfo(tid)
+		if !exists {
+			cclog.Warnf("Tag does not exist (ID): %d", tid)
+			return nil, fmt.Errorf("tag does not exist (ID): %d", tid)
 		}
+
+		// Test Access: Admins && Admin Tag OR Support/Admin and Global Tag OR Everyone && Private Tag
+		if user.HasRole(schema.RoleAdmin) && tscope == "admin" ||
+			user.HasAnyRole([]schema.Role{schema.RoleAdmin, schema.RoleSupport}) && tscope == "global" ||
+			user.Username == tscope {
+			// Remove from Job
+			if tags, err = r.Repo.RemoveTag(user, jid, tid); err != nil {
+				cclog.Warn("Error while removing tag")
+				return nil, err
+			}
+		} else {
+			cclog.Warnf("Not authorized to remove tag: %d", tid)
+			return nil, fmt.Errorf("not authorized to remove tag: %d", tid)
+		}
+
 	}
 
 	return tags, nil
 }
 
+// RemoveTagFromList is the resolver for the removeTagFromList field.
+func (r *mutationResolver) RemoveTagFromList(ctx context.Context, tagIds []string) ([]int, error) {
+	// Needs Contextuser
+	user := repository.GetUserFromContext(ctx)
+	if user == nil {
+		return nil, fmt.Errorf("no user in context")
+	}
+
+	tags := []int{}
+	for _, tagId := range tagIds {
+		// Get ID
+		tid, err := strconv.ParseInt(tagId, 10, 64)
+		if err != nil {
+			cclog.Warn("Error while parsing tag id for removal")
+			return nil, err
+		}
+
+		// Test Exists
+		_, _, tscope, exists := r.Repo.TagInfo(tid)
+		if !exists {
+			cclog.Warnf("Tag does not exist (ID): %d", tid)
+			return nil, fmt.Errorf("tag does not exist (ID): %d", tid)
+		}
+
+		// Test Access: Admins && Admin Tag OR Everyone && Private Tag
+		if user.HasRole(schema.RoleAdmin) && (tscope == "global" || tscope == "admin") || user.Username == tscope {
+			// Remove from DB
+			if err = r.Repo.RemoveTagById(tid); err != nil {
+				cclog.Warn("Error while removing tag")
+				return nil, err
+			} else {
+				tags = append(tags, int(tid))
+			}
+		} else {
+			cclog.Warnf("Not authorized to remove tag: %d", tid)
+			return nil, fmt.Errorf("not authorized to remove tag: %d", tid)
+		}
+	}
+	return tags, nil
+}
+
 // UpdateConfiguration is the resolver for the updateConfiguration field.
 func (r *mutationResolver) UpdateConfiguration(ctx context.Context, name string, value string) (*string, error) {
 	if err := repository.GetUserCfgRepo().UpdateConfig(name, value, repository.GetUserFromContext(ctx)); err != nil {
-		log.Warn("Error while updating user config")
+		cclog.Warn("Error while updating user config")
 		return nil, err
 	}
 
 	return nil, nil
 }
 
+// NodeState is the resolver for the nodeState field.
+func (r *nodeResolver) NodeState(ctx context.Context, obj *schema.Node) (string, error) {
+	panic(fmt.Errorf("not implemented: NodeState - nodeState"))
+}
+
+// HealthState is the resolver for the HealthState field.
+func (r *nodeResolver) HealthState(ctx context.Context, obj *schema.Node) (schema.NodeState, error) {
+	panic(fmt.Errorf("not implemented: HealthState - HealthState"))
+}
+
+// MetaData is the resolver for the metaData field.
+func (r *nodeResolver) MetaData(ctx context.Context, obj *schema.Node) (any, error) {
+	panic(fmt.Errorf("not implemented: MetaData - metaData"))
+}
+
 // Clusters is the resolver for the clusters field.
 func (r *queryResolver) Clusters(ctx context.Context) ([]*schema.Cluster, error) {
 	return archive.Clusters, nil
@@ -136,7 +327,12 @@ func (r *queryResolver) Clusters(ctx context.Context) ([]*schema.Cluster, error)
 
 // Tags is the resolver for the tags field.
 func (r *queryResolver) Tags(ctx context.Context) ([]*schema.Tag, error) {
-	return r.Repo.GetTags(nil)
+	return r.Repo.GetTags(repository.GetUserFromContext(ctx), nil)
+}
+
+// GlobalMetrics is the resolver for the globalMetrics field.
+func (r *queryResolver) GlobalMetrics(ctx context.Context) ([]*schema.GlobalMetricListItem, error) {
+	return archive.GlobalMetricList, nil
 }
 
 // User is the resolver for the user field.
@@ -148,7 +344,7 @@ func (r *queryResolver) User(ctx context.Context, username string) (*model.User,
 func (r *queryResolver) AllocatedNodes(ctx context.Context, cluster string) ([]*model.Count, error) {
 	data, err := r.Repo.AllocatedNodes(cluster)
 	if err != nil {
-		log.Warn("Error while fetching allocated nodes")
+		cclog.Warn("Error while fetching allocated nodes")
 		return nil, err
 	}
 
@@ -163,17 +359,41 @@ func (r *queryResolver) AllocatedNodes(ctx context.Context, cluster string) ([]*
 	return counts, nil
 }
 
+// Node is the resolver for the node field.
+func (r *queryResolver) Node(ctx context.Context, id string) (*schema.Node, error) {
+	repo := repository.GetNodeRepository()
+	numericId, err := strconv.ParseInt(id, 10, 64)
+	if err != nil {
+		cclog.Warn("Error while parsing job id")
+		return nil, err
+	}
+	return repo.GetNode(numericId, false)
+}
+
+// Nodes is the resolver for the nodes field.
+func (r *queryResolver) Nodes(ctx context.Context, filter []*model.NodeFilter, order *model.OrderByInput) (*model.NodeStateResultList, error) {
+	repo := repository.GetNodeRepository()
+	nodes, err := repo.QueryNodes(ctx, filter, order)
+	count := len(nodes)
+	return &model.NodeStateResultList{Items: nodes, Count: &count}, err
+}
+
+// NodeStats is the resolver for the nodeStats field.
+func (r *queryResolver) NodeStats(ctx context.Context, filter []*model.NodeFilter) ([]*model.NodeStats, error) {
+	panic(fmt.Errorf("not implemented: NodeStats - nodeStats"))
+}
+
 // Job is the resolver for the job field.
 func (r *queryResolver) Job(ctx context.Context, id string) (*schema.Job, error) {
 	numericId, err := strconv.ParseInt(id, 10, 64)
 	if err != nil {
-		log.Warn("Error while parsing job id")
+		cclog.Warn("Error while parsing job id")
 		return nil, err
 	}
 
-	job, err := r.Repo.FindById(numericId)
+	job, err := r.Repo.FindById(ctx, numericId)
 	if err != nil {
-		log.Warn("Error while finding job by id")
+		cclog.Warn("Error while finding job by id")
 		return nil, err
 	}
 
@@ -187,16 +407,26 @@ func (r *queryResolver) Job(ctx context.Context, id string) (*schema.Job, error)
 }
 
 // JobMetrics is the resolver for the jobMetrics field.
-func (r *queryResolver) JobMetrics(ctx context.Context, id string, metrics []string, scopes []schema.MetricScope) ([]*model.JobMetricWithName, error) {
+func (r *queryResolver) JobMetrics(ctx context.Context, id string, metrics []string, scopes []schema.MetricScope, resolution *int) ([]*model.JobMetricWithName, error) {
+	if resolution == nil { // Load from Config
+		if config.Keys.EnableResampling != nil {
+			defaultRes := slices.Max(config.Keys.EnableResampling.Resolutions)
+			resolution = &defaultRes
+		} else { // Set 0 (Loads configured metric timestep)
+			defaultRes := 0
+			resolution = &defaultRes
+		}
+	}
+
 	job, err := r.Query().Job(ctx, id)
 	if err != nil {
-		log.Warn("Error while querying job for metrics")
+		cclog.Warn("Error while querying job for metrics")
 		return nil, err
 	}
 
-	data, err := metricdata.LoadData(job, metrics, scopes, ctx)
+	data, err := metricDataDispatcher.LoadData(job, metrics, scopes, ctx, *resolution)
 	if err != nil {
-		log.Warn("Error while loading job data")
+		cclog.Warn("Error while loading job data")
 		return nil, err
 	}
 
@@ -214,9 +444,67 @@ func (r *queryResolver) JobMetrics(ctx context.Context, id string, metrics []str
 	return res, err
 }
 
-// JobsFootprints is the resolver for the jobsFootprints field.
-func (r *queryResolver) JobsFootprints(ctx context.Context, filter []*model.JobFilter, metrics []string) (*model.Footprints, error) {
-	return r.jobsFootprints(ctx, filter, metrics)
+// JobStats is the resolver for the jobStats field.
+func (r *queryResolver) JobStats(ctx context.Context, id string, metrics []string) ([]*model.NamedStats, error) {
+	job, err := r.Query().Job(ctx, id)
+	if err != nil {
+		cclog.Warnf("Error while querying job %s for metadata", id)
+		return nil, err
+	}
+
+	data, err := metricDataDispatcher.LoadJobStats(job, metrics, ctx)
+	if err != nil {
+		cclog.Warnf("Error while loading jobStats data for job id %s", id)
+		return nil, err
+	}
+
+	res := []*model.NamedStats{}
+	for name, md := range data {
+		res = append(res, &model.NamedStats{
+			Name: name,
+			Data: &md,
+		})
+	}
+
+	return res, err
+}
+
+// ScopedJobStats is the resolver for the scopedJobStats field.
+func (r *queryResolver) ScopedJobStats(ctx context.Context, id string, metrics []string, scopes []schema.MetricScope) ([]*model.NamedStatsWithScope, error) {
+	job, err := r.Query().Job(ctx, id)
+	if err != nil {
+		cclog.Warnf("Error while querying job %s for metadata", id)
+		return nil, err
+	}
+
+	data, err := metricDataDispatcher.LoadScopedJobStats(job, metrics, scopes, ctx)
+	if err != nil {
+		cclog.Warnf("Error while loading scopedJobStats data for job id %s", id)
+		return nil, err
+	}
+
+	res := make([]*model.NamedStatsWithScope, 0)
+	for name, scoped := range data {
+		for scope, stats := range scoped {
+
+			mdlStats := make([]*model.ScopedStats, 0)
+			for _, stat := range stats {
+				mdlStats = append(mdlStats, &model.ScopedStats{
+					Hostname: stat.Hostname,
+					ID:       stat.Id,
+					Data:     stat.Data,
+				})
+			}
+
+			res = append(res, &model.NamedStatsWithScope{
+				Name:  name,
+				Scope: scope,
+				Stats: mdlStats,
+			})
+		}
+	}
+
+	return res, nil
 }
 
 // Jobs is the resolver for the jobs field.
@@ -230,24 +518,46 @@ func (r *queryResolver) Jobs(ctx context.Context, filter []*model.JobFilter, pag
 
 	jobs, err := r.Repo.QueryJobs(ctx, filter, page, order)
 	if err != nil {
-		log.Warn("Error while querying jobs")
+		cclog.Warn("Error while querying jobs")
 		return nil, err
 	}
 
 	count, err := r.Repo.CountJobs(ctx, filter)
 	if err != nil {
-		log.Warn("Error while counting jobs")
+		cclog.Warn("Error while counting jobs")
 		return nil, err
 	}
 
-	return &model.JobResultList{Items: jobs, Count: &count}, nil
+	// Note: Even if App-Default 'config.Keys.UiDefaults["job_list_usePaging"]' is set, always return hasNextPage boolean.
+	// Users can decide in frontend to use continuous scroll, even if app-default is paging!
+	/*
+	  Example Page 4 @ 10 IpP : Does item 41 exist?
+	  Minimal Page 41 @ 1 IpP : If len(result) is 1, Page 5 @ 10 IpP exists.
+	*/
+	nextPage := &model.PageRequest{
+		ItemsPerPage: 1,
+		Page:         ((page.Page * page.ItemsPerPage) + 1),
+	}
+	nextJobs, err := r.Repo.QueryJobs(ctx, filter, nextPage, order)
+	if err != nil {
+		cclog.Warn("Error while querying next jobs")
+		return nil, err
+	}
+
+	hasNextPage := len(nextJobs) == 1
+
+	return &model.JobResultList{Items: jobs, Count: &count, HasNextPage: &hasNextPage}, nil
 }
 
 // JobsStatistics is the resolver for the jobsStatistics field.
-func (r *queryResolver) JobsStatistics(ctx context.Context, filter []*model.JobFilter, page *model.PageRequest, sortBy *model.SortByAggregate, groupBy *model.Aggregate) ([]*model.JobsStatistics, error) {
+func (r *queryResolver) JobsStatistics(ctx context.Context, filter []*model.JobFilter, metrics []string, page *model.PageRequest, sortBy *model.SortByAggregate, groupBy *model.Aggregate, numDurationBins *string, numMetricBins *int) ([]*model.JobsStatistics, error) {
 	var err error
 	var stats []*model.JobsStatistics
 
+	// Top Level Defaults
+	defaultDurationBins := "1h"
+	defaultMetricBins := 10
+
 	if requireField(ctx, "totalJobs") || requireField(ctx, "totalWalltime") || requireField(ctx, "totalNodes") || requireField(ctx, "totalCores") ||
 		requireField(ctx, "totalAccs") || requireField(ctx, "totalNodeHours") || requireField(ctx, "totalCoreHours") || requireField(ctx, "totalAccHours") {
 		if groupBy == nil {
@@ -281,8 +591,13 @@ func (r *queryResolver) JobsStatistics(ctx context.Context, filter []*model.JobF
 	}
 
 	if requireField(ctx, "histDuration") || requireField(ctx, "histNumNodes") || requireField(ctx, "histNumCores") || requireField(ctx, "histNumAccs") {
+
+		if numDurationBins == nil {
+			numDurationBins = &defaultDurationBins
+		}
+
 		if groupBy == nil {
-			stats[0], err = r.Repo.AddHistograms(ctx, filter, stats[0])
+			stats[0], err = r.Repo.AddHistograms(ctx, filter, stats[0], numDurationBins)
 			if err != nil {
 				return nil, err
 			}
@@ -291,9 +606,81 @@ func (r *queryResolver) JobsStatistics(ctx context.Context, filter []*model.JobF
 		}
 	}
 
+	if requireField(ctx, "histMetrics") {
+
+		if numMetricBins == nil {
+			numMetricBins = &defaultMetricBins
+		}
+
+		if groupBy == nil {
+			stats[0], err = r.Repo.AddMetricHistograms(ctx, filter, metrics, stats[0], numMetricBins)
+			if err != nil {
+				return nil, err
+			}
+		} else {
+			return nil, errors.New("metric histograms only implemented without groupBy argument")
+		}
+	}
+
 	return stats, nil
 }
 
+// JobsMetricStats is the resolver for the jobsMetricStats field.
+func (r *queryResolver) JobsMetricStats(ctx context.Context, filter []*model.JobFilter, metrics []string) ([]*model.JobStats, error) {
+	// No Paging, Fixed Order by StartTime ASC
+	order := &model.OrderByInput{
+		Field: "startTime",
+		Type:  "col",
+		Order: "ASC",
+	}
+
+	jobs, err := r.Repo.QueryJobs(ctx, filter, nil, order)
+	if err != nil {
+		cclog.Warn("Error while querying jobs for comparison")
+		return nil, err
+	}
+
+	res := []*model.JobStats{}
+	for _, job := range jobs {
+		data, err := metricDataDispatcher.LoadJobStats(job, metrics, ctx)
+		if err != nil {
+			cclog.Warnf("Error while loading comparison jobStats data for job id %d", job.JobID)
+			continue
+			// return nil, err
+		}
+
+		sres := []*model.NamedStats{}
+		for name, md := range data {
+			sres = append(sres, &model.NamedStats{
+				Name: name,
+				Data: &md,
+			})
+		}
+
+		numThreadsInt := int(job.NumHWThreads)
+		numAccsInt := int(job.NumAcc)
+		res = append(res, &model.JobStats{
+			ID:              int(*job.ID),
+			JobID:           strconv.Itoa(int(job.JobID)),
+			StartTime:       int(job.StartTime),
+			Duration:        int(job.Duration),
+			Cluster:         job.Cluster,
+			SubCluster:      job.SubCluster,
+			NumNodes:        int(job.NumNodes),
+			NumHWThreads:    &numThreadsInt,
+			NumAccelerators: &numAccsInt,
+			Stats:           sres,
+		})
+	}
+	return res, err
+}
+
+// JobsFootprints is the resolver for the jobsFootprints field.
+func (r *queryResolver) JobsFootprints(ctx context.Context, filter []*model.JobFilter, metrics []string) (*model.Footprints, error) {
+	// NOTE: Legacy Naming! This resolver is for normalized histograms in analysis view only - *Not* related to DB "footprint" column!
+	return r.jobsFootprints(ctx, filter, metrics)
+}
+
 // RooflineHeatmap is the resolver for the rooflineHeatmap field.
 func (r *queryResolver) RooflineHeatmap(ctx context.Context, filter []*model.JobFilter, rows int, cols int, minX float64, minY float64, maxX float64, maxY float64) ([][]float64, error) {
 	return r.rooflineHeatmap(ctx, filter, rows, cols, minX, minY, maxX, maxY)
@@ -302,8 +689,8 @@ func (r *queryResolver) RooflineHeatmap(ctx context.Context, filter []*model.Job
 // NodeMetrics is the resolver for the nodeMetrics field.
 func (r *queryResolver) NodeMetrics(ctx context.Context, cluster string, nodes []string, scopes []schema.MetricScope, metrics []string, from time.Time, to time.Time) ([]*model.NodeMetrics, error) {
 	user := repository.GetUserFromContext(ctx)
-	if user != nil && !user.HasRole(schema.RoleAdmin) {
-		return nil, errors.New("you need to be an administrator for this query")
+	if user != nil && !user.HasAnyRole([]schema.Role{schema.RoleAdmin, schema.RoleSupport}) {
+		return nil, errors.New("you need to be administrator or support staff for this query")
 	}
 
 	if metrics == nil {
@@ -312,9 +699,9 @@ func (r *queryResolver) NodeMetrics(ctx context.Context, cluster string, nodes [
 		}
 	}
 
-	data, err := metricdata.LoadNodeData(cluster, metrics, nodes, scopes, from, to, ctx)
+	data, err := metricDataDispatcher.LoadNodeData(cluster, metrics, nodes, scopes, from, to, ctx)
 	if err != nil {
-		log.Warn("Error while loading node data")
+		cclog.Warn("error while loading node data")
 		return nil, err
 	}
 
@@ -324,7 +711,10 @@ func (r *queryResolver) NodeMetrics(ctx context.Context, cluster string, nodes [
 			Host:    hostname,
 			Metrics: make([]*model.JobMetricWithName, 0, len(metrics)*len(scopes)),
 		}
-		host.SubCluster, _ = archive.GetSubClusterByNode(cluster, hostname)
+		host.SubCluster, err = archive.GetSubClusterByNode(cluster, hostname)
+		if err != nil {
+			cclog.Warnf("error in nodeMetrics resolver: %s", err)
+		}
 
 		for metric, scopedMetrics := range metrics {
 			for _, scopedMetric := range scopedMetrics {
@@ -342,6 +732,68 @@ func (r *queryResolver) NodeMetrics(ctx context.Context, cluster string, nodes [
 	return nodeMetrics, nil
 }
 
+// NodeMetricsList is the resolver for the nodeMetricsList field.
+func (r *queryResolver) NodeMetricsList(ctx context.Context, cluster string, subCluster string, nodeFilter string, scopes []schema.MetricScope, metrics []string, from time.Time, to time.Time, page *model.PageRequest, resolution *int) (*model.NodesResultList, error) {
+	if resolution == nil { // Load from Config
+		if config.Keys.EnableResampling != nil {
+			defaultRes := slices.Max(config.Keys.EnableResampling.Resolutions)
+			resolution = &defaultRes
+		} else { // Set 0 (Loads configured metric timestep)
+			defaultRes := 0
+			resolution = &defaultRes
+		}
+	}
+
+	user := repository.GetUserFromContext(ctx)
+	if user != nil && !user.HasAnyRole([]schema.Role{schema.RoleAdmin, schema.RoleSupport}) {
+		return nil, errors.New("you need to be administrator or support staff for this query")
+	}
+
+	if metrics == nil {
+		for _, mc := range archive.GetCluster(cluster).MetricConfig {
+			metrics = append(metrics, mc.Name)
+		}
+	}
+
+	data, totalNodes, hasNextPage, err := metricDataDispatcher.LoadNodeListData(cluster, subCluster, nodeFilter, metrics, scopes, *resolution, from, to, page, ctx)
+	if err != nil {
+		cclog.Warn("error while loading node data")
+		return nil, err
+	}
+
+	nodeMetricsList := make([]*model.NodeMetrics, 0, len(data))
+	for hostname, metrics := range data {
+		host := &model.NodeMetrics{
+			Host:    hostname,
+			Metrics: make([]*model.JobMetricWithName, 0, len(metrics)*len(scopes)),
+		}
+		host.SubCluster, err = archive.GetSubClusterByNode(cluster, hostname)
+		if err != nil {
+			cclog.Warnf("error in nodeMetrics resolver: %s", err)
+		}
+
+		for metric, scopedMetrics := range metrics {
+			for scope, scopedMetric := range scopedMetrics {
+				host.Metrics = append(host.Metrics, &model.JobMetricWithName{
+					Name:   metric,
+					Scope:  scope,
+					Metric: scopedMetric,
+				})
+			}
+		}
+
+		nodeMetricsList = append(nodeMetricsList, host)
+	}
+
+	nodeMetricsListResult := &model.NodesResultList{
+		Items:       nodeMetricsList,
+		TotalNodes:  &totalNodes,
+		HasNextPage: &hasNextPage,
+	}
+
+	return nodeMetricsListResult, nil
+}
+
 // NumberOfNodes is the resolver for the numberOfNodes field.
 func (r *subClusterResolver) NumberOfNodes(ctx context.Context, obj *schema.SubCluster) (int, error) {
 	nodeList, err := archive.ParseNodeList(obj.Nodes)
@@ -357,9 +809,15 @@ func (r *Resolver) Cluster() generated.ClusterResolver { return &clusterResolver
 // Job returns generated.JobResolver implementation.
 func (r *Resolver) Job() generated.JobResolver { return &jobResolver{r} }
 
+// MetricValue returns generated.MetricValueResolver implementation.
+func (r *Resolver) MetricValue() generated.MetricValueResolver { return &metricValueResolver{r} }
+
 // Mutation returns generated.MutationResolver implementation.
 func (r *Resolver) Mutation() generated.MutationResolver { return &mutationResolver{r} }
 
+// Node returns generated.NodeResolver implementation.
+func (r *Resolver) Node() generated.NodeResolver { return &nodeResolver{r} }
+
 // Query returns generated.QueryResolver implementation.
 func (r *Resolver) Query() generated.QueryResolver { return &queryResolver{r} }
 
@@ -368,6 +826,8 @@ func (r *Resolver) SubCluster() generated.SubClusterResolver { return &subCluste
 
 type clusterResolver struct{ *Resolver }
 type jobResolver struct{ *Resolver }
+type metricValueResolver struct{ *Resolver }
 type mutationResolver struct{ *Resolver }
+type nodeResolver struct{ *Resolver }
 type queryResolver struct{ *Resolver }
 type subClusterResolver struct{ *Resolver }

internal/graph/util.go

@@ -1,5 +1,5 @@
-// Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved.
+// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
+// All rights reserved. This file is part of cc-backend.
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 package graph
@@ -11,10 +11,9 @@ import (
 
 	"github.com/99designs/gqlgen/graphql"
 	"github.com/ClusterCockpit/cc-backend/internal/graph/model"
-	"github.com/ClusterCockpit/cc-backend/internal/metricdata"
-	"github.com/ClusterCockpit/cc-backend/pkg/log"
-	"github.com/ClusterCockpit/cc-backend/pkg/schema"
-	// "github.com/ClusterCockpit/cc-backend/pkg/archive"
+	"github.com/ClusterCockpit/cc-backend/internal/metricDataDispatcher"
+	cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
+	"github.com/ClusterCockpit/cc-lib/schema"
 )
 
 const MAX_JOBS_FOR_ANALYSIS = 500
@@ -24,11 +23,11 @@ func (r *queryResolver) rooflineHeatmap(
 	ctx context.Context,
 	filter []*model.JobFilter,
 	rows int, cols int,
-	minX float64, minY float64, maxX float64, maxY float64) ([][]float64, error) {
-
+	minX float64, minY float64, maxX float64, maxY float64,
+) ([][]float64, error) {
 	jobs, err := r.Repo.QueryJobs(ctx, filter, &model.PageRequest{Page: 1, ItemsPerPage: MAX_JOBS_FOR_ANALYSIS + 1}, nil)
 	if err != nil {
-		log.Error("Error while querying jobs for roofline")
+		cclog.Error("Error while querying jobs for roofline")
 		return nil, err
 	}
 	if len(jobs) > MAX_JOBS_FOR_ANALYSIS {
@@ -47,15 +46,22 @@ func (r *queryResolver) rooflineHeatmap(
 			continue
 		}
 
-		jobdata, err := metricdata.LoadData(job, []string{"flops_any", "mem_bw"}, []schema.MetricScope{schema.MetricScopeNode}, ctx)
+		// metricConfigs := archive.GetCluster(job.Cluster).MetricConfig
+		// resolution := 0
+
+		// for _, mc := range metricConfigs {
+		// 	resolution = max(resolution, mc.Timestep)
+		// }
+
+		jobdata, err := metricDataDispatcher.LoadData(job, []string{"flops_any", "mem_bw"}, []schema.MetricScope{schema.MetricScopeNode}, ctx, 0)
 		if err != nil {
-			log.Errorf("Error while loading roofline metrics for job %d", job.ID)
+			cclog.Errorf("Error while loading roofline metrics for job %d", job.ID)
 			return nil, err
 		}
 
 		flops_, membw_ := jobdata["flops_any"], jobdata["mem_bw"]
 		if flops_ == nil && membw_ == nil {
-			log.Infof("rooflineHeatmap(): 'flops_any' or 'mem_bw' missing for job %d", job.ID)
+			cclog.Infof("rooflineHeatmap(): 'flops_any' or 'mem_bw' missing for job %d", job.ID)
 			continue
 			// return nil, fmt.Errorf("GRAPH/UTIL > 'flops_any' or 'mem_bw' missing for job %d", job.ID)
 		}
@@ -63,7 +69,7 @@ func (r *queryResolver) rooflineHeatmap(
 		flops, ok1 := flops_["node"]
 		membw, ok2 := membw_["node"]
 		if !ok1 || !ok2 {
-			log.Info("rooflineHeatmap() query not implemented for where flops_any or mem_bw not available at 'node' level")
+			cclog.Info("rooflineHeatmap() query not implemented for where flops_any or mem_bw not available at 'node' level")
 			continue
 			// TODO/FIXME:
 			// return nil, errors.New("GRAPH/UTIL > todo: rooflineHeatmap() query not implemented for where flops_any or mem_bw not available at 'node' level")
@@ -98,7 +104,7 @@ func (r *queryResolver) rooflineHeatmap(
 func (r *queryResolver) jobsFootprints(ctx context.Context, filter []*model.JobFilter, metrics []string) (*model.Footprints, error) {
 	jobs, err := r.Repo.QueryJobs(ctx, filter, &model.PageRequest{Page: 1, ItemsPerPage: MAX_JOBS_FOR_ANALYSIS + 1}, nil)
 	if err != nil {
-		log.Error("Error while querying jobs for footprint")
+		cclog.Error("Error while querying jobs for footprint")
 		return nil, err
 	}
 	if len(jobs) > MAX_JOBS_FOR_ANALYSIS {
@@ -120,8 +126,8 @@ func (r *queryResolver) jobsFootprints(ctx context.Context, filter []*model.JobF
 			continue
 		}
 
-		if err := metricdata.LoadAverages(job, metrics, avgs, ctx); err != nil {
-			log.Error("Error while loading averages for footprint")
+		if err := metricDataDispatcher.LoadAverages(job, metrics, avgs, ctx); err != nil {
+			cclog.Error("Error while loading averages for footprint")
 			return nil, err
 		}
 

internal/importer/handleImport.go

@@ -1,5 +1,5 @@
-// Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved.
+// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
+// All rights reserved. This file is part of cc-backend.
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 package importer
@@ -8,15 +8,15 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
+	"math"
 	"os"
 	"strings"
-	"time"
 
 	"github.com/ClusterCockpit/cc-backend/internal/config"
 	"github.com/ClusterCockpit/cc-backend/internal/repository"
 	"github.com/ClusterCockpit/cc-backend/pkg/archive"
-	"github.com/ClusterCockpit/cc-backend/pkg/log"
-	"github.com/ClusterCockpit/cc-backend/pkg/schema"
+	cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
+	"github.com/ClusterCockpit/cc-lib/schema"
 )
 
 // Import all jobs specified as `<path-to-meta.json>:<path-to-data.json>,...`
@@ -31,7 +31,7 @@ func HandleImportFlag(flag string) error {
 
 		raw, err := os.ReadFile(files[0])
 		if err != nil {
-			log.Warn("Error while reading metadata file for import")
+			cclog.Warn("Error while reading metadata file for import")
 			return err
 		}
 
@@ -42,15 +42,18 @@ func HandleImportFlag(flag string) error {
 		}
 		dec := json.NewDecoder(bytes.NewReader(raw))
 		dec.DisallowUnknownFields()
-		jobMeta := schema.JobMeta{BaseJob: schema.JobDefaults}
-		if err = dec.Decode(&jobMeta); err != nil {
-			log.Warn("Error while decoding raw json metadata for import")
+		job := schema.Job{
+			Exclusive:        1,
+			MonitoringStatus: schema.MonitoringStatusRunningOrArchiving,
+		}
+		if err = dec.Decode(&job); err != nil {
+			cclog.Warn("Error while decoding raw json metadata for import")
 			return err
 		}
 
 		raw, err = os.ReadFile(files[1])
 		if err != nil {
-			log.Warn("Error while reading jobdata file for import")
+			cclog.Warn("Error while reading jobdata file for import")
 			return err
 		}
 
@@ -63,72 +66,108 @@ func HandleImportFlag(flag string) error {
 		dec.DisallowUnknownFields()
 		jobData := schema.JobData{}
 		if err = dec.Decode(&jobData); err != nil {
-			log.Warn("Error while decoding raw json jobdata for import")
+			cclog.Warn("Error while decoding raw json jobdata for import")
 			return err
 		}
 
-		// checkJobData(&jobData)
+		job.MonitoringStatus = schema.MonitoringStatusArchivingSuccessful
 
-		jobMeta.MonitoringStatus = schema.MonitoringStatusArchivingSuccessful
-
-		// if _, err = r.Find(&jobMeta.JobID, &jobMeta.Cluster, &jobMeta.StartTime); err != sql.ErrNoRows {
-		// 	if err != nil {
-		// 		log.Warn("Error while finding job in jobRepository")
-		// 		return err
-		// 	}
-		//
-		// 	return fmt.Errorf("REPOSITORY/INIT > a job with that jobId, cluster and startTime does already exist")
-		// }
-		//
-		job := schema.Job{
-			BaseJob:       jobMeta.BaseJob,
-			StartTime:     time.Unix(jobMeta.StartTime, 0),
-			StartTimeUnix: jobMeta.StartTime,
+		sc, err := archive.GetSubCluster(job.Cluster, job.SubCluster)
+		if err != nil {
+			cclog.Errorf("cannot get subcluster: %s", err.Error())
+			return err
 		}
 
-		// TODO: Other metrics...
-		job.LoadAvg = loadJobStat(&jobMeta, "cpu_load")
-		job.FlopsAnyAvg = loadJobStat(&jobMeta, "flops_any")
-		job.MemUsedMax = loadJobStat(&jobMeta, "mem_used")
-		job.MemBwAvg = loadJobStat(&jobMeta, "mem_bw")
-		job.NetBwAvg = loadJobStat(&jobMeta, "net_bw")
-		job.FileBwAvg = loadJobStat(&jobMeta, "file_bw")
+		job.Footprint = make(map[string]float64)
+
+		for _, fp := range sc.Footprint {
+			statType := "avg"
+
+			if i, err := archive.MetricIndex(sc.MetricConfig, fp); err != nil {
+				statType = sc.MetricConfig[i].Footprint
+			}
+
+			name := fmt.Sprintf("%s_%s", fp, statType)
+
+			job.Footprint[name] = repository.LoadJobStat(&job, fp, statType)
+		}
+
+		job.RawFootprint, err = json.Marshal(job.Footprint)
+		if err != nil {
+			cclog.Warn("Error while marshaling job footprint")
+			return err
+		}
+
+		job.EnergyFootprint = make(map[string]float64)
+
+		// Total Job Energy Outside Loop
+		totalEnergy := 0.0
+		for _, fp := range sc.EnergyFootprint {
+			// Always Init Metric Energy Inside Loop
+			metricEnergy := 0.0
+			if i, err := archive.MetricIndex(sc.MetricConfig, fp); err == nil {
+				// Note: For DB data, calculate and save as kWh
+				if sc.MetricConfig[i].Energy == "energy" { // this metric has energy as unit (Joules)
+					cclog.Warnf("Update EnergyFootprint for Job %d and Metric %s on cluster %s: Set to 'energy' in cluster.json: Not implemented, will return 0.0", job.JobID, job.Cluster, fp)
+					// FIXME: Needs sum as stats type
+				} else if sc.MetricConfig[i].Energy == "power" { // this metric has power as unit (Watt)
+					// Energy: Power (in Watts) * Time (in Seconds)
+					// Unit: (W * (s / 3600)) / 1000 = kWh
+					// Round 2 Digits: round(Energy * 100) / 100
+					// Here: (All-Node Metric Average * Number of Nodes) * (Job Duration in Seconds / 3600) / 1000
+					// Note: Shared Jobs handled correctly since "Node Average" is based on partial resources, while "numNodes" factor is 1
+					rawEnergy := ((repository.LoadJobStat(&job, fp, "avg") * float64(job.NumNodes)) * (float64(job.Duration) / 3600.0)) / 1000.0
+					metricEnergy = math.Round(rawEnergy*100.0) / 100.0
+				}
+			} else {
+				cclog.Warnf("Error while collecting energy metric %s for job, DB ID '%v', return '0.0'", fp, job.ID)
+			}
+
+			job.EnergyFootprint[fp] = metricEnergy
+			totalEnergy += metricEnergy
+		}
+
+		job.Energy = (math.Round(totalEnergy*100.0) / 100.0)
+		if job.RawEnergyFootprint, err = json.Marshal(job.EnergyFootprint); err != nil {
+			cclog.Warnf("Error while marshaling energy footprint for job INTO BYTES, DB ID '%v'", job.ID)
+			return err
+		}
 
 		job.RawResources, err = json.Marshal(job.Resources)
 		if err != nil {
-			log.Warn("Error while marshaling job resources")
+			cclog.Warn("Error while marshaling job resources")
 			return err
 		}
 		job.RawMetaData, err = json.Marshal(job.MetaData)
 		if err != nil {
-			log.Warn("Error while marshaling job metadata")
+			cclog.Warn("Error while marshaling job metadata")
 			return err
 		}
 
-		if err = SanityChecks(&job.BaseJob); err != nil {
-			log.Warn("BaseJob SanityChecks failed")
+		if err = SanityChecks(&job); err != nil {
+			cclog.Warn("BaseJob SanityChecks failed")
 			return err
 		}
 
-		if err = archive.GetHandle().ImportJob(&jobMeta, &jobData); err != nil {
-			log.Error("Error while importing job")
+		if err = archive.GetHandle().ImportJob(&job, &jobData); err != nil {
+			cclog.Error("Error while importing job")
 			return err
 		}
 
 		id, err := r.InsertJob(&job)
 		if err != nil {
-			log.Warn("Error while job db insert")
+			cclog.Warn("Error while job db insert")
 			return err
 		}
 
 		for _, tag := range job.Tags {
-			if _, err := r.AddTagOrCreate(id, tag.Type, tag.Name); err != nil {
-				log.Error("Error while adding or creating tag")
+			if err := r.ImportTag(id, tag.Type, tag.Name, tag.Scope); err != nil {
+				cclog.Error("Error while adding or creating tag on import")
 				return err
 			}
 		}
 
-		log.Infof("successfully imported a new job (jobId: %d, cluster: %s, dbid: %d)", job.JobID, job.Cluster, id)
+		cclog.Infof("successfully imported a new job (jobId: %d, cluster: %s, dbid: %d)", job.JobID, job.Cluster, id)
 	}
 	return nil
 }

internal/importer/importer_test.go

@@ -1,5 +1,5 @@
-// Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved.
+// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
+// All rights reserved. This file is part of cc-backend.
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 package importer_test
@@ -16,7 +16,7 @@ import (
 	"github.com/ClusterCockpit/cc-backend/internal/importer"
 	"github.com/ClusterCockpit/cc-backend/internal/repository"
 	"github.com/ClusterCockpit/cc-backend/pkg/archive"
-	"github.com/ClusterCockpit/cc-backend/pkg/log"
+	cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
 )
 
 func copyFile(s string, d string) error {
@@ -45,6 +45,9 @@ func setup(t *testing.T) *repository.JobRepository {
     "jwts": {
         "max-age": "2m"
     },
+  "apiAllowedIPs": [
+    "*"
+  ],
 	"clusters": [
 	{
 	   "name": "testcluster",
@@ -75,14 +78,14 @@ func setup(t *testing.T) *repository.JobRepository {
 	 }
 	]}`
 
-	log.Init("info", true)
+	cclog.Init("info", true)
 	tmpdir := t.TempDir()
 
 	jobarchive := filepath.Join(tmpdir, "job-archive")
 	if err := os.Mkdir(jobarchive, 0777); err != nil {
 		t.Fatal(err)
 	}
-	if err := os.WriteFile(filepath.Join(jobarchive, "version.txt"), []byte(fmt.Sprintf("%d", 1)), 0666); err != nil {
+	if err := os.WriteFile(filepath.Join(jobarchive, "version.txt"), []byte(fmt.Sprintf("%d", 2)), 0666); err != nil {
 		t.Fatal(err)
 	}
 	fritzArchive := filepath.Join(tmpdir, "job-archive", "fritz")
@@ -163,7 +166,7 @@ func TestHandleImportFlag(t *testing.T) {
 			}
 
 			result := readResult(t, testname)
-			job, err := r.Find(&result.JobId, &result.Cluster, &result.StartTime)
+			job, err := r.FindCached(&result.JobId, &result.Cluster, &result.StartTime)
 			if err != nil {
 				t.Fatal(err)
 			}

internal/importer/initDB.go

@@ -1,5 +1,5 @@
-// Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved.
+// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
+// All rights reserved. This file is part of cc-backend.
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 package importer
@@ -7,13 +7,19 @@ package importer
 import (
 	"encoding/json"
 	"fmt"
+	"math"
 	"strings"
 	"time"
 
 	"github.com/ClusterCockpit/cc-backend/internal/repository"
 	"github.com/ClusterCockpit/cc-backend/pkg/archive"
-	"github.com/ClusterCockpit/cc-backend/pkg/log"
-	"github.com/ClusterCockpit/cc-backend/pkg/schema"
+	cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
+	"github.com/ClusterCockpit/cc-lib/schema"
+)
+
+const (
+	addTagQuery = "INSERT INTO tag (tag_name, tag_type) VALUES (?, ?)"
+	setTagQuery = "INSERT INTO jobtag (job_id, tag_id) VALUES (?, ?)"
 )
 
 // Delete the tables "job", "tag" and "jobtag" from the database and
@@ -21,15 +27,15 @@ import (
 func InitDB() error {
 	r := repository.GetJobRepository()
 	if err := r.Flush(); err != nil {
-		log.Errorf("repository initDB(): %v", err)
+		cclog.Errorf("repository initDB(): %v", err)
 		return err
 	}
 	starttime := time.Now()
-	log.Print("Building job table...")
+	cclog.Print("Building job table...")
 
 	t, err := r.TransactionInit()
 	if err != nil {
-		log.Warn("Error while initializing SQL transactions")
+		cclog.Warn("Error while initializing SQL transactions")
 		return err
 	}
 	tags := make(map[string]int64)
@@ -54,61 +60,114 @@ func InitDB() error {
 		}
 
 		jobMeta.MonitoringStatus = schema.MonitoringStatusArchivingSuccessful
-		job := schema.Job{
-			BaseJob:       jobMeta.BaseJob,
-			StartTime:     time.Unix(jobMeta.StartTime, 0),
-			StartTimeUnix: jobMeta.StartTime,
-		}
 
-		// TODO: Other metrics...
-		job.LoadAvg = loadJobStat(jobMeta, "cpu_load")
-		job.FlopsAnyAvg = loadJobStat(jobMeta, "flops_any")
-		job.MemUsedMax = loadJobStat(jobMeta, "mem_used")
-		job.MemBwAvg = loadJobStat(jobMeta, "mem_bw")
-		job.NetBwAvg = loadJobStat(jobMeta, "net_bw")
-		job.FileBwAvg = loadJobStat(jobMeta, "file_bw")
-
-		job.RawResources, err = json.Marshal(job.Resources)
+		sc, err := archive.GetSubCluster(jobMeta.Cluster, jobMeta.SubCluster)
 		if err != nil {
-			log.Errorf("repository initDB(): %v", err)
-			errorOccured++
-			continue
+			cclog.Errorf("cannot get subcluster: %s", err.Error())
+			return err
 		}
 
-		job.RawMetaData, err = json.Marshal(job.MetaData)
+		jobMeta.Footprint = make(map[string]float64)
+
+		for _, fp := range sc.Footprint {
+			statType := "avg"
+
+			if i, err := archive.MetricIndex(sc.MetricConfig, fp); err != nil {
+				statType = sc.MetricConfig[i].Footprint
+			}
+
+			name := fmt.Sprintf("%s_%s", fp, statType)
+
+			jobMeta.Footprint[name] = repository.LoadJobStat(jobMeta, fp, statType)
+		}
+
+		jobMeta.RawFootprint, err = json.Marshal(jobMeta.Footprint)
 		if err != nil {
-			log.Errorf("repository initDB(): %v", err)
-			errorOccured++
-			continue
+			cclog.Warn("Error while marshaling job footprint")
+			return err
 		}
 
-		if err := SanityChecks(&job.BaseJob); err != nil {
-			log.Errorf("repository initDB(): %v", err)
-			errorOccured++
-			continue
+		jobMeta.EnergyFootprint = make(map[string]float64)
+
+		// Total Job Energy Outside Loop
+		totalEnergy := 0.0
+		for _, fp := range sc.EnergyFootprint {
+			// Always Init Metric Energy Inside Loop
+			metricEnergy := 0.0
+			if i, err := archive.MetricIndex(sc.MetricConfig, fp); err == nil {
+				// Note: For DB data, calculate and save as kWh
+				if sc.MetricConfig[i].Energy == "energy" { // this metric has energy as unit (Joules)
+					cclog.Warnf("Update EnergyFootprint for Job %d and Metric %s on cluster %s: Set to 'energy' in cluster.json: Not implemented, will return 0.0", jobMeta.JobID, jobMeta.Cluster, fp)
+					// FIXME: Needs sum as stats type
+				} else if sc.MetricConfig[i].Energy == "power" { // this metric has power as unit (Watt)
+					// Energy: Power (in Watts) * Time (in Seconds)
+					// Unit: (W * (s / 3600)) / 1000 = kWh
+					// Round 2 Digits: round(Energy * 100) / 100
+					// Here: (All-Node Metric Average * Number of Nodes) * (Job Duration in Seconds / 3600) / 1000
+					// Note: Shared Jobs handled correctly since "Node Average" is based on partial resources, while "numNodes" factor is 1
+					rawEnergy := ((repository.LoadJobStat(jobMeta, fp, "avg") * float64(jobMeta.NumNodes)) * (float64(jobMeta.Duration) / 3600.0)) / 1000.0
+					metricEnergy = math.Round(rawEnergy*100.0) / 100.0
+				}
+			} else {
+				cclog.Warnf("Error while collecting energy metric %s for job, DB ID '%v', return '0.0'", fp, jobMeta.ID)
+			}
+
+			jobMeta.EnergyFootprint[fp] = metricEnergy
+			totalEnergy += metricEnergy
 		}
 
-		id, err := r.TransactionAdd(t, job)
+		jobMeta.Energy = (math.Round(totalEnergy*100.0) / 100.0)
+		if jobMeta.RawEnergyFootprint, err = json.Marshal(jobMeta.EnergyFootprint); err != nil {
+			cclog.Warnf("Error while marshaling energy footprint for job INTO BYTES, DB ID '%v'", jobMeta.ID)
+			return err
+		}
+
+		jobMeta.RawResources, err = json.Marshal(jobMeta.Resources)
 		if err != nil {
-			log.Errorf("repository initDB(): %v", err)
+			cclog.Errorf("repository initDB(): %v", err)
 			errorOccured++
 			continue
 		}
 
-		for _, tag := range job.Tags {
+		jobMeta.RawMetaData, err = json.Marshal(jobMeta.MetaData)
+		if err != nil {
+			cclog.Errorf("repository initDB(): %v", err)
+			errorOccured++
+			continue
+		}
+
+		if err := SanityChecks(jobMeta); err != nil {
+			cclog.Errorf("repository initDB(): %v", err)
+			errorOccured++
+			continue
+		}
+
+		id, err := r.TransactionAddNamed(t,
+			repository.NamedJobInsert, jobMeta)
+		if err != nil {
+			cclog.Errorf("repository initDB(): %v", err)
+			errorOccured++
+			continue
+		}
+
+		for _, tag := range jobMeta.Tags {
 			tagstr := tag.Name + ":" + tag.Type
 			tagId, ok := tags[tagstr]
 			if !ok {
-				tagId, err = r.TransactionAddTag(t, tag)
+				tagId, err = r.TransactionAdd(t,
+					addTagQuery,
+					tag.Name, tag.Type)
 				if err != nil {
-					log.Errorf("Error adding tag: %v", err)
+					cclog.Errorf("Error adding tag: %v", err)
 					errorOccured++
 					continue
 				}
 				tags[tagstr] = tagId
 			}
 
-			r.TransactionSetTag(t, id, tagId)
+			r.TransactionAdd(t,
+				setTagQuery,
+				id, tagId)
 		}
 
 		if err == nil {
@@ -117,21 +176,21 @@ func InitDB() error {
 	}
 
 	if errorOccured > 0 {
-		log.Warnf("Error in import of %d jobs!", errorOccured)
+		cclog.Warnf("Error in import of %d jobs!", errorOccured)
 	}
 
 	r.TransactionEnd(t)
-	log.Printf("A total of %d jobs have been registered in %.3f seconds.\n", i, time.Since(starttime).Seconds())
+	cclog.Printf("A total of %d jobs have been registered in %.3f seconds.\n", i, time.Since(starttime).Seconds())
 	return nil
 }
 
 // This function also sets the subcluster if necessary!
-func SanityChecks(job *schema.BaseJob) error {
+func SanityChecks(job *schema.Job) error {
 	if c := archive.GetCluster(job.Cluster); c == nil {
 		return fmt.Errorf("no such cluster: %v", job.Cluster)
 	}
 	if err := archive.AssignSubCluster(job); err != nil {
-		log.Warn("Error while assigning subcluster to job")
+		cclog.Warn("Error while assigning subcluster to job")
 		return err
 	}
 	if !job.State.Valid() {
@@ -150,18 +209,6 @@ func SanityChecks(job *schema.BaseJob) error {
 	return nil
 }
 
-func loadJobStat(job *schema.JobMeta, metric string) float64 {
-	if stats, ok := job.Statistics[metric]; ok {
-		if metric == "mem_used" {
-			return stats.Max
-		} else {
-			return stats.Avg
-		}
-	}
-
-	return 0.0
-}
-
 func checkJobData(d *schema.JobData) error {
 	for _, scopes := range *d {
 		// var newUnit schema.Unit

internal/importer/normalize.go

@@ -1,5 +1,5 @@
-// Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved.
+// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
+// All rights reserved. This file is part of cc-backend.
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 package importer
@@ -7,7 +7,7 @@ package importer
 import (
 	"math"
 
-	ccunits "github.com/ClusterCockpit/cc-units"
+	ccunits "github.com/ClusterCockpit/cc-lib/ccUnits"
 )
 
 func getNormalizationFactor(v float64) (float64, int) {

internal/importer/normalize_test.go

@@ -1,5 +1,5 @@
-// Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved.
+// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
+// All rights reserved. This file is part of cc-backend.
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 package importer
@@ -8,7 +8,7 @@ import (
 	"fmt"
 	"testing"
 
-	ccunits "github.com/ClusterCockpit/cc-units"
+	ccunits "github.com/ClusterCockpit/cc-lib/ccUnits"
 )
 
 func TestNormalizeFactor(t *testing.T) {

internal/metricDataDispatcher/dataLoader.go

@@ -0,0 +1,382 @@
+// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
+// All rights reserved. This file is part of cc-backend.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+package metricDataDispatcher
+
+import (
+	"context"
+	"fmt"
+	"math"
+	"time"
+
+	"github.com/ClusterCockpit/cc-backend/internal/config"
+	"github.com/ClusterCockpit/cc-backend/internal/graph/model"
+	"github.com/ClusterCockpit/cc-backend/internal/metricdata"
+	"github.com/ClusterCockpit/cc-backend/pkg/archive"
+	cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
+	"github.com/ClusterCockpit/cc-lib/lrucache"
+	"github.com/ClusterCockpit/cc-lib/resampler"
+	"github.com/ClusterCockpit/cc-lib/schema"
+)
+
+var cache *lrucache.Cache = lrucache.New(128 * 1024 * 1024)
+
+func cacheKey(
+	job *schema.Job,
+	metrics []string,
+	scopes []schema.MetricScope,
+	resolution int,
+) string {
+	// Duration and StartTime do not need to be in the cache key as StartTime is less unique than
+	// job.ID and the TTL of the cache entry makes sure it does not stay there forever.
+	return fmt.Sprintf("%d(%s):[%v],[%v]-%d",
+		job.ID, job.State, metrics, scopes, resolution)
+}
+
+// Fetches the metric data for a job.
+func LoadData(job *schema.Job,
+	metrics []string,
+	scopes []schema.MetricScope,
+	ctx context.Context,
+	resolution int,
+) (schema.JobData, error) {
+	data := cache.Get(cacheKey(job, metrics, scopes, resolution), func() (_ interface{}, ttl time.Duration, size int) {
+		var jd schema.JobData
+		var err error
+
+		if job.State == schema.JobStateRunning ||
+			job.MonitoringStatus == schema.MonitoringStatusRunningOrArchiving ||
+			config.Keys.DisableArchive {
+
+			repo, err := metricdata.GetMetricDataRepo(job.Cluster)
+			if err != nil {
+				return fmt.Errorf("METRICDATA/METRICDATA > no metric data repository configured for '%s'", job.Cluster), 0, 0
+			}
+
+			if scopes == nil {
+				scopes = append(scopes, schema.MetricScopeNode)
+			}
+
+			if metrics == nil {
+				cluster := archive.GetCluster(job.Cluster)
+				for _, mc := range cluster.MetricConfig {
+					metrics = append(metrics, mc.Name)
+				}
+			}
+
+			jd, err = repo.LoadData(job, metrics, scopes, ctx, resolution)
+			if err != nil {
+				if len(jd) != 0 {
+					cclog.Warnf("partial error: %s", err.Error())
+					// return err, 0, 0 // Reactivating will block archiving on one partial error
+				} else {
+					cclog.Error("Error while loading job data from metric repository")
+					return err, 0, 0
+				}
+			}
+			size = jd.Size()
+		} else {
+			var jd_temp schema.JobData
+			jd_temp, err = archive.GetHandle().LoadJobData(job)
+			if err != nil {
+				cclog.Error("Error while loading job data from archive")
+				return err, 0, 0
+			}
+
+			// Deep copy the cached archive hashmap
+			jd = metricdata.DeepCopy(jd_temp)
+
+			// Resampling for archived data.
+			// Pass the resolution from frontend here.
+			for _, v := range jd {
+				for _, v_ := range v {
+					timestep := 0
+					for i := 0; i < len(v_.Series); i += 1 {
+						v_.Series[i].Data, timestep, err = resampler.LargestTriangleThreeBucket(v_.Series[i].Data, v_.Timestep, resolution)
+						if err != nil {
+							return err, 0, 0
+						}
+					}
+					v_.Timestep = timestep
+				}
+			}
+
+			// Avoid sending unrequested data to the client:
+			if metrics != nil || scopes != nil {
+				if metrics == nil {
+					metrics = make([]string, 0, len(jd))
+					for k := range jd {
+						metrics = append(metrics, k)
+					}
+				}
+
+				res := schema.JobData{}
+				for _, metric := range metrics {
+					if perscope, ok := jd[metric]; ok {
+						if len(perscope) > 1 {
+							subset := make(map[schema.MetricScope]*schema.JobMetric)
+							for _, scope := range scopes {
+								if jm, ok := perscope[scope]; ok {
+									subset[scope] = jm
+								}
+							}
+
+							if len(subset) > 0 {
+								perscope = subset
+							}
+						}
+
+						res[metric] = perscope
+					}
+				}
+				jd = res
+			}
+			size = jd.Size()
+		}
+
+		ttl = 5 * time.Hour
+		if job.State == schema.JobStateRunning {
+			ttl = 2 * time.Minute
+		}
+
+		// FIXME: Review: Is this really necessary or correct.
+		// Note: Lines 147-170 formerly known as prepareJobData(jobData, scopes)
+		// For /monitoring/job/<job> and some other places, flops_any and mem_bw need
+		// to be available at the scope 'node'. If a job has a lot of nodes,
+		// statisticsSeries should be available so that a min/median/max Graph can be
+		// used instead of a lot of single lines.
+		// NOTE: New StatsSeries will always be calculated as 'min/median/max'
+		//       Existing (archived) StatsSeries can be 'min/mean/max'!
+		const maxSeriesSize int = 15
+		for _, scopes := range jd {
+			for _, jm := range scopes {
+				if jm.StatisticsSeries != nil || len(jm.Series) <= maxSeriesSize {
+					continue
+				}
+
+				jm.AddStatisticsSeries()
+			}
+		}
+
+		nodeScopeRequested := false
+		for _, scope := range scopes {
+			if scope == schema.MetricScopeNode {
+				nodeScopeRequested = true
+			}
+		}
+
+		if nodeScopeRequested {
+			jd.AddNodeScope("flops_any")
+			jd.AddNodeScope("mem_bw")
+		}
+
+		// Round Resulting Stat Values
+		jd.RoundMetricStats()
+
+		return jd, ttl, size
+	})
+
+	if err, ok := data.(error); ok {
+		cclog.Error("Error in returned dataset")
+		return nil, err
+	}
+
+	return data.(schema.JobData), nil
+}
+
+// Used for the jobsFootprint GraphQL-Query. TODO: Rename/Generalize.
+func LoadAverages(
+	job *schema.Job,
+	metrics []string,
+	data [][]schema.Float,
+	ctx context.Context,
+) error {
+	if job.State != schema.JobStateRunning && !config.Keys.DisableArchive {
+		return archive.LoadAveragesFromArchive(job, metrics, data) // #166 change also here?
+	}
+
+	repo, err := metricdata.GetMetricDataRepo(job.Cluster)
+	if err != nil {
+		return fmt.Errorf("METRICDATA/METRICDATA > no metric data repository configured for '%s'", job.Cluster)
+	}
+
+	stats, err := repo.LoadStats(job, metrics, ctx) // #166 how to handle stats for acc normalizazion?
+	if err != nil {
+		cclog.Errorf("Error while loading statistics for job %v (User %v, Project %v)", job.JobID, job.User, job.Project)
+		return err
+	}
+
+	for i, m := range metrics {
+		nodes, ok := stats[m]
+		if !ok {
+			data[i] = append(data[i], schema.NaN)
+			continue
+		}
+
+		sum := 0.0
+		for _, node := range nodes {
+			sum += node.Avg
+		}
+		data[i] = append(data[i], schema.Float(sum))
+	}
+
+	return nil
+}
+
+// Used for statsTable in frontend: Return scoped statistics by metric.
+func LoadScopedJobStats(
+	job *schema.Job,
+	metrics []string,
+	scopes []schema.MetricScope,
+	ctx context.Context,
+) (schema.ScopedJobStats, error) {
+	if job.State != schema.JobStateRunning && !config.Keys.DisableArchive {
+		return archive.LoadScopedStatsFromArchive(job, metrics, scopes)
+	}
+
+	repo, err := metricdata.GetMetricDataRepo(job.Cluster)
+	if err != nil {
+		return nil, fmt.Errorf("job %d: no metric data repository configured for '%s'", job.JobID, job.Cluster)
+	}
+
+	scopedStats, err := repo.LoadScopedStats(job, metrics, scopes, ctx)
+	if err != nil {
+		cclog.Errorf("error while loading scoped statistics for job %d (User %s, Project %s)", job.JobID, job.User, job.Project)
+		return nil, err
+	}
+
+	return scopedStats, nil
+}
+
+// Used for polar plots in frontend: Aggregates statistics for all nodes to single values for job per metric.
+func LoadJobStats(
+	job *schema.Job,
+	metrics []string,
+	ctx context.Context,
+) (map[string]schema.MetricStatistics, error) {
+	if job.State != schema.JobStateRunning && !config.Keys.DisableArchive {
+		return archive.LoadStatsFromArchive(job, metrics)
+	}
+
+	data := make(map[string]schema.MetricStatistics, len(metrics))
+	repo, err := metricdata.GetMetricDataRepo(job.Cluster)
+	if err != nil {
+		return data, fmt.Errorf("job %d: no metric data repository configured for '%s'", job.JobID, job.Cluster)
+	}
+
+	stats, err := repo.LoadStats(job, metrics, ctx)
+	if err != nil {
+		cclog.Errorf("error while loading statistics for job %d (User %s, Project %s)", job.JobID, job.User, job.Project)
+		return data, err
+	}
+
+	for _, m := range metrics {
+		sum, avg, min, max := 0.0, 0.0, 0.0, 0.0
+		nodes, ok := stats[m]
+		if !ok {
+			data[m] = schema.MetricStatistics{Min: min, Avg: avg, Max: max}
+			continue
+		}
+
+		for _, node := range nodes {
+			sum += node.Avg
+			min = math.Min(min, node.Min)
+			max = math.Max(max, node.Max)
+		}
+
+		data[m] = schema.MetricStatistics{
+			Avg: (math.Round((sum/float64(job.NumNodes))*100) / 100),
+			Min: (math.Round(min*100) / 100),
+			Max: (math.Round(max*100) / 100),
+		}
+	}
+
+	return data, nil
+}
+
+// Used for the classic node/system view. Returns a map of nodes to a map of metrics.
+func LoadNodeData(
+	cluster string,
+	metrics, nodes []string,
+	scopes []schema.MetricScope,
+	from, to time.Time,
+	ctx context.Context,
+) (map[string]map[string][]*schema.JobMetric, error) {
+	repo, err := metricdata.GetMetricDataRepo(cluster)
+	if err != nil {
+		return nil, fmt.Errorf("METRICDATA/METRICDATA > no metric data repository configured for '%s'", cluster)
+	}
+
+	if metrics == nil {
+		for _, m := range archive.GetCluster(cluster).MetricConfig {
+			metrics = append(metrics, m.Name)
+		}
+	}
+
+	data, err := repo.LoadNodeData(cluster, metrics, nodes, scopes, from, to, ctx)
+	if err != nil {
+		if len(data) != 0 {
+			cclog.Warnf("partial error: %s", err.Error())
+		} else {
+			cclog.Error("Error while loading node data from metric repository")
+			return nil, err
+		}
+	}
+
+	if data == nil {
+		return nil, fmt.Errorf("METRICDATA/METRICDATA > the metric data repository for '%s' does not support this query", cluster)
+	}
+
+	return data, nil
+}
+
+func LoadNodeListData(
+	cluster, subCluster, nodeFilter string,
+	metrics []string,
+	scopes []schema.MetricScope,
+	resolution int,
+	from, to time.Time,
+	page *model.PageRequest,
+	ctx context.Context,
+) (map[string]schema.JobData, int, bool, error) {
+	repo, err := metricdata.GetMetricDataRepo(cluster)
+	if err != nil {
+		return nil, 0, false, fmt.Errorf("METRICDATA/METRICDATA > no metric data repository configured for '%s'", cluster)
+	}
+
+	if metrics == nil {
+		for _, m := range archive.GetCluster(cluster).MetricConfig {
+			metrics = append(metrics, m.Name)
+		}
+	}
+
+	data, totalNodes, hasNextPage, err := repo.LoadNodeListData(cluster, subCluster, nodeFilter, metrics, scopes, resolution, from, to, page, ctx)
+	if err != nil {
+		if len(data) != 0 {
+			cclog.Warnf("partial error: %s", err.Error())
+		} else {
+			cclog.Error("Error while loading node data from metric repository")
+			return nil, totalNodes, hasNextPage, err
+		}
+	}
+
+	// NOTE: New StatsSeries will always be calculated as 'min/median/max'
+	const maxSeriesSize int = 8
+	for _, jd := range data {
+		for _, scopes := range jd {
+			for _, jm := range scopes {
+				if jm.StatisticsSeries != nil || len(jm.Series) < maxSeriesSize {
+					continue
+				}
+				jm.AddStatisticsSeries()
+			}
+		}
+	}
+
+	if data == nil {
+		return nil, totalNodes, hasNextPage, fmt.Errorf("METRICDATA/METRICDATA > the metric data repository for '%s' does not support this query", cluster)
+	}
+
+	return data, totalNodes, hasNextPage, nil
+}

internal/metricdata/influxdb-v2.go

@@ -1,313 +0,0 @@
-// Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved.
-// Use of this source code is governed by a MIT-style
-// license that can be found in the LICENSE file.
-package metricdata
-
-import (
-	"context"
-	"crypto/tls"
-	"encoding/json"
-	"errors"
-	"fmt"
-	"strings"
-	"time"
-
-	"github.com/ClusterCockpit/cc-backend/pkg/archive"
-	"github.com/ClusterCockpit/cc-backend/pkg/log"
-	"github.com/ClusterCockpit/cc-backend/pkg/schema"
-	influxdb2 "github.com/influxdata/influxdb-client-go/v2"
-	influxdb2Api "github.com/influxdata/influxdb-client-go/v2/api"
-)
-
-type InfluxDBv2DataRepositoryConfig struct {
-	Url     string `json:"url"`
-	Token   string `json:"token"`
-	Bucket  string `json:"bucket"`
-	Org     string `json:"org"`
-	SkipTls bool   `json:"skiptls"`
-}
-
-type InfluxDBv2DataRepository struct {
-	client              influxdb2.Client
-	queryClient         influxdb2Api.QueryAPI
-	bucket, measurement string
-}
-
-func (idb *InfluxDBv2DataRepository) Init(rawConfig json.RawMessage) error {
-	var config InfluxDBv2DataRepositoryConfig
-	if err := json.Unmarshal(rawConfig, &config); err != nil {
-		log.Warn("Error while unmarshaling raw json config")
-		return err
-	}
-
-	idb.client = influxdb2.NewClientWithOptions(config.Url, config.Token, influxdb2.DefaultOptions().SetTLSConfig(&tls.Config{InsecureSkipVerify: config.SkipTls}))
-	idb.queryClient = idb.client.QueryAPI(config.Org)
-	idb.bucket = config.Bucket
-
-	return nil
-}
-
-func (idb *InfluxDBv2DataRepository) formatTime(t time.Time) string {
-	return t.Format(time.RFC3339) // Like “2006-01-02T15:04:05Z07:00”
-}
-
-func (idb *InfluxDBv2DataRepository) epochToTime(epoch int64) time.Time {
-	return time.Unix(epoch, 0)
-}
-
-func (idb *InfluxDBv2DataRepository) LoadData(
-	job *schema.Job,
-	metrics []string,
-	scopes []schema.MetricScope,
-	ctx context.Context) (schema.JobData, error) {
-
-	measurementsConds := make([]string, 0, len(metrics))
-	for _, m := range metrics {
-		measurementsConds = append(measurementsConds, fmt.Sprintf(`r["_measurement"] == "%s"`, m))
-	}
-	measurementsCond := strings.Join(measurementsConds, " or ")
-
-	hostsConds := make([]string, 0, len(job.Resources))
-	for _, h := range job.Resources {
-		if h.HWThreads != nil || h.Accelerators != nil {
-			// TODO
-			return nil, errors.New("METRICDATA/INFLUXV2 > the InfluxDB metric data repository does not yet support HWThreads or Accelerators")
-		}
-		hostsConds = append(hostsConds, fmt.Sprintf(`r["hostname"] == "%s"`, h.Hostname))
-	}
-	hostsCond := strings.Join(hostsConds, " or ")
-
-	jobData := make(schema.JobData) // Empty Schema: map[<string>FIELD]map[<MetricScope>SCOPE]<*JobMetric>METRIC
-	// Requested Scopes
-	for _, scope := range scopes {
-		query := ""
-		switch scope {
-		case "node":
-			// Get Finest Granularity, Groupy By Measurement and Hostname (== Metric / Node), Calculate Mean for 60s windows
-			// log.Info("Scope 'node' requested. ")
-			query = fmt.Sprintf(`
-								from(bucket: "%s")
-								|> range(start: %s, stop: %s)
-								|> filter(fn: (r) => (%s) and (%s) )
-								|> drop(columns: ["_start", "_stop"])
-								|> group(columns: ["hostname", "_measurement"])
-		            |> aggregateWindow(every: 60s, fn: mean)
-								|> drop(columns: ["_time"])`,
-				idb.bucket,
-				idb.formatTime(job.StartTime), idb.formatTime(idb.epochToTime(job.StartTimeUnix+int64(job.Duration)+int64(1))),
-				measurementsCond, hostsCond)
-		case "socket":
-			log.Info("Scope 'socket' requested, but not yet supported: Will return 'node' scope only. ")
-			continue
-		case "core":
-			log.Info(" Scope 'core' requested, but not yet supported: Will return 'node' scope only. ")
-			continue
-			// Get Finest Granularity only, Set NULL to 0.0
-			// query = fmt.Sprintf(`
-			//  	from(bucket: "%s")
-			//  	|> range(start: %s, stop: %s)
-			//  	|> filter(fn: (r) => %s )
-			//  	|> filter(fn: (r) => %s )
-			//  	|> drop(columns: ["_start", "_stop", "cluster"])
-			//  	|> map(fn: (r) => (if exists r._value then {r with _value: r._value} else {r with _value: 0.0}))`,
-			//  	idb.bucket,
-			//  	idb.formatTime(job.StartTime), idb.formatTime(idb.epochToTime(job.StartTimeUnix + int64(job.Duration) + int64(1) )),
-			//  	measurementsCond, hostsCond)
-		default:
-			log.Infof("Unknown scope '%s' requested: Will return 'node' scope.", scope)
-			continue
-			// return nil, errors.New("METRICDATA/INFLUXV2 > the InfluxDB metric data repository does not yet support other scopes than 'node'")
-		}
-
-		rows, err := idb.queryClient.Query(ctx, query)
-		if err != nil {
-			log.Error("Error while performing query")
-			return nil, err
-		}
-
-		// Init Metrics: Only Node level now -> TODO: Matching /check on scope level ...
-		for _, metric := range metrics {
-			jobMetric, ok := jobData[metric]
-			if !ok {
-				mc := archive.GetMetricConfig(job.Cluster, metric)
-				jobMetric = map[schema.MetricScope]*schema.JobMetric{
-					scope: { // uses scope var from above!
-						Unit:             mc.Unit,
-						Timestep:         mc.Timestep,
-						Series:           make([]schema.Series, 0, len(job.Resources)),
-						StatisticsSeries: nil, // Should be: &schema.StatsSeries{},
-					},
-				}
-			}
-			jobData[metric] = jobMetric
-		}
-
-		// Process Result: Time-Data
-		field, host, hostSeries := "", "", schema.Series{}
-		// typeId := 0
-		switch scope {
-		case "node":
-			for rows.Next() {
-				row := rows.Record()
-				if host == "" || host != row.ValueByKey("hostname").(string) || rows.TableChanged() {
-					if host != "" {
-						// Append Series before reset
-						jobData[field][scope].Series = append(jobData[field][scope].Series, hostSeries)
-					}
-					field, host = row.Measurement(), row.ValueByKey("hostname").(string)
-					hostSeries = schema.Series{
-						Hostname:   host,
-						Statistics: schema.MetricStatistics{}, //TODO Add Statistics
-						Data:       make([]schema.Float, 0),
-					}
-				}
-				val, ok := row.Value().(float64)
-				if ok {
-					hostSeries.Data = append(hostSeries.Data, schema.Float(val))
-				} else {
-					hostSeries.Data = append(hostSeries.Data, schema.Float(0))
-				}
-			}
-		case "socket":
-			continue
-		case "core":
-			continue
-			// Include Series.Id in hostSeries
-			// for rows.Next() {
-			// 		row := rows.Record()
-			// 		if ( host == "" || host != row.ValueByKey("hostname").(string) || typeId != row.ValueByKey("type-id").(int) || rows.TableChanged() ) {
-			// 		 		if ( host != "" ) {
-			// 						// Append Series before reset
-			// 		 		  	jobData[field][scope].Series = append(jobData[field][scope].Series, hostSeries)
-			// 		 		}
-			// 		 		field, host, typeId = row.Measurement(), row.ValueByKey("hostname").(string), row.ValueByKey("type-id").(int)
-			// 		 		hostSeries  = schema.Series{
-			// 		 				Hostname:   host,
-			// 						Id:					&typeId,
-			// 		 				Statistics: nil,
-			// 		 				Data:       make([]schema.Float, 0),
-			// 		 		}
-			// 		}
-			// 		val := row.Value().(float64)
-			// 		hostSeries.Data = append(hostSeries.Data, schema.Float(val))
-			// }
-		default:
-			log.Infof("Unknown scope '%s' requested: Will return 'node' scope.", scope)
-			continue
-			// return nil, errors.New("the InfluxDB metric data repository does not yet support other scopes than 'node, core'")
-		}
-		// Append last Series
-		jobData[field][scope].Series = append(jobData[field][scope].Series, hostSeries)
-	}
-
-	// Get Stats
-	stats, err := idb.LoadStats(job, metrics, ctx)
-	if err != nil {
-		log.Warn("Error while loading statistics")
-		return nil, err
-	}
-
-	for _, scope := range scopes {
-		if scope == "node" { // No 'socket/core' support yet
-			for metric, nodes := range stats {
-				for node, stats := range nodes {
-					for index, _ := range jobData[metric][scope].Series {
-						if jobData[metric][scope].Series[index].Hostname == node {
-							jobData[metric][scope].Series[index].Statistics = schema.MetricStatistics{Avg: stats.Avg, Min: stats.Min, Max: stats.Max}
-						}
-					}
-				}
-			}
-		}
-	}
-
-	return jobData, nil
-}
-
-func (idb *InfluxDBv2DataRepository) LoadStats(
-	job *schema.Job,
-	metrics []string,
-	ctx context.Context) (map[string]map[string]schema.MetricStatistics, error) {
-
-	stats := map[string]map[string]schema.MetricStatistics{}
-
-	hostsConds := make([]string, 0, len(job.Resources))
-	for _, h := range job.Resources {
-		if h.HWThreads != nil || h.Accelerators != nil {
-			// TODO
-			return nil, errors.New("METRICDATA/INFLUXV2 > the InfluxDB metric data repository does not yet support HWThreads or Accelerators")
-		}
-		hostsConds = append(hostsConds, fmt.Sprintf(`r["hostname"] == "%s"`, h.Hostname))
-	}
-	hostsCond := strings.Join(hostsConds, " or ")
-
-	// lenMet := len(metrics)
-
-	for _, metric := range metrics {
-		// log.Debugf("<< You are here: %s (Index %d of %d metrics)", metric, index, lenMet)
-
-		query := fmt.Sprintf(`
-				  data = from(bucket: "%s")
-				  |> range(start: %s, stop: %s)
-				  |> filter(fn: (r) => r._measurement == "%s" and r._field == "value" and (%s))
-				  union(tables: [data |> mean(column: "_value") |> set(key: "_field", value: "avg"),
-				                 data |>  min(column: "_value") |> set(key: "_field", value: "min"),
-				                 data |>  max(column: "_value") |> set(key: "_field", value: "max")])
-				  |> pivot(rowKey: ["hostname"], columnKey: ["_field"], valueColumn: "_value")
-				  |> group()`,
-			idb.bucket,
-			idb.formatTime(job.StartTime), idb.formatTime(idb.epochToTime(job.StartTimeUnix+int64(job.Duration)+int64(1))),
-			metric, hostsCond)
-
-		rows, err := idb.queryClient.Query(ctx, query)
-		if err != nil {
-			log.Error("Error while performing query")
-			return nil, err
-		}
-
-		nodes := map[string]schema.MetricStatistics{}
-		for rows.Next() {
-			row := rows.Record()
-			host := row.ValueByKey("hostname").(string)
-
-			avg, avgok := row.ValueByKey("avg").(float64)
-			if !avgok {
-				// log.Debugf(">> Assertion error for metric %s, statistic AVG. Expected 'float64', got %v", metric, avg)
-				avg = 0.0
-			}
-			min, minok := row.ValueByKey("min").(float64)
-			if !minok {
-				// log.Debugf(">> Assertion error for metric %s, statistic MIN. Expected 'float64', got %v", metric, min)
-				min = 0.0
-			}
-			max, maxok := row.ValueByKey("max").(float64)
-			if !maxok {
-				// log.Debugf(">> Assertion error for metric %s, statistic MAX. Expected 'float64', got %v", metric, max)
-				max = 0.0
-			}
-
-			nodes[host] = schema.MetricStatistics{
-				Avg: avg,
-				Min: min,
-				Max: max,
-			}
-		}
-		stats[metric] = nodes
-	}
-
-	return stats, nil
-}
-
-func (idb *InfluxDBv2DataRepository) LoadNodeData(
-	cluster string,
-	metrics, nodes []string,
-	scopes []schema.MetricScope,
-	from, to time.Time,
-	ctx context.Context) (map[string]map[string][]*schema.JobMetric, error) {
-
-	// TODO : Implement to be used in Analysis- und System/Node-View
-	log.Infof("LoadNodeData unimplemented for InfluxDBv2DataRepository, Args: cluster %s, metrics %v, nodes %v, scopes %v", cluster, metrics, nodes, scopes)
-
-	return nil, errors.New("METRICDATA/INFLUXV2 > unimplemented for InfluxDBv2DataRepository")
-}

internal/metricdata/metricdata.go

@@ -1,5 +1,5 @@
-// Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved.
+// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
+// All rights reserved. This file is part of cc-backend.
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 package metricdata
@@ -8,14 +8,12 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"math"
 	"time"
 
 	"github.com/ClusterCockpit/cc-backend/internal/config"
-	"github.com/ClusterCockpit/cc-backend/pkg/archive"
-	"github.com/ClusterCockpit/cc-backend/pkg/log"
-	"github.com/ClusterCockpit/cc-backend/pkg/lrucache"
-	"github.com/ClusterCockpit/cc-backend/pkg/schema"
+	"github.com/ClusterCockpit/cc-backend/internal/graph/model"
+	cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
+	"github.com/ClusterCockpit/cc-lib/schema"
 )
 
 type MetricDataRepository interface {
@@ -24,29 +22,31 @@ type MetricDataRepository interface {
 	Init(rawConfig json.RawMessage) error
 
 	// Return the JobData for the given job, only with the requested metrics.
-	LoadData(job *schema.Job, metrics []string, scopes []schema.MetricScope, ctx context.Context) (schema.JobData, error)
+	LoadData(job *schema.Job, metrics []string, scopes []schema.MetricScope, ctx context.Context, resolution int) (schema.JobData, error)
 
-	// Return a map of metrics to a map of nodes to the metric statistics of the job. node scope assumed for now.
+	// Return a map of metrics to a map of nodes to the metric statistics of the job. node scope only.
 	LoadStats(job *schema.Job, metrics []string, ctx context.Context) (map[string]map[string]schema.MetricStatistics, error)
 
-	// Return a map of hosts to a map of metrics at the requested scopes for that node.
+	// Return a map of metrics to a map of scopes to the scoped metric statistics of the job.
+	LoadScopedStats(job *schema.Job, metrics []string, scopes []schema.MetricScope, ctx context.Context) (schema.ScopedJobStats, error)
+
+	// Return a map of hosts to a map of metrics at the requested scopes (currently only node) for that node.
 	LoadNodeData(cluster string, metrics, nodes []string, scopes []schema.MetricScope, from, to time.Time, ctx context.Context) (map[string]map[string][]*schema.JobMetric, error)
+
+	// Return a map of hosts to a map of metrics to a map of scopes for multiple nodes.
+	LoadNodeListData(cluster, subCluster, nodeFilter string, metrics []string, scopes []schema.MetricScope, resolution int, from, to time.Time, page *model.PageRequest, ctx context.Context) (map[string]schema.JobData, int, bool, error)
 }
 
 var metricDataRepos map[string]MetricDataRepository = map[string]MetricDataRepository{}
 
-var useArchive bool
-
-func Init(disableArchive bool) error {
-
-	useArchive = !disableArchive
+func Init() error {
 	for _, cluster := range config.Keys.Clusters {
 		if cluster.MetricDataRepository != nil {
 			var kind struct {
 				Kind string `json:"kind"`
 			}
 			if err := json.Unmarshal(cluster.MetricDataRepository, &kind); err != nil {
-				log.Warn("Error while unmarshaling raw json MetricDataRepository")
+				cclog.Warn("Error while unmarshaling raw json MetricDataRepository")
 				return err
 			}
 
@@ -54,8 +54,6 @@ func Init(disableArchive bool) error {
 			switch kind.Kind {
 			case "cc-metric-store":
 				mdr = &CCMetricStore{}
-			case "influxdb":
-				mdr = &InfluxDBv2DataRepository{}
 			case "prometheus":
 				mdr = &PrometheusDataRepository{}
 			case "test":
@@ -65,7 +63,7 @@ func Init(disableArchive bool) error {
 			}
 
 			if err := mdr.Init(cluster.MetricDataRepository); err != nil {
-				log.Errorf("Error initializing MetricDataRepository %v for cluster %v", kind.Kind, cluster.Name)
+				cclog.Errorf("Error initializing MetricDataRepository %v for cluster %v", kind.Kind, cluster.Name)
 				return err
 			}
 			metricDataRepos[cluster.Name] = mdr
@@ -74,283 +72,13 @@ func Init(disableArchive bool) error {
 	return nil
 }
 
-var cache *lrucache.Cache = lrucache.New(128 * 1024 * 1024)
-
-// Fetches the metric data for a job.
-func LoadData(job *schema.Job,
-	metrics []string,
-	scopes []schema.MetricScope,
-	ctx context.Context) (schema.JobData, error) {
-	data := cache.Get(cacheKey(job, metrics, scopes), func() (_ interface{}, ttl time.Duration, size int) {
-		var jd schema.JobData
-		var err error
-
-		if job.State == schema.JobStateRunning ||
-			job.MonitoringStatus == schema.MonitoringStatusRunningOrArchiving ||
-			!useArchive {
-
-			repo, ok := metricDataRepos[job.Cluster]
-
-			if !ok {
-				return fmt.Errorf("METRICDATA/METRICDATA > no metric data repository configured for '%s'", job.Cluster), 0, 0
-			}
-
-			if scopes == nil {
-				scopes = append(scopes, schema.MetricScopeNode)
-			}
-
-			if metrics == nil {
-				cluster := archive.GetCluster(job.Cluster)
-				for _, mc := range cluster.MetricConfig {
-					metrics = append(metrics, mc.Name)
-				}
-			}
-
-			jd, err = repo.LoadData(job, metrics, scopes, ctx)
-			if err != nil {
-				if len(jd) != 0 {
-					log.Warnf("partial error: %s", err.Error())
-				} else {
-					log.Error("Error while loading job data from metric repository")
-					return err, 0, 0
-				}
-			}
-			size = jd.Size()
-		} else {
-			jd, err = archive.GetHandle().LoadJobData(job)
-			if err != nil {
-				log.Error("Error while loading job data from archive")
-				return err, 0, 0
-			}
-
-			// Avoid sending unrequested data to the client:
-			if metrics != nil || scopes != nil {
-				if metrics == nil {
-					metrics = make([]string, 0, len(jd))
-					for k := range jd {
-						metrics = append(metrics, k)
-					}
-				}
-
-				res := schema.JobData{}
-				for _, metric := range metrics {
-					if perscope, ok := jd[metric]; ok {
-						if len(perscope) > 1 {
-							subset := make(map[schema.MetricScope]*schema.JobMetric)
-							for _, scope := range scopes {
-								if jm, ok := perscope[scope]; ok {
-									subset[scope] = jm
-								}
-							}
-
-							if len(subset) > 0 {
-								perscope = subset
-							}
-						}
-
-						res[metric] = perscope
-					}
-				}
-				jd = res
-			}
-			size = jd.Size()
-		}
-
-		ttl = 5 * time.Hour
-		if job.State == schema.JobStateRunning {
-			ttl = 2 * time.Minute
-		}
-
-		prepareJobData(job, jd, scopes)
-
-		return jd, ttl, size
-	})
-
-	if err, ok := data.(error); ok {
-		log.Error("Error in returned dataset")
-		return nil, err
-	}
-
-	return data.(schema.JobData), nil
-}
-
-// Used for the jobsFootprint GraphQL-Query. TODO: Rename/Generalize.
-func LoadAverages(
-	job *schema.Job,
-	metrics []string,
-	data [][]schema.Float,
-	ctx context.Context) error {
-
-	if job.State != schema.JobStateRunning && useArchive {
-		return archive.LoadAveragesFromArchive(job, metrics, data) // #166 change also here?
-	}
-
-	repo, ok := metricDataRepos[job.Cluster]
-	if !ok {
-		return fmt.Errorf("METRICDATA/METRICDATA > no metric data repository configured for '%s'", job.Cluster)
-	}
-
-	stats, err := repo.LoadStats(job, metrics, ctx) // #166 how to handle stats for acc normalizazion?
-	if err != nil {
-		log.Errorf("Error while loading statistics for job %v (User %v, Project %v)", job.JobID, job.User, job.Project)
-		return err
-	}
-
-	for i, m := range metrics {
-		nodes, ok := stats[m]
-		if !ok {
-			data[i] = append(data[i], schema.NaN)
-			continue
-		}
-
-		sum := 0.0
-		for _, node := range nodes {
-			sum += node.Avg
-		}
-		data[i] = append(data[i], schema.Float(sum))
-	}
-
-	return nil
-}
-
-// Used for the node/system view. Returns a map of nodes to a map of metrics.
-func LoadNodeData(
-	cluster string,
-	metrics, nodes []string,
-	scopes []schema.MetricScope,
-	from, to time.Time,
-	ctx context.Context) (map[string]map[string][]*schema.JobMetric, error) {
-
+func GetMetricDataRepo(cluster string) (MetricDataRepository, error) {
+	var err error
 	repo, ok := metricDataRepos[cluster]
+
 	if !ok {
-		return nil, fmt.Errorf("METRICDATA/METRICDATA > no metric data repository configured for '%s'", cluster)
+		err = fmt.Errorf("METRICDATA/METRICDATA > no metric data repository configured for '%s'", cluster)
 	}
 
-	if metrics == nil {
-		for _, m := range archive.GetCluster(cluster).MetricConfig {
-			metrics = append(metrics, m.Name)
-		}
-	}
-
-	data, err := repo.LoadNodeData(cluster, metrics, nodes, scopes, from, to, ctx)
-	if err != nil {
-		if len(data) != 0 {
-			log.Warnf("partial error: %s", err.Error())
-		} else {
-			log.Error("Error while loading node data from metric repository")
-			return nil, err
-		}
-	}
-
-	if data == nil {
-		return nil, fmt.Errorf("METRICDATA/METRICDATA > the metric data repository for '%s' does not support this query", cluster)
-	}
-
-	return data, nil
-}
-
-func cacheKey(
-	job *schema.Job,
-	metrics []string,
-	scopes []schema.MetricScope) string {
-
-	// Duration and StartTime do not need to be in the cache key as StartTime is less unique than
-	// job.ID and the TTL of the cache entry makes sure it does not stay there forever.
-	return fmt.Sprintf("%d(%s):[%v],[%v]",
-		job.ID, job.State, metrics, scopes)
-}
-
-// For /monitoring/job/<job> and some other places, flops_any and mem_bw need
-// to be available at the scope 'node'. If a job has a lot of nodes,
-// statisticsSeries should be available so that a min/mean/max Graph can be
-// used instead of a lot of single lines.
-func prepareJobData(
-	job *schema.Job,
-	jobData schema.JobData,
-	scopes []schema.MetricScope) {
-
-	const maxSeriesSize int = 15
-	for _, scopes := range jobData {
-		for _, jm := range scopes {
-			if jm.StatisticsSeries != nil || len(jm.Series) <= maxSeriesSize {
-				continue
-			}
-
-			jm.AddStatisticsSeries()
-		}
-	}
-
-	nodeScopeRequested := false
-	for _, scope := range scopes {
-		if scope == schema.MetricScopeNode {
-			nodeScopeRequested = true
-		}
-	}
-
-	if nodeScopeRequested {
-		jobData.AddNodeScope("flops_any")
-		jobData.AddNodeScope("mem_bw")
-	}
-}
-
-// Writes a running job to the job-archive
-func ArchiveJob(job *schema.Job, ctx context.Context) (*schema.JobMeta, error) {
-
-	allMetrics := make([]string, 0)
-	metricConfigs := archive.GetCluster(job.Cluster).MetricConfig
-	for _, mc := range metricConfigs {
-		allMetrics = append(allMetrics, mc.Name)
-	}
-
-	// TODO: Talk about this! What resolutions to store data at...
-	scopes := []schema.MetricScope{schema.MetricScopeNode}
-	if job.NumNodes <= 8 {
-		scopes = append(scopes, schema.MetricScopeCore)
-	}
-
-	jobData, err := LoadData(job, allMetrics, scopes, ctx)
-	if err != nil {
-		log.Error("Error wile loading job data for archiving")
-		return nil, err
-	}
-
-	jobMeta := &schema.JobMeta{
-		BaseJob:    job.BaseJob,
-		StartTime:  job.StartTime.Unix(),
-		Statistics: make(map[string]schema.JobStatistics),
-	}
-
-	for metric, data := range jobData {
-		avg, min, max := 0.0, math.MaxFloat32, -math.MaxFloat32
-		nodeData, ok := data["node"]
-		if !ok {
-			// TODO/FIXME: Calc average for non-node metrics as well!
-			continue
-		}
-
-		for _, series := range nodeData.Series {
-			avg += series.Statistics.Avg
-			min = math.Min(min, series.Statistics.Min)
-			max = math.Max(max, series.Statistics.Max)
-		}
-
-		jobMeta.Statistics[metric] = schema.JobStatistics{
-			Unit: schema.Unit{
-				Prefix: archive.GetMetricConfig(job.Cluster, metric).Unit.Prefix,
-				Base:   archive.GetMetricConfig(job.Cluster, metric).Unit.Base,
-			},
-			Avg: avg / float64(job.NumNodes),
-			Min: min,
-			Max: max,
-		}
-	}
-
-	// If the file based archive is disabled,
-	// only return the JobMeta structure as the
-	// statistics in there are needed.
-	if !useArchive {
-		return jobMeta, nil
-	}
-
-	return jobMeta, archive.GetHandle().ImportJob(jobMeta, &jobData)
+	return repo, err
 }

internal/metricdata/prometheus.go

@@ -1,5 +1,5 @@
 // Copyright (C) 2022 DKRZ
-// All rights reserved.
+// All rights reserved. This file is part of cc-backend.
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 package metricdata
@@ -20,9 +20,10 @@ import (
 	"text/template"
 	"time"
 
+	"github.com/ClusterCockpit/cc-backend/internal/graph/model"
 	"github.com/ClusterCockpit/cc-backend/pkg/archive"
-	"github.com/ClusterCockpit/cc-backend/pkg/log"
-	"github.com/ClusterCockpit/cc-backend/pkg/schema"
+	cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
+	"github.com/ClusterCockpit/cc-lib/schema"
 	promapi "github.com/prometheus/client_golang/api"
 	promv1 "github.com/prometheus/client_golang/api/prometheus/v1"
 	promcfg "github.com/prometheus/common/config"
@@ -159,17 +160,17 @@ func (pdb *PrometheusDataRepository) Init(rawConfig json.RawMessage) error {
 	var config PrometheusDataRepositoryConfig
 	// parse config
 	if err := json.Unmarshal(rawConfig, &config); err != nil {
-		log.Warn("Error while unmarshaling raw json config")
+		cclog.Warn("Error while unmarshaling raw json config")
 		return err
 	}
 	// support basic authentication
 	var rt http.RoundTripper = nil
 	if prom_pw := os.Getenv("PROMETHEUS_PASSWORD"); prom_pw != "" && config.Username != "" {
 		prom_pw := promcfg.Secret(prom_pw)
-		rt = promcfg.NewBasicAuthRoundTripper(config.Username, prom_pw, "", promapi.DefaultRoundTripper)
+		rt = promcfg.NewBasicAuthRoundTripper(promcfg.NewInlineSecret(config.Username), promcfg.NewInlineSecret(string(prom_pw)), promapi.DefaultRoundTripper)
 	} else {
 		if config.Username != "" {
-			return errors.New("METRICDATA/PROMETHEUS > Prometheus username provided, but PROMETHEUS_PASSWORD not set.")
+			return errors.New("METRICDATA/PROMETHEUS > Prometheus username provided, but PROMETHEUS_PASSWORD not set")
 		}
 	}
 	// init client
@@ -178,7 +179,7 @@ func (pdb *PrometheusDataRepository) Init(rawConfig json.RawMessage) error {
 		RoundTripper: rt,
 	})
 	if err != nil {
-		log.Error("Error while initializing new prometheus client")
+		cclog.Error("Error while initializing new prometheus client")
 		return err
 	}
 	// init query client
@@ -191,9 +192,9 @@ func (pdb *PrometheusDataRepository) Init(rawConfig json.RawMessage) error {
 	for metric, templ := range config.Templates {
 		pdb.templates[metric], err = template.New(metric).Parse(templ)
 		if err == nil {
-			log.Debugf("Added PromQL template for %s: %s", metric, templ)
+			cclog.Debugf("Added PromQL template for %s: %s", metric, templ)
 		} else {
-			log.Warnf("Failed to parse PromQL template %s for metric %s", templ, metric)
+			cclog.Warnf("Failed to parse PromQL template %s for metric %s", templ, metric)
 		}
 	}
 	return nil
@@ -204,8 +205,8 @@ func (pdb *PrometheusDataRepository) FormatQuery(
 	metric string,
 	scope schema.MetricScope,
 	nodes []string,
-	cluster string) (string, error) {
-
+	cluster string,
+) (string, error) {
 	args := PromQLArgs{}
 	if len(nodes) > 0 {
 		args.Nodes = fmt.Sprintf("(%s)%s", nodeRegex(nodes), pdb.suffix)
@@ -220,7 +221,7 @@ func (pdb *PrometheusDataRepository) FormatQuery(
 			return "", errors.New(fmt.Sprintf("METRICDATA/PROMETHEUS > Error compiling template %v", templ))
 		} else {
 			query := buf.String()
-			log.Debugf("PromQL: %s", query)
+			cclog.Debugf("PromQL: %s", query)
 			return query, nil
 		}
 	} else {
@@ -233,12 +234,13 @@ func (pdb *PrometheusDataRepository) RowToSeries(
 	from time.Time,
 	step int64,
 	steps int64,
-	row *promm.SampleStream) schema.Series {
+	row *promm.SampleStream,
+) schema.Series {
 	ts := from.Unix()
 	hostname := strings.TrimSuffix(string(row.Metric["exported_instance"]), pdb.suffix)
 	// init array of expected length with NaN
 	values := make([]schema.Float, steps+1)
-	for i, _ := range values {
+	for i := range values {
 		values[i] = schema.NaN
 	}
 	// copy recorded values from prom sample pair
@@ -263,8 +265,9 @@ func (pdb *PrometheusDataRepository) LoadData(
 	job *schema.Job,
 	metrics []string,
 	scopes []schema.MetricScope,
-	ctx context.Context) (schema.JobData, error) {
-
+	ctx context.Context,
+	resolution int,
+) (schema.JobData, error) {
 	// TODO respect requested scope
 	if len(scopes) == 0 || !contains(scopes, schema.MetricScopeNode) {
 		scopes = append(scopes, schema.MetricScopeNode)
@@ -276,13 +279,13 @@ func (pdb *PrometheusDataRepository) LoadData(
 	for i, resource := range job.Resources {
 		nodes[i] = resource.Hostname
 	}
-	from := job.StartTime
-	to := job.StartTime.Add(time.Duration(job.Duration) * time.Second)
+	from := time.Unix(job.StartTime, 0)
+	to := time.Unix(job.StartTime+int64(job.Duration), 0)
 
 	for _, scope := range scopes {
 		if scope != schema.MetricScopeNode {
 			logOnce.Do(func() {
-				log.Infof("Scope '%s' requested, but not yet supported: Will return 'node' scope only.", scope)
+				cclog.Infof("Scope '%s' requested, but not yet supported: Will return 'node' scope only.", scope)
 			})
 			continue
 		}
@@ -290,12 +293,12 @@ func (pdb *PrometheusDataRepository) LoadData(
 		for _, metric := range metrics {
 			metricConfig := archive.GetMetricConfig(job.Cluster, metric)
 			if metricConfig == nil {
-				log.Warnf("Error in LoadData: Metric %s for cluster %s not configured", metric, job.Cluster)
+				cclog.Warnf("Error in LoadData: Metric %s for cluster %s not configured", metric, job.Cluster)
 				return nil, errors.New("Prometheus config error")
 			}
 			query, err := pdb.FormatQuery(metric, scope, nodes, job.Cluster)
 			if err != nil {
-				log.Warn("Error while formatting prometheus query")
+				cclog.Warn("Error while formatting prometheus query")
 				return nil, err
 			}
 
@@ -306,13 +309,12 @@ func (pdb *PrometheusDataRepository) LoadData(
 				Step:  time.Duration(metricConfig.Timestep * 1e9),
 			}
 			result, warnings, err := pdb.queryClient.QueryRange(ctx, query, r)
-
 			if err != nil {
-				log.Errorf("Prometheus query error in LoadData: %v\nQuery: %s", err, query)
+				cclog.Errorf("Prometheus query error in LoadData: %v\nQuery: %s", err, query)
 				return nil, errors.New("Prometheus query error")
 			}
 			if len(warnings) > 0 {
-				log.Warnf("Warnings: %v\n", warnings)
+				cclog.Warnf("Warnings: %v\n", warnings)
 			}
 
 			// init data structures
@@ -335,7 +337,7 @@ func (pdb *PrometheusDataRepository) LoadData(
 					pdb.RowToSeries(from, step, steps, row))
 			}
 			// only add metric if at least one host returned data
-			if !ok && len(jobMetric.Series) > 0{
+			if !ok && len(jobMetric.Series) > 0 {
 				jobData[metric][scope] = jobMetric
 			}
 			// sort by hostname to get uniform coloring
@@ -351,14 +353,14 @@ func (pdb *PrometheusDataRepository) LoadData(
 func (pdb *PrometheusDataRepository) LoadStats(
 	job *schema.Job,
 	metrics []string,
-	ctx context.Context) (map[string]map[string]schema.MetricStatistics, error) {
-
+	ctx context.Context,
+) (map[string]map[string]schema.MetricStatistics, error) {
 	// map of metrics of nodes of stats
 	stats := map[string]map[string]schema.MetricStatistics{}
 
-	data, err := pdb.LoadData(job, metrics, []schema.MetricScope{schema.MetricScopeNode}, ctx)
+	data, err := pdb.LoadData(job, metrics, []schema.MetricScope{schema.MetricScopeNode}, ctx, 0 /*resolution here*/)
 	if err != nil {
-		log.Warn("Error while loading job for stats")
+		cclog.Warn("Error while loading job for stats")
 		return nil, err
 	}
 	for metric, metricData := range data {
@@ -376,7 +378,8 @@ func (pdb *PrometheusDataRepository) LoadNodeData(
 	metrics, nodes []string,
 	scopes []schema.MetricScope,
 	from, to time.Time,
-	ctx context.Context) (map[string]map[string][]*schema.JobMetric, error) {
+	ctx context.Context,
+) (map[string]map[string][]*schema.JobMetric, error) {
 	t0 := time.Now()
 	// Map of hosts of metrics of value slices
 	data := make(map[string]map[string][]*schema.JobMetric)
@@ -388,19 +391,19 @@ func (pdb *PrometheusDataRepository) LoadNodeData(
 	for _, scope := range scopes {
 		if scope != schema.MetricScopeNode {
 			logOnce.Do(func() {
-				log.Infof("Note: Scope '%s' requested, but not yet supported: Will return 'node' scope only.", scope)
+				cclog.Infof("Note: Scope '%s' requested, but not yet supported: Will return 'node' scope only.", scope)
 			})
 			continue
 		}
 		for _, metric := range metrics {
 			metricConfig := archive.GetMetricConfig(cluster, metric)
 			if metricConfig == nil {
-				log.Warnf("Error in LoadNodeData: Metric %s for cluster %s not configured", metric, cluster)
+				cclog.Warnf("Error in LoadNodeData: Metric %s for cluster %s not configured", metric, cluster)
 				return nil, errors.New("Prometheus config error")
 			}
 			query, err := pdb.FormatQuery(metric, scope, nodes, cluster)
 			if err != nil {
-				log.Warn("Error while formatting prometheus query")
+				cclog.Warn("Error while formatting prometheus query")
 				return nil, err
 			}
 
@@ -411,13 +414,12 @@ func (pdb *PrometheusDataRepository) LoadNodeData(
 				Step:  time.Duration(metricConfig.Timestep * 1e9),
 			}
 			result, warnings, err := pdb.queryClient.QueryRange(ctx, query, r)
-
 			if err != nil {
-				log.Errorf("Prometheus query error in LoadNodeData: %v\n", err)
+				cclog.Errorf("Prometheus query error in LoadNodeData: %v\n", err)
 				return nil, errors.New("Prometheus query error")
 			}
 			if len(warnings) > 0 {
-				log.Warnf("Warnings: %v\n", warnings)
+				cclog.Warnf("Warnings: %v\n", warnings)
 			}
 
 			step := int64(metricConfig.Timestep)
@@ -442,6 +444,190 @@ func (pdb *PrometheusDataRepository) LoadNodeData(
 		}
 	}
 	t1 := time.Since(t0)
-	log.Debugf("LoadNodeData of %v nodes took %s", len(data), t1)
+	cclog.Debugf("LoadNodeData of %v nodes took %s", len(data), t1)
 	return data, nil
 }
+
+// Implemented by NHR@FAU; Used in Job-View StatsTable
+func (pdb *PrometheusDataRepository) LoadScopedStats(
+	job *schema.Job,
+	metrics []string,
+	scopes []schema.MetricScope,
+	ctx context.Context,
+) (schema.ScopedJobStats, error) {
+	// Assumption: pdb.loadData() only returns series node-scope - use node scope for statsTable
+	scopedJobStats := make(schema.ScopedJobStats)
+	data, err := pdb.LoadData(job, metrics, []schema.MetricScope{schema.MetricScopeNode}, ctx, 0 /*resolution here*/)
+	if err != nil {
+		cclog.Warn("Error while loading job for scopedJobStats")
+		return nil, err
+	}
+
+	for metric, metricData := range data {
+		for _, scope := range scopes {
+			if scope != schema.MetricScopeNode {
+				logOnce.Do(func() {
+					cclog.Infof("Note: Scope '%s' requested, but not yet supported: Will return 'node' scope only.", scope)
+				})
+				continue
+			}
+
+			if _, ok := scopedJobStats[metric]; !ok {
+				scopedJobStats[metric] = make(map[schema.MetricScope][]*schema.ScopedStats)
+			}
+
+			if _, ok := scopedJobStats[metric][scope]; !ok {
+				scopedJobStats[metric][scope] = make([]*schema.ScopedStats, 0)
+			}
+
+			for _, series := range metricData[scope].Series {
+				scopedJobStats[metric][scope] = append(scopedJobStats[metric][scope], &schema.ScopedStats{
+					Hostname: series.Hostname,
+					Data:     &series.Statistics,
+				})
+			}
+		}
+	}
+
+	return scopedJobStats, nil
+}
+
+// Implemented by NHR@FAU; Used in NodeList-View
+func (pdb *PrometheusDataRepository) LoadNodeListData(
+	cluster, subCluster, nodeFilter string,
+	metrics []string,
+	scopes []schema.MetricScope,
+	resolution int,
+	from, to time.Time,
+	page *model.PageRequest,
+	ctx context.Context,
+) (map[string]schema.JobData, int, bool, error) {
+	// Assumption: pdb.loadData() only returns series node-scope - use node scope for NodeList
+
+	// 0) Init additional vars
+	var totalNodes int = 0
+	var hasNextPage bool = false
+
+	// 1) Get list of all nodes
+	var nodes []string
+	if subCluster != "" {
+		scNodes := archive.NodeLists[cluster][subCluster]
+		nodes = scNodes.PrintList()
+	} else {
+		subClusterNodeLists := archive.NodeLists[cluster]
+		for _, nodeList := range subClusterNodeLists {
+			nodes = append(nodes, nodeList.PrintList()...)
+		}
+	}
+
+	// 2) Filter nodes
+	if nodeFilter != "" {
+		filteredNodes := []string{}
+		for _, node := range nodes {
+			if strings.Contains(node, nodeFilter) {
+				filteredNodes = append(filteredNodes, node)
+			}
+		}
+		nodes = filteredNodes
+	}
+
+	// 2.1) Count total nodes && Sort nodes -> Sorting invalidated after return ...
+	totalNodes = len(nodes)
+	sort.Strings(nodes)
+
+	// 3) Apply paging
+	if len(nodes) > page.ItemsPerPage {
+		start := (page.Page - 1) * page.ItemsPerPage
+		end := start + page.ItemsPerPage
+		if end >= len(nodes) {
+			end = len(nodes)
+			hasNextPage = false
+		} else {
+			hasNextPage = true
+		}
+		nodes = nodes[start:end]
+	}
+
+	// 4) Fetch Data, based on pdb.LoadNodeData()
+
+	t0 := time.Now()
+	// Map of hosts of jobData
+	data := make(map[string]schema.JobData)
+
+	// query db for each metric
+	// TODO: scopes seems to be always empty
+	if len(scopes) == 0 || !contains(scopes, schema.MetricScopeNode) {
+		scopes = append(scopes, schema.MetricScopeNode)
+	}
+
+	for _, scope := range scopes {
+		if scope != schema.MetricScopeNode {
+			logOnce.Do(func() {
+				cclog.Infof("Note: Scope '%s' requested, but not yet supported: Will return 'node' scope only.", scope)
+			})
+			continue
+		}
+
+		for _, metric := range metrics {
+			metricConfig := archive.GetMetricConfig(cluster, metric)
+			if metricConfig == nil {
+				cclog.Warnf("Error in LoadNodeListData: Metric %s for cluster %s not configured", metric, cluster)
+				return nil, totalNodes, hasNextPage, errors.New("Prometheus config error")
+			}
+			query, err := pdb.FormatQuery(metric, scope, nodes, cluster)
+			if err != nil {
+				cclog.Warn("Error while formatting prometheus query")
+				return nil, totalNodes, hasNextPage, err
+			}
+
+			// ranged query over all nodes
+			r := promv1.Range{
+				Start: from,
+				End:   to,
+				Step:  time.Duration(metricConfig.Timestep * 1e9),
+			}
+			result, warnings, err := pdb.queryClient.QueryRange(ctx, query, r)
+			if err != nil {
+				cclog.Errorf("Prometheus query error in LoadNodeData: %v\n", err)
+				return nil, totalNodes, hasNextPage, errors.New("Prometheus query error")
+			}
+			if len(warnings) > 0 {
+				cclog.Warnf("Warnings: %v\n", warnings)
+			}
+
+			step := int64(metricConfig.Timestep)
+			steps := int64(to.Sub(from).Seconds()) / step
+
+			// iter rows of host, metric, values
+			for _, row := range result.(promm.Matrix) {
+				hostname := strings.TrimSuffix(string(row.Metric["exported_instance"]), pdb.suffix)
+
+				hostdata, ok := data[hostname]
+				if !ok {
+					hostdata = make(schema.JobData)
+					data[hostname] = hostdata
+				}
+
+				metricdata, ok := hostdata[metric]
+				if !ok {
+					metricdata = make(map[schema.MetricScope]*schema.JobMetric)
+					data[hostname][metric] = metricdata
+				}
+
+				// output per host, metric and scope
+				scopeData, ok := metricdata[scope]
+				if !ok {
+					scopeData = &schema.JobMetric{
+						Unit:     metricConfig.Unit,
+						Timestep: metricConfig.Timestep,
+						Series:   []schema.Series{pdb.RowToSeries(from, step, steps, row)},
+					}
+					data[hostname][metric][scope] = scopeData
+				}
+			}
+		}
+	}
+	t1 := time.Since(t0)
+	cclog.Debugf("LoadNodeListData of %v nodes took %s", len(data), t1)
+	return data, totalNodes, hasNextPage, nil
+}

internal/metricdata/utils.go

@@ -1,5 +1,5 @@
-// Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved.
+// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
+// All rights reserved. This file is part of cc-backend.
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 package metricdata
@@ -9,10 +9,11 @@ import (
 	"encoding/json"
 	"time"
 
-	"github.com/ClusterCockpit/cc-backend/pkg/schema"
+	"github.com/ClusterCockpit/cc-backend/internal/graph/model"
+	"github.com/ClusterCockpit/cc-lib/schema"
 )
 
-var TestLoadDataCallback func(job *schema.Job, metrics []string, scopes []schema.MetricScope, ctx context.Context) (schema.JobData, error) = func(job *schema.Job, metrics []string, scopes []schema.MetricScope, ctx context.Context) (schema.JobData, error) {
+var TestLoadDataCallback func(job *schema.Job, metrics []string, scopes []schema.MetricScope, ctx context.Context, resolution int) (schema.JobData, error) = func(job *schema.Job, metrics []string, scopes []schema.MetricScope, ctx context.Context, resolution int) (schema.JobData, error) {
 	panic("TODO")
 }
 
@@ -27,15 +28,26 @@ func (tmdr *TestMetricDataRepository) LoadData(
 	job *schema.Job,
 	metrics []string,
 	scopes []schema.MetricScope,
-	ctx context.Context) (schema.JobData, error) {
-
-	return TestLoadDataCallback(job, metrics, scopes, ctx)
+	ctx context.Context,
+	resolution int,
+) (schema.JobData, error) {
+	return TestLoadDataCallback(job, metrics, scopes, ctx, resolution)
 }
 
 func (tmdr *TestMetricDataRepository) LoadStats(
 	job *schema.Job,
-	metrics []string, ctx context.Context) (map[string]map[string]schema.MetricStatistics, error) {
+	metrics []string,
+	ctx context.Context,
+) (map[string]map[string]schema.MetricStatistics, error) {
+	panic("TODO")
+}
 
+func (tmdr *TestMetricDataRepository) LoadScopedStats(
+	job *schema.Job,
+	metrics []string,
+	scopes []schema.MetricScope,
+	ctx context.Context,
+) (schema.ScopedJobStats, error) {
 	panic("TODO")
 }
 
@@ -44,7 +56,65 @@ func (tmdr *TestMetricDataRepository) LoadNodeData(
 	metrics, nodes []string,
 	scopes []schema.MetricScope,
 	from, to time.Time,
-	ctx context.Context) (map[string]map[string][]*schema.JobMetric, error) {
-
+	ctx context.Context,
+) (map[string]map[string][]*schema.JobMetric, error) {
 	panic("TODO")
 }
+
+func (tmdr *TestMetricDataRepository) LoadNodeListData(
+	cluster, subCluster, nodeFilter string,
+	metrics []string,
+	scopes []schema.MetricScope,
+	resolution int,
+	from, to time.Time,
+	page *model.PageRequest,
+	ctx context.Context,
+) (map[string]schema.JobData, int, bool, error) {
+	panic("TODO")
+}
+
+func DeepCopy(jd_temp schema.JobData) schema.JobData {
+	var jd schema.JobData
+
+	jd = make(schema.JobData, len(jd_temp))
+	for k, v := range jd_temp {
+		jd[k] = make(map[schema.MetricScope]*schema.JobMetric, len(jd_temp[k]))
+		for k_, v_ := range v {
+			jd[k][k_] = new(schema.JobMetric)
+			jd[k][k_].Series = make([]schema.Series, len(v_.Series))
+			for i := 0; i < len(v_.Series); i += 1 {
+				jd[k][k_].Series[i].Data = make([]schema.Float, len(v_.Series[i].Data))
+				copy(jd[k][k_].Series[i].Data, v_.Series[i].Data)
+				jd[k][k_].Series[i].Hostname = v_.Series[i].Hostname
+				jd[k][k_].Series[i].Id = v_.Series[i].Id
+				jd[k][k_].Series[i].Statistics.Avg = v_.Series[i].Statistics.Avg
+				jd[k][k_].Series[i].Statistics.Min = v_.Series[i].Statistics.Min
+				jd[k][k_].Series[i].Statistics.Max = v_.Series[i].Statistics.Max
+			}
+			jd[k][k_].Timestep = v_.Timestep
+			jd[k][k_].Unit.Base = v_.Unit.Base
+			jd[k][k_].Unit.Prefix = v_.Unit.Prefix
+			if v_.StatisticsSeries != nil {
+				// Init Slices
+				jd[k][k_].StatisticsSeries = new(schema.StatsSeries)
+				jd[k][k_].StatisticsSeries.Max = make([]schema.Float, len(v_.StatisticsSeries.Max))
+				jd[k][k_].StatisticsSeries.Min = make([]schema.Float, len(v_.StatisticsSeries.Min))
+				jd[k][k_].StatisticsSeries.Median = make([]schema.Float, len(v_.StatisticsSeries.Median))
+				jd[k][k_].StatisticsSeries.Mean = make([]schema.Float, len(v_.StatisticsSeries.Mean))
+				// Copy Data
+				copy(jd[k][k_].StatisticsSeries.Max, v_.StatisticsSeries.Max)
+				copy(jd[k][k_].StatisticsSeries.Min, v_.StatisticsSeries.Min)
+				copy(jd[k][k_].StatisticsSeries.Median, v_.StatisticsSeries.Median)
+				copy(jd[k][k_].StatisticsSeries.Mean, v_.StatisticsSeries.Mean)
+				// Handle Percentiles
+				for k__, v__ := range v_.StatisticsSeries.Percentiles {
+					jd[k][k_].StatisticsSeries.Percentiles[k__] = make([]schema.Float, len(v__))
+					copy(jd[k][k_].StatisticsSeries.Percentiles[k__], v__)
+				}
+			} else {
+				jd[k][k_].StatisticsSeries = v_.StatisticsSeries
+			}
+		}
+	}
+	return jd
+}

internal/repository/dbConnection.go

@@ -1,15 +1,17 @@
-// Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved.
+// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
+// All rights reserved. This file is part of cc-backend.
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 package repository
 
 import (
 	"database/sql"
+	"fmt"
+	"net/url"
 	"sync"
 	"time"
 
-	"github.com/ClusterCockpit/cc-backend/pkg/log"
+	cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
 	"github.com/jmoiron/sqlx"
 	"github.com/mattn/go-sqlite3"
 	"github.com/qustavo/sqlhooks/v2"
@@ -33,6 +35,27 @@ type DatabaseOptions struct {
 	ConnectionMaxIdleTime time.Duration
 }
 
+func setupSqlite(db *sql.DB) (err error) {
+	pragmas := []string{
+		// "journal_mode = WAL",
+		// "busy_timeout = 5000",
+		// "synchronous = NORMAL",
+		// "cache_size = 1000000000", // 1GB
+		// "foreign_keys = true",
+		"temp_store = memory",
+		// "mmap_size = 3000000000",
+	}
+
+	for _, pragma := range pragmas {
+		_, err = db.Exec("PRAGMA " + pragma)
+		if err != nil {
+			return
+		}
+	}
+
+	return nil
+}
+
 func Connect(driver string, db string) {
 	var err error
 	var dbHandle *sqlx.DB
@@ -48,28 +71,34 @@ func Connect(driver string, db string) {
 
 		switch driver {
 		case "sqlite3":
-			// - Set WAL mode (not strictly necessary each time because it's persisted in the database, but good for first run)
-			// - Set busy timeout, so concurrent writers wait on each other instead of erroring immediately
-			// - Enable foreign key checks
-			opts.URL += "?_journal=WAL&_timeout=5000&_fk=true"
+			// TODO: Have separate DB handles for Writes and Reads
+			// Optimize SQLite connection: https://kerkour.com/sqlite-for-servers
+			connectionUrlParams := make(url.Values)
+			connectionUrlParams.Add("_txlock", "immediate")
+			connectionUrlParams.Add("_journal_mode", "WAL")
+			connectionUrlParams.Add("_busy_timeout", "5000")
+			connectionUrlParams.Add("_synchronous", "NORMAL")
+			connectionUrlParams.Add("_cache_size", "1000000000")
+			connectionUrlParams.Add("_foreign_keys", "true")
+			opts.URL = fmt.Sprintf("file:%s?%s", opts.URL, connectionUrlParams.Encode())
 
-			if log.Loglevel() == "debug" {
+			if cclog.Loglevel() == "debug" {
 				sql.Register("sqlite3WithHooks", sqlhooks.Wrap(&sqlite3.SQLiteDriver{}, &Hooks{}))
 				dbHandle, err = sqlx.Open("sqlite3WithHooks", opts.URL)
 			} else {
 				dbHandle, err = sqlx.Open("sqlite3", opts.URL)
 			}
-			if err != nil {
-				log.Fatal(err)
-			}
+
+			setupSqlite(dbHandle.DB)
 		case "mysql":
 			opts.URL += "?multiStatements=true"
 			dbHandle, err = sqlx.Open("mysql", opts.URL)
-			if err != nil {
-				log.Fatalf("sqlx.Open() error: %v", err)
-			}
 		default:
-			log.Fatalf("unsupported database driver: %s", driver)
+			cclog.Abortf("DB Connection: Unsupported database driver '%s'.\n", driver)
+		}
+
+		if err != nil {
+			cclog.Abortf("DB Connection: Could not connect to '%s' database with sqlx.Open().\nError: %s\n", driver, err.Error())
 		}
 
 		dbHandle.SetMaxOpenConns(opts.MaxOpenConnections)
@@ -80,14 +109,14 @@ func Connect(driver string, db string) {
 		dbConnInstance = &DBConnection{DB: dbHandle, Driver: driver}
 		err = checkDBVersion(driver, dbHandle.DB)
 		if err != nil {
-			log.Fatal(err)
+			cclog.Abortf("DB Connection: Failed DB version check.\nError: %s\n", err.Error())
 		}
 	})
 }
 
 func GetConnection() *DBConnection {
 	if dbConnInstance == nil {
-		log.Fatalf("Database connection not initialized!")
+		cclog.Fatalf("Database connection not initialized!")
 	}
 
 	return dbConnInstance

internal/repository/hooks.go

@@ -1,5 +1,5 @@
-// Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved.
+// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
+// All rights reserved. This file is part of cc-backend.
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 package repository
@@ -8,21 +8,21 @@ import (
 	"context"
 	"time"
 
-	"github.com/ClusterCockpit/cc-backend/pkg/log"
+	cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
 )
 
 // Hooks satisfies the sqlhook.Hooks interface
 type Hooks struct{}
 
 // Before hook will print the query with it's args and return the context with the timestamp
-func (h *Hooks) Before(ctx context.Context, query string, args ...interface{}) (context.Context, error) {
-	log.Debugf("SQL query %s %q", query, args)
+func (h *Hooks) Before(ctx context.Context, query string, args ...any) (context.Context, error) {
+	cclog.Debugf("SQL query %s %q", query, args)
 	return context.WithValue(ctx, "begin", time.Now()), nil
 }
 
 // After hook will get the timestamp registered on the Before hook and print the elapsed time
-func (h *Hooks) After(ctx context.Context, query string, args ...interface{}) (context.Context, error) {
+func (h *Hooks) After(ctx context.Context, query string, args ...any) (context.Context, error) {
 	begin := ctx.Value("begin").(time.Time)
-	log.Debugf("Took: %s\n", time.Since(begin))
+	cclog.Debugf("Took: %s\n", time.Since(begin))
 	return ctx, nil
 }

internal/repository/job.go

@@ -1,24 +1,24 @@
-// Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved.
+// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
+// All rights reserved. This file is part of cc-backend.
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 package repository
 
 import (
-	"context"
 	"database/sql"
 	"encoding/json"
 	"errors"
 	"fmt"
+	"maps"
+	"math"
 	"strconv"
 	"sync"
 	"time"
 
-	"github.com/ClusterCockpit/cc-backend/internal/graph/model"
-	"github.com/ClusterCockpit/cc-backend/internal/metricdata"
-	"github.com/ClusterCockpit/cc-backend/pkg/log"
-	"github.com/ClusterCockpit/cc-backend/pkg/lrucache"
-	"github.com/ClusterCockpit/cc-backend/pkg/schema"
+	"github.com/ClusterCockpit/cc-backend/pkg/archive"
+	cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
+	"github.com/ClusterCockpit/cc-lib/lrucache"
+	"github.com/ClusterCockpit/cc-lib/schema"
 	sq "github.com/Masterminds/squirrel"
 	"github.com/jmoiron/sqlx"
 )
@@ -29,14 +29,11 @@ var (
 )
 
 type JobRepository struct {
-	DB     *sqlx.DB
-	driver string
-
+	DB        *sqlx.DB
 	stmtCache *sq.StmtCache
 	cache     *lrucache.Cache
-
-	archiveChannel chan *schema.Job
-	archivePending sync.WaitGroup
+	driver    string
+	Mutex     sync.Mutex
 }
 
 func GetJobRepository() *JobRepository {
@@ -47,47 +44,59 @@ func GetJobRepository() *JobRepository {
 			DB:     db.DB,
 			driver: db.Driver,
 
-			stmtCache:      sq.NewStmtCache(db.DB),
-			cache:          lrucache.New(1024 * 1024),
-			archiveChannel: make(chan *schema.Job, 128),
+			stmtCache: sq.NewStmtCache(db.DB),
+			cache:     lrucache.New(1024 * 1024),
 		}
-		// start archiving worker
-		go jobRepoInstance.archivingWorker()
 	})
 	return jobRepoInstance
 }
 
 var jobColumns []string = []string{
-	"job.id", "job.job_id", "job.user", "job.project", "job.cluster", "job.subcluster", "job.start_time", "job.partition", "job.array_job_id",
-	"job.num_nodes", "job.num_hwthreads", "job.num_acc", "job.exclusive", "job.monitoring_status", "job.smt", "job.job_state",
-	"job.duration", "job.walltime", "job.resources", // "job.meta_data",
+	"job.id", "job.job_id", "job.hpc_user", "job.project", "job.cluster", "job.subcluster",
+	"job.start_time", "job.cluster_partition", "job.array_job_id", "job.num_nodes",
+	"job.num_hwthreads", "job.num_acc", "job.exclusive", "job.monitoring_status",
+	"job.smt", "job.job_state", "job.duration", "job.walltime", "job.resources",
+	"job.footprint", "job.energy",
 }
 
-func scanJob(row interface{ Scan(...interface{}) error }) (*schema.Job, error) {
+var jobCacheColumns []string = []string{
+	"job_cache.id", "job_cache.job_id", "job_cache.hpc_user", "job_cache.project", "job_cache.cluster",
+	"job_cache.subcluster", "job_cache.start_time", "job_cache.cluster_partition",
+	"job_cache.array_job_id", "job_cache.num_nodes", "job_cache.num_hwthreads",
+	"job_cache.num_acc", "job_cache.exclusive", "job_cache.monitoring_status", "job_cache.smt",
+	"job_cache.job_state", "job_cache.duration", "job_cache.walltime", "job_cache.resources",
+	"job_cache.footprint", "job_cache.energy",
+}
+
+func scanJob(row interface{ Scan(...any) error }) (*schema.Job, error) {
 	job := &schema.Job{}
+
 	if err := row.Scan(
-		&job.ID, &job.JobID, &job.User, &job.Project, &job.Cluster, &job.SubCluster, &job.StartTimeUnix, &job.Partition, &job.ArrayJobId,
-		&job.NumNodes, &job.NumHWThreads, &job.NumAcc, &job.Exclusive, &job.MonitoringStatus, &job.SMT, &job.State,
-		&job.Duration, &job.Walltime, &job.RawResources /*&job.RawMetaData*/); err != nil {
-		log.Warnf("Error while scanning rows (Job): %v", err)
+		&job.ID, &job.JobID, &job.User, &job.Project, &job.Cluster, &job.SubCluster,
+		&job.StartTime, &job.Partition, &job.ArrayJobId, &job.NumNodes, &job.NumHWThreads,
+		&job.NumAcc, &job.Exclusive, &job.MonitoringStatus, &job.SMT, &job.State,
+		&job.Duration, &job.Walltime, &job.RawResources, &job.RawFootprint, &job.Energy); err != nil {
+		cclog.Warnf("Error while scanning rows (Job): %v", err)
 		return nil, err
 	}
 
 	if err := json.Unmarshal(job.RawResources, &job.Resources); err != nil {
-		log.Warn("Error while unmarhsaling raw resources json")
+		cclog.Warn("Error while unmarshaling raw resources json")
 		return nil, err
 	}
+	job.RawResources = nil
 
-	// if err := json.Unmarshal(job.RawMetaData, &job.MetaData); err != nil {
-	// 	return nil, err
-	// }
+	if err := json.Unmarshal(job.RawFootprint, &job.Footprint); err != nil {
+		cclog.Warnf("Error while unmarshaling raw footprint json: %v", err)
+		return nil, err
+	}
+	job.RawFootprint = nil
 
-	job.StartTime = time.Unix(job.StartTimeUnix, 0)
-	if job.Duration == 0 && job.State == schema.JobStateRunning {
-		job.Duration = int32(time.Since(job.StartTime).Seconds())
+	// Always ensure accurate duration for running jobs
+	if job.State == schema.JobStateRunning {
+		job.Duration = int32(time.Now().Unix() - job.StartTime)
 	}
 
-	job.RawResources = nil
 	return job, nil
 }
 
@@ -100,7 +109,7 @@ func (r *JobRepository) Optimize() error {
 			return err
 		}
 	case "mysql":
-		log.Info("Optimize currently not supported for mysql driver")
+		cclog.Info("Optimize currently not supported for mysql driver")
 	}
 
 	return nil
@@ -141,17 +150,6 @@ func (r *JobRepository) Flush() error {
 	return nil
 }
 
-func scanJobLink(row interface{ Scan(...interface{}) error }) (*model.JobLink, error) {
-	jobLink := &model.JobLink{}
-	if err := row.Scan(
-		&jobLink.ID, &jobLink.JobID); err != nil {
-		log.Warn("Error while scanning rows (jobLink)")
-		return nil, err
-	}
-
-	return jobLink, nil
-}
-
 func (r *JobRepository) FetchMetadata(job *schema.Job) (map[string]string, error) {
 	start := time.Now()
 	cachekey := fmt.Sprintf("metadata:%d", job.ID)
@@ -162,7 +160,7 @@ func (r *JobRepository) FetchMetadata(job *schema.Job) (map[string]string, error
 
 	if err := sq.Select("job.meta_data").From("job").Where("job.id = ?", job.ID).
 		RunWith(r.stmtCache).QueryRow().Scan(&job.RawMetaData); err != nil {
-		log.Warn("Error while scanning for job metadata")
+		cclog.Warn("Error while scanning for job metadata")
 		return nil, err
 	}
 
@@ -171,12 +169,12 @@ func (r *JobRepository) FetchMetadata(job *schema.Job) (map[string]string, error
 	}
 
 	if err := json.Unmarshal(job.RawMetaData, &job.MetaData); err != nil {
-		log.Warn("Error while unmarshaling raw metadata json")
+		cclog.Warn("Error while unmarshaling raw metadata json")
 		return nil, err
 	}
 
 	r.cache.Put(cachekey, job.MetaData, len(job.RawMetaData), 24*time.Hour)
-	log.Debugf("Timer FetchMetadata %s", time.Since(start))
+	cclog.Debugf("Timer FetchMetadata %s", time.Since(start))
 	return job.MetaData, nil
 }
 
@@ -185,16 +183,14 @@ func (r *JobRepository) UpdateMetadata(job *schema.Job, key, val string) (err er
 	r.cache.Del(cachekey)
 	if job.MetaData == nil {
 		if _, err = r.FetchMetadata(job); err != nil {
-			log.Warnf("Error while fetching metadata for job, DB ID '%v'", job.ID)
+			cclog.Warnf("Error while fetching metadata for job, DB ID '%v'", job.ID)
 			return err
 		}
 	}
 
 	if job.MetaData != nil {
 		cpy := make(map[string]string, len(job.MetaData)+1)
-		for k, v := range job.MetaData {
-			cpy[k] = v
-		}
+		maps.Copy(cpy, job.MetaData)
 		cpy[key] = val
 		job.MetaData = cpy
 	} else {
@@ -202,371 +198,117 @@ func (r *JobRepository) UpdateMetadata(job *schema.Job, key, val string) (err er
 	}
 
 	if job.RawMetaData, err = json.Marshal(job.MetaData); err != nil {
-		log.Warnf("Error while marshaling metadata for job, DB ID '%v'", job.ID)
+		cclog.Warnf("Error while marshaling metadata for job, DB ID '%v'", job.ID)
 		return err
 	}
 
-	if _, err = sq.Update("job").Set("meta_data", job.RawMetaData).Where("job.id = ?", job.ID).RunWith(r.stmtCache).Exec(); err != nil {
-		log.Warnf("Error while updating metadata for job, DB ID '%v'", job.ID)
+	if _, err = sq.Update("job").
+		Set("meta_data", job.RawMetaData).
+		Where("job.id = ?", job.ID).
+		RunWith(r.stmtCache).Exec(); err != nil {
+		cclog.Warnf("Error while updating metadata for job, DB ID '%v'", job.ID)
 		return err
 	}
 
 	r.cache.Put(cachekey, job.MetaData, len(job.RawMetaData), 24*time.Hour)
-	return nil
+	return archive.UpdateMetadata(job, job.MetaData)
 }
 
-// Find executes a SQL query to find a specific batch job.
-// The job is queried using the batch job id, the cluster name,
-// and the start time of the job in UNIX epoch time seconds.
-// It returns a pointer to a schema.Job data structure and an error variable.
-// To check if no job was found test err == sql.ErrNoRows
-func (r *JobRepository) Find(
-	jobId *int64,
-	cluster *string,
-	startTime *int64) (*schema.Job, error) {
-
+func (r *JobRepository) FetchFootprint(job *schema.Job) (map[string]float64, error) {
 	start := time.Now()
-	q := sq.Select(jobColumns...).From("job").
-		Where("job.job_id = ?", *jobId)
 
-	if cluster != nil {
-		q = q.Where("job.cluster = ?", *cluster)
-	}
-	if startTime != nil {
-		q = q.Where("job.start_time = ?", *startTime)
-	}
-
-	log.Debugf("Timer Find %s", time.Since(start))
-	return scanJob(q.RunWith(r.stmtCache).QueryRow())
-}
-
-// Find executes a SQL query to find a specific batch job.
-// The job is queried using the batch job id, the cluster name,
-// and the start time of the job in UNIX epoch time seconds.
-// It returns a pointer to a schema.Job data structure and an error variable.
-// To check if no job was found test err == sql.ErrNoRows
-func (r *JobRepository) FindAll(
-	jobId *int64,
-	cluster *string,
-	startTime *int64) ([]*schema.Job, error) {
-
-	start := time.Now()
-	q := sq.Select(jobColumns...).From("job").
-		Where("job.job_id = ?", *jobId)
-
-	if cluster != nil {
-		q = q.Where("job.cluster = ?", *cluster)
-	}
-	if startTime != nil {
-		q = q.Where("job.start_time = ?", *startTime)
-	}
-
-	rows, err := q.RunWith(r.stmtCache).Query()
-	if err != nil {
-		log.Error("Error while running query")
+	if err := sq.Select("job.footprint").From("job").Where("job.id = ?", job.ID).
+		RunWith(r.stmtCache).QueryRow().Scan(&job.RawFootprint); err != nil {
+		cclog.Warn("Error while scanning for job footprint")
 		return nil, err
 	}
 
-	jobs := make([]*schema.Job, 0, 10)
-	for rows.Next() {
-		job, err := scanJob(rows)
-		if err != nil {
-			log.Warn("Error while scanning rows")
-			return nil, err
-		}
-		jobs = append(jobs, job)
-	}
-	log.Debugf("Timer FindAll %s", time.Since(start))
-	return jobs, nil
-}
-
-// FindById executes a SQL query to find a specific batch job.
-// The job is queried using the database id.
-// It returns a pointer to a schema.Job data structure and an error variable.
-// To check if no job was found test err == sql.ErrNoRows
-func (r *JobRepository) FindById(jobId int64) (*schema.Job, error) {
-	q := sq.Select(jobColumns...).
-		From("job").Where("job.id = ?", jobId)
-	return scanJob(q.RunWith(r.stmtCache).QueryRow())
-}
-
-func (r *JobRepository) FindConcurrentJobs(
-	ctx context.Context,
-	job *schema.Job) (*model.JobLinkResultList, error) {
-	if job == nil {
+	if len(job.RawFootprint) == 0 {
 		return nil, nil
 	}
 
-	query, qerr := SecurityCheck(ctx, sq.Select("job.id", "job.job_id", "job.start_time").From("job"))
-	if qerr != nil {
-		return nil, qerr
-	}
-
-	query = query.Where("cluster = ?", job.Cluster)
-	var startTime int64
-	var stopTime int64
-
-	startTime = job.StartTimeUnix
-	hostname := job.Resources[0].Hostname
-
-	if job.State == schema.JobStateRunning {
-		stopTime = time.Now().Unix()
-	} else {
-		stopTime = startTime + int64(job.Duration)
-	}
-
-	// Add 200s overlap for jobs start time at the end
-	startTimeTail := startTime + 10
-	stopTimeTail := stopTime - 200
-	startTimeFront := startTime + 200
-
-	queryRunning := query.Where("job.job_state = ?").Where("(job.start_time BETWEEN ? AND ? OR job.start_time < ?)",
-		"running", startTimeTail, stopTimeTail, startTime)
-	queryRunning = queryRunning.Where("job.resources LIKE ?", fmt.Sprint("%", hostname, "%"))
-
-	query = query.Where("job.job_state != ?").Where("((job.start_time BETWEEN ? AND ?) OR (job.start_time + job.duration) BETWEEN ? AND ? OR (job.start_time < ?) AND (job.start_time + job.duration) > ?)",
-		"running", startTimeTail, stopTimeTail, startTimeFront, stopTimeTail, startTime, stopTime)
-	query = query.Where("job.resources LIKE ?", fmt.Sprint("%", hostname, "%"))
-
-	rows, err := query.RunWith(r.stmtCache).Query()
-	if err != nil {
-		log.Errorf("Error while running query: %v", err)
+	if err := json.Unmarshal(job.RawFootprint, &job.Footprint); err != nil {
+		cclog.Warn("Error while unmarshaling raw footprint json")
 		return nil, err
 	}
 
-	items := make([]*model.JobLink, 0, 10)
-	queryString := fmt.Sprintf("cluster=%s", job.Cluster)
+	cclog.Debugf("Timer FetchFootprint %s", time.Since(start))
+	return job.Footprint, nil
+}
 
-	for rows.Next() {
-		var id, jobId, startTime sql.NullInt64
-
-		if err = rows.Scan(&id, &jobId, &startTime); err != nil {
-			log.Warn("Error while scanning rows")
-			return nil, err
-		}
-
-		if id.Valid {
-			queryString += fmt.Sprintf("&jobId=%d", int(jobId.Int64))
-			items = append(items,
-				&model.JobLink{
-					ID:    fmt.Sprint(id.Int64),
-					JobID: int(jobId.Int64),
-				})
-		}
+func (r *JobRepository) FetchEnergyFootprint(job *schema.Job) (map[string]float64, error) {
+	start := time.Now()
+	cachekey := fmt.Sprintf("energyFootprint:%d", job.ID)
+	if cached := r.cache.Get(cachekey, nil); cached != nil {
+		job.EnergyFootprint = cached.(map[string]float64)
+		return job.EnergyFootprint, nil
 	}
 
-	rows, err = queryRunning.RunWith(r.stmtCache).Query()
-	if err != nil {
-		log.Errorf("Error while running query: %v", err)
+	if err := sq.Select("job.energy_footprint").From("job").Where("job.id = ?", job.ID).
+		RunWith(r.stmtCache).QueryRow().Scan(&job.RawEnergyFootprint); err != nil {
+		cclog.Warn("Error while scanning for job energy_footprint")
 		return nil, err
 	}
 
-	for rows.Next() {
-		var id, jobId, startTime sql.NullInt64
-
-		if err := rows.Scan(&id, &jobId, &startTime); err != nil {
-			log.Warn("Error while scanning rows")
-			return nil, err
-		}
-
-		if id.Valid {
-			queryString += fmt.Sprintf("&jobId=%d", int(jobId.Int64))
-			items = append(items,
-				&model.JobLink{
-					ID:    fmt.Sprint(id.Int64),
-					JobID: int(jobId.Int64),
-				})
-		}
+	if len(job.RawEnergyFootprint) == 0 {
+		return nil, nil
 	}
 
-	cnt := len(items)
-
-	return &model.JobLinkResultList{
-		ListQuery: &queryString,
-		Items:     items,
-		Count:     &cnt,
-	}, nil
-}
-
-// Start inserts a new job in the table, returning the unique job ID.
-// Statistics are not transfered!
-func (r *JobRepository) Start(job *schema.JobMeta) (id int64, err error) {
-	job.RawResources, err = json.Marshal(job.Resources)
-	if err != nil {
-		return -1, fmt.Errorf("REPOSITORY/JOB > encoding resources field failed: %w", err)
+	if err := json.Unmarshal(job.RawEnergyFootprint, &job.EnergyFootprint); err != nil {
+		cclog.Warn("Error while unmarshaling raw energy footprint json")
+		return nil, err
 	}
 
-	job.RawMetaData, err = json.Marshal(job.MetaData)
-	if err != nil {
-		return -1, fmt.Errorf("REPOSITORY/JOB > encoding metaData field failed: %w", err)
-	}
-
-	res, err := r.DB.NamedExec(`INSERT INTO job (
-		job_id, user, project, cluster, subcluster, `+"`partition`"+`, array_job_id, num_nodes, num_hwthreads, num_acc,
-		exclusive, monitoring_status, smt, job_state, start_time, duration, walltime, resources, meta_data
-	) VALUES (
-		:job_id, :user, :project, :cluster, :subcluster, :partition, :array_job_id, :num_nodes, :num_hwthreads, :num_acc,
-		:exclusive, :monitoring_status, :smt, :job_state, :start_time, :duration, :walltime, :resources, :meta_data
-	);`, job)
-	if err != nil {
-		return -1, err
-	}
-
-	return res.LastInsertId()
-}
-
-// Stop updates the job with the database id jobId using the provided arguments.
-func (r *JobRepository) Stop(
-	jobId int64,
-	duration int32,
-	state schema.JobState,
-	monitoringStatus int32) (err error) {
-
-	stmt := sq.Update("job").
-		Set("job_state", state).
-		Set("duration", duration).
-		Set("monitoring_status", monitoringStatus).
-		Where("job.id = ?", jobId)
-
-	_, err = stmt.RunWith(r.stmtCache).Exec()
-	return
+	r.cache.Put(cachekey, job.EnergyFootprint, len(job.EnergyFootprint), 24*time.Hour)
+	cclog.Debugf("Timer FetchEnergyFootprint %s", time.Since(start))
+	return job.EnergyFootprint, nil
 }
 
 func (r *JobRepository) DeleteJobsBefore(startTime int64) (int, error) {
 	var cnt int
-	qs := fmt.Sprintf("SELECT count(*) FROM job WHERE job.start_time < %d", startTime)
-	err := r.DB.Get(&cnt, qs) //ignore error as it will also occur in delete statement
-	_, err = r.DB.Exec(`DELETE FROM job WHERE job.start_time < ?`, startTime)
+	q := sq.Select("count(*)").From("job").Where("job.start_time < ?", startTime)
+	q.RunWith(r.DB).QueryRow().Scan(cnt)
+	qd := sq.Delete("job").Where("job.start_time < ?", startTime)
+	_, err := qd.RunWith(r.DB).Exec()
+
 	if err != nil {
-		log.Errorf(" DeleteJobsBefore(%d): error %#v", startTime, err)
+		s, _, _ := qd.ToSql()
+		cclog.Errorf(" DeleteJobsBefore(%d) with %s: error %#v", startTime, s, err)
 	} else {
-		log.Debugf("DeleteJobsBefore(%d): Deleted %d jobs", startTime, cnt)
+		cclog.Debugf("DeleteJobsBefore(%d): Deleted %d jobs", startTime, cnt)
 	}
 	return cnt, err
 }
 
 func (r *JobRepository) DeleteJobById(id int64) error {
-	_, err := r.DB.Exec(`DELETE FROM job WHERE job.id = ?`, id)
+	qd := sq.Delete("job").Where("job.id = ?", id)
+	_, err := qd.RunWith(r.DB).Exec()
+
 	if err != nil {
-		log.Errorf("DeleteJobById(%d): error %#v", id, err)
+		s, _, _ := qd.ToSql()
+		cclog.Errorf("DeleteJobById(%d) with %s : error %#v", id, s, err)
 	} else {
-		log.Debugf("DeleteJobById(%d): Success", id)
+		cclog.Debugf("DeleteJobById(%d): Success", id)
 	}
 	return err
 }
 
-func (r *JobRepository) UpdateMonitoringStatus(job int64, monitoringStatus int32) (err error) {
-	stmt := sq.Update("job").
-		Set("monitoring_status", monitoringStatus).
-		Where("job.id = ?", job)
-
-	_, err = stmt.RunWith(r.stmtCache).Exec()
-	return
-}
-
-// Stop updates the job with the database id jobId using the provided arguments.
-func (r *JobRepository) MarkArchived(
-	jobId int64,
-	monitoringStatus int32,
-	metricStats map[string]schema.JobStatistics) error {
-
-	stmt := sq.Update("job").
-		Set("monitoring_status", monitoringStatus).
-		Where("job.id = ?", jobId)
-
-	for metric, stats := range metricStats {
-		switch metric {
-		case "flops_any":
-			stmt = stmt.Set("flops_any_avg", stats.Avg)
-		case "mem_used":
-			stmt = stmt.Set("mem_used_max", stats.Max)
-		case "mem_bw":
-			stmt = stmt.Set("mem_bw_avg", stats.Avg)
-		case "load":
-		case "cpu_load":
-			stmt = stmt.Set("load_avg", stats.Avg)
-		case "net_bw":
-			stmt = stmt.Set("net_bw_avg", stats.Avg)
-		case "file_bw":
-			stmt = stmt.Set("file_bw_avg", stats.Avg)
-		default:
-			log.Debugf("MarkArchived() Metric '%v' unknown", metric)
-		}
-	}
-
-	if _, err := stmt.RunWith(r.stmtCache).Exec(); err != nil {
-		log.Warn("Error while marking job as archived")
-		return err
-	}
-	return nil
-}
-
-// Archiving worker thread
-func (r *JobRepository) archivingWorker() {
-	for {
-		select {
-		case job, ok := <-r.archiveChannel:
-			if !ok {
-				break
-			}
-			start := time.Now()
-			// not using meta data, called to load JobMeta into Cache?
-			// will fail if job meta not in repository
-			if _, err := r.FetchMetadata(job); err != nil {
-				log.Errorf("archiving job (dbid: %d) failed: %s", job.ID, err.Error())
-				r.UpdateMonitoringStatus(job.ID, schema.MonitoringStatusArchivingFailed)
-				continue
-			}
-
-			// metricdata.ArchiveJob will fetch all the data from a MetricDataRepository and push into configured archive backend
-			// TODO: Maybe use context with cancel/timeout here
-			jobMeta, err := metricdata.ArchiveJob(job, context.Background())
-			if err != nil {
-				log.Errorf("archiving job (dbid: %d) failed: %s", job.ID, err.Error())
-				r.UpdateMonitoringStatus(job.ID, schema.MonitoringStatusArchivingFailed)
-				continue
-			}
-
-			// Update the jobs database entry one last time:
-			if err := r.MarkArchived(job.ID, schema.MonitoringStatusArchivingSuccessful, jobMeta.Statistics); err != nil {
-				log.Errorf("archiving job (dbid: %d) failed: %s", job.ID, err.Error())
-				continue
-			}
-			log.Debugf("archiving job %d took %s", job.JobID, time.Since(start))
-			log.Printf("archiving job (dbid: %d) successful", job.ID)
-			r.archivePending.Done()
-		}
-	}
-}
-
-// Trigger async archiving
-func (r *JobRepository) TriggerArchiving(job *schema.Job) {
-	r.archivePending.Add(1)
-	r.archiveChannel <- job
-}
-
-// Wait for background thread to finish pending archiving operations
-func (r *JobRepository) WaitForArchiving() {
-	// close channel and wait for worker to process remaining jobs
-	r.archivePending.Wait()
-}
-
 func (r *JobRepository) FindUserOrProjectOrJobname(user *schema.User, searchterm string) (jobid string, username string, project string, jobname string) {
 	if _, err := strconv.Atoi(searchterm); err == nil { // Return empty on successful conversion: parent method will redirect for integer jobId
 		return searchterm, "", "", ""
 	} else { // Has to have letters and logged-in user for other guesses
 		if user != nil {
-			// Find username in jobs (match)
-			uresult, _ := r.FindColumnValue(user, searchterm, "job", "user", "user", false)
+			// Find username by username in job table (match)
+			uresult, _ := r.FindColumnValue(user, searchterm, "job", "hpc_user", "hpc_user", false)
 			if uresult != "" {
 				return "", uresult, "", ""
 			}
-			// Find username by name (like)
-			nresult, _ := r.FindColumnValue(user, searchterm, "user", "username", "name", true)
+			// Find username by real name in hpc_user table (like)
+			nresult, _ := r.FindColumnValue(user, searchterm, "hpc_user", "username", "name", true)
 			if nresult != "" {
 				return "", nresult, "", ""
 			}
-			// Find projectId in jobs (match)
+			// Find projectId by projectId in job table (match)
 			presult, _ := r.FindColumnValue(user, searchterm, "job", "project", "project", false)
 			if presult != "" {
 				return "", "", presult, ""
@@ -577,8 +319,10 @@ func (r *JobRepository) FindUserOrProjectOrJobname(user *schema.User, searchterm
 	}
 }
 
-var ErrNotFound = errors.New("no such jobname, project or user")
-var ErrForbidden = errors.New("not authorized")
+var (
+	ErrNotFound  = errors.New("no such jobname, project or user")
+	ErrForbidden = errors.New("not authorized")
+)
 
 func (r *JobRepository) FindColumnValue(user *schema.User, searchterm string, table string, selectColumn string, whereColumn string, isLike bool) (result string, err error) {
 	compareStr := " = ?"
@@ -607,7 +351,7 @@ func (r *JobRepository) FindColumnValue(user *schema.User, searchterm string, ta
 		}
 		return "", ErrNotFound
 	} else {
-		log.Infof("Non-Admin User %s : Requested Query '%s' on table '%s' : Forbidden", user.Name, query, table)
+		cclog.Infof("Non-Admin User %s : Requested Query '%s' on table '%s' : Forbidden", user.Name, query, table)
 		return "", ErrForbidden
 	}
 }
@@ -626,7 +370,7 @@ func (r *JobRepository) FindColumnValues(user *schema.User, query string, table
 				err := rows.Scan(&result)
 				if err != nil {
 					rows.Close()
-					log.Warnf("Error while scanning rows: %v", err)
+					cclog.Warnf("Error while scanning rows: %v", err)
 					return emptyResult, err
 				}
 				results = append(results, result)
@@ -636,7 +380,7 @@ func (r *JobRepository) FindColumnValues(user *schema.User, query string, table
 		return emptyResult, ErrNotFound
 
 	} else {
-		log.Infof("Non-Admin User %s : Requested Query '%s' on table '%s' : Forbidden", user.Name, query, table)
+		cclog.Infof("Non-Admin User %s : Requested Query '%s' on table '%s' : Forbidden", user.Name, query, table)
 		return emptyResult, ErrForbidden
 	}
 }
@@ -644,9 +388,9 @@ func (r *JobRepository) FindColumnValues(user *schema.User, query string, table
 func (r *JobRepository) Partitions(cluster string) ([]string, error) {
 	var err error
 	start := time.Now()
-	partitions := r.cache.Get("partitions:"+cluster, func() (interface{}, time.Duration, int) {
+	partitions := r.cache.Get("partitions:"+cluster, func() (any, time.Duration, int) {
 		parts := []string{}
-		if err = r.DB.Select(&parts, `SELECT DISTINCT job.partition FROM job WHERE job.cluster = ?;`, cluster); err != nil {
+		if err = r.DB.Select(&parts, `SELECT DISTINCT job.cluster_partition FROM job WHERE job.cluster = ?;`, cluster); err != nil {
 			return nil, 0, 1000
 		}
 
@@ -655,14 +399,13 @@ func (r *JobRepository) Partitions(cluster string) ([]string, error) {
 	if err != nil {
 		return nil, err
 	}
-	log.Debugf("Timer Partitions %s", time.Since(start))
+	cclog.Debugf("Timer Partitions %s", time.Since(start))
 	return partitions.([]string), nil
 }
 
 // AllocatedNodes returns a map of all subclusters to a map of hostnames to the amount of jobs running on that host.
 // Hosts with zero jobs running on them will not show up!
 func (r *JobRepository) AllocatedNodes(cluster string) (map[string]map[string]int, error) {
-
 	start := time.Now()
 	subclusters := make(map[string]map[string]int)
 	rows, err := sq.Select("resources", "subcluster").From("job").
@@ -670,7 +413,7 @@ func (r *JobRepository) AllocatedNodes(cluster string) (map[string]map[string]in
 		Where("job.cluster = ?", cluster).
 		RunWith(r.stmtCache).Query()
 	if err != nil {
-		log.Error("Error while running query")
+		cclog.Error("Error while running query")
 		return nil, err
 	}
 
@@ -681,11 +424,11 @@ func (r *JobRepository) AllocatedNodes(cluster string) (map[string]map[string]in
 		var resources []*schema.Resource
 		var subcluster string
 		if err := rows.Scan(&raw, &subcluster); err != nil {
-			log.Warn("Error while scanning rows")
+			cclog.Warn("Error while scanning rows")
 			return nil, err
 		}
 		if err := json.Unmarshal(raw, &resources); err != nil {
-			log.Warn("Error while unmarshaling raw resources json")
+			cclog.Warn("Error while unmarshaling raw resources json")
 			return nil, err
 		}
 
@@ -700,12 +443,12 @@ func (r *JobRepository) AllocatedNodes(cluster string) (map[string]map[string]in
 		}
 	}
 
-	log.Debugf("Timer AllocatedNodes %s", time.Since(start))
+	cclog.Debugf("Timer AllocatedNodes %s", time.Since(start))
 	return subclusters, nil
 }
 
+// FIXME: Set duration to requested walltime?
 func (r *JobRepository) StopJobsExceedingWalltimeBy(seconds int) error {
-
 	start := time.Now()
 	res, err := sq.Update("job").
 		Set("monitoring_status", schema.MonitoringStatusArchivingFailed).
@@ -716,44 +459,59 @@ func (r *JobRepository) StopJobsExceedingWalltimeBy(seconds int) error {
 		Where(fmt.Sprintf("(%d - job.start_time) > (job.walltime + %d)", time.Now().Unix(), seconds)).
 		RunWith(r.DB).Exec()
 	if err != nil {
-		log.Warn("Error while stopping jobs exceeding walltime")
+		cclog.Warn("Error while stopping jobs exceeding walltime")
 		return err
 	}
 
 	rowsAffected, err := res.RowsAffected()
 	if err != nil {
-		log.Warn("Error while fetching affected rows after stopping due to exceeded walltime")
+		cclog.Warn("Error while fetching affected rows after stopping due to exceeded walltime")
 		return err
 	}
 
 	if rowsAffected > 0 {
-		log.Infof("%d jobs have been marked as failed due to running too long", rowsAffected)
+		cclog.Infof("%d jobs have been marked as failed due to running too long", rowsAffected)
 	}
-	log.Debugf("Timer StopJobsExceedingWalltimeBy %s", time.Since(start))
+	cclog.Debugf("Timer StopJobsExceedingWalltimeBy %s", time.Since(start))
 	return nil
 }
 
-func (r *JobRepository) FindJobsBetween(startTimeBegin int64, startTimeEnd int64) ([]*schema.Job, error) {
+func (r *JobRepository) FindJobIdsByTag(tagId int64) ([]int64, error) {
+	query := sq.Select("job.id").From("job").
+		Join("jobtag ON jobtag.job_id = job.id").
+		Where(sq.Eq{"jobtag.tag_id": tagId}).Distinct()
+	rows, err := query.RunWith(r.stmtCache).Query()
+	if err != nil {
+		cclog.Error("Error while running query")
+		return nil, err
+	}
+	jobIds := make([]int64, 0, 100)
 
-	var query sq.SelectBuilder
+	for rows.Next() {
+		var jobId int64
 
-	if startTimeBegin == startTimeEnd || startTimeBegin > startTimeEnd {
-		return nil, errors.New("startTimeBegin is equal or larger startTimeEnd")
+		if err := rows.Scan(&jobId); err != nil {
+			rows.Close()
+			cclog.Warn("Error while scanning rows")
+			return nil, err
+		}
+
+		jobIds = append(jobIds, jobId)
 	}
 
-	if startTimeBegin == 0 {
-		log.Infof("Find jobs before %d", startTimeEnd)
-		query = sq.Select(jobColumns...).From("job").Where(fmt.Sprintf(
-			"job.start_time < %d", startTimeEnd))
-	} else {
-		log.Infof("Find jobs between %d and %d", startTimeBegin, startTimeEnd)
-		query = sq.Select(jobColumns...).From("job").Where(fmt.Sprintf(
-			"job.start_time BETWEEN %d AND %d", startTimeBegin, startTimeEnd))
-	}
+	return jobIds, nil
+}
+
+// FIXME: Reconsider filtering short jobs with harcoded threshold
+func (r *JobRepository) FindRunningJobs(cluster string) ([]*schema.Job, error) {
+	query := sq.Select(jobColumns...).From("job").
+		Where(fmt.Sprintf("job.cluster = '%s'", cluster)).
+		Where("job.job_state = 'running'").
+		Where("job.duration > 600")
 
 	rows, err := query.RunWith(r.stmtCache).Query()
 	if err != nil {
-		log.Error("Error while running query")
+		cclog.Error("Error while running query")
 		return nil, err
 	}
 
@@ -762,37 +520,179 @@ func (r *JobRepository) FindJobsBetween(startTimeBegin int64, startTimeEnd int64
 		job, err := scanJob(rows)
 		if err != nil {
 			rows.Close()
-			log.Warn("Error while scanning rows")
+			cclog.Warn("Error while scanning rows")
 			return nil, err
 		}
 		jobs = append(jobs, job)
 	}
 
-	log.Infof("Return job count %d", len(jobs))
+	cclog.Infof("Return job count %d", len(jobs))
 	return jobs, nil
 }
 
-const NamedJobInsert string = `INSERT INTO job (
-	job_id, user, project, cluster, subcluster, ` + "`partition`" + `, array_job_id, num_nodes, num_hwthreads, num_acc,
-	exclusive, monitoring_status, smt, job_state, start_time, duration, walltime, resources, meta_data,
-	mem_used_max, flops_any_avg, mem_bw_avg, load_avg, net_bw_avg, net_data_vol_total, file_bw_avg, file_data_vol_total
-) VALUES (
-	:job_id, :user, :project, :cluster, :subcluster, :partition, :array_job_id, :num_nodes, :num_hwthreads, :num_acc,
-	:exclusive, :monitoring_status, :smt, :job_state, :start_time, :duration, :walltime, :resources, :meta_data,
-	:mem_used_max, :flops_any_avg, :mem_bw_avg, :load_avg, :net_bw_avg, :net_data_vol_total, :file_bw_avg, :file_data_vol_total
-);`
+func (r *JobRepository) UpdateDuration() error {
+	stmnt := sq.Update("job").
+		Set("duration", sq.Expr("? - job.start_time", time.Now().Unix())).
+		Where("job_state = 'running'")
 
-func (r *JobRepository) InsertJob(job *schema.Job) (int64, error) {
-	res, err := r.DB.NamedExec(NamedJobInsert, job)
+	_, err := stmnt.RunWith(r.stmtCache).Exec()
 	if err != nil {
-		log.Warn("Error while NamedJobInsert")
-		return 0, err
-	}
-	id, err := res.LastInsertId()
-	if err != nil {
-		log.Warn("Error while getting last insert ID")
-		return 0, err
+		return err
 	}
 
-	return id, nil
+	return nil
+}
+
+func (r *JobRepository) FindJobsBetween(startTimeBegin int64, startTimeEnd int64) ([]*schema.Job, error) {
+	var query sq.SelectBuilder
+
+	if startTimeBegin == startTimeEnd || startTimeBegin > startTimeEnd {
+		return nil, errors.New("startTimeBegin is equal or larger startTimeEnd")
+	}
+
+	if startTimeBegin == 0 {
+		cclog.Infof("Find jobs before %d", startTimeEnd)
+		query = sq.Select(jobColumns...).From("job").Where(fmt.Sprintf(
+			"job.start_time < %d", startTimeEnd))
+	} else {
+		cclog.Infof("Find jobs between %d and %d", startTimeBegin, startTimeEnd)
+		query = sq.Select(jobColumns...).From("job").Where(fmt.Sprintf(
+			"job.start_time BETWEEN %d AND %d", startTimeBegin, startTimeEnd))
+	}
+
+	rows, err := query.RunWith(r.stmtCache).Query()
+	if err != nil {
+		cclog.Error("Error while running query")
+		return nil, err
+	}
+
+	jobs := make([]*schema.Job, 0, 50)
+	for rows.Next() {
+		job, err := scanJob(rows)
+		if err != nil {
+			rows.Close()
+			cclog.Warn("Error while scanning rows")
+			return nil, err
+		}
+		jobs = append(jobs, job)
+	}
+
+	cclog.Infof("Return job count %d", len(jobs))
+	return jobs, nil
+}
+
+func (r *JobRepository) UpdateMonitoringStatus(job int64, monitoringStatus int32) (err error) {
+	stmt := sq.Update("job").
+		Set("monitoring_status", monitoringStatus).
+		Where("job.id = ?", job)
+
+	_, err = stmt.RunWith(r.stmtCache).Exec()
+	return
+}
+
+func (r *JobRepository) Execute(stmt sq.UpdateBuilder) error {
+	if _, err := stmt.RunWith(r.stmtCache).Exec(); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (r *JobRepository) MarkArchived(
+	stmt sq.UpdateBuilder,
+	monitoringStatus int32,
+) sq.UpdateBuilder {
+	return stmt.Set("monitoring_status", monitoringStatus)
+}
+
+func (r *JobRepository) UpdateEnergy(
+	stmt sq.UpdateBuilder,
+	jobMeta *schema.Job,
+) (sq.UpdateBuilder, error) {
+	/* Note: Only Called for Running Jobs during Intermediate Update or on Archiving */
+	sc, err := archive.GetSubCluster(jobMeta.Cluster, jobMeta.SubCluster)
+	if err != nil {
+		cclog.Errorf("cannot get subcluster: %s", err.Error())
+		return stmt, err
+	}
+	energyFootprint := make(map[string]float64)
+
+	// Total Job Energy Outside Loop
+	totalEnergy := 0.0
+	for _, fp := range sc.EnergyFootprint {
+		// Always Init Metric Energy Inside Loop
+		metricEnergy := 0.0
+		if i, err := archive.MetricIndex(sc.MetricConfig, fp); err == nil {
+			// Note: For DB data, calculate and save as kWh
+			if sc.MetricConfig[i].Energy == "energy" { // this metric has energy as unit (Joules or Wh)
+				cclog.Warnf("Update EnergyFootprint for Job %d and Metric %s on cluster %s: Set to 'energy' in cluster.json: Not implemented, will return 0.0", jobMeta.JobID, jobMeta.Cluster, fp)
+				// FIXME: Needs sum as stats type
+			} else if sc.MetricConfig[i].Energy == "power" { // this metric has power as unit (Watt)
+				// Energy: Power (in Watts) * Time (in Seconds)
+				// Unit: (W * (s / 3600)) / 1000 = kWh
+				// Round 2 Digits: round(Energy * 100) / 100
+				// Here: (All-Node Metric Average * Number of Nodes) * (Job Duration in Seconds / 3600) / 1000
+				// Note: Shared Jobs handled correctly since "Node Average" is based on partial resources, while "numNodes" factor is 1
+				rawEnergy := ((LoadJobStat(jobMeta, fp, "avg") * float64(jobMeta.NumNodes)) * (float64(jobMeta.Duration) / 3600.0)) / 1000.0
+				metricEnergy = math.Round(rawEnergy*100.0) / 100.0
+			}
+		} else {
+			cclog.Warnf("Error while collecting energy metric %s for job, DB ID '%v', return '0.0'", fp, jobMeta.ID)
+		}
+
+		energyFootprint[fp] = metricEnergy
+		totalEnergy += metricEnergy
+
+		// cclog.Infof("Metric %s Average %f -> %f kWh | Job %d Total -> %f kWh", fp, LoadJobStat(jobMeta, fp, "avg"), energy, jobMeta.JobID, totalEnergy)
+	}
+
+	var rawFootprint []byte
+	if rawFootprint, err = json.Marshal(energyFootprint); err != nil {
+		cclog.Warnf("Error while marshaling energy footprint for job INTO BYTES, DB ID '%v'", jobMeta.ID)
+		return stmt, err
+	}
+
+	return stmt.Set("energy_footprint", string(rawFootprint)).Set("energy", (math.Round(totalEnergy*100.0) / 100.0)), nil
+}
+
+func (r *JobRepository) UpdateFootprint(
+	stmt sq.UpdateBuilder,
+	jobMeta *schema.Job,
+) (sq.UpdateBuilder, error) {
+	/* Note: Only Called for Running Jobs during Intermediate Update or on Archiving */
+	sc, err := archive.GetSubCluster(jobMeta.Cluster, jobMeta.SubCluster)
+	if err != nil {
+		cclog.Errorf("cannot get subcluster: %s", err.Error())
+		return stmt, err
+	}
+	footprint := make(map[string]float64)
+
+	for _, fp := range sc.Footprint {
+		var statType string
+		for _, gm := range archive.GlobalMetricList {
+			if gm.Name == fp {
+				statType = gm.Footprint
+			}
+		}
+
+		if statType != "avg" && statType != "min" && statType != "max" {
+			cclog.Warnf("unknown statType for footprint update: %s", statType)
+			return stmt, fmt.Errorf("unknown statType for footprint update: %s", statType)
+		}
+
+		if i, err := archive.MetricIndex(sc.MetricConfig, fp); err != nil {
+			statType = sc.MetricConfig[i].Footprint
+		}
+
+		name := fmt.Sprintf("%s_%s", fp, statType)
+		footprint[name] = LoadJobStat(jobMeta, fp, statType)
+	}
+
+	var rawFootprint []byte
+	if rawFootprint, err = json.Marshal(footprint); err != nil {
+		cclog.Warnf("Error while marshaling footprint for job INTO BYTES, DB ID '%v'", jobMeta.ID)
+		return stmt, err
+	}
+
+	return stmt.Set("footprint", string(rawFootprint)), nil
 }

internal/repository/jobCreate.go

@@ -0,0 +1,140 @@
+// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
+// All rights reserved. This file is part of cc-backend.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+package repository
+
+import (
+	"encoding/json"
+	"fmt"
+
+	cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
+	"github.com/ClusterCockpit/cc-lib/schema"
+	sq "github.com/Masterminds/squirrel"
+)
+
+const NamedJobCacheInsert string = `INSERT INTO job_cache (
+	job_id, hpc_user, project, cluster, subcluster, cluster_partition, array_job_id, num_nodes, num_hwthreads, num_acc,
+	exclusive, monitoring_status, smt, job_state, start_time, duration, walltime, footprint, energy, energy_footprint, resources, meta_data
+) VALUES (
+	:job_id, :hpc_user, :project, :cluster, :subcluster, :cluster_partition, :array_job_id, :num_nodes, :num_hwthreads, :num_acc,
+  :exclusive, :monitoring_status, :smt, :job_state, :start_time, :duration, :walltime, :footprint,  :energy, :energy_footprint, :resources, :meta_data
+);`
+
+const NamedJobInsert string = `INSERT INTO job (
+	job_id, hpc_user, project, cluster, subcluster, cluster_partition, array_job_id, num_nodes, num_hwthreads, num_acc,
+	exclusive, monitoring_status, smt, job_state, start_time, duration, walltime, footprint, energy, energy_footprint, resources, meta_data
+) VALUES (
+	:job_id, :hpc_user, :project, :cluster, :subcluster, :cluster_partition, :array_job_id, :num_nodes, :num_hwthreads, :num_acc,
+  :exclusive, :monitoring_status, :smt, :job_state, :start_time, :duration, :walltime, :footprint,  :energy, :energy_footprint, :resources, :meta_data
+);`
+
+func (r *JobRepository) InsertJob(job *schema.Job) (int64, error) {
+	r.Mutex.Lock()
+	res, err := r.DB.NamedExec(NamedJobCacheInsert, job)
+	r.Mutex.Unlock()
+	if err != nil {
+		cclog.Warn("Error while NamedJobInsert")
+		return 0, err
+	}
+	id, err := res.LastInsertId()
+	if err != nil {
+		cclog.Warn("Error while getting last insert ID")
+		return 0, err
+	}
+
+	return id, nil
+}
+
+func (r *JobRepository) SyncJobs() ([]*schema.Job, error) {
+	r.Mutex.Lock()
+	defer r.Mutex.Unlock()
+
+	query := sq.Select(jobCacheColumns...).From("job_cache")
+
+	rows, err := query.RunWith(r.stmtCache).Query()
+	if err != nil {
+		cclog.Errorf("Error while running query %v", err)
+		return nil, err
+	}
+
+	jobs := make([]*schema.Job, 0, 50)
+	for rows.Next() {
+		job, err := scanJob(rows)
+		if err != nil {
+			rows.Close()
+			cclog.Warn("Error while scanning rows")
+			return nil, err
+		}
+		jobs = append(jobs, job)
+	}
+
+	_, err = r.DB.Exec(
+		"INSERT INTO job (job_id, cluster, subcluster, start_time, hpc_user, project, cluster_partition, array_job_id, num_nodes, num_hwthreads, num_acc, exclusive, monitoring_status, smt, job_state, duration, walltime, footprint, energy, energy_footprint, resources, meta_data) SELECT job_id, cluster, subcluster, start_time, hpc_user, project, cluster_partition, array_job_id, num_nodes, num_hwthreads, num_acc, exclusive, monitoring_status, smt, job_state, duration, walltime, footprint, energy, energy_footprint, resources, meta_data FROM job_cache")
+	if err != nil {
+		cclog.Warnf("Error while Job sync: %v", err)
+		return nil, err
+	}
+
+	_, err = r.DB.Exec("DELETE FROM job_cache")
+	if err != nil {
+		cclog.Warnf("Error while Job cache clean: %v", err)
+		return nil, err
+	}
+
+	return jobs, nil
+}
+
+// Start inserts a new job in the table, returning the unique job ID.
+// Statistics are not transfered!
+func (r *JobRepository) Start(job *schema.Job) (id int64, err error) {
+	job.RawFootprint, err = json.Marshal(job.Footprint)
+	if err != nil {
+		return -1, fmt.Errorf("REPOSITORY/JOB > encoding footprint field failed: %w", err)
+	}
+
+	job.RawResources, err = json.Marshal(job.Resources)
+	if err != nil {
+		return -1, fmt.Errorf("REPOSITORY/JOB > encoding resources field failed: %w", err)
+	}
+
+	job.RawMetaData, err = json.Marshal(job.MetaData)
+	if err != nil {
+		return -1, fmt.Errorf("REPOSITORY/JOB > encoding metaData field failed: %w", err)
+	}
+
+	return r.InsertJob(job)
+}
+
+// Stop updates the job with the database id jobId using the provided arguments.
+func (r *JobRepository) Stop(
+	jobId int64,
+	duration int32,
+	state schema.JobState,
+	monitoringStatus int32,
+) (err error) {
+	stmt := sq.Update("job").
+		Set("job_state", state).
+		Set("duration", duration).
+		Set("monitoring_status", monitoringStatus).
+		Where("job.id = ?", jobId)
+
+	_, err = stmt.RunWith(r.stmtCache).Exec()
+	return
+}
+
+func (r *JobRepository) StopCached(
+	jobId int64,
+	duration int32,
+	state schema.JobState,
+	monitoringStatus int32,
+) (err error) {
+	stmt := sq.Update("job_cache").
+		Set("job_state", state).
+		Set("duration", duration).
+		Set("monitoring_status", monitoringStatus).
+		Where("job.id = ?", jobId)
+
+	_, err = stmt.RunWith(r.stmtCache).Exec()
+	return
+}

internal/repository/jobFind.go

@@ -0,0 +1,312 @@
+// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
+// All rights reserved. This file is part of cc-backend.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+package repository
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+	"time"
+
+	"github.com/ClusterCockpit/cc-backend/internal/graph/model"
+	cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
+	"github.com/ClusterCockpit/cc-lib/schema"
+	sq "github.com/Masterminds/squirrel"
+)
+
+// Find executes a SQL query to find a specific batch job.
+// The job is queried using the batch job id, the cluster name,
+// and the start time of the job in UNIX epoch time seconds.
+// It returns a pointer to a schema.Job data structure and an error variable.
+// To check if no job was found test err == sql.ErrNoRows
+func (r *JobRepository) Find(
+	jobId *int64,
+	cluster *string,
+	startTime *int64,
+) (*schema.Job, error) {
+	start := time.Now()
+	q := sq.Select(jobColumns...).From("job").
+		Where("job.job_id = ?", *jobId)
+
+	if cluster != nil {
+		q = q.Where("job.cluster = ?", *cluster)
+	}
+	if startTime != nil {
+		q = q.Where("job.start_time = ?", *startTime)
+	}
+
+	q = q.OrderBy("job.id DESC") // always use newest matching job by db id if more than one match
+
+	cclog.Debugf("Timer Find %s", time.Since(start))
+	return scanJob(q.RunWith(r.stmtCache).QueryRow())
+}
+
+func (r *JobRepository) FindCached(
+	jobId *int64,
+	cluster *string,
+	startTime *int64,
+) (*schema.Job, error) {
+	q := sq.Select(jobCacheColumns...).From("job_cache").
+		Where("job_cache.job_id = ?", *jobId)
+
+	if cluster != nil {
+		q = q.Where("job_cache.cluster = ?", *cluster)
+	}
+	if startTime != nil {
+		q = q.Where("job_cache.start_time = ?", *startTime)
+	}
+
+	q = q.OrderBy("job_cache.id DESC") // always use newest matching job by db id if more than one match
+
+	return scanJob(q.RunWith(r.stmtCache).QueryRow())
+}
+
+// Find executes a SQL query to find a specific batch job.
+// The job is queried using the batch job id, the cluster name,
+// and the start time of the job in UNIX epoch time seconds.
+// It returns a pointer to a schema.Job data structure and an error variable.
+// To check if no job was found test err == sql.ErrNoRows
+func (r *JobRepository) FindAll(
+	jobId *int64,
+	cluster *string,
+	startTime *int64,
+) ([]*schema.Job, error) {
+	start := time.Now()
+	q := sq.Select(jobColumns...).From("job").
+		Where("job.job_id = ?", *jobId)
+
+	if cluster != nil {
+		q = q.Where("job.cluster = ?", *cluster)
+	}
+	if startTime != nil {
+		q = q.Where("job.start_time = ?", *startTime)
+	}
+
+	rows, err := q.RunWith(r.stmtCache).Query()
+	if err != nil {
+		cclog.Error("Error while running query")
+		return nil, err
+	}
+
+	jobs := make([]*schema.Job, 0, 10)
+	for rows.Next() {
+		job, err := scanJob(rows)
+		if err != nil {
+			cclog.Warn("Error while scanning rows")
+			return nil, err
+		}
+		jobs = append(jobs, job)
+	}
+	cclog.Debugf("Timer FindAll %s", time.Since(start))
+	return jobs, nil
+}
+
+// Get complete joblist only consisting of db ids.
+// This is useful to process large job counts and intended to be used
+// together with FindById to process jobs one by one
+func (r *JobRepository) GetJobList() ([]int64, error) {
+	query := sq.Select("id").From("job").
+		Where("job.job_state != 'running'")
+
+	rows, err := query.RunWith(r.stmtCache).Query()
+	if err != nil {
+		cclog.Error("Error while running query")
+		return nil, err
+	}
+
+	jl := make([]int64, 0, 1000)
+	for rows.Next() {
+		var id int64
+		err := rows.Scan(&id)
+		if err != nil {
+			rows.Close()
+			cclog.Warn("Error while scanning rows")
+			return nil, err
+		}
+		jl = append(jl, id)
+	}
+
+	cclog.Infof("Return job count %d", len(jl))
+	return jl, nil
+}
+
+// FindById executes a SQL query to find a specific batch job.
+// The job is queried using the database id.
+// It returns a pointer to a schema.Job data structure and an error variable.
+// To check if no job was found test err == sql.ErrNoRows
+func (r *JobRepository) FindById(ctx context.Context, jobId int64) (*schema.Job, error) {
+	q := sq.Select(jobColumns...).
+		From("job").Where("job.id = ?", jobId)
+
+	q, qerr := SecurityCheck(ctx, q)
+	if qerr != nil {
+		return nil, qerr
+	}
+
+	return scanJob(q.RunWith(r.stmtCache).QueryRow())
+}
+
+// FindByIdWithUser executes a SQL query to find a specific batch job.
+// The job is queried using the database id. The user is passed directly,
+// instead as part of the context.
+// It returns a pointer to a schema.Job data structure and an error variable.
+// To check if no job was found test err == sql.ErrNoRows
+func (r *JobRepository) FindByIdWithUser(user *schema.User, jobId int64) (*schema.Job, error) {
+	q := sq.Select(jobColumns...).
+		From("job").Where("job.id = ?", jobId)
+
+	q, qerr := SecurityCheckWithUser(user, q)
+	if qerr != nil {
+		return nil, qerr
+	}
+
+	return scanJob(q.RunWith(r.stmtCache).QueryRow())
+}
+
+// FindByIdDirect executes a SQL query to find a specific batch job.
+// The job is queried using the database id.
+// It returns a pointer to a schema.Job data structure and an error variable.
+// To check if no job was found test err == sql.ErrNoRows
+func (r *JobRepository) FindByIdDirect(jobId int64) (*schema.Job, error) {
+	q := sq.Select(jobColumns...).
+		From("job").Where("job.id = ?", jobId)
+	return scanJob(q.RunWith(r.stmtCache).QueryRow())
+}
+
+// FindByJobId executes a SQL query to find a specific batch job.
+// The job is queried using the slurm id and the clustername.
+// It returns a pointer to a schema.Job data structure and an error variable.
+// To check if no job was found test err == sql.ErrNoRows
+func (r *JobRepository) FindByJobId(ctx context.Context, jobId int64, startTime int64, cluster string) (*schema.Job, error) {
+	q := sq.Select(jobColumns...).
+		From("job").
+		Where("job.job_id = ?", jobId).
+		Where("job.cluster = ?", cluster).
+		Where("job.start_time = ?", startTime)
+
+	q, qerr := SecurityCheck(ctx, q)
+	if qerr != nil {
+		return nil, qerr
+	}
+
+	return scanJob(q.RunWith(r.stmtCache).QueryRow())
+}
+
+// IsJobOwner executes a SQL query to find a specific batch job.
+// The job is queried using the slurm id,a username and the cluster.
+// It returns a bool.
+// If job was found, user is owner: test err != sql.ErrNoRows
+func (r *JobRepository) IsJobOwner(jobId int64, startTime int64, user string, cluster string) bool {
+	q := sq.Select("id").
+		From("job").
+		Where("job.job_id = ?", jobId).
+		Where("job.hpc_user = ?", user).
+		Where("job.cluster = ?", cluster).
+		Where("job.start_time = ?", startTime)
+
+	_, err := scanJob(q.RunWith(r.stmtCache).QueryRow())
+	return err != sql.ErrNoRows
+}
+
+func (r *JobRepository) FindConcurrentJobs(
+	ctx context.Context,
+	job *schema.Job,
+) (*model.JobLinkResultList, error) {
+	if job == nil {
+		return nil, nil
+	}
+
+	query, qerr := SecurityCheck(ctx, sq.Select("job.id", "job.job_id", "job.start_time").From("job"))
+	if qerr != nil {
+		return nil, qerr
+	}
+
+	query = query.Where("cluster = ?", job.Cluster)
+	var startTime int64
+	var stopTime int64
+
+	startTime = job.StartTime
+	hostname := job.Resources[0].Hostname
+
+	if job.State == schema.JobStateRunning {
+		stopTime = time.Now().Unix()
+	} else {
+		stopTime = startTime + int64(job.Duration)
+	}
+
+	// Add 200s overlap for jobs start time at the end
+	startTimeTail := startTime + 10
+	stopTimeTail := stopTime - 200
+	startTimeFront := startTime + 200
+
+	queryRunning := query.Where("job.job_state = ?").Where("(job.start_time BETWEEN ? AND ? OR job.start_time < ?)",
+		"running", startTimeTail, stopTimeTail, startTime)
+	// Get At Least One Exact Hostname Match from JSON Resources Array in Database
+	queryRunning = queryRunning.Where("EXISTS (SELECT 1 FROM json_each(job.resources) WHERE json_extract(value, '$.hostname') = ?)", hostname)
+
+	query = query.Where("job.job_state != ?").Where("((job.start_time BETWEEN ? AND ?) OR (job.start_time + job.duration) BETWEEN ? AND ? OR (job.start_time < ?) AND (job.start_time + job.duration) > ?)",
+		"running", startTimeTail, stopTimeTail, startTimeFront, stopTimeTail, startTime, stopTime)
+	// Get At Least One Exact Hostname Match from JSON Resources Array in Database
+	query = query.Where("EXISTS (SELECT 1 FROM json_each(job.resources) WHERE json_extract(value, '$.hostname') = ?)", hostname)
+
+	rows, err := query.RunWith(r.stmtCache).Query()
+	if err != nil {
+		cclog.Errorf("Error while running query: %v", err)
+		return nil, err
+	}
+
+	items := make([]*model.JobLink, 0, 10)
+	queryString := fmt.Sprintf("cluster=%s", job.Cluster)
+
+	for rows.Next() {
+		var id, jobId, startTime sql.NullInt64
+
+		if err = rows.Scan(&id, &jobId, &startTime); err != nil {
+			cclog.Warn("Error while scanning rows")
+			return nil, err
+		}
+
+		if id.Valid {
+			queryString += fmt.Sprintf("&jobId=%d", int(jobId.Int64))
+			items = append(items,
+				&model.JobLink{
+					ID:    fmt.Sprint(id.Int64),
+					JobID: int(jobId.Int64),
+				})
+		}
+	}
+
+	rows, err = queryRunning.RunWith(r.stmtCache).Query()
+	if err != nil {
+		cclog.Errorf("Error while running query: %v", err)
+		return nil, err
+	}
+
+	for rows.Next() {
+		var id, jobId, startTime sql.NullInt64
+
+		if err := rows.Scan(&id, &jobId, &startTime); err != nil {
+			cclog.Warn("Error while scanning rows")
+			return nil, err
+		}
+
+		if id.Valid {
+			queryString += fmt.Sprintf("&jobId=%d", int(jobId.Int64))
+			items = append(items,
+				&model.JobLink{
+					ID:    fmt.Sprint(id.Int64),
+					JobID: int(jobId.Int64),
+				})
+		}
+	}
+
+	cnt := len(items)
+
+	return &model.JobLinkResultList{
+		ListQuery: &queryString,
+		Items:     items,
+		Count:     &cnt,
+	}, nil
+}

internal/repository/jobHooks.go

@@ -0,0 +1,57 @@
+// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
+// All rights reserved. This file is part of cc-backend.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+package repository
+
+import (
+	"sync"
+
+	"github.com/ClusterCockpit/cc-lib/schema"
+)
+
+type JobHook interface {
+	JobStartCallback(job *schema.Job)
+	JobStopCallback(job *schema.Job)
+}
+
+var (
+	initOnce sync.Once
+	hooks    []JobHook
+)
+
+func RegisterJobJook(hook JobHook) {
+	initOnce.Do(func() {
+		hooks = make([]JobHook, 0)
+	})
+
+	if hook != nil {
+		hooks = append(hooks, hook)
+	}
+}
+
+func CallJobStartHooks(jobs []*schema.Job) {
+	if hooks == nil {
+		return
+	}
+
+	for _, hook := range hooks {
+		if hook != nil {
+			for _, job := range jobs {
+				hook.JobStartCallback(job)
+			}
+		}
+	}
+}
+
+func CallJobStopHooks(job *schema.Job) {
+	if hooks == nil {
+		return
+	}
+
+	for _, hook := range hooks {
+		if hook != nil {
+			hook.JobStopCallback(job)
+		}
+	}
+}

internal/repository/jobQuery.go

@@ -0,0 +1,347 @@
+// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
+// All rights reserved. This file is part of cc-backend.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+package repository
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"regexp"
+	"strings"
+	"time"
+
+	"github.com/ClusterCockpit/cc-backend/internal/graph/model"
+	cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
+	"github.com/ClusterCockpit/cc-lib/schema"
+	sq "github.com/Masterminds/squirrel"
+)
+
+func (r *JobRepository) QueryJobs(
+	ctx context.Context,
+	filters []*model.JobFilter,
+	page *model.PageRequest,
+	order *model.OrderByInput,
+) ([]*schema.Job, error) {
+	query, qerr := SecurityCheck(ctx, sq.Select(jobColumns...).From("job"))
+	if qerr != nil {
+		return nil, qerr
+	}
+
+	if order != nil {
+		field := toSnakeCase(order.Field)
+		if order.Type == "col" {
+			// "col": Fixed column name query
+			switch order.Order {
+			case model.SortDirectionEnumAsc:
+				query = query.OrderBy(fmt.Sprintf("job.%s ASC", field))
+			case model.SortDirectionEnumDesc:
+				query = query.OrderBy(fmt.Sprintf("job.%s DESC", field))
+			default:
+				return nil, errors.New("REPOSITORY/QUERY > invalid sorting order for column")
+			}
+		} else {
+			// "foot": Order by footprint JSON field values
+			// Verify and Search Only in Valid Jsons
+			query = query.Where("JSON_VALID(meta_data)")
+			switch order.Order {
+			case model.SortDirectionEnumAsc:
+				query = query.OrderBy(fmt.Sprintf("JSON_EXTRACT(footprint, \"$.%s\") ASC", field))
+			case model.SortDirectionEnumDesc:
+				query = query.OrderBy(fmt.Sprintf("JSON_EXTRACT(footprint, \"$.%s\") DESC", field))
+			default:
+				return nil, errors.New("REPOSITORY/QUERY > invalid sorting order for footprint")
+			}
+		}
+	}
+
+	if page != nil && page.ItemsPerPage != -1 {
+		limit := uint64(page.ItemsPerPage)
+		query = query.Offset((uint64(page.Page) - 1) * limit).Limit(limit)
+	}
+
+	for _, f := range filters {
+		query = BuildWhereClause(f, query)
+	}
+
+	rows, err := query.RunWith(r.stmtCache).Query()
+	if err != nil {
+		queryString, queryVars, _ := query.ToSql()
+		cclog.Errorf("Error while running query '%s' %v: %v", queryString, queryVars, err)
+		return nil, err
+	}
+
+	jobs := make([]*schema.Job, 0, 50)
+	for rows.Next() {
+		job, err := scanJob(rows)
+		if err != nil {
+			rows.Close()
+			cclog.Warn("Error while scanning rows (Jobs)")
+			return nil, err
+		}
+		jobs = append(jobs, job)
+	}
+
+	return jobs, nil
+}
+
+func (r *JobRepository) CountJobs(
+	ctx context.Context,
+	filters []*model.JobFilter,
+) (int, error) {
+	// DISTICT count for tags filters, does not affect other queries
+	query, qerr := SecurityCheck(ctx, sq.Select("count(DISTINCT job.id)").From("job"))
+	if qerr != nil {
+		return 0, qerr
+	}
+
+	for _, f := range filters {
+		query = BuildWhereClause(f, query)
+	}
+
+	var count int
+	if err := query.RunWith(r.DB).Scan(&count); err != nil {
+		return 0, err
+	}
+
+	return count, nil
+}
+
+func SecurityCheckWithUser(user *schema.User, query sq.SelectBuilder) (sq.SelectBuilder, error) {
+	if user == nil {
+		var qnil sq.SelectBuilder
+		return qnil, fmt.Errorf("user context is nil")
+	}
+
+	switch {
+	case len(user.Roles) == 1 && user.HasRole(schema.RoleApi): // API-User : All jobs
+		return query, nil
+	case user.HasAnyRole([]schema.Role{schema.RoleAdmin, schema.RoleSupport}): // Admin & Support : All jobs
+		return query, nil
+	case user.HasRole(schema.RoleManager): // Manager : Add filter for managed projects' jobs only + personal jobs
+		if len(user.Projects) != 0 {
+			return query.Where(sq.Or{sq.Eq{"job.project": user.Projects}, sq.Eq{"job.hpc_user": user.Username}}), nil
+		} else {
+			cclog.Debugf("Manager-User '%s' has no defined projects to lookup! Query only personal jobs ...", user.Username)
+			return query.Where("job.hpc_user = ?", user.Username), nil
+		}
+	case user.HasRole(schema.RoleUser): // User : Only personal jobs
+		return query.Where("job.hpc_user = ?", user.Username), nil
+	default: // No known Role, return error
+		var qnil sq.SelectBuilder
+		return qnil, fmt.Errorf("user has no or unknown roles")
+	}
+}
+
+func SecurityCheck(ctx context.Context, query sq.SelectBuilder) (sq.SelectBuilder, error) {
+	user := GetUserFromContext(ctx)
+
+	return SecurityCheckWithUser(user, query)
+}
+
+// Build a sq.SelectBuilder out of a schema.JobFilter.
+func BuildWhereClause(filter *model.JobFilter, query sq.SelectBuilder) sq.SelectBuilder {
+	if filter.Tags != nil {
+		// This is an OR-Logic query: Returns all distinct jobs with at least one of the requested tags; TODO: AND-Logic query?
+		query = query.Join("jobtag ON jobtag.job_id = job.id").Where(sq.Eq{"jobtag.tag_id": filter.Tags}).Distinct()
+	}
+	if filter.DbID != nil {
+		dbIDs := make([]string, len(filter.DbID))
+		copy(dbIDs, filter.DbID)
+		query = query.Where(sq.Eq{"job.id": dbIDs})
+	}
+	if filter.JobID != nil {
+		query = buildStringCondition("job.job_id", filter.JobID, query)
+	}
+	if filter.ArrayJobID != nil {
+		query = query.Where("job.array_job_id = ?", *filter.ArrayJobID)
+	}
+	if filter.User != nil {
+		query = buildStringCondition("job.hpc_user", filter.User, query)
+	}
+	if filter.Project != nil {
+		query = buildStringCondition("job.project", filter.Project, query)
+	}
+	if filter.JobName != nil {
+		query = buildMetaJsonCondition("jobName", filter.JobName, query)
+	}
+	if filter.Cluster != nil {
+		query = buildStringCondition("job.cluster", filter.Cluster, query)
+	}
+	if filter.Partition != nil {
+		query = buildStringCondition("job.cluster_partition", filter.Partition, query)
+	}
+	if filter.StartTime != nil {
+		query = buildTimeCondition("job.start_time", filter.StartTime, query)
+	}
+	if filter.Duration != nil {
+		query = buildIntCondition("job.duration", filter.Duration, query)
+	}
+	if filter.MinRunningFor != nil {
+		now := time.Now().Unix() // There does not seam to be a portable way to get the current unix timestamp accross different DBs.
+		query = query.Where("(job.job_state != 'running' OR (? - job.start_time) > ?)", now, *filter.MinRunningFor)
+	}
+	if filter.Exclusive != nil {
+		query = query.Where("job.exclusive = ?", *filter.Exclusive)
+	}
+	if filter.State != nil {
+		states := make([]string, len(filter.State))
+		for i, val := range filter.State {
+			states[i] = string(val)
+		}
+
+		query = query.Where(sq.Eq{"job.job_state": states})
+	}
+	if filter.NumNodes != nil {
+		query = buildIntCondition("job.num_nodes", filter.NumNodes, query)
+	}
+	if filter.NumAccelerators != nil {
+		query = buildIntCondition("job.num_acc", filter.NumAccelerators, query)
+	}
+	if filter.NumHWThreads != nil {
+		query = buildIntCondition("job.num_hwthreads", filter.NumHWThreads, query)
+	}
+	if filter.Node != nil {
+		query = buildResourceJsonCondition("hostname", filter.Node, query)
+	}
+	if filter.Energy != nil {
+		query = buildFloatCondition("job.energy", filter.Energy, query)
+	}
+	if filter.MetricStats != nil {
+		for _, ms := range filter.MetricStats {
+			query = buildFloatJsonCondition(ms.MetricName, ms.Range, query)
+		}
+	}
+	return query
+}
+
+func buildIntCondition(field string, cond *schema.IntRange, query sq.SelectBuilder) sq.SelectBuilder {
+	return query.Where(field+" BETWEEN ? AND ?", cond.From, cond.To)
+}
+
+func buildFloatCondition(field string, cond *model.FloatRange, query sq.SelectBuilder) sq.SelectBuilder {
+	return query.Where(field+" BETWEEN ? AND ?", cond.From, cond.To)
+}
+
+func buildTimeCondition(field string, cond *schema.TimeRange, query sq.SelectBuilder) sq.SelectBuilder {
+	if cond.From != nil && cond.To != nil {
+		return query.Where(field+" BETWEEN ? AND ?", cond.From.Unix(), cond.To.Unix())
+	} else if cond.From != nil {
+		return query.Where("? <= "+field, cond.From.Unix())
+	} else if cond.To != nil {
+		return query.Where(field+" <= ?", cond.To.Unix())
+	} else if cond.Range != "" {
+		now := time.Now().Unix()
+		var then int64
+		switch cond.Range {
+		case "last6h":
+			then = now - (60 * 60 * 6)
+		case "last24h":
+			then = now - (60 * 60 * 24)
+		case "last7d":
+			then = now - (60 * 60 * 24 * 7)
+		case "last30d":
+			then = now - (60 * 60 * 24 * 30)
+		default:
+			cclog.Debugf("No known named timeRange: startTime.range = %s", cond.Range)
+			return query
+		}
+		return query.Where(field+" BETWEEN ? AND ?", then, now)
+	} else {
+		return query
+	}
+}
+
+func buildFloatJsonCondition(condName string, condRange *model.FloatRange, query sq.SelectBuilder) sq.SelectBuilder {
+	// Verify and Search Only in Valid Jsons
+	query = query.Where("JSON_VALID(footprint)")
+	return query.Where("JSON_EXTRACT(footprint, \"$."+condName+"\") BETWEEN ? AND ?", condRange.From, condRange.To)
+}
+
+func buildStringCondition(field string, cond *model.StringInput, query sq.SelectBuilder) sq.SelectBuilder {
+	if cond.Eq != nil {
+		return query.Where(field+" = ?", *cond.Eq)
+	}
+	if cond.Neq != nil {
+		return query.Where(field+" != ?", *cond.Neq)
+	}
+	if cond.StartsWith != nil {
+		return query.Where(field+" LIKE ?", fmt.Sprint(*cond.StartsWith, "%"))
+	}
+	if cond.EndsWith != nil {
+		return query.Where(field+" LIKE ?", fmt.Sprint("%", *cond.EndsWith))
+	}
+	if cond.Contains != nil {
+		return query.Where(field+" LIKE ?", fmt.Sprint("%", *cond.Contains, "%"))
+	}
+	if cond.In != nil {
+		queryElements := make([]string, len(cond.In))
+		copy(queryElements, cond.In)
+		return query.Where(sq.Or{sq.Eq{field: queryElements}})
+	}
+	return query
+}
+
+func buildMetaJsonCondition(jsonField string, cond *model.StringInput, query sq.SelectBuilder) sq.SelectBuilder {
+	// Verify and Search Only in Valid Jsons
+	query = query.Where("JSON_VALID(meta_data)")
+	// add "AND" Sql query Block for field match
+	if cond.Eq != nil {
+		return query.Where("JSON_EXTRACT(meta_data, \"$."+jsonField+"\") = ?", *cond.Eq)
+	}
+	if cond.Neq != nil {
+		return query.Where("JSON_EXTRACT(meta_data, \"$."+jsonField+"\") != ?", *cond.Neq)
+	}
+	if cond.StartsWith != nil {
+		return query.Where("JSON_EXTRACT(meta_data, \"$."+jsonField+"\") LIKE ?", fmt.Sprint(*cond.StartsWith, "%"))
+	}
+	if cond.EndsWith != nil {
+		return query.Where("JSON_EXTRACT(meta_data, \"$."+jsonField+"\") LIKE ?", fmt.Sprint("%", *cond.EndsWith))
+	}
+	if cond.Contains != nil {
+		return query.Where("JSON_EXTRACT(meta_data, \"$."+jsonField+"\") LIKE ?", fmt.Sprint("%", *cond.Contains, "%"))
+	}
+	return query
+}
+
+func buildResourceJsonCondition(jsonField string, cond *model.StringInput, query sq.SelectBuilder) sq.SelectBuilder {
+	// Verify and Search Only in Valid Jsons
+	query = query.Where("JSON_VALID(resources)")
+	// add "AND" Sql query Block for field match
+	if cond.Eq != nil {
+		return query.Where("EXISTS (SELECT 1 FROM json_each(job.resources) WHERE json_extract(value, \"$."+jsonField+"\") = ?)", *cond.Eq)
+	}
+	if cond.Neq != nil { // Currently Unused
+		return query.Where("EXISTS (SELECT 1 FROM json_each(job.resources) WHERE json_extract(value, \"$."+jsonField+"\") != ?)", *cond.Neq)
+	}
+	if cond.StartsWith != nil { // Currently Unused
+		return query.Where("EXISTS (SELECT 1 FROM json_each(job.resources) WHERE json_extract(value, \"$."+jsonField+"\")) LIKE ?)", fmt.Sprint(*cond.StartsWith, "%"))
+	}
+	if cond.EndsWith != nil { // Currently Unused
+		return query.Where("EXISTS (SELECT 1 FROM json_each(job.resources) WHERE json_extract(value, \"$."+jsonField+"\") LIKE ?)", fmt.Sprint("%", *cond.EndsWith))
+	}
+	if cond.Contains != nil {
+		return query.Where("EXISTS (SELECT 1 FROM json_each(job.resources) WHERE json_extract(value, \"$."+jsonField+"\") LIKE ?)", fmt.Sprint("%", *cond.Contains, "%"))
+	}
+	return query
+}
+
+var (
+	matchFirstCap = regexp.MustCompile("(.)([A-Z][a-z]+)")
+	matchAllCap   = regexp.MustCompile("([a-z0-9])([A-Z])")
+)
+
+func toSnakeCase(str string) string {
+	for _, c := range str {
+		if c == '\'' || c == '\\' {
+			cclog.Panic("toSnakeCase() attack vector!")
+		}
+	}
+
+	str = strings.ReplaceAll(str, "'", "")
+	str = strings.ReplaceAll(str, "\\", "")
+	snake := matchFirstCap.ReplaceAllString(str, "${1}_${2}")
+	snake = matchAllCap.ReplaceAllString(snake, "${1}_${2}")
+	return strings.ToLower(snake)
+}

internal/repository/job_test.go

@@ -1,13 +1,15 @@
-// Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved.
+// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
+// All rights reserved. This file is part of cc-backend.
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 package repository
 
 import (
+	"context"
 	"fmt"
 	"testing"
 
+	"github.com/ClusterCockpit/cc-lib/schema"
 	_ "github.com/mattn/go-sqlite3"
 )
 
@@ -22,7 +24,7 @@ func TestFind(t *testing.T) {
 
 	// fmt.Printf("%+v", job)
 
-	if job.ID != 5 {
+	if *job.ID != 5 {
 		t.Errorf("wrong summary for diagnostic 3\ngot: %d \nwant: 1366", job.JobID)
 	}
 }
@@ -30,7 +32,7 @@ func TestFind(t *testing.T) {
 func TestFindById(t *testing.T) {
 	r := setup(t)
 
-	job, err := r.FindById(5)
+	job, err := r.FindById(getContext(t), 5)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -45,7 +47,19 @@ func TestFindById(t *testing.T) {
 func TestGetTags(t *testing.T) {
 	r := setup(t)
 
-	tags, counts, err := r.CountTags(nil)
+	const contextUserKey ContextKey = "user"
+	contextUserValue := &schema.User{
+		Username:   "testuser",
+		Projects:   make([]string, 0),
+		Roles:      []string{"user"},
+		AuthType:   0,
+		AuthSource: 2,
+	}
+
+	ctx := context.WithValue(getContext(t), contextUserKey, contextUserValue)
+
+	// Test Tag has Scope "global"
+	tags, counts, err := r.CountTags(GetUserFromContext(ctx))
 	if err != nil {
 		t.Fatal(err)
 	}

internal/repository/migration.go

@@ -1,5 +1,5 @@
-// Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved.
+// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
+// All rights reserved. This file is part of cc-backend.
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 package repository
@@ -9,14 +9,14 @@ import (
 	"embed"
 	"fmt"
 
-	"github.com/ClusterCockpit/cc-backend/pkg/log"
+	cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
 	"github.com/golang-migrate/migrate/v4"
 	"github.com/golang-migrate/migrate/v4/database/mysql"
 	"github.com/golang-migrate/migrate/v4/database/sqlite3"
 	"github.com/golang-migrate/migrate/v4/source/iofs"
 )
 
-const Version uint = 6
+const Version uint = 10
 
 //go:embed migrations/*
 var migrationFiles embed.FS
@@ -54,13 +54,13 @@ func checkDBVersion(backend string, db *sql.DB) error {
 			return err
 		}
 	default:
-		log.Fatalf("unsupported database backend: %s", backend)
+		cclog.Abortf("Migration: Unsupported database backend '%s'.\n", backend)
 	}
 
-	v, _, err := m.Version()
+	v, dirty, err := m.Version()
 	if err != nil {
 		if err == migrate.ErrNilVersion {
-			log.Warn("Legacy database without version or missing database file!")
+			cclog.Warn("Legacy database without version or missing database file!")
 		} else {
 			return err
 		}
@@ -68,46 +68,72 @@ func checkDBVersion(backend string, db *sql.DB) error {
 
 	if v < Version {
 		return fmt.Errorf("unsupported database version %d, need %d.\nPlease backup your database file and run cc-backend -migrate-db", v, Version)
+	} else if v > Version {
+		return fmt.Errorf("unsupported database version %d, need %d.\nPlease refer to documentation how to downgrade db with external migrate tool", v, Version)
 	}
 
-	if v > Version {
-		return fmt.Errorf("unsupported database version %d, need %d.\nPlease refer to documentation how to downgrade db with external migrate tool", v, Version)
+	if dirty {
+		return fmt.Errorf("last migration to version %d has failed, please fix the db manually and force version with -force-db flag", Version)
 	}
 
 	return nil
 }
 
-func MigrateDB(backend string, db string) error {
-	var m *migrate.Migrate
-
+func getMigrateInstance(backend string, db string) (m *migrate.Migrate, err error) {
 	switch backend {
 	case "sqlite3":
 		d, err := iofs.New(migrationFiles, "migrations/sqlite3")
 		if err != nil {
-			log.Fatal(err)
+			cclog.Fatal(err)
 		}
 
 		m, err = migrate.NewWithSourceInstance("iofs", d, fmt.Sprintf("sqlite3://%s?_foreign_keys=on", db))
 		if err != nil {
-			return err
+			return m, err
 		}
 	case "mysql":
 		d, err := iofs.New(migrationFiles, "migrations/mysql")
 		if err != nil {
-			return err
+			return m, err
 		}
 
 		m, err = migrate.NewWithSourceInstance("iofs", d, fmt.Sprintf("mysql://%s?multiStatements=true", db))
 		if err != nil {
-			return err
+			return m, err
 		}
 	default:
-		log.Fatalf("unsupported database backend: %s", backend)
+		cclog.Abortf("Migration: Unsupported database backend '%s'.\n", backend)
+	}
+
+	return m, nil
+}
+
+func MigrateDB(backend string, db string) error {
+	m, err := getMigrateInstance(backend, db)
+	if err != nil {
+		return err
+	}
+
+	v, dirty, err := m.Version()
+	if err != nil {
+		if err == migrate.ErrNilVersion {
+			cclog.Warn("Legacy database without version or missing database file!")
+		} else {
+			return err
+		}
+	}
+
+	if v < Version {
+		cclog.Infof("unsupported database version %d, need %d.\nPlease backup your database file and run cc-backend -migrate-db", v, Version)
+	}
+
+	if dirty {
+		return fmt.Errorf("last migration to version %d has failed, please fix the db manually and force version with -force-db flag", Version)
 	}
 
 	if err := m.Up(); err != nil {
 		if err == migrate.ErrNoChange {
-			log.Info("DB already up to date!")
+			cclog.Info("DB already up to date!")
 		} else {
 			return err
 		}
@@ -116,3 +142,35 @@ func MigrateDB(backend string, db string) error {
 	m.Close()
 	return nil
 }
+
+func RevertDB(backend string, db string) error {
+	m, err := getMigrateInstance(backend, db)
+	if err != nil {
+		return err
+	}
+
+	if err := m.Migrate(Version - 1); err != nil {
+		if err == migrate.ErrNoChange {
+			cclog.Info("DB already up to date!")
+		} else {
+			return err
+		}
+	}
+
+	m.Close()
+	return nil
+}
+
+func ForceDB(backend string, db string) error {
+	m, err := getMigrateInstance(backend, db)
+	if err != nil {
+		return err
+	}
+
+	if err := m.Force(int(Version)); err != nil {
+		return err
+	}
+
+	m.Close()
+	return nil
+}

internal/repository/migrations/mysql/07_fix-tag-id.down.sql

@@ -0,0 +1,3 @@
+SET FOREIGN_KEY_CHECKS = 0;
+ALTER TABLE tag MODIFY id INTEGER;
+SET FOREIGN_KEY_CHECKS = 1;

internal/repository/migrations/mysql/07_fix-tag-id.up.sql

@@ -0,0 +1,3 @@
+SET FOREIGN_KEY_CHECKS = 0;
+ALTER TABLE tag MODIFY id INTEGER AUTO_INCREMENT;
+SET FOREIGN_KEY_CHECKS = 1;

internal/repository/migrations/mysql/08_add-footprint.down.sql

@@ -0,0 +1,83 @@
+ALTER TABLE job DROP energy;
+ALTER TABLE job DROP energy_footprint;
+ALTER TABLE job ADD COLUMN flops_any_avg;
+ALTER TABLE job ADD COLUMN mem_bw_avg;
+ALTER TABLE job ADD COLUMN mem_used_max;
+ALTER TABLE job ADD COLUMN load_avg;
+ALTER TABLE job ADD COLUMN net_bw_avg;
+ALTER TABLE job ADD COLUMN net_data_vol_total;
+ALTER TABLE job ADD COLUMN file_bw_avg;
+ALTER TABLE job ADD COLUMN file_data_vol_total;
+
+UPDATE job SET flops_any_avg = json_extract(footprint, '$.flops_any_avg');
+UPDATE job SET mem_bw_avg = json_extract(footprint, '$.mem_bw_avg');
+UPDATE job SET mem_used_max = json_extract(footprint, '$.mem_used_max');
+UPDATE job SET load_avg = json_extract(footprint, '$.cpu_load_avg');
+UPDATE job SET net_bw_avg = json_extract(footprint, '$.net_bw_avg');
+UPDATE job SET net_data_vol_total = json_extract(footprint, '$.net_data_vol_total');
+UPDATE job SET file_bw_avg = json_extract(footprint, '$.file_bw_avg');
+UPDATE job SET file_data_vol_total = json_extract(footprint, '$.file_data_vol_total');
+
+ALTER TABLE job DROP footprint;
+-- Do not use reserved keywords anymore
+RENAME TABLE hpc_user TO `user`;
+ALTER TABLE job RENAME COLUMN hpc_user TO `user`;
+ALTER TABLE job RENAME COLUMN cluster_partition TO `partition`;
+
+DROP INDEX IF EXISTS jobs_cluster;
+DROP INDEX IF EXISTS jobs_cluster_user;
+DROP INDEX IF EXISTS jobs_cluster_project;
+DROP INDEX IF EXISTS jobs_cluster_subcluster;
+DROP INDEX IF EXISTS jobs_cluster_starttime;
+DROP INDEX IF EXISTS jobs_cluster_duration;
+DROP INDEX IF EXISTS jobs_cluster_numnodes;
+
+DROP INDEX IF EXISTS jobs_cluster_partition;
+DROP INDEX IF EXISTS jobs_cluster_partition_starttime;
+DROP INDEX IF EXISTS jobs_cluster_partition_duration;
+DROP INDEX IF EXISTS jobs_cluster_partition_numnodes;
+
+DROP INDEX IF EXISTS jobs_cluster_partition_jobstate;
+DROP INDEX IF EXISTS jobs_cluster_partition_jobstate_user;
+DROP INDEX IF EXISTS jobs_cluster_partition_jobstate_project;
+DROP INDEX IF EXISTS jobs_cluster_partition_jobstate_starttime;
+DROP INDEX IF EXISTS jobs_cluster_partition_jobstate_duration;
+DROP INDEX IF EXISTS jobs_cluster_partition_jobstate_numnodes;
+
+DROP INDEX IF EXISTS jobs_cluster_jobstate;
+DROP INDEX IF EXISTS jobs_cluster_jobstate_user;
+DROP INDEX IF EXISTS jobs_cluster_jobstate_project;
+
+DROP INDEX IF EXISTS jobs_cluster_jobstate_starttime;
+DROP INDEX IF EXISTS jobs_cluster_jobstate_duration;
+DROP INDEX IF EXISTS jobs_cluster_jobstate_numnodes;
+
+DROP INDEX IF EXISTS jobs_user;
+DROP INDEX IF EXISTS jobs_user_starttime;
+DROP INDEX IF EXISTS jobs_user_duration;
+DROP INDEX IF EXISTS jobs_user_numnodes;
+
+DROP INDEX IF EXISTS jobs_project;
+DROP INDEX IF EXISTS jobs_project_user;
+DROP INDEX IF EXISTS jobs_project_starttime;
+DROP INDEX IF EXISTS jobs_project_duration;
+DROP INDEX IF EXISTS jobs_project_numnodes;
+
+DROP INDEX IF EXISTS jobs_jobstate;
+DROP INDEX IF EXISTS jobs_jobstate_user;
+DROP INDEX IF EXISTS jobs_jobstate_project;
+DROP INDEX IF EXISTS jobs_jobstate_starttime;
+DROP INDEX IF EXISTS jobs_jobstate_duration;
+DROP INDEX IF EXISTS jobs_jobstate_numnodes;
+
+DROP INDEX IF EXISTS jobs_arrayjobid_starttime;
+DROP INDEX IF EXISTS jobs_cluster_arrayjobid_starttime;
+
+DROP INDEX IF EXISTS jobs_starttime;
+DROP INDEX IF EXISTS jobs_duration;
+DROP INDEX IF EXISTS jobs_numnodes;
+
+DROP INDEX IF EXISTS jobs_duration_starttime;
+DROP INDEX IF EXISTS jobs_numnodes_starttime;
+DROP INDEX IF EXISTS jobs_numacc_starttime;
+DROP INDEX IF EXISTS jobs_energy_starttime;

internal/repository/migrations/mysql/08_add-footprint.up.sql

@@ -0,0 +1,123 @@
+DROP INDEX IF EXISTS job_stats ON job;
+DROP INDEX IF EXISTS job_by_user ON job;
+DROP INDEX IF EXISTS job_by_starttime ON job;
+DROP INDEX IF EXISTS job_by_job_id ON job;
+DROP INDEX IF EXISTS job_list ON job;
+DROP INDEX IF EXISTS job_list_user ON job;
+DROP INDEX IF EXISTS job_list_users ON job;
+DROP INDEX IF EXISTS job_list_users_start ON job;
+
+ALTER TABLE job ADD COLUMN energy REAL NOT NULL DEFAULT 0.0;
+ALTER TABLE job ADD COLUMN energy_footprint JSON;
+
+ALTER TABLE job ADD COLUMN footprint JSON;
+ALTER TABLE tag ADD COLUMN tag_scope TEXT NOT NULL DEFAULT 'global';
+
+-- Do not use reserved keywords anymore
+RENAME TABLE `user` TO hpc_user;
+ALTER TABLE job RENAME COLUMN `user` TO hpc_user;
+ALTER TABLE job RENAME COLUMN `partition` TO cluster_partition;
+
+ALTER TABLE job MODIFY COLUMN cluster VARCHAR(50);
+ALTER TABLE job MODIFY COLUMN hpc_user VARCHAR(50);
+ALTER TABLE job MODIFY COLUMN subcluster VARCHAR(50);
+ALTER TABLE job MODIFY COLUMN project VARCHAR(50);
+ALTER TABLE job MODIFY COLUMN cluster_partition VARCHAR(50);
+ALTER TABLE job MODIFY COLUMN job_state VARCHAR(25);
+
+UPDATE job SET footprint = '{"flops_any_avg": 0.0}';
+UPDATE job SET footprint = json_replace(footprint, '$.flops_any_avg', job.flops_any_avg);
+UPDATE job SET footprint = json_insert(footprint, '$.mem_bw_avg', job.mem_bw_avg);
+UPDATE job SET footprint = json_insert(footprint, '$.mem_used_max', job.mem_used_max);
+UPDATE job SET footprint = json_insert(footprint, '$.cpu_load_avg', job.load_avg);
+UPDATE job SET footprint = json_insert(footprint, '$.net_bw_avg', job.net_bw_avg) WHERE job.net_bw_avg != 0;
+UPDATE job SET footprint = json_insert(footprint, '$.net_data_vol_total', job.net_data_vol_total) WHERE job.net_data_vol_total != 0;
+UPDATE job SET footprint = json_insert(footprint, '$.file_bw_avg', job.file_bw_avg) WHERE job.file_bw_avg != 0;
+UPDATE job SET footprint = json_insert(footprint, '$.file_data_vol_total', job.file_data_vol_total) WHERE job.file_data_vol_total != 0;
+
+ALTER TABLE job DROP flops_any_avg;
+ALTER TABLE job DROP mem_bw_avg;
+ALTER TABLE job DROP mem_used_max;
+ALTER TABLE job DROP load_avg;
+ALTER TABLE job DROP net_bw_avg;
+ALTER TABLE job DROP net_data_vol_total;
+ALTER TABLE job DROP file_bw_avg;
+ALTER TABLE job DROP file_data_vol_total;
+
+-- Indices for: Single filters, combined filters, sorting, sorting with filters
+-- Cluster Filter
+CREATE INDEX IF NOT EXISTS jobs_cluster ON job (cluster);
+CREATE INDEX IF NOT EXISTS jobs_cluster_user ON job (cluster, hpc_user);
+CREATE INDEX IF NOT EXISTS jobs_cluster_project ON job (cluster, project);
+CREATE INDEX IF NOT EXISTS jobs_cluster_subcluster ON job (cluster, subcluster);
+-- Cluster Filter Sorting
+CREATE INDEX IF NOT EXISTS jobs_cluster_starttime ON job (cluster, start_time);
+CREATE INDEX IF NOT EXISTS jobs_cluster_duration ON job (cluster, duration);
+CREATE INDEX IF NOT EXISTS jobs_cluster_numnodes ON job (cluster, num_nodes);
+
+-- Cluster+Partition Filter
+CREATE INDEX IF NOT EXISTS jobs_cluster_partition ON job (cluster, cluster_partition);
+-- Cluster+Partition Filter Sorting
+CREATE INDEX IF NOT EXISTS jobs_cluster_partition_starttime ON job (cluster, cluster_partition, start_time);
+CREATE INDEX IF NOT EXISTS jobs_cluster_partition_duration ON job (cluster, cluster_partition, duration);
+CREATE INDEX IF NOT EXISTS jobs_cluster_partition_numnodes ON job (cluster, cluster_partition, num_nodes);
+
+-- Cluster+Partition+Jobstate Filter
+CREATE INDEX IF NOT EXISTS jobs_cluster_partition_jobstate ON job (cluster, cluster_partition, job_state);
+CREATE INDEX IF NOT EXISTS jobs_cluster_partition_jobstate_user ON job (cluster, cluster_partition, job_state, hpc_user);
+CREATE INDEX IF NOT EXISTS jobs_cluster_partition_jobstate_project ON job (cluster, cluster_partition, job_state, project);
+-- Cluster+Partition+Jobstate Filter Sorting
+CREATE INDEX IF NOT EXISTS jobs_cluster_partition_jobstate_starttime ON job (cluster, cluster_partition, job_state, start_time);
+CREATE INDEX IF NOT EXISTS jobs_cluster_partition_jobstate_duration ON job (cluster, cluster_partition, job_state, duration);
+CREATE INDEX IF NOT EXISTS jobs_cluster_partition_jobstate_numnodes ON job (cluster, cluster_partition, job_state, num_nodes);
+
+-- Cluster+JobState Filter
+CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate ON job (cluster, job_state);
+CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_user ON job (cluster, job_state, hpc_user);
+CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_project ON job (cluster, job_state, project);
+-- Cluster+JobState Filter Sorting
+CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_starttime ON job (cluster, job_state, start_time);
+CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_duration ON job (cluster, job_state, duration);
+CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_numnodes ON job (cluster, job_state, num_nodes);
+
+-- User Filter
+CREATE INDEX IF NOT EXISTS jobs_user ON job (hpc_user);
+-- User Filter Sorting
+CREATE INDEX IF NOT EXISTS jobs_user_starttime ON job (hpc_user, start_time);
+CREATE INDEX IF NOT EXISTS jobs_user_duration ON job (hpc_user, duration);
+CREATE INDEX IF NOT EXISTS jobs_user_numnodes ON job (hpc_user, num_nodes);
+
+-- Project Filter
+CREATE INDEX IF NOT EXISTS jobs_project ON job (project);
+CREATE INDEX IF NOT EXISTS jobs_project_user ON job (project, hpc_user);
+-- Project Filter Sorting
+CREATE INDEX IF NOT EXISTS jobs_project_starttime ON job (project, start_time);
+CREATE INDEX IF NOT EXISTS jobs_project_duration ON job (project, duration);
+CREATE INDEX IF NOT EXISTS jobs_project_numnodes ON job (project, num_nodes);
+
+-- JobState Filter
+CREATE INDEX IF NOT EXISTS jobs_jobstate ON job (job_state);
+CREATE INDEX IF NOT EXISTS jobs_jobstate_user ON job (job_state, hpc_user);
+CREATE INDEX IF NOT EXISTS jobs_jobstate_project ON job (job_state, project);
+CREATE INDEX IF NOT EXISTS jobs_jobstate_cluster ON job (job_state, cluster);
+-- JobState Filter Sorting
+CREATE INDEX IF NOT EXISTS jobs_jobstate_starttime ON job (job_state, start_time);
+CREATE INDEX IF NOT EXISTS jobs_jobstate_duration ON job (job_state, duration);
+CREATE INDEX IF NOT EXISTS jobs_jobstate_numnodes ON job (job_state, num_nodes);
+
+-- ArrayJob Filter
+CREATE INDEX IF NOT EXISTS jobs_arrayjobid_starttime ON job (array_job_id, start_time);
+CREATE INDEX IF NOT EXISTS jobs_cluster_arrayjobid_starttime ON job (cluster, array_job_id, start_time);
+
+-- Sorting without active filters
+CREATE INDEX IF NOT EXISTS jobs_starttime ON job (start_time);
+CREATE INDEX IF NOT EXISTS jobs_duration ON job (duration);
+CREATE INDEX IF NOT EXISTS jobs_numnodes ON job (num_nodes);
+
+-- Single filters with default starttime sorting
+CREATE INDEX IF NOT EXISTS jobs_duration_starttime ON job (duration, start_time);
+CREATE INDEX IF NOT EXISTS jobs_numnodes_starttime ON job (num_nodes, start_time);
+CREATE INDEX IF NOT EXISTS jobs_numacc_starttime ON job (num_acc, start_time);
+CREATE INDEX IF NOT EXISTS jobs_energy_starttime ON job (energy, start_time);
+
+-- Optimize DB index usage

internal/repository/migrations/sqlite3/08_add-footprint.down.sql

@@ -0,0 +1,103 @@
+ALTER TABLE job DROP energy;
+ALTER TABLE job DROP energy_footprint;
+ALTER TABLE job ADD COLUMN flops_any_avg;
+ALTER TABLE job ADD COLUMN mem_bw_avg;
+ALTER TABLE job ADD COLUMN mem_used_max;
+ALTER TABLE job ADD COLUMN load_avg;
+ALTER TABLE job ADD COLUMN net_bw_avg;
+ALTER TABLE job ADD COLUMN net_data_vol_total;
+ALTER TABLE job ADD COLUMN file_bw_avg;
+ALTER TABLE job ADD COLUMN file_data_vol_total;
+
+UPDATE job SET flops_any_avg = json_extract(footprint, '$.flops_any_avg');
+UPDATE job SET mem_bw_avg = json_extract(footprint, '$.mem_bw_avg');
+UPDATE job SET mem_used_max = json_extract(footprint, '$.mem_used_max');
+UPDATE job SET load_avg = json_extract(footprint, '$.cpu_load_avg');
+UPDATE job SET net_bw_avg = json_extract(footprint, '$.net_bw_avg');
+UPDATE job SET net_data_vol_total = json_extract(footprint, '$.net_data_vol_total');
+UPDATE job SET file_bw_avg = json_extract(footprint, '$.file_bw_avg');
+UPDATE job SET file_data_vol_total = json_extract(footprint, '$.file_data_vol_total');
+
+ALTER TABLE job DROP footprint;
+
+DROP INDEX IF EXISTS jobs_cluster;
+DROP INDEX IF EXISTS jobs_cluster_user;
+DROP INDEX IF EXISTS jobs_cluster_project;
+DROP INDEX IF EXISTS jobs_cluster_subcluster;
+DROP INDEX IF EXISTS jobs_cluster_starttime;
+DROP INDEX IF EXISTS jobs_cluster_duration;
+DROP INDEX IF EXISTS jobs_cluster_numnodes;
+DROP INDEX IF EXISTS jobs_cluster_numhwthreads;
+DROP INDEX IF EXISTS jobs_cluster_numacc;
+DROP INDEX IF EXISTS jobs_cluster_energy;
+
+DROP INDEX IF EXISTS jobs_cluster_partition;
+DROP INDEX IF EXISTS jobs_cluster_partition_starttime;
+DROP INDEX IF EXISTS jobs_cluster_partition_duration;
+DROP INDEX IF EXISTS jobs_cluster_partition_numnodes;
+DROP INDEX IF EXISTS jobs_cluster_partition_numhwthreads;
+DROP INDEX IF EXISTS jobs_cluster_partition_numacc;
+DROP INDEX IF EXISTS jobs_cluster_partition_energy;
+
+DROP INDEX IF EXISTS jobs_cluster_partition_jobstate;
+DROP INDEX IF EXISTS jobs_cluster_partition_jobstate_user;
+DROP INDEX IF EXISTS jobs_cluster_partition_jobstate_project;
+DROP INDEX IF EXISTS jobs_cluster_partition_jobstate_starttime;
+DROP INDEX IF EXISTS jobs_cluster_partition_jobstate_duration;
+DROP INDEX IF EXISTS jobs_cluster_partition_jobstate_numnodes;
+DROP INDEX IF EXISTS jobs_cluster_partition_jobstate_numhwthreads;
+DROP INDEX IF EXISTS jobs_cluster_partition_jobstate_numacc;
+DROP INDEX IF EXISTS jobs_cluster_partition_jobstate_energy;
+
+DROP INDEX IF EXISTS jobs_cluster_jobstate;
+DROP INDEX IF EXISTS jobs_cluster_jobstate_user;
+DROP INDEX IF EXISTS jobs_cluster_jobstate_project;
+
+DROP INDEX IF EXISTS jobs_cluster_jobstate_starttime;
+DROP INDEX IF EXISTS jobs_cluster_jobstate_duration;
+DROP INDEX IF EXISTS jobs_cluster_jobstate_numnodes;
+DROP INDEX IF EXISTS jobs_cluster_jobstate_numhwthreads;
+DROP INDEX IF EXISTS jobs_cluster_jobstate_numacc;
+DROP INDEX IF EXISTS jobs_cluster_jobstate_energy;
+
+DROP INDEX IF EXISTS jobs_user;
+DROP INDEX IF EXISTS jobs_user_starttime;
+DROP INDEX IF EXISTS jobs_user_duration;
+DROP INDEX IF EXISTS jobs_user_numnodes;
+DROP INDEX IF EXISTS jobs_user_numhwthreads;
+DROP INDEX IF EXISTS jobs_user_numacc;
+DROP INDEX IF EXISTS jobs_user_energy;
+
+DROP INDEX IF EXISTS jobs_project;
+DROP INDEX IF EXISTS jobs_project_user;
+DROP INDEX IF EXISTS jobs_project_starttime;
+DROP INDEX IF EXISTS jobs_project_duration;
+DROP INDEX IF EXISTS jobs_project_numnodes;
+DROP INDEX IF EXISTS jobs_project_numhwthreads;
+DROP INDEX IF EXISTS jobs_project_numacc;
+DROP INDEX IF EXISTS jobs_project_energy;
+
+DROP INDEX IF EXISTS jobs_jobstate;
+DROP INDEX IF EXISTS jobs_jobstate_user;
+DROP INDEX IF EXISTS jobs_jobstate_project;
+DROP INDEX IF EXISTS jobs_jobstate_starttime;
+DROP INDEX IF EXISTS jobs_jobstate_duration;
+DROP INDEX IF EXISTS jobs_jobstate_numnodes;
+DROP INDEX IF EXISTS jobs_jobstate_numhwthreads;
+DROP INDEX IF EXISTS jobs_jobstate_numacc;
+
+DROP INDEX IF EXISTS jobs_arrayjobid_starttime;
+DROP INDEX IF EXISTS jobs_cluster_arrayjobid_starttime;
+
+DROP INDEX IF EXISTS jobs_starttime;
+DROP INDEX IF EXISTS jobs_duration;
+DROP INDEX IF EXISTS jobs_numnodes;
+DROP INDEX IF EXISTS jobs_numhwthreads;
+DROP INDEX IF EXISTS jobs_numacc;
+DROP INDEX IF EXISTS jobs_energy;
+
+DROP INDEX IF EXISTS jobs_duration_starttime;
+DROP INDEX IF EXISTS jobs_numnodes_starttime;
+DROP INDEX IF EXISTS jobs_numhwthreads_starttime;
+DROP INDEX IF EXISTS jobs_numacc_starttime;
+DROP INDEX IF EXISTS jobs_energy_starttime;

internal/repository/migrations/sqlite3/08_add-footprint.up.sql

@@ -0,0 +1,142 @@
+DROP INDEX IF EXISTS job_stats;
+DROP INDEX IF EXISTS job_by_user;
+DROP INDEX IF EXISTS job_by_starttime;
+DROP INDEX IF EXISTS job_by_job_id;
+DROP INDEX IF EXISTS job_list;
+DROP INDEX IF EXISTS job_list_user;
+DROP INDEX IF EXISTS job_list_users;
+DROP INDEX IF EXISTS job_list_users_start;
+
+ALTER TABLE job ADD COLUMN energy REAL NOT NULL DEFAULT 0.0;
+ALTER TABLE job ADD COLUMN energy_footprint TEXT DEFAULT NULL;
+
+ALTER TABLE job ADD COLUMN footprint TEXT DEFAULT NULL;
+ALTER TABLE tag ADD COLUMN tag_scope TEXT NOT NULL DEFAULT 'global';
+
+-- Do not use reserved keywords anymore
+ALTER TABLE "user" RENAME TO hpc_user;
+ALTER TABLE job RENAME COLUMN "user" TO hpc_user;
+ALTER TABLE job RENAME COLUMN "partition" TO cluster_partition;
+
+UPDATE job SET footprint = '{"flops_any_avg": 0.0}';
+UPDATE job SET footprint = json_replace(footprint, '$.flops_any_avg', job.flops_any_avg);
+UPDATE job SET footprint = json_insert(footprint, '$.mem_bw_avg', job.mem_bw_avg);
+UPDATE job SET footprint = json_insert(footprint, '$.mem_used_max', job.mem_used_max);
+UPDATE job SET footprint = json_insert(footprint, '$.cpu_load_avg', job.load_avg);
+UPDATE job SET footprint = json_insert(footprint, '$.net_bw_avg', job.net_bw_avg) WHERE job.net_bw_avg != 0;
+UPDATE job SET footprint = json_insert(footprint, '$.net_data_vol_total', job.net_data_vol_total) WHERE job.net_data_vol_total != 0;
+UPDATE job SET footprint = json_insert(footprint, '$.file_bw_avg', job.file_bw_avg) WHERE job.file_bw_avg != 0;
+UPDATE job SET footprint = json_insert(footprint, '$.file_data_vol_total', job.file_data_vol_total) WHERE job.file_data_vol_total != 0;
+
+ALTER TABLE job DROP flops_any_avg;
+ALTER TABLE job DROP mem_bw_avg;
+ALTER TABLE job DROP mem_used_max;
+ALTER TABLE job DROP load_avg;
+ALTER TABLE job DROP net_bw_avg;
+ALTER TABLE job DROP net_data_vol_total;
+ALTER TABLE job DROP file_bw_avg;
+ALTER TABLE job DROP file_data_vol_total;
+
+-- Indices for: Single filters, combined filters, sorting, sorting with filters
+-- Cluster Filter
+CREATE INDEX IF NOT EXISTS jobs_cluster ON job (cluster);
+CREATE INDEX IF NOT EXISTS jobs_cluster_user ON job (cluster, hpc_user);
+CREATE INDEX IF NOT EXISTS jobs_cluster_project ON job (cluster, project);
+CREATE INDEX IF NOT EXISTS jobs_cluster_subcluster ON job (cluster, subcluster);
+-- Cluster Filter Sorting
+CREATE INDEX IF NOT EXISTS jobs_cluster_starttime ON job (cluster, start_time);
+CREATE INDEX IF NOT EXISTS jobs_cluster_duration ON job (cluster, duration);
+CREATE INDEX IF NOT EXISTS jobs_cluster_numnodes ON job (cluster, num_nodes);
+CREATE INDEX IF NOT EXISTS jobs_cluster_numhwthreads ON job (cluster, num_hwthreads);
+CREATE INDEX IF NOT EXISTS jobs_cluster_numacc ON job (cluster, num_acc);
+CREATE INDEX IF NOT EXISTS jobs_cluster_energy ON job (cluster, energy);
+
+-- Cluster+Partition Filter
+CREATE INDEX IF NOT EXISTS jobs_cluster_partition ON job (cluster, cluster_partition);
+-- Cluster+Partition Filter Sorting
+CREATE INDEX IF NOT EXISTS jobs_cluster_partition_starttime ON job (cluster, cluster_partition, start_time);
+CREATE INDEX IF NOT EXISTS jobs_cluster_partition_duration ON job (cluster, cluster_partition, duration);
+CREATE INDEX IF NOT EXISTS jobs_cluster_partition_numnodes ON job (cluster, cluster_partition, num_nodes);
+CREATE INDEX IF NOT EXISTS jobs_cluster_partition_numhwthreads ON job (cluster, cluster_partition, num_hwthreads);
+CREATE INDEX IF NOT EXISTS jobs_cluster_partition_numacc ON job (cluster, cluster_partition, num_acc);
+CREATE INDEX IF NOT EXISTS jobs_cluster_partition_energy ON job (cluster, cluster_partition, energy);
+
+-- Cluster+Partition+Jobstate Filter
+CREATE INDEX IF NOT EXISTS jobs_cluster_partition_jobstate ON job (cluster, cluster_partition, job_state);
+CREATE INDEX IF NOT EXISTS jobs_cluster_partition_jobstate_user ON job (cluster, cluster_partition, job_state, hpc_user);
+CREATE INDEX IF NOT EXISTS jobs_cluster_partition_jobstate_project ON job (cluster, cluster_partition, job_state, project);
+-- Cluster+Partition+Jobstate Filter Sorting
+CREATE INDEX IF NOT EXISTS jobs_cluster_partition_jobstate_starttime ON job (cluster, cluster_partition, job_state, start_time);
+CREATE INDEX IF NOT EXISTS jobs_cluster_partition_jobstate_duration ON job (cluster, cluster_partition, job_state, duration);
+CREATE INDEX IF NOT EXISTS jobs_cluster_partition_jobstate_numnodes ON job (cluster, cluster_partition, job_state, num_nodes);
+CREATE INDEX IF NOT EXISTS jobs_cluster_partition_jobstate_numhwthreads ON job (cluster, cluster_partition, job_state, num_hwthreads);
+CREATE INDEX IF NOT EXISTS jobs_cluster_partition_jobstate_numacc ON job (cluster, cluster_partition, job_state, num_acc);
+CREATE INDEX IF NOT EXISTS jobs_cluster_partition_jobstate_energy ON job (cluster, cluster_partition, job_state, energy);
+
+-- Cluster+JobState Filter
+CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate ON job (cluster, job_state);
+CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_user ON job (cluster, job_state, hpc_user);
+CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_project ON job (cluster, job_state, project);
+-- Cluster+JobState Filter Sorting
+CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_starttime ON job (cluster, job_state, start_time);
+CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_duration ON job (cluster, job_state, duration);
+CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_numnodes ON job (cluster, job_state, num_nodes);
+CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_numhwthreads ON job (cluster, job_state, num_hwthreads);
+CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_numacc ON job (cluster, job_state, num_acc);
+CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_energy ON job (cluster, job_state, energy);
+
+-- User Filter
+CREATE INDEX IF NOT EXISTS jobs_user ON job (hpc_user);
+-- User Filter Sorting
+CREATE INDEX IF NOT EXISTS jobs_user_starttime ON job (hpc_user, start_time);
+CREATE INDEX IF NOT EXISTS jobs_user_duration ON job (hpc_user, duration);
+CREATE INDEX IF NOT EXISTS jobs_user_numnodes ON job (hpc_user, num_nodes);
+CREATE INDEX IF NOT EXISTS jobs_user_numhwthreads ON job (hpc_user, num_hwthreads);
+CREATE INDEX IF NOT EXISTS jobs_user_numacc ON job (hpc_user, num_acc);
+CREATE INDEX IF NOT EXISTS jobs_user_energy ON job (hpc_user, energy);
+
+-- Project Filter
+CREATE INDEX IF NOT EXISTS jobs_project ON job (project);
+CREATE INDEX IF NOT EXISTS jobs_project_user ON job (project, hpc_user);
+-- Project Filter Sorting
+CREATE INDEX IF NOT EXISTS jobs_project_starttime ON job (project, start_time);
+CREATE INDEX IF NOT EXISTS jobs_project_duration ON job (project, duration);
+CREATE INDEX IF NOT EXISTS jobs_project_numnodes ON job (project, num_nodes);
+CREATE INDEX IF NOT EXISTS jobs_project_numhwthreads ON job (project, num_hwthreads);
+CREATE INDEX IF NOT EXISTS jobs_project_numacc ON job (project, num_acc);
+CREATE INDEX IF NOT EXISTS jobs_project_energy ON job (project, energy);
+
+-- JobState Filter
+CREATE INDEX IF NOT EXISTS jobs_jobstate ON job (job_state);
+CREATE INDEX IF NOT EXISTS jobs_jobstate_user ON job (job_state, hpc_user);
+CREATE INDEX IF NOT EXISTS jobs_jobstate_project ON job (job_state, project);
+CREATE INDEX IF NOT EXISTS jobs_jobstate_cluster ON job (job_state, cluster);
+-- JobState Filter Sorting
+CREATE INDEX IF NOT EXISTS jobs_jobstate_starttime ON job (job_state, start_time);
+CREATE INDEX IF NOT EXISTS jobs_jobstate_duration ON job (job_state, duration);
+CREATE INDEX IF NOT EXISTS jobs_jobstate_numnodes ON job (job_state, num_nodes);
+CREATE INDEX IF NOT EXISTS jobs_jobstate_numhwthreads ON job (job_state, num_hwthreads);
+CREATE INDEX IF NOT EXISTS jobs_jobstate_numacc ON job (job_state, num_acc);
+CREATE INDEX IF NOT EXISTS jobs_jobstate_energy ON job (job_state, energy);
+
+-- ArrayJob Filter
+CREATE INDEX IF NOT EXISTS jobs_arrayjobid_starttime ON job (array_job_id, start_time);
+CREATE INDEX IF NOT EXISTS jobs_cluster_arrayjobid_starttime ON job (cluster, array_job_id, start_time);
+
+-- Sorting without active filters
+CREATE INDEX IF NOT EXISTS jobs_starttime ON job (start_time);
+CREATE INDEX IF NOT EXISTS jobs_duration ON job (duration);
+CREATE INDEX IF NOT EXISTS jobs_numnodes ON job (num_nodes);
+CREATE INDEX IF NOT EXISTS jobs_numhwthreads ON job (num_hwthreads);
+CREATE INDEX IF NOT EXISTS jobs_numacc ON job (num_acc);
+CREATE INDEX IF NOT EXISTS jobs_energy ON job (energy);
+
+-- Single filters with default starttime sorting
+CREATE INDEX IF NOT EXISTS jobs_duration_starttime ON job (duration, start_time);
+CREATE INDEX IF NOT EXISTS jobs_numnodes_starttime ON job (num_nodes, start_time);
+CREATE INDEX IF NOT EXISTS jobs_numhwthreads_starttime ON job (num_hwthreads, start_time);
+CREATE INDEX IF NOT EXISTS jobs_numacc_starttime ON job (num_acc, start_time);
+CREATE INDEX IF NOT EXISTS jobs_energy_starttime ON job (energy, start_time);
+
+-- Optimize DB index usage
+PRAGMA optimize;

internal/repository/migrations/sqlite3/09_add-job-cache.down.sql

@@ -0,0 +1 @@
+DROP TABLE IF EXISTS job_cache;

internal/repository/migrations/sqlite3/09_add-job-cache.up.sql

@@ -0,0 +1,31 @@
+CREATE TABLE "job_cache" (
+    id INTEGER PRIMARY KEY,
+    job_id BIGINT NOT NULL,
+    cluster VARCHAR(255) NOT NULL,
+    subcluster VARCHAR(255) NOT NULL,
+    start_time BIGINT NOT NULL, -- Unix timestamp
+    hpc_user VARCHAR(255) NOT NULL,
+    project VARCHAR(255) NOT NULL,
+    cluster_partition VARCHAR(255),
+    array_job_id BIGINT,
+    duration INT NOT NULL,
+    walltime INT NOT NULL,
+    job_state VARCHAR(255) NOT NULL
+    CHECK (job_state IN (
+        'running', 'completed', 'failed', 'cancelled',
+        'stopped', 'timeout', 'preempted', 'out_of_memory'
+    )),
+    meta_data TEXT,          -- JSON
+    resources TEXT NOT NULL, -- JSON
+    num_nodes INT NOT NULL,
+    num_hwthreads INT,
+    num_acc INT,
+    smt TINYINT NOT NULL DEFAULT 1 CHECK (smt IN (0, 1)),
+    exclusive TINYINT NOT NULL DEFAULT 1 CHECK (exclusive IN (0, 1, 2)),
+    monitoring_status TINYINT NOT NULL DEFAULT 1
+    CHECK (monitoring_status IN (0, 1, 2, 3)),
+    energy REAL NOT NULL DEFAULT 0.0,
+    energy_footprint TEXT DEFAULT NULL,
+    footprint TEXT DEFAULT NULL,
+    UNIQUE (job_id, cluster, start_time)
+);

internal/repository/migrations/sqlite3/10_node-table.down.sql

@@ -0,0 +1 @@
+DROP TABLE IF EXISTS node;