Backup commit

Initially submit every job received via rest on NATS

.github/dependabot.yml

@@ -1,15 +0,0 @@
-# To get started with Dependabot version updates, you'll need to specify which
-# package ecosystems to update and where the package manifests are located.
-# Please see the documentation for all configuration options:
-# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file
-
-version: 2
-updates:
-  - package-ecosystem: "gomod"
-    directory: "/"
-    schedule:
-      interval: "weekly"
-  - package-ecosystem: "npm"
-    directory: "/web/frontend"
-    schedule:
-      interval: "weekly"

.github/workflows/Release.yml

@@ -0,0 +1,331 @@
+# See: https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions
+
+# Workflow name
+name: Release
+
+# Run on tag push
+on:
+ push:
+  tags:
+  - '**'
+
+jobs:
+
+  #
+  # Build on AlmaLinux 8.5 using golang-1.18.2
+  #
+  AlmaLinux-RPM-build:
+    runs-on: ubuntu-latest
+    # See: https://hub.docker.com/_/almalinux
+    container: almalinux:8.5
+    # The job outputs link to the outputs of the 'rpmrename' step
+    # Only job outputs can be used in child jobs
+    outputs:
+      rpm : ${{steps.rpmrename.outputs.RPM}}
+      srpm : ${{steps.rpmrename.outputs.SRPM}}
+    steps:
+
+    # Use dnf to install development packages
+    - name: Install development packages
+      run: |
+          dnf --assumeyes group install "Development Tools" "RPM Development Tools"
+          dnf --assumeyes install wget openssl-devel diffutils delve which npm
+          dnf --assumeyes install 'dnf-command(builddep)'
+
+    # Checkout git repository and submodules
+    # fetch-depth must be 0 to use git describe
+    # See: https://github.com/marketplace/actions/checkout
+    - name: Checkout
+      uses: actions/checkout@v2
+      with:
+        submodules: recursive
+        fetch-depth: 0
+
+    # Use dnf to install build dependencies
+    - name: Install build dependencies
+      run: |
+          wget -q http://mirror.centos.org/centos/8-stream/AppStream/x86_64/os/Packages/golang-1.18.2-1.module_el8.7.0+1173+5d37c0fd.x86_64.rpm \
+                  http://mirror.centos.org/centos/8-stream/AppStream/x86_64/os/Packages/golang-bin-1.18.2-1.module_el8.7.0+1173+5d37c0fd.x86_64.rpm \
+                  http://mirror.centos.org/centos/8-stream/AppStream/x86_64/os/Packages/golang-src-1.18.2-1.module_el8.7.0+1173+5d37c0fd.noarch.rpm \
+                  http://mirror.centos.org/centos/8-stream/AppStream/x86_64/os/Packages/go-toolset-1.18.2-1.module_el8.7.0+1173+5d37c0fd.x86_64.rpm
+          rpm -i go*.rpm
+          npm install --global yarn rollup svelte rollup-plugin-svelte
+          #dnf --assumeyes builddep build/package/cc-backend.spec
+
+    - name: RPM build ClusterCockpit
+      id: rpmbuild
+      run: make RPM
+
+    # AlmaLinux 8.5 is a derivate of RedHat Enterprise Linux 8 (UBI8),
+    # so the created RPM both contain the substring 'el8' in the RPM file names
+    # This step replaces the substring 'el8' to 'alma85'. It uses the move operation
+    # because it is unclear whether the default AlmaLinux 8.5 container contains the 
+    # 'rename' command. This way we also get the new names for output.
+    - name: Rename RPMs (s/el8/alma85/)
+      id: rpmrename
+      run: |
+        OLD_RPM="${{steps.rpmbuild.outputs.RPM}}"
+        OLD_SRPM="${{steps.rpmbuild.outputs.SRPM}}"
+        NEW_RPM="${OLD_RPM/el8/alma85}"
+        NEW_SRPM=${OLD_SRPM/el8/alma85}
+        mv "${OLD_RPM}" "${NEW_RPM}"
+        mv "${OLD_SRPM}" "${NEW_SRPM}"
+        echo "::set-output name=SRPM::${NEW_SRPM}"
+        echo "::set-output name=RPM::${NEW_RPM}"
+
+    # See: https://github.com/actions/upload-artifact
+    - name: Save RPM as artifact
+      uses: actions/upload-artifact@v2
+      with:
+        name: cc-backend RPM for AlmaLinux 8.5
+        path: ${{ steps.rpmrename.outputs.RPM }}
+    - name: Save SRPM as artifact
+      uses: actions/upload-artifact@v2
+      with:
+        name: cc-backend SRPM for AlmaLinux 8.5
+        path: ${{ steps.rpmrename.outputs.SRPM }}
+
+  #
+  # Build on UBI 8 using golang-1.18.2
+  #
+  UBI-8-RPM-build:
+    runs-on: ubuntu-latest
+    # See: https://catalog.redhat.com/software/containers/ubi8/ubi/5c359854d70cc534b3a3784e?container-tabs=gti
+    container: registry.access.redhat.com/ubi8/ubi:8.5-226.1645809065
+    # The job outputs link to the outputs of the 'rpmbuild' step
+    outputs:
+      rpm : ${{steps.rpmbuild.outputs.RPM}}
+      srpm : ${{steps.rpmbuild.outputs.SRPM}}
+    steps:
+
+    # Use dnf to install development packages
+    - name: Install development packages
+      run: dnf --assumeyes --disableplugin=subscription-manager install rpm-build go-srpm-macros rpm-build-libs rpm-libs gcc make python38 git wget openssl-devel diffutils delve which
+
+    # Checkout git repository and submodules
+    # fetch-depth must be 0 to use git describe
+    # See: https://github.com/marketplace/actions/checkout
+    - name: Checkout
+      uses: actions/checkout@v2
+      with:
+        submodules: recursive
+        fetch-depth: 0
+
+    # Use dnf to install build dependencies
+    - name: Install build dependencies
+      run: |
+          wget -q http://mirror.centos.org/centos/8-stream/AppStream/x86_64/os/Packages/golang-1.18.2-1.module_el8.7.0+1173+5d37c0fd.x86_64.rpm \
+                  http://mirror.centos.org/centos/8-stream/AppStream/x86_64/os/Packages/golang-bin-1.18.2-1.module_el8.7.0+1173+5d37c0fd.x86_64.rpm \
+                  http://mirror.centos.org/centos/8-stream/AppStream/x86_64/os/Packages/golang-src-1.18.2-1.module_el8.7.0+1173+5d37c0fd.noarch.rpm \
+                  http://mirror.centos.org/centos/8-stream/AppStream/x86_64/os/Packages/go-toolset-1.18.2-1.module_el8.7.0+1173+5d37c0fd.x86_64.rpm
+          rpm -i go*.rpm
+          dnf --assumeyes --disableplugin=subscription-manager install npm
+          npm install --global yarn rollup svelte rollup-plugin-svelte
+          #dnf --assumeyes builddep build/package/cc-backend.spec
+
+    - name: RPM build ClusterCockpit
+      id: rpmbuild
+      run: make RPM
+
+    # See: https://github.com/actions/upload-artifact
+    - name: Save RPM as artifact
+      uses: actions/upload-artifact@v2
+      with:
+        name: cc-backend RPM for UBI 8
+        path: ${{ steps.rpmbuild.outputs.RPM }}
+    - name: Save SRPM as artifact
+      uses: actions/upload-artifact@v2
+      with:
+        name: cc-backend SRPM for UBI 8
+        path: ${{ steps.rpmbuild.outputs.SRPM }}
+
+  #
+  # Build on Ubuntu 20.04 using official go 1.19.1 package
+  #
+  Ubuntu-focal-build:
+    runs-on: ubuntu-latest
+    container: ubuntu:20.04
+    # The job outputs link to the outputs of the 'debrename' step
+    # Only job outputs can be used in child jobs
+    outputs:
+      deb : ${{steps.debrename.outputs.DEB}}
+    steps:
+    # Use apt to install development packages
+    - name: Install development packages
+      run: |
+          apt update && apt --assume-yes upgrade
+          apt --assume-yes install build-essential sed git wget bash
+          apt --assume-yes install npm
+          npm install --global yarn rollup svelte rollup-plugin-svelte
+    # Checkout git repository and submodules
+    # fetch-depth must be 0 to use git describe
+    # See: https://github.com/marketplace/actions/checkout
+    - name: Checkout
+      uses: actions/checkout@v2
+      with:
+        submodules: recursive
+        fetch-depth: 0
+    # Use official golang package
+    - name: Install Golang
+      run: |
+          wget -q https://go.dev/dl/go1.19.1.linux-amd64.tar.gz
+          tar -C /usr/local -xzf go1.19.1.linux-amd64.tar.gz
+          export PATH=/usr/local/go/bin:/usr/local/go/pkg/tool/linux_amd64:$PATH
+          go version
+    - name: DEB build ClusterCockpit
+      id: dpkg-build
+      run: |
+          ls -la
+          pwd
+          env
+          export PATH=/usr/local/go/bin:/usr/local/go/pkg/tool/linux_amd64:$PATH
+          git config --global --add safe.directory $(pwd)
+          make DEB
+    - name: Rename DEB (add '_ubuntu20.04')
+      id: debrename
+      run: |
+        OLD_DEB_NAME=$(echo "${{steps.dpkg-build.outputs.DEB}}" | rev | cut -d '.' -f 2- | rev)
+        NEW_DEB_FILE="${OLD_DEB_NAME}_ubuntu20.04.deb"
+        mv "${{steps.dpkg-build.outputs.DEB}}" "${NEW_DEB_FILE}"
+        echo "::set-output name=DEB::${NEW_DEB_FILE}"
+    # See: https://github.com/actions/upload-artifact
+    - name: Save DEB as artifact
+      uses: actions/upload-artifact@v2
+      with:
+        name: cc-backend DEB for Ubuntu 20.04
+        path: ${{ steps.debrename.outputs.DEB }}
+
+  #
+  # Build on Ubuntu 20.04 using official go 1.19.1 package
+  #
+  Ubuntu-jammy-build:
+    runs-on: ubuntu-latest
+    container: ubuntu:22.04
+    # The job outputs link to the outputs of the 'debrename' step
+    # Only job outputs can be used in child jobs
+    outputs:
+      deb : ${{steps.debrename.outputs.DEB}}
+    steps:
+    # Use apt to install development packages
+    - name: Install development packages
+      run: |
+          apt update && apt --assume-yes upgrade
+          apt --assume-yes install build-essential sed git wget bash npm
+          npm install --global yarn rollup svelte rollup-plugin-svelte
+    # Checkout git repository and submodules
+    # fetch-depth must be 0 to use git describe
+    # See: https://github.com/marketplace/actions/checkout
+    - name: Checkout
+      uses: actions/checkout@v2
+      with:
+        submodules: recursive
+        fetch-depth: 0
+    # Use official golang package
+    - name: Install Golang
+      run: |
+          wget -q https://go.dev/dl/go1.19.1.linux-amd64.tar.gz
+          tar -C /usr/local -xzf go1.19.1.linux-amd64.tar.gz
+          export PATH=/usr/local/go/bin:/usr/local/go/pkg/tool/linux_amd64:$PATH
+          go version
+    - name: DEB build ClusterCockpit
+      id: dpkg-build
+      run: |
+          ls -la
+          pwd
+          env
+          export PATH=/usr/local/go/bin:/usr/local/go/pkg/tool/linux_amd64:$PATH
+          git config --global --add safe.directory $(pwd)
+          make DEB
+    - name: Rename DEB (add '_ubuntu22.04')
+      id: debrename
+      run: |
+        OLD_DEB_NAME=$(echo "${{steps.dpkg-build.outputs.DEB}}" | rev | cut -d '.' -f 2- | rev)
+        NEW_DEB_FILE="${OLD_DEB_NAME}_ubuntu22.04.deb"
+        mv "${{steps.dpkg-build.outputs.DEB}}" "${NEW_DEB_FILE}"
+        echo "::set-output name=DEB::${NEW_DEB_FILE}"
+    # See: https://github.com/actions/upload-artifact
+    - name: Save DEB as artifact
+      uses: actions/upload-artifact@v2
+      with:
+        name: cc-backend DEB for Ubuntu 22.04
+        path: ${{ steps.debrename.outputs.DEB }}
+
+  #
+  # Create release with fresh RPMs
+  #
+  Release:
+    runs-on: ubuntu-latest
+    # We need the RPMs, so add dependency
+    needs: [AlmaLinux-RPM-build, UBI-8-RPM-build, Ubuntu-focal-build, Ubuntu-jammy-build]
+
+    steps:
+    # See: https://github.com/actions/download-artifact
+    - name: Download AlmaLinux 8.5 RPM
+      uses: actions/download-artifact@v2
+      with:
+        name: cc-backend RPM for AlmaLinux 8.5
+    - name: Download AlmaLinux 8.5 SRPM
+      uses: actions/download-artifact@v2
+      with:
+        name: cc-backend SRPM for AlmaLinux 8.5
+    
+    - name: Download UBI 8 RPM
+      uses: actions/download-artifact@v2
+      with:
+        name: cc-backend RPM for UBI 8
+    - name: Download UBI 8 SRPM
+      uses: actions/download-artifact@v2
+      with:
+        name: cc-backend SRPM for UBI 8
+
+    - name: Download Ubuntu 20.04 DEB
+      uses: actions/download-artifact@v2
+      with:
+        name: cc-backend DEB for Ubuntu 20.04
+
+    - name: Download Ubuntu 22.04 DEB
+      uses: actions/download-artifact@v2
+      with:
+        name: cc-backend DEB for Ubuntu 22.04
+
+    # The download actions do not publish the name of the downloaded file,
+    # so we re-use the job outputs of the parent jobs. The files are all
+    # downloaded to the current folder.
+    # The gh-release action afterwards does not accept file lists but all
+    # files have to be listed at 'files'. The step creates one output per
+    # RPM package (2 per distro)
+    - name: Set RPM variables
+      id: files
+      run: |
+        ALMA_85_RPM=$(basename "${{ needs.AlmaLinux-RPM-build.outputs.rpm}}")
+        ALMA_85_SRPM=$(basename "${{ needs.AlmaLinux-RPM-build.outputs.srpm}}")
+        UBI_8_RPM=$(basename "${{ needs.UBI-8-RPM-build.outputs.rpm}}")
+        UBI_8_SRPM=$(basename "${{ needs.UBI-8-RPM-build.outputs.srpm}}")
+        U_2004_DEB=$(basename "${{ needs.Ubuntu-focal-build.outputs.deb}}")
+        U_2204_DEB=$(basename "${{ needs.Ubuntu-jammy-build.outputs.deb}}")
+        echo "ALMA_85_RPM::${ALMA_85_RPM}"
+        echo "ALMA_85_SRPM::${ALMA_85_SRPM}"
+        echo "UBI_8_RPM::${UBI_8_RPM}"
+        echo "UBI_8_SRPM::${UBI_8_SRPM}"
+        echo "U_2004_DEB::${U_2004_DEB}"
+        echo "U_2204_DEB::${U_2204_DEB}"
+        echo "::set-output name=ALMA_85_RPM::${ALMA_85_RPM}"
+        echo "::set-output name=ALMA_85_SRPM::${ALMA_85_SRPM}"
+        echo "::set-output name=UBI_8_RPM::${UBI_8_RPM}"
+        echo "::set-output name=UBI_8_SRPM::${UBI_8_SRPM}"
+        echo "::set-output name=U_2004_DEB::${U_2004_DEB}"
+        echo "::set-output name=U_2204_DEB::${U_2204_DEB}"
+
+    # See: https://github.com/softprops/action-gh-release
+    - name: Release
+      uses: softprops/action-gh-release@v1
+      if: startsWith(github.ref, 'refs/tags/')
+      with:
+        name: cc-backend-${{github.ref_name}}
+        files: |
+         ${{ steps.files.outputs.ALMA_85_RPM }}
+         ${{ steps.files.outputs.ALMA_85_SRPM }}
+         ${{ steps.files.outputs.UBI_8_RPM }}
+         ${{ steps.files.outputs.UBI_8_SRPM }}
+         ${{ steps.files.outputs.U_2004_DEB }}
+         ${{ steps.files.outputs.U_2204_DEB }}

.github/workflows/test.yml

@@ -7,7 +7,7 @@ jobs:
       - name: Install Go
         uses: actions/setup-go@v4
         with:
-          go-version: 1.25.x
+          go-version: 1.22.x
       - name: Checkout code
         uses: actions/checkout@v3
       - name: Build, Vet & Test

.gitignore

@@ -1,29 +1,21 @@
 /cc-backend
-/.env
-/config.json
-/uiConfig.json
 
 /var/job-archive
-/var/machine-state
-/var/job.db-shm
-/var/job.db-wal
 /var/*.db
-/var/*.txt
+/var/machine-state
 
-/var/checkpoints*
-
-migrateTimestamps.pl
-test_ccms_write_api.sh
+/.env
+/config.json
 
 /web/frontend/public/build
 /web/frontend/node_modules
-
+/.vscode/*
 /archive-migration
 /archive-manager
+var/job.db-shm
+var/job.db-wal
 
-/internal/repository/testdata/job.db-shm
-/internal/repository/testdata/job.db-wal
-
-/.vscode/*
 dist/
 *.db
+internal/repository/testdata/job.db-shm
+internal/repository/testdata/job.db-wal

Makefile

@@ -2,7 +2,7 @@ TARGET = ./cc-backend
 VAR = ./var
 CFG = config.json .env
 FRONTEND = ./web/frontend
-VERSION = 1.4.4
+VERSION = 1.4.2
 GIT_HASH := $(shell git rev-parse --short HEAD || echo 'development')
 CURRENT_TIME = $(shell date +"%Y-%m-%d:T%H:%M:%S")
 LD_FLAGS = '-s -X main.date=${CURRENT_TIME} -X main.version=${VERSION} -X main.commit=${GIT_HASH}'
@@ -52,7 +52,7 @@ frontend:
 
 swagger:
 	$(info ===>  GENERATE swagger)
-	@go run github.com/swaggo/swag/cmd/swag init  --parseDependency -d ./internal/api -g rest.go -o ./api
+	@go run github.com/swaggo/swag/cmd/swag init -d ./internal/api,./pkg/schema -g rest.go -o ./api
 	@mv ./api/docs.go ./internal/api/docs.go
 
 graphql:
@@ -82,7 +82,7 @@ tags:
 	@ctags -R
 
 $(VAR):
-	@mkdir -p $(VAR)
+	@mkdir $(VAR)
 
 config.json:
 	$(info ===>  Initialize config.json file)

README.md

@@ -1,8 +1,5 @@
 # NOTE
 
-While we do our best to keep the master branch in a usable state, there is no guarantee the master branch works.
-Please do not use it for production!
-
 Please have a look at the [Release
 Notes](https://github.com/ClusterCockpit/cc-backend/blob/master/ReleaseNotes.md)
 for breaking changes!

ReleaseNotes.md

@@ -1,27 +1,13 @@
-# `cc-backend` version 1.4.4
+# `cc-backend` version 1.4.2
 
 Supports job archive version 2 and database version 8.
 
-This is a bug fix release of `cc-backend`, the API backend and frontend
+This is a small bug fix release of `cc-backend`, the API backend and frontend
 implementation of ClusterCockpit.
 For release specific notes visit the [ClusterCockpit Documentation](https://clusterockpit.org/docs/release/).
 
 ## Breaking changes
 
-The option `apiAllowedIPs` is now a required configuration attribute in
-`config.json`. This option restricts access to the admin API.
-
-To retain the previous behavior that the API is per default accessible from
-everywhere set:
-
-```json
-  "apiAllowedIPs": [
-    "*"
-  ]
-```
-
-## Breaking changes for minor release 1.4.x
-
 - You need to perform a database migration. Depending on your database size the
   migration might require several hours!
 - You need to adapt the `cluster.json` configuration files in the job-archive,
@@ -36,7 +22,20 @@ everywhere set:
 
 ## New features
 
-- Enable to delete tags from the web interface
+- Tags have a scope now. Tags created by a basic user are only visible by that
+  user. Tags created by an admin/support role can be configured to be visible by
+  all users (global scope) or only be admin/support role.
+- Re-sampling support for running (requires a recent `cc-metric-store`) and
+  archived jobs. This greatly speeds up loading of large or very long jobs. You
+  need to add the new configuration key `enable-resampling` to the `config.json`
+  file.
+- For finished jobs a total job energy is shown in the job view.
+- Continuous scrolling in job lists is default now.
+- All database queries (especially for sqlite) were optimized resulting in
+  dramatically faster load times.
+- A performance and energy footprint can be freely configured on a per
+  subcluster base. One can filter for footprint statistics for running and
+  finished jobs.
 
 ## Known issues
 

api/schema.graphqls

@@ -4,89 +4,61 @@ scalar Any
 scalar NullableFloat
 scalar MetricScope
 scalar JobState
-scalar SchedulerState
-scalar MonitoringState
-
-type Node {
-  id: ID!
-  hostname: String!
-  cluster: String!
-  subCluster: String!
-  jobsRunning: Int!
-  cpusAllocated: Int
-  memoryAllocated: Int
-  gpusAllocated: Int
-  schedulerState: SchedulerState!
-  healthState: MonitoringState!
-  metaData: Any
-}
-
-type NodeStates {
-  state: String!
-  count: Int!
-}
-
-type NodeStatesTimed {
-  state: String!
-  type: String!
-  count: Int!
-  time: Int!
-}
 
 type Job {
-  id: ID!
-  jobId: Int!
-  user: String!
-  project: String!
-  cluster: String!
-  subCluster: String!
-  startTime: Time!
-  duration: Int!
-  walltime: Int!
-  numNodes: Int!
-  numHWThreads: Int!
-  numAcc: Int!
-  energy: Float!
-  SMT: Int!
-  shared: String!
-  partition: String!
-  arrayJobId: Int!
+  id:               ID!
+  jobId:            Int!
+  user:             String!
+  project:          String!
+  cluster:          String!
+  subCluster:       String!
+  startTime:        Time!
+  duration:         Int!
+  walltime:         Int!
+  numNodes:         Int!
+  numHWThreads:     Int!
+  numAcc:           Int!
+  energy:           Float!
+  SMT:              Int!
+  exclusive:        Int!
+  partition:        String!
+  arrayJobId:       Int!
   monitoringStatus: Int!
-  state: JobState!
-  tags: [Tag!]!
-  resources: [Resource!]!
-  concurrentJobs: JobLinkResultList
-  footprint: [FootprintValue]
-  energyFootprint: [EnergyFootprintValue]
-  metaData: Any
-  userData: User
+  state:            JobState!
+  tags:             [Tag!]!
+  resources:        [Resource!]!
+  concurrentJobs:   JobLinkResultList
+  footprint:        [FootprintValue]
+  energyFootprint:  [EnergyFootprintValue]
+  metaData:         Any
+  userData:         User
 }
 
 type JobLink {
-  id: ID!
-  jobId: Int!
+  id:               ID!
+  jobId:            Int!
 }
 
 type Cluster {
-  name: String!
-  partitions: [String!]! # Slurm partitions
-  subClusters: [SubCluster!]! # Hardware partitions/subclusters
+  name:         String!
+  partitions:   [String!]!        # Slurm partitions
+  subClusters:  [SubCluster!]!    # Hardware partitions/subclusters
 }
 
 type SubCluster {
-  name: String!
-  nodes: String!
-  numberOfNodes: Int!
-  processorType: String!
-  socketsPerNode: Int!
-  coresPerSocket: Int!
-  threadsPerCore: Int!
-  flopRateScalar: MetricValue!
-  flopRateSimd: MetricValue!
+  name:            String!
+  nodes:           String!
+  numberOfNodes:   Int!
+  processorType:   String!
+  socketsPerNode:  Int!
+  coresPerSocket:  Int!
+  threadsPerCore:  Int!
+  flopRateScalar:  MetricValue!
+  flopRateSimd:    MetricValue!
   memoryBandwidth: MetricValue!
-  topology: Topology!
-  metricConfig: [MetricConfig!]!
-  footprint: [String!]!
+  topology:        Topology!
+  metricConfig:    [MetricConfig!]!
+  footprint:       [String!]!
 }
 
 type FootprintValue {
@@ -108,112 +80,75 @@ type MetricValue {
 }
 
 type Topology {
-  node: [Int!]
-  socket: [[Int!]!]
+  node:         [Int!]
+  socket:       [[Int!]!]
   memoryDomain: [[Int!]!]
-  die: [[Int!]!]
-  core: [[Int!]!]
+  die:          [[Int!]!]
+  core:         [[Int!]!]
   accelerators: [Accelerator!]
 }
 
 type Accelerator {
-  id: String!
-  type: String!
+  id:    String!
+  type:  String!
   model: String!
 }
 
 type SubClusterConfig {
-  name: String!
-  peak: Float
-  normal: Float
+  name:    String!
+  peak:    Float
+  normal:  Float
   caution: Float
-  alert: Float
-  remove: Boolean
+  alert:   Float
+  remove:  Boolean
 }
 
 type MetricConfig {
-  name: String!
-  unit: Unit!
-  scope: MetricScope!
+  name:        String!
+  unit:        Unit!
+  scope:       MetricScope!
   aggregation: String!
-  timestep: Int!
-  peak: Float!
-  normal: Float
+  timestep:    Int!
+  peak:    Float!
+  normal:  Float
   caution: Float!
-  alert: Float!
+  alert:   Float!
   lowerIsBetter: Boolean
   subClusters: [SubClusterConfig!]!
 }
 
 type Tag {
-  id: ID!
+  id:   ID!
   type: String!
   name: String!
   scope: String!
 }
 
 type Resource {
-  hostname: String!
-  hwthreads: [Int!]
-  accelerators: [String!]
+  hostname:      String!
+  hwthreads:     [Int!]
+  accelerators:  [String!]
   configuration: String
 }
 
 type JobMetricWithName {
-  name: String!
-  scope: MetricScope!
+  name:   String!
+  scope:  MetricScope!
   metric: JobMetric!
 }
 
 type JobMetric {
-  unit: Unit
-  timestep: Int!
-  series: [Series!]
+  unit:             Unit
+  timestep:         Int!
+  series:           [Series!]
   statisticsSeries: StatsSeries
 }
 
 type Series {
-  hostname: String!
-  id: String
+  hostname:   String!
+  id:         String
   statistics: MetricStatistics
-  data: [NullableFloat!]!
-}
-
-type StatsSeries {
-  mean: [NullableFloat!]!
-  median: [NullableFloat!]!
-  min: [NullableFloat!]!
-  max: [NullableFloat!]!
-}
-
-type NamedStatsWithScope {
-  name: String!
-  scope: MetricScope!
-  stats: [ScopedStats!]!
-}
-
-type ScopedStats {
-  hostname: String!
-  id: String
-  data: MetricStatistics!
-}
-
-type JobStats {
-  id: Int!
-  jobId: String!
-  startTime: Int!
-  duration: Int!
-  cluster: String!
-  subCluster: String!
-  numNodes: Int!
-  numHWThreads: Int
-  numAccelerators: Int
-  stats: [NamedStats!]!
-}
-
-type NamedStats {
-  name: String!
-  data: MetricStatistics!
+  data:       [NullableFloat!]!
 }
 
 type Unit {
@@ -227,14 +162,21 @@ type MetricStatistics {
   max: Float!
 }
 
+type StatsSeries {
+  mean:   [NullableFloat!]!
+  median: [NullableFloat!]!
+  min:    [NullableFloat!]!
+  max:    [NullableFloat!]!
+}
+
 type MetricFootprints {
   metric: String!
-  data: [NullableFloat!]!
+  data:   [NullableFloat!]!
 }
 
 type Footprints {
   timeWeights: TimeWeights!
-  metrics: [MetricFootprints!]!
+  metrics:   [MetricFootprints!]!
 }
 
 type TimeWeights {
@@ -243,35 +185,20 @@ type TimeWeights {
   coreHours: [NullableFloat!]!
 }
 
-enum Aggregate {
-  USER
-  PROJECT
-  CLUSTER
-  SUBCLUSTER
-}
-enum SortByAggregate {
-  TOTALWALLTIME
-  TOTALJOBS
-  TOTALUSERS
-  TOTALNODES
-  TOTALNODEHOURS
-  TOTALCORES
-  TOTALCOREHOURS
-  TOTALACCS
-  TOTALACCHOURS
-}
+enum Aggregate { USER, PROJECT, CLUSTER }
+enum SortByAggregate { TOTALWALLTIME, TOTALJOBS, TOTALNODES, TOTALNODEHOURS, TOTALCORES, TOTALCOREHOURS, TOTALACCS, TOTALACCHOURS }
 
 type NodeMetrics {
-  host: String!
+  host:       String!
   subCluster: String!
-  metrics: [JobMetricWithName!]!
+  metrics:    [JobMetricWithName!]!
 }
 
 type NodesResultList {
-  items: [NodeMetrics!]!
+  items:  [NodeMetrics!]!
   offset: Int
-  limit: Int
-  count: Int
+  limit:  Int
+  count:  Int
   totalNodes: Int
   hasNextPage: Boolean
 }
@@ -290,14 +217,14 @@ type GlobalMetricListItem {
 }
 
 type Count {
-  name: String!
+  name:  String!
   count: Int!
 }
 
 type User {
   username: String!
-  name: String!
-  email: String!
+  name:     String!
+  email:    String!
 }
 
 input MetricStatItem {
@@ -306,84 +233,24 @@ input MetricStatItem {
 }
 
 type Query {
-  clusters: [Cluster!]! # List of all clusters
-  tags: [Tag!]! # List of all tags
-  globalMetrics: [GlobalMetricListItem!]!
+  clusters:     [Cluster!]!   # List of all clusters
+  tags:         [Tag!]!       # List of all tags
+  globalMetrics:   [GlobalMetricListItem!]!
 
   user(username: String!): User
   allocatedNodes(cluster: String!): [Count!]!
 
-  ## Node Queries New
-  node(id: ID!): Node
-  nodes(filter: [NodeFilter!], order: OrderByInput): NodeStateResultList!
-  nodeStates(filter: [NodeFilter!]): [NodeStates!]!
-  nodeStatesTimed(filter: [NodeFilter!]): [NodeStatesTimed!]!
-
   job(id: ID!): Job
-  jobMetrics(
-    id: ID!
-    metrics: [String!]
-    scopes: [MetricScope!]
-    resolution: Int
-  ): [JobMetricWithName!]!
-
-  jobStats(id: ID!, metrics: [String!]): [NamedStats!]!
-
-  scopedJobStats(
-    id: ID!
-    metrics: [String!]
-    scopes: [MetricScope!]
-  ): [NamedStatsWithScope!]!
-
-  jobs(
-    filter: [JobFilter!]
-    page: PageRequest
-    order: OrderByInput
-  ): JobResultList!
-
-  jobsStatistics(
-    filter: [JobFilter!]
-    metrics: [String!]
-    page: PageRequest
-    sortBy: SortByAggregate
-    groupBy: Aggregate
-    numDurationBins: String
-    numMetricBins: Int
-  ): [JobsStatistics!]!
-
-  jobsMetricStats(filter: [JobFilter!], metrics: [String!]): [JobStats!]!
+  jobMetrics(id: ID!, metrics: [String!], scopes: [MetricScope!], resolution: Int): [JobMetricWithName!]!
   jobsFootprints(filter: [JobFilter!], metrics: [String!]!): Footprints
 
-  rooflineHeatmap(
-    filter: [JobFilter!]!
-    rows: Int!
-    cols: Int!
-    minX: Float!
-    minY: Float!
-    maxX: Float!
-    maxY: Float!
-  ): [[Float!]!]!
+  jobs(filter: [JobFilter!], page: PageRequest, order: OrderByInput): JobResultList!
+  jobsStatistics(filter: [JobFilter!], metrics: [String!], page: PageRequest, sortBy: SortByAggregate, groupBy: Aggregate, numDurationBins: String, numMetricBins: Int): [JobsStatistics!]!
 
-  nodeMetrics(
-    cluster: String!
-    nodes: [String!]
-    scopes: [MetricScope!]
-    metrics: [String!]
-    from: Time!
-    to: Time!
-  ): [NodeMetrics!]!
+  rooflineHeatmap(filter: [JobFilter!]!, rows: Int!, cols: Int!, minX: Float!, minY: Float!, maxX: Float!, maxY: Float!): [[Float!]!]!
 
-  nodeMetricsList(
-    cluster: String!
-    subCluster: String!
-    nodeFilter: String!
-    scopes: [MetricScope!]
-    metrics: [String!]
-    from: Time!
-    to: Time!
-    page: PageRequest
-    resolution: Int
-  ): NodesResultList!
+  nodeMetrics(cluster: String!, nodes: [String!], scopes: [MetricScope!], metrics: [String!], from: Time!, to: Time!): [NodeMetrics!]!
+  nodeMetricsList(cluster: String!, subCluster: String!, nodeFilter: String!, scopes: [MetricScope!], metrics: [String!], from: Time!, to: Time!, page: PageRequest, resolution: Int): NodesResultList!
 }
 
 type Mutation {
@@ -391,59 +258,41 @@ type Mutation {
   deleteTag(id: ID!): ID!
   addTagsToJob(job: ID!, tagIds: [ID!]!): [Tag!]!
   removeTagsFromJob(job: ID!, tagIds: [ID!]!): [Tag!]!
-  removeTagFromList(tagIds: [ID!]!): [Int!]!
 
   updateConfiguration(name: String!, value: String!): String
 }
 
-type IntRangeOutput {
-  from: Int!
-  to: Int!
-}
-type TimeRangeOutput {
-  range: String
-  from: Time!
-  to: Time!
-}
-
-input NodeFilter {
-  hostname: StringInput
-  cluster: StringInput
-  subcluster: StringInput
-  schedulerState: SchedulerState
-  healthState: MonitoringState
-  timeStart: Int
-}
+type IntRangeOutput { from: Int!, to: Int! }
+type TimeRangeOutput { range: String, from: Time!, to: Time! }
 
 input JobFilter {
-  tags: [ID!]
-  dbId: [ID!]
-  jobId: StringInput
-  arrayJobId: Int
-  user: StringInput
-  project: StringInput
-  jobName: StringInput
-  cluster: StringInput
-  partition: StringInput
-  duration: IntRange
-  energy: FloatRange
+  tags:        [ID!]
+  jobId:       StringInput
+  arrayJobId:  Int
+  user:        StringInput
+  project:     StringInput
+  jobName:     StringInput
+  cluster:     StringInput
+  partition:   StringInput
+  duration:    IntRange
+  energy:      FloatRange
 
   minRunningFor: Int
 
-  numNodes: IntRange
+  numNodes:        IntRange
   numAccelerators: IntRange
-  numHWThreads: IntRange
+  numHWThreads:    IntRange
 
-  startTime: TimeRange
-  state: [JobState!]
+  startTime:   TimeRange
+  state:       [JobState!]
   metricStats: [MetricStatItem!]
-  shared: String
-  node: StringInput
+  exclusive:     Int
+  node:    StringInput
 }
 
 input OrderByInput {
   field: String!
-  type: String!
+  type: String!,
   order: SortDirectionEnum! = ASC
 }
 
@@ -453,46 +302,34 @@ enum SortDirectionEnum {
 }
 
 input StringInput {
-  eq: String
-  neq: String
-  contains: String
+  eq:         String
+  neq:        String
+  contains:   String
   startsWith: String
-  endsWith: String
-  in: [String!]
+  endsWith:   String
+  in:         [String!]
 }
 
-input IntRange {
-  from: Int!
-  to: Int!
-}
-input TimeRange {
-  range: String
-  from: Time
-  to: Time
-}
+input IntRange   { from: Int!, to: Int! }
+input TimeRange  { range: String, from: Time, to: Time }
 
 input FloatRange {
   from: Float!
   to: Float!
 }
 
-type NodeStateResultList {
-  items: [Node!]!
-  count: Int
-}
-
 type JobResultList {
-  items: [Job!]!
+  items:  [Job!]!
   offset: Int
-  limit: Int
-  count: Int
+  limit:  Int
+  count:  Int
   hasNextPage: Boolean
 }
 
 type JobLinkResultList {
   listQuery: String
-  items: [JobLink!]!
-  count: Int
+  items:  [JobLink!]!
+  count:  Int
 }
 
 type HistoPoint {
@@ -514,28 +351,27 @@ type MetricHistoPoint {
   max: Int
 }
 
-type JobsStatistics {
-  id: ID! # If `groupBy` was used, ID of the user/project/cluster/subcluster
-  name: String! # if User-Statistics: Given Name of Account (ID) Owner
-  totalUsers: Int! # if *not* User-Statistics: Number of active users (based on running jobs)
-  totalJobs: Int! # Number of jobs
-  runningJobs: Int! # Number of running jobs
-  shortJobs: Int! # Number of jobs with a duration of less than config'd ShortRunningJobsDuration
-  totalWalltime: Int! # Sum of the duration of all matched jobs in hours
-  totalNodes: Int! # Sum of the nodes of all matched jobs
-  totalNodeHours: Int! # Sum of the node hours of all matched jobs
-  totalCores: Int! # Sum of the cores of all matched jobs
-  totalCoreHours: Int! # Sum of the core hours of all matched jobs
-  totalAccs: Int! # Sum of the accs of all matched jobs
-  totalAccHours: Int! # Sum of the gpu hours of all matched jobs
-  histDuration: [HistoPoint!]! # value: hour, count: number of jobs with a rounded duration of value
-  histNumNodes: [HistoPoint!]! # value: number of nodes, count: number of jobs with that number of nodes
-  histNumCores: [HistoPoint!]! # value: number of cores, count: number of jobs with that number of cores
-  histNumAccs: [HistoPoint!]! # value: number of accs, count: number of jobs with that number of accs
-  histMetrics: [MetricHistoPoints!]! # metric: metricname, data array of histopoints: value: metric average bin, count: number of jobs with that metric average
+type JobsStatistics  {
+  id:             ID!            # If `groupBy` was used, ID of the user/project/cluster
+  name:           String!        # if User-Statistics: Given Name of Account (ID) Owner
+  totalJobs:      Int!           # Number of jobs
+  runningJobs:    Int!           # Number of running jobs
+  shortJobs:      Int!           # Number of jobs with a duration of less than duration
+  totalWalltime:  Int!           # Sum of the duration of all matched jobs in hours
+  totalNodes:     Int!           # Sum of the nodes of all matched jobs
+  totalNodeHours: Int!           # Sum of the node hours of all matched jobs
+  totalCores:     Int!           # Sum of the cores of all matched jobs
+  totalCoreHours: Int!           # Sum of the core hours of all matched jobs
+  totalAccs:      Int!         # Sum of the accs of all matched jobs
+  totalAccHours:  Int!           # Sum of the gpu hours of all matched jobs
+  histDuration:   [HistoPoint!]! # value: hour, count: number of jobs with a rounded duration of value
+  histNumNodes:   [HistoPoint!]! # value: number of nodes, count: number of jobs with that number of nodes
+  histNumCores:   [HistoPoint!]! # value: number of cores, count: number of jobs with that number of cores
+  histNumAccs:    [HistoPoint!]! # value: number of accs, count: number of jobs with that number of accs
+  histMetrics:    [MetricHistoPoints!]! # metric: metricname, data array of histopoints: value: metric average bin, count: number of jobs with that metric average
 }
 
 input PageRequest {
   itemsPerPage: Int!
-  page: Int!
+  page:         Int!
 }

api/swagger.json

@@ -15,8 +15,9 @@
         "version": "1.0.0"
     },
     "host": "localhost:8080",
+    "basePath": "/api",
     "paths": {
-        "/api/clusters/": {
+        "/clusters/": {
             "get": {
                 "security": [
                     {
@@ -73,7 +74,7 @@
                 }
             }
         },
-        "/api/jobs/": {
+        "/jobs/": {
             "get": {
                 "security": [
                     {
@@ -168,7 +169,7 @@
                 }
             }
         },
-        "/api/jobs/delete_job/": {
+        "/jobs/delete_job/": {
             "delete": {
                 "security": [
                     {
@@ -201,7 +202,7 @@
                     "200": {
                         "description": "Success message",
                         "schema": {
-                            "$ref": "#/definitions/api.DefaultApiResponse"
+                            "$ref": "#/definitions/api.DeleteJobApiResponse"
                         }
                     },
                     "400": {
@@ -243,7 +244,7 @@
                 }
             }
         },
-        "/api/jobs/delete_job/{id}": {
+        "/jobs/delete_job/{id}": {
             "delete": {
                 "security": [
                     {
@@ -271,7 +272,7 @@
                     "200": {
                         "description": "Success message",
                         "schema": {
-                            "$ref": "#/definitions/api.DefaultApiResponse"
+                            "$ref": "#/definitions/api.DeleteJobApiResponse"
                         }
                     },
                     "400": {
@@ -313,7 +314,7 @@
                 }
             }
         },
-        "/api/jobs/delete_job_before/{ts}": {
+        "/jobs/delete_job_before/{ts}": {
             "delete": {
                 "security": [
                     {
@@ -341,7 +342,7 @@
                     "200": {
                         "description": "Success message",
                         "schema": {
-                            "$ref": "#/definitions/api.DefaultApiResponse"
+                            "$ref": "#/definitions/api.DeleteJobApiResponse"
                         }
                     },
                     "400": {
@@ -383,7 +384,7 @@
                 }
             }
         },
-        "/api/jobs/edit_meta/{id}": {
+        "/jobs/edit_meta/{id}": {
             "post": {
                 "security": [
                     {
@@ -453,14 +454,14 @@
                 }
             }
         },
-        "/api/jobs/start_job/": {
+        "/jobs/start_job/": {
             "post": {
                 "security": [
                     {
                         "ApiKeyAuth": []
                     }
                 ],
-                "description": "Job specified in request body will be saved to database as \"running\" with new DB ID.\nJob specifications follow the 'Job' scheme, API will fail to execute if requirements are not met.",
+                "description": "Job specified in request body will be saved to database as \"running\" with new DB ID.\nJob specifications follow the 'JobMeta' scheme, API will fail to execute if requirements are not met.",
                 "consumes": [
                     "application/json"
                 ],
@@ -478,7 +479,7 @@
                         "in": "body",
                         "required": true,
                         "schema": {
-                            "$ref": "#/definitions/schema.Job"
+                            "$ref": "#/definitions/schema.JobMeta"
                         }
                     }
                 ],
@@ -486,7 +487,7 @@
                     "201": {
                         "description": "Job added successfully",
                         "schema": {
-                            "$ref": "#/definitions/api.DefaultApiResponse"
+                            "$ref": "#/definitions/api.StartJobApiResponse"
                         }
                     },
                     "400": {
@@ -522,14 +523,14 @@
                 }
             }
         },
-        "/api/jobs/stop_job/": {
+        "/jobs/stop_job/": {
             "post": {
                 "security": [
                     {
                         "ApiKeyAuth": []
                     }
                 ],
-                "description": "Job to stop is specified by request body. All fields are required in this case.\nReturns full job resource information according to 'Job' scheme.",
+                "description": "Job to stop is specified by request body. All fields are required in this case.\nReturns full job resource information according to 'JobMeta' scheme.",
                 "produces": [
                     "application/json"
                 ],
@@ -552,7 +553,7 @@
                     "200": {
                         "description": "Success message",
                         "schema": {
-                            "$ref": "#/definitions/schema.Job"
+                            "$ref": "#/definitions/schema.JobMeta"
                         }
                     },
                     "400": {
@@ -580,7 +581,7 @@
                         }
                     },
                     "422": {
-                        "description": "Unprocessable Entity: job has already been stopped",
+                        "description": "Unprocessable Entity: finding job failed: sql: no rows in result set",
                         "schema": {
                             "$ref": "#/definitions/api.ErrorResponse"
                         }
@@ -594,7 +595,7 @@
                 }
             }
         },
-        "/api/jobs/tag_job/{id}": {
+        "/jobs/tag_job/{id}": {
             "post": {
                 "security": [
                     {
@@ -667,14 +668,14 @@
                 }
             }
         },
-        "/api/jobs/{id}": {
+        "/jobs/{id}": {
             "get": {
                 "security": [
                     {
                         "ApiKeyAuth": []
                     }
                 ],
-                "description": "Job to get is specified by database ID\nReturns full job resource information according to 'Job' scheme and all metrics according to 'JobData'.",
+                "description": "Job to get is specified by database ID\nReturns full job resource information according to 'JobMeta' scheme and all metrics according to 'JobData'.",
                 "produces": [
                     "application/json"
                 ],
@@ -748,7 +749,7 @@
                         "ApiKeyAuth": []
                     }
                 ],
-                "description": "Job to get is specified by database ID\nReturns full job resource information according to 'Job' scheme and all metrics according to 'JobData'.",
+                "description": "Job to get is specified by database ID\nReturns full job resource information according to 'JobMeta' scheme and all metrics according to 'JobData'.",
                 "consumes": [
                     "application/json"
                 ],
@@ -826,74 +827,119 @@
                 }
             }
         },
-        "/api/nodestats/": {
+        "/user/{id}": {
             "post": {
                 "security": [
                     {
                         "ApiKeyAuth": []
                     }
                 ],
-                "description": "Returns a JSON-encoded list of users.\nRequired query-parameter defines if all users or only users with additional special roles are returned.",
+                "description": "Modifies user defined by username (id) in one of four possible ways.\nIf more than one formValue is set then only the highest priority field is used.\nOnly accessible from IPs registered with apiAllowedIPs configuration option.",
+                "consumes": [
+                    "multipart/form-data"
+                ],
                 "produces": [
-                    "application/json"
+                    "text/plain"
                 ],
                 "tags": [
-                    "Nodestates"
+                    "User"
                 ],
-                "summary": "Deliver updated Slurm node states",
+                "summary": "Updates an existing user",
                 "parameters": [
                     {
-                        "description": "Request body containing nodes and their states",
-                        "name": "request",
-                        "in": "body",
-                        "required": true,
-                        "schema": {
-                            "$ref": "#/definitions/api.UpdateNodeStatesRequest"
-                        }
+                        "type": "string",
+                        "description": "Database ID of User",
+                        "name": "id",
+                        "in": "path",
+                        "required": true
+                    },
+                    {
+                        "enum": [
+                            "admin",
+                            "support",
+                            "manager",
+                            "user",
+                            "api"
+                        ],
+                        "type": "string",
+                        "description": "Priority 1: Role to add",
+                        "name": "add-role",
+                        "in": "formData"
+                    },
+                    {
+                        "enum": [
+                            "admin",
+                            "support",
+                            "manager",
+                            "user",
+                            "api"
+                        ],
+                        "type": "string",
+                        "description": "Priority 2: Role to remove",
+                        "name": "remove-role",
+                        "in": "formData"
+                    },
+                    {
+                        "type": "string",
+                        "description": "Priority 3: Project to add",
+                        "name": "add-project",
+                        "in": "formData"
+                    },
+                    {
+                        "type": "string",
+                        "description": "Priority 4: Project to remove",
+                        "name": "remove-project",
+                        "in": "formData"
                     }
                 ],
                 "responses": {
                     "200": {
-                        "description": "Success message",
+                        "description": "Success Response Message",
                         "schema": {
-                            "$ref": "#/definitions/api.DefaultApiResponse"
+                            "type": "string"
                         }
                     },
                     "400": {
                         "description": "Bad Request",
                         "schema": {
-                            "$ref": "#/definitions/api.ErrorResponse"
+                            "type": "string"
                         }
                     },
                     "401": {
                         "description": "Unauthorized",
                         "schema": {
-                            "$ref": "#/definitions/api.ErrorResponse"
+                            "type": "string"
                         }
                     },
                     "403": {
                         "description": "Forbidden",
                         "schema": {
-                            "$ref": "#/definitions/api.ErrorResponse"
+                            "type": "string"
+                        }
+                    },
+                    "422": {
+                        "description": "Unprocessable Entity: The user could not be updated",
+                        "schema": {
+                            "type": "string"
                         }
                     },
                     "500": {
                         "description": "Internal Server Error",
                         "schema": {
-                            "$ref": "#/definitions/api.ErrorResponse"
+                            "type": "string"
                         }
                     }
                 }
             }
         },
-        "/api/users/": {
+        "/users/": {
             "get": {
                 "security": [
                     {
                         "ApiKeyAuth": []
                     }
                 ],
-                "description": "Returns a JSON-encoded list of users.\nRequired query-parameter defines if all users or only users with additional special roles are returned.",
+                "description": "Returns a JSON-encoded list of users.\nRequired query-parameter defines if all users or only users with additional special roles are returned.\nOnly accessible from IPs registered with apiAllowedIPs configuration option.",
                 "produces": [
                     "application/json"
                 ],
@@ -945,111 +991,70 @@
                         }
                     }
                 }
-            }
-        },
-        "/jobs/tag_job/{id}": {
-            "delete": {
+            },
+            "post": {
                 "security": [
                     {
                         "ApiKeyAuth": []
                     }
                 ],
-                "description": "Removes tag(s) from a job specified by DB ID. Name and Type of Tag(s) must match.\nTag Scope is required for matching, options: \"global\", \"admin\". Private tags can not be deleted via API.\nIf tagged job is already finished: Tag will be removed from respective archive files.",
+                "description": "User specified in form data will be saved to database.\nOnly accessible from IPs registered with apiAllowedIPs configuration option.",
                 "consumes": [
-                    "application/json"
-                ],
-                "produces": [
-                    "application/json"
-                ],
-                "tags": [
-                    "Job add and modify"
-                ],
-                "summary": "Removes one or more tags from a job",
-                "parameters": [
-                    {
-                        "type": "integer",
-                        "description": "Job Database ID",
-                        "name": "id",
-                        "in": "path",
-                        "required": true
-                    },
-                    {
-                        "description": "Array of tag-objects to remove",
-                        "name": "request",
-                        "in": "body",
-                        "required": true,
-                        "schema": {
-                            "type": "array",
-                            "items": {
-                                "$ref": "#/definitions/api.ApiTag"
-                            }
-                        }
-                    }
-                ],
-                "responses": {
-                    "200": {
-                        "description": "Updated job resource",
-                        "schema": {
-                            "$ref": "#/definitions/schema.Job"
-                        }
-                    },
-                    "400": {
-                        "description": "Bad Request",
-                        "schema": {
-                            "$ref": "#/definitions/api.ErrorResponse"
-                        }
-                    },
-                    "401": {
-                        "description": "Unauthorized",
-                        "schema": {
-                            "$ref": "#/definitions/api.ErrorResponse"
-                        }
-                    },
-                    "404": {
-                        "description": "Job or tag does not exist",
-                        "schema": {
-                            "$ref": "#/definitions/api.ErrorResponse"
-                        }
-                    },
-                    "500": {
-                        "description": "Internal Server Error",
-                        "schema": {
-                            "$ref": "#/definitions/api.ErrorResponse"
-                        }
-                    }
-                }
-            }
-        },
-        "/tags/": {
-            "delete": {
-                "security": [
-                    {
-                        "ApiKeyAuth": []
-                    }
-                ],
-                "description": "Removes tags by type and name. Name and Type of Tag(s) must match.\nTag Scope is required for matching, options: \"global\", \"admin\". Private tags can not be deleted via API.\nTag wills be removed from respective archive files.",
-                "consumes": [
-                    "application/json"
+                    "multipart/form-data"
                 ],
                 "produces": [
                     "text/plain"
                 ],
                 "tags": [
-                    "Tag remove"
+                    "User"
                 ],
-                "summary": "Removes all tags and job-relations for type:name tuple",
+                "summary": "Adds a new user",
                 "parameters": [
                     {
-                        "description": "Array of tag-objects to remove",
-                        "name": "request",
-                        "in": "body",
-                        "required": true,
-                        "schema": {
-                            "type": "array",
-                            "items": {
-                                "$ref": "#/definitions/api.ApiTag"
-                            }
-                        }
+                        "type": "string",
+                        "description": "Unique user ID",
+                        "name": "username",
+                        "in": "formData",
+                        "required": true
+                    },
+                    {
+                        "type": "string",
+                        "description": "User password",
+                        "name": "password",
+                        "in": "formData",
+                        "required": true
+                    },
+                    {
+                        "enum": [
+                            "admin",
+                            "support",
+                            "manager",
+                            "user",
+                            "api"
+                        ],
+                        "type": "string",
+                        "description": "User role",
+                        "name": "role",
+                        "in": "formData",
+                        "required": true
+                    },
+                    {
+                        "type": "string",
+                        "description": "Managed project, required for new manager role user",
+                        "name": "project",
+                        "in": "formData"
+                    },
+                    {
+                        "type": "string",
+                        "description": "Users name",
+                        "name": "name",
+                        "in": "formData"
+                    },
+                    {
+                        "type": "string",
+                        "description": "Users email",
+                        "name": "email",
+                        "in": "formData"
                     }
                 ],
                 "responses": {
@@ -1062,25 +1067,93 @@
                     "400": {
                         "description": "Bad Request",
                         "schema": {
-                            "$ref": "#/definitions/api.ErrorResponse"
+                            "type": "string"
                         }
                     },
                     "401": {
                         "description": "Unauthorized",
                         "schema": {
-                            "$ref": "#/definitions/api.ErrorResponse"
+                            "type": "string"
                         }
                     },
-                    "404": {
-                        "description": "Job or tag does not exist",
+                    "403": {
+                        "description": "Forbidden",
                         "schema": {
-                            "$ref": "#/definitions/api.ErrorResponse"
+                            "type": "string"
+                        }
+                    },
+                    "422": {
+                        "description": "Unprocessable Entity: creating user failed",
+                        "schema": {
+                            "type": "string"
                         }
                     },
                     "500": {
                         "description": "Internal Server Error",
                         "schema": {
-                            "$ref": "#/definitions/api.ErrorResponse"
+                            "type": "string"
+                        }
+                    }
+                }
+            },
+            "delete": {
+                "security": [
+                    {
+                        "ApiKeyAuth": []
+                    }
+                ],
+                "description": "User defined by username in form data will be deleted from database.\nOnly accessible from IPs registered with apiAllowedIPs configuration option.",
+                "consumes": [
+                    "multipart/form-data"
+                ],
+                "produces": [
+                    "text/plain"
+                ],
+                "tags": [
+                    "User"
+                ],
+                "summary": "Deletes a user",
+                "parameters": [
+                    {
+                        "type": "string",
+                        "description": "User ID to delete",
+                        "name": "username",
+                        "in": "formData",
+                        "required": true
+                    }
+                ],
+                "responses": {
+                    "200": {
+                        "description": "User deleted successfully"
+                    },
+                    "400": {
+                        "description": "Bad Request",
+                        "schema": {
+                            "type": "string"
+                        }
+                    },
+                    "401": {
+                        "description": "Unauthorized",
+                        "schema": {
+                            "type": "string"
+                        }
+                    },
+                    "403": {
+                        "description": "Forbidden",
+                        "schema": {
+                            "type": "string"
+                        }
+                    },
+                    "422": {
+                        "description": "Unprocessable Entity: deleting user failed",
+                        "schema": {
+                            "type": "string"
+                        }
+                    },
+                    "500": {
+                        "description": "Internal Server Error",
+                        "schema": {
+                            "type": "string"
                         }
                     }
                 }
@@ -1134,14 +1207,6 @@
                 }
             }
         },
-        "api.DefaultApiResponse": {
-            "type": "object",
-            "properties": {
-                "msg": {
-                    "type": "string"
-                }
-            }
-        },
         "api.DeleteJobApiRequest": {
             "type": "object",
             "required": [
@@ -1165,6 +1230,14 @@
                 }
             }
         },
+        "api.DeleteJobApiResponse": {
+            "type": "object",
+            "properties": {
+                "msg": {
+                    "type": "string"
+                }
+            }
+        },
         "api.EditMetaRequest": {
             "type": "object",
             "properties": {
@@ -1228,7 +1301,7 @@
                     "description": "Array of jobs",
                     "type": "array",
                     "items": {
-                        "$ref": "#/definitions/schema.Job"
+                        "$ref": "#/definitions/schema.JobMeta"
                     }
                 },
                 "page": {
@@ -1251,35 +1324,11 @@
                 }
             }
         },
-        "api.Node": {
+        "api.StartJobApiResponse": {
             "type": "object",
             "properties": {
-                "cpusAllocated": {
-                    "type": "integer"
-                },
-                "cpusTotal": {
-                    "type": "integer"
-                },
-                "gpusAllocated": {
-                    "type": "integer"
-                },
-                "gpusTotal": {
-                    "type": "integer"
-                },
-                "hostname": {
+                "msg": {
                     "type": "string"
-                },
-                "memoryAllocated": {
-                    "type": "integer"
-                },
-                "memoryTotal": {
-                    "type": "integer"
-                },
-                "states": {
-                    "type": "array",
-                    "items": {
-                        "type": "string"
-                    }
                 }
             }
         },
@@ -1316,21 +1365,6 @@
                 }
             }
         },
-        "api.UpdateNodeStatesRequest": {
-            "type": "object",
-            "properties": {
-                "cluster": {
-                    "type": "string",
-                    "example": "fritz"
-                },
-                "nodes": {
-                    "type": "array",
-                    "items": {
-                        "$ref": "#/definitions/api.Node"
-                    }
-                }
-            }
-        },
         "schema.Accelerator": {
             "type": "object",
             "properties": {
@@ -1366,6 +1400,7 @@
             }
         },
         "schema.Job": {
+            "description": "Information of a HPC job.",
             "type": "object",
             "properties": {
                 "arrayJobId": {
@@ -1390,15 +1425,19 @@
                 "energyFootprint": {
                     "type": "object",
                     "additionalProperties": {
-                        "type": "number",
-                        "format": "float64"
+                        "type": "number"
                     }
                 },
+                "exclusive": {
+                    "type": "integer",
+                    "maximum": 2,
+                    "minimum": 0,
+                    "example": 1
+                },
                 "footprint": {
                     "type": "object",
                     "additionalProperties": {
-                        "type": "number",
-                        "format": "float64"
+                        "type": "number"
                     }
                 },
                 "id": {
@@ -1410,18 +1449,12 @@
                 },
                 "jobState": {
                     "enum": [
-                        "boot_fail",
-                        "cancelled",
                         "completed",
-                        "deadline",
                         "failed",
-                        "node_fail",
-                        "out-of-memory",
-                        "pending",
-                        "preempted",
-                        "running",
-                        "suspended",
-                        "timeout"
+                        "cancelled",
+                        "stopped",
+                        "timeout",
+                        "out_of_memory"
                     ],
                     "allOf": [
                         {
@@ -1465,48 +1498,23 @@
                     "type": "string",
                     "example": "abcd200"
                 },
-                "requestedMemory": {
-                    "description": "in MB",
-                    "type": "integer",
-                    "minimum": 1,
-                    "example": 128000
-                },
                 "resources": {
                     "type": "array",
                     "items": {
                         "$ref": "#/definitions/schema.Resource"
                     }
                 },
-                "shared": {
-                    "type": "string",
-                    "enum": [
-                        "none",
-                        "single_user",
-                        "multi_user"
-                    ]
-                },
                 "smt": {
                     "type": "integer",
                     "example": 4
                 },
                 "startTime": {
-                    "type": "integer",
-                    "example": 1649723812
-                },
-                "statistics": {
-                    "type": "object",
-                    "additionalProperties": {
-                        "$ref": "#/definitions/schema.JobStatistics"
-                    }
+                    "type": "string"
                 },
                 "subCluster": {
                     "type": "string",
                     "example": "main"
                 },
-                "submitTime": {
-                    "type": "integer",
-                    "example": 1649723812
-                },
                 "tags": {
                     "type": "array",
                     "items": {
@@ -1549,6 +1557,147 @@
                 }
             }
         },
+        "schema.JobMeta": {
+            "description": "Meta data information of a HPC job.",
+            "type": "object",
+            "properties": {
+                "arrayJobId": {
+                    "type": "integer",
+                    "example": 123000
+                },
+                "cluster": {
+                    "type": "string",
+                    "example": "fritz"
+                },
+                "concurrentJobs": {
+                    "$ref": "#/definitions/schema.JobLinkResultList"
+                },
+                "duration": {
+                    "type": "integer",
+                    "minimum": 1,
+                    "example": 43200
+                },
+                "energy": {
+                    "type": "number"
+                },
+                "energyFootprint": {
+                    "type": "object",
+                    "additionalProperties": {
+                        "type": "number"
+                    }
+                },
+                "exclusive": {
+                    "type": "integer",
+                    "maximum": 2,
+                    "minimum": 0,
+                    "example": 1
+                },
+                "footprint": {
+                    "type": "object",
+                    "additionalProperties": {
+                        "type": "number"
+                    }
+                },
+                "id": {
+                    "type": "integer"
+                },
+                "jobId": {
+                    "type": "integer",
+                    "example": 123000
+                },
+                "jobState": {
+                    "enum": [
+                        "completed",
+                        "failed",
+                        "cancelled",
+                        "stopped",
+                        "timeout",
+                        "out_of_memory"
+                    ],
+                    "allOf": [
+                        {
+                            "$ref": "#/definitions/schema.JobState"
+                        }
+                    ],
+                    "example": "completed"
+                },
+                "metaData": {
+                    "type": "object",
+                    "additionalProperties": {
+                        "type": "string"
+                    }
+                },
+                "monitoringStatus": {
+                    "type": "integer",
+                    "maximum": 3,
+                    "minimum": 0,
+                    "example": 1
+                },
+                "numAcc": {
+                    "type": "integer",
+                    "minimum": 1,
+                    "example": 2
+                },
+                "numHwthreads": {
+                    "type": "integer",
+                    "minimum": 1,
+                    "example": 20
+                },
+                "numNodes": {
+                    "type": "integer",
+                    "minimum": 1,
+                    "example": 2
+                },
+                "partition": {
+                    "type": "string",
+                    "example": "main"
+                },
+                "project": {
+                    "type": "string",
+                    "example": "abcd200"
+                },
+                "resources": {
+                    "type": "array",
+                    "items": {
+                        "$ref": "#/definitions/schema.Resource"
+                    }
+                },
+                "smt": {
+                    "type": "integer",
+                    "example": 4
+                },
+                "startTime": {
+                    "type": "integer",
+                    "minimum": 1,
+                    "example": 1649723812
+                },
+                "statistics": {
+                    "type": "object",
+                    "additionalProperties": {
+                        "$ref": "#/definitions/schema.JobStatistics"
+                    }
+                },
+                "subCluster": {
+                    "type": "string",
+                    "example": "main"
+                },
+                "tags": {
+                    "type": "array",
+                    "items": {
+                        "$ref": "#/definitions/schema.Tag"
+                    }
+                },
+                "user": {
+                    "type": "string",
+                    "example": "abcd100h"
+                },
+                "walltime": {
+                    "type": "integer",
+                    "minimum": 1,
+                    "example": 86400
+                }
+            }
+        },
         "schema.JobMetric": {
             "type": "object",
             "properties": {
@@ -1572,32 +1721,24 @@
         "schema.JobState": {
             "type": "string",
             "enum": [
-                "boot_fail",
-                "cancelled",
-                "completed",
-                "deadline",
-                "failed",
-                "node_fail",
-                "out_of_memory",
-                "pending",
-                "preempted",
                 "running",
-                "suspended",
-                "timeout"
+                "completed",
+                "failed",
+                "cancelled",
+                "stopped",
+                "timeout",
+                "preempted",
+                "out_of_memory"
             ],
             "x-enum-varnames": [
-                "JobStateBootFail",
-                "JobStateCancelled",
-                "JobStateCompleted",
-                "JobStateDeadline",
-                "JobStateFailed",
-                "JobStateNodeFail",
-                "JobStateOutOfMemory",
-                "JobStatePending",
-                "JobStatePreempted",
                 "JobStateRunning",
-                "JobStateSuspended",
-                "JobStateTimeout"
+                "JobStateCompleted",
+                "JobStateFailed",
+                "JobStateCancelled",
+                "JobStateStopped",
+                "JobStateTimeout",
+                "JobStatePreempted",
+                "JobStateOutOfMemory"
             ]
         },
         "schema.JobStatistics": {
@@ -1796,8 +1937,7 @@
                     "additionalProperties": {
                         "type": "array",
                         "items": {
-                            "type": "number",
-                            "format": "float64"
+                            "type": "number"
                         }
                     }
                 }
@@ -1885,9 +2025,6 @@
                 },
                 "remove": {
                     "type": "boolean"
-                },
-                "unit": {
-                    "$ref": "#/definitions/schema.Unit"
                 }
             }
         },

api/swagger.yaml

@@ -1,3 +1,4 @@
+basePath: /api
 definitions:
   api.ApiReturnedUser:
     properties:
@@ -31,11 +32,6 @@ definitions:
         example: Debug
         type: string
     type: object
-  api.DefaultApiResponse:
-    properties:
-      msg:
-        type: string
-    type: object
   api.DeleteJobApiRequest:
     properties:
       cluster:
@@ -53,6 +49,11 @@ definitions:
     required:
     - jobId
     type: object
+  api.DeleteJobApiResponse:
+    properties:
+      msg:
+        type: string
+    type: object
   api.EditMetaRequest:
     properties:
       key:
@@ -96,7 +97,7 @@ definitions:
       jobs:
         description: Array of jobs
         items:
-          $ref: '#/definitions/schema.Job'
+          $ref: '#/definitions/schema.JobMeta'
         type: array
       page:
         description: Page id returned
@@ -111,26 +112,10 @@ definitions:
       scope:
         $ref: '#/definitions/schema.MetricScope'
     type: object
-  api.Node:
+  api.StartJobApiResponse:
     properties:
-      cpusAllocated:
-        type: integer
-      cpusTotal:
-        type: integer
-      gpusAllocated:
-        type: integer
-      gpusTotal:
-        type: integer
-      hostname:
+      msg:
         type: string
-      memoryAllocated:
-        type: integer
-      memoryTotal:
-        type: integer
-      states:
-        items:
-          type: string
-        type: array
     type: object
   api.StopJobApiRequest:
     properties:
@@ -154,16 +139,6 @@ definitions:
     - jobState
     - stopTime
     type: object
-  api.UpdateNodeStatesRequest:
-    properties:
-      cluster:
-        example: fritz
-        type: string
-      nodes:
-        items:
-          $ref: '#/definitions/api.Node'
-        type: array
-    type: object
   schema.Accelerator:
     properties:
       id:
@@ -187,6 +162,7 @@ definitions:
         type: array
     type: object
   schema.Job:
+    description: Information of a HPC job.
     properties:
       arrayJobId:
         example: 123000
@@ -204,12 +180,15 @@ definitions:
         type: number
       energyFootprint:
         additionalProperties:
-          format: float64
           type: number
         type: object
+      exclusive:
+        example: 1
+        maximum: 2
+        minimum: 0
+        type: integer
       footprint:
         additionalProperties:
-          format: float64
           type: number
         type: object
       id:
@@ -221,18 +200,12 @@ definitions:
         allOf:
         - $ref: '#/definitions/schema.JobState'
         enum:
-        - boot_fail
-        - cancelled
         - completed
-        - deadline
         - failed
-        - node_fail
-        - out-of-memory
-        - pending
-        - preempted
-        - running
-        - suspended
+        - cancelled
+        - stopped
         - timeout
+        - out_of_memory
         example: completed
       metaData:
         additionalProperties:
@@ -261,37 +234,18 @@ definitions:
       project:
         example: abcd200
         type: string
-      requestedMemory:
-        description: in MB
-        example: 128000
-        minimum: 1
-        type: integer
       resources:
         items:
           $ref: '#/definitions/schema.Resource'
         type: array
-      shared:
-        enum:
-        - none
-        - single_user
-        - multi_user
-        type: string
       smt:
         example: 4
         type: integer
       startTime:
-        example: 1649723812
-        type: integer
-      statistics:
-        additionalProperties:
-          $ref: '#/definitions/schema.JobStatistics'
-        type: object
+        type: string
       subCluster:
         example: main
         type: string
-      submitTime:
-        example: 1649723812
-        type: integer
       tags:
         items:
           $ref: '#/definitions/schema.Tag'
@@ -320,6 +274,109 @@ definitions:
           $ref: '#/definitions/schema.JobLink'
         type: array
     type: object
+  schema.JobMeta:
+    description: Meta data information of a HPC job.
+    properties:
+      arrayJobId:
+        example: 123000
+        type: integer
+      cluster:
+        example: fritz
+        type: string
+      concurrentJobs:
+        $ref: '#/definitions/schema.JobLinkResultList'
+      duration:
+        example: 43200
+        minimum: 1
+        type: integer
+      energy:
+        type: number
+      energyFootprint:
+        additionalProperties:
+          type: number
+        type: object
+      exclusive:
+        example: 1
+        maximum: 2
+        minimum: 0
+        type: integer
+      footprint:
+        additionalProperties:
+          type: number
+        type: object
+      id:
+        type: integer
+      jobId:
+        example: 123000
+        type: integer
+      jobState:
+        allOf:
+        - $ref: '#/definitions/schema.JobState'
+        enum:
+        - completed
+        - failed
+        - cancelled
+        - stopped
+        - timeout
+        - out_of_memory
+        example: completed
+      metaData:
+        additionalProperties:
+          type: string
+        type: object
+      monitoringStatus:
+        example: 1
+        maximum: 3
+        minimum: 0
+        type: integer
+      numAcc:
+        example: 2
+        minimum: 1
+        type: integer
+      numHwthreads:
+        example: 20
+        minimum: 1
+        type: integer
+      numNodes:
+        example: 2
+        minimum: 1
+        type: integer
+      partition:
+        example: main
+        type: string
+      project:
+        example: abcd200
+        type: string
+      resources:
+        items:
+          $ref: '#/definitions/schema.Resource'
+        type: array
+      smt:
+        example: 4
+        type: integer
+      startTime:
+        example: 1649723812
+        minimum: 1
+        type: integer
+      statistics:
+        additionalProperties:
+          $ref: '#/definitions/schema.JobStatistics'
+        type: object
+      subCluster:
+        example: main
+        type: string
+      tags:
+        items:
+          $ref: '#/definitions/schema.Tag'
+        type: array
+      user:
+        example: abcd100h
+        type: string
+      walltime:
+        example: 86400
+        minimum: 1
+        type: integer
+    type: object
   schema.JobMetric:
     properties:
       series:
@@ -335,32 +392,24 @@ definitions:
     type: object
   schema.JobState:
     enum:
-    - boot_fail
-    - cancelled
-    - completed
-    - deadline
-    - failed
-    - node_fail
-    - out_of_memory
-    - pending
-    - preempted
     - running
-    - suspended
+    - completed
+    - failed
+    - cancelled
+    - stopped
     - timeout
+    - preempted
+    - out_of_memory
     type: string
     x-enum-varnames:
-    - JobStateBootFail
-    - JobStateCancelled
-    - JobStateCompleted
-    - JobStateDeadline
-    - JobStateFailed
-    - JobStateNodeFail
-    - JobStateOutOfMemory
-    - JobStatePending
-    - JobStatePreempted
     - JobStateRunning
-    - JobStateSuspended
+    - JobStateCompleted
+    - JobStateFailed
+    - JobStateCancelled
+    - JobStateStopped
     - JobStateTimeout
+    - JobStatePreempted
+    - JobStateOutOfMemory
   schema.JobStatistics:
     description: Specification for job metric statistics.
     properties:
@@ -497,7 +546,6 @@ definitions:
       percentiles:
         additionalProperties:
           items:
-            format: float64
             type: number
           type: array
         type: object
@@ -557,8 +605,6 @@ definitions:
         type: number
       remove:
         type: boolean
-      unit:
-        $ref: '#/definitions/schema.Unit'
     type: object
   schema.Tag:
     description: Defines a tag using name and type.
@@ -630,7 +676,7 @@ info:
   title: ClusterCockpit REST API
   version: 1.0.0
 paths:
-  /api/clusters/:
+  /clusters/:
     get:
       description: Get a list of all cluster configs. Specific cluster can be requested
         using query parameter.
@@ -667,7 +713,7 @@ paths:
       summary: Lists all cluster configs
       tags:
       - Cluster query
-  /api/jobs/:
+  /jobs/:
     get:
       description: |-
         Get a list of all jobs. Filters can be applied using query parameters.
@@ -732,11 +778,11 @@ paths:
       summary: Lists all jobs
       tags:
       - Job query
-  /api/jobs/{id}:
+  /jobs/{id}:
     get:
       description: |-
         Job to get is specified by database ID
-        Returns full job resource information according to 'Job' scheme and all metrics according to 'JobData'.
+        Returns full job resource information according to 'JobMeta' scheme and all metrics according to 'JobData'.
       parameters:
       - description: Database ID of Job
         in: path
@@ -789,7 +835,7 @@ paths:
       - application/json
       description: |-
         Job to get is specified by database ID
-        Returns full job resource information according to 'Job' scheme and all metrics according to 'JobData'.
+        Returns full job resource information according to 'JobMeta' scheme and all metrics according to 'JobData'.
       parameters:
       - description: Database ID of Job
         in: path
@@ -841,7 +887,7 @@ paths:
       summary: Get job meta and configurable metric data
       tags:
       - Job query
-  /api/jobs/delete_job/:
+  /jobs/delete_job/:
     delete:
       consumes:
       - application/json
@@ -860,7 +906,7 @@ paths:
         "200":
           description: Success message
           schema:
-            $ref: '#/definitions/api.DefaultApiResponse'
+            $ref: '#/definitions/api.DeleteJobApiResponse'
         "400":
           description: Bad Request
           schema:
@@ -891,7 +937,7 @@ paths:
       summary: Remove a job from the sql database
       tags:
       - Job remove
-  /api/jobs/delete_job/{id}:
+  /jobs/delete_job/{id}:
     delete:
       description: Job to remove is specified by database ID. This will not remove
         the job from the job archive.
@@ -907,7 +953,7 @@ paths:
         "200":
           description: Success message
           schema:
-            $ref: '#/definitions/api.DefaultApiResponse'
+            $ref: '#/definitions/api.DeleteJobApiResponse'
         "400":
           description: Bad Request
           schema:
@@ -938,7 +984,7 @@ paths:
       summary: Remove a job from the sql database
       tags:
       - Job remove
-  /api/jobs/delete_job_before/{ts}:
+  /jobs/delete_job_before/{ts}:
     delete:
       description: Remove all jobs with start time before timestamp. The jobs will
         not be removed from the job archive.
@@ -954,7 +1000,7 @@ paths:
         "200":
           description: Success message
           schema:
-            $ref: '#/definitions/api.DefaultApiResponse'
+            $ref: '#/definitions/api.DeleteJobApiResponse'
         "400":
           description: Bad Request
           schema:
@@ -985,7 +1031,7 @@ paths:
       summary: Remove a job from the sql database
       tags:
       - Job remove
-  /api/jobs/edit_meta/{id}:
+  /jobs/edit_meta/{id}:
     post:
       consumes:
       - application/json
@@ -1032,27 +1078,27 @@ paths:
       summary: Edit meta-data json
       tags:
       - Job add and modify
-  /api/jobs/start_job/:
+  /jobs/start_job/:
     post:
       consumes:
       - application/json
       description: |-
         Job specified in request body will be saved to database as "running" with new DB ID.
-        Job specifications follow the 'Job' scheme, API will fail to execute if requirements are not met.
+        Job specifications follow the 'JobMeta' scheme, API will fail to execute if requirements are not met.
       parameters:
       - description: Job to add
         in: body
         name: request
         required: true
         schema:
-          $ref: '#/definitions/schema.Job'
+          $ref: '#/definitions/schema.JobMeta'
       produces:
       - application/json
       responses:
         "201":
           description: Job added successfully
           schema:
-            $ref: '#/definitions/api.DefaultApiResponse'
+            $ref: '#/definitions/api.StartJobApiResponse'
         "400":
           description: Bad Request
           schema:
@@ -1079,11 +1125,11 @@ paths:
       summary: Adds a new job as "running"
       tags:
       - Job add and modify
-  /api/jobs/stop_job/:
+  /jobs/stop_job/:
     post:
       description: |-
         Job to stop is specified by request body. All fields are required in this case.
-        Returns full job resource information according to 'Job' scheme.
+        Returns full job resource information according to 'JobMeta' scheme.
       parameters:
       - description: All fields required
         in: body
@@ -1097,7 +1143,7 @@ paths:
         "200":
           description: Success message
           schema:
-            $ref: '#/definitions/schema.Job'
+            $ref: '#/definitions/schema.JobMeta'
         "400":
           description: Bad Request
           schema:
@@ -1115,7 +1161,8 @@ paths:
           schema:
             $ref: '#/definitions/api.ErrorResponse'
         "422":
-          description: 'Unprocessable Entity: job has already been stopped'
+          description: 'Unprocessable Entity: finding job failed: sql: no rows in
+            result set'
           schema:
             $ref: '#/definitions/api.ErrorResponse'
         "500":
@@ -1127,7 +1174,7 @@ paths:
       summary: Marks job as completed and triggers archiving
       tags:
       - Job add and modify
-  /api/jobs/tag_job/{id}:
+  /jobs/tag_job/{id}:
     post:
       consumes:
       - application/json
@@ -1177,51 +1224,128 @@ paths:
       summary: Adds one or more tags to a job
       tags:
       - Job add and modify
-  /api/nodestats/:
+  /user/{id}:
     post:
+      consumes:
+      - multipart/form-data
       description: |-
-        Returns a JSON-encoded list of users.
-        Required query-parameter defines if all users or only users with additional special roles are returned.
+        Modifies user defined by username (id) in one of four possible ways.
+        If more than one formValue is set then only the highest priority field is used.
+        Only accessible from IPs registered with apiAllowedIPs configuration option.
       parameters:
-      - description: Request body containing nodes and their states
-        in: body
-        name: request
+      - description: Database ID of User
+        in: path
+        name: id
         required: true
-        schema:
-          $ref: '#/definitions/api.UpdateNodeStatesRequest'
+        type: string
+      - description: 'Priority 1: Role to add'
+        enum:
+        - admin
+        - support
+        - manager
+        - user
+        - api
+        in: formData
+        name: add-role
+        type: string
+      - description: 'Priority 2: Role to remove'
+        enum:
+        - admin
+        - support
+        - manager
+        - user
+        - api
+        in: formData
+        name: remove-role
+        type: string
+      - description: 'Priority 3: Project to add'
+        in: formData
+        name: add-project
+        type: string
+      - description: 'Priority 4: Project to remove'
+        in: formData
+        name: remove-project
+        type: string
       produces:
-      - application/json
+      - text/plain
       responses:
         "200":
-          description: Success message
+          description: Success Response Message
           schema:
-            $ref: '#/definitions/api.DefaultApiResponse'
+            type: string
         "400":
           description: Bad Request
           schema:
-            $ref: '#/definitions/api.ErrorResponse'
+            type: string
         "401":
           description: Unauthorized
           schema:
-            $ref: '#/definitions/api.ErrorResponse'
+            type: string
         "403":
           description: Forbidden
           schema:
-            $ref: '#/definitions/api.ErrorResponse'
+            type: string
+        "422":
+          description: 'Unprocessable Entity: The user could not be updated'
+          schema:
+            type: string
         "500":
           description: Internal Server Error
           schema:
-            $ref: '#/definitions/api.ErrorResponse'
+            type: string
       security:
       - ApiKeyAuth: []
-      summary: Deliver updated Slurm node states
+      summary: Updates an existing user
       tags:
-      - Nodestates
-  /api/users/:
+      - User
+  /users/:
+    delete:
+      consumes:
+      - multipart/form-data
+      description: |-
+        User defined by username in form data will be deleted from database.
+        Only accessible from IPs registered with apiAllowedIPs configuration option.
+      parameters:
+      - description: User ID to delete
+        in: formData
+        name: username
+        required: true
+        type: string
+      produces:
+      - text/plain
+      responses:
+        "200":
+          description: User deleted successfully
+        "400":
+          description: Bad Request
+          schema:
+            type: string
+        "401":
+          description: Unauthorized
+          schema:
+            type: string
+        "403":
+          description: Forbidden
+          schema:
+            type: string
+        "422":
+          description: 'Unprocessable Entity: deleting user failed'
+          schema:
+            type: string
+        "500":
+          description: Internal Server Error
+          schema:
+            type: string
+      security:
+      - ApiKeyAuth: []
+      summary: Deletes a user
+      tags:
+      - User
     get:
       description: |-
         Returns a JSON-encoded list of users.
         Required query-parameter defines if all users or only users with additional special roles are returned.
+        Only accessible from IPs registered with apiAllowedIPs configuration option.
       parameters:
       - description: If returned list should contain all users or only users with
           additional special roles
@@ -1259,73 +1383,46 @@ paths:
       summary: Returns a list of users
       tags:
       - User
-  /jobs/tag_job/{id}:
-    delete:
+    post:
       consumes:
-      - application/json
+      - multipart/form-data
       description: |-
-        Removes tag(s) from a job specified by DB ID. Name and Type of Tag(s) must match.
-        Tag Scope is required for matching, options: "global", "admin". Private tags can not be deleted via API.
-        If tagged job is already finished: Tag will be removed from respective archive files.
+        User specified in form data will be saved to database.
+        Only accessible from IPs registered with apiAllowedIPs configuration option.
       parameters:
-      - description: Job Database ID
-        in: path
-        name: id
+      - description: Unique user ID
+        in: formData
+        name: username
         required: true
-        type: integer
-      - description: Array of tag-objects to remove
-        in: body
-        name: request
+        type: string
+      - description: User password
+        in: formData
+        name: password
         required: true
-        schema:
-          items:
-            $ref: '#/definitions/api.ApiTag'
-          type: array
-      produces:
-      - application/json
-      responses:
-        "200":
-          description: Updated job resource
-          schema:
-            $ref: '#/definitions/schema.Job'
-        "400":
-          description: Bad Request
-          schema:
-            $ref: '#/definitions/api.ErrorResponse'
-        "401":
-          description: Unauthorized
-          schema:
-            $ref: '#/definitions/api.ErrorResponse'
-        "404":
-          description: Job or tag does not exist
-          schema:
-            $ref: '#/definitions/api.ErrorResponse'
-        "500":
-          description: Internal Server Error
-          schema:
-            $ref: '#/definitions/api.ErrorResponse'
-      security:
-      - ApiKeyAuth: []
-      summary: Removes one or more tags from a job
-      tags:
-      - Job add and modify
-  /tags/:
-    delete:
-      consumes:
-      - application/json
-      description: |-
-        Removes tags by type and name. Name and Type of Tag(s) must match.
-        Tag Scope is required for matching, options: "global", "admin". Private tags can not be deleted via API.
-        Tag wills be removed from respective archive files.
-      parameters:
-      - description: Array of tag-objects to remove
-        in: body
-        name: request
+        type: string
+      - description: User role
+        enum:
+        - admin
+        - support
+        - manager
+        - user
+        - api
+        in: formData
+        name: role
         required: true
-        schema:
-          items:
-            $ref: '#/definitions/api.ApiTag'
-          type: array
+        type: string
+      - description: Managed project, required for new manager role user
+        in: formData
+        name: project
+        type: string
+      - description: Users name
+        in: formData
+        name: name
+        type: string
+      - description: Users email
+        in: formData
+        name: email
+        type: string
       produces:
       - text/plain
       responses:
@@ -1336,24 +1433,28 @@ paths:
         "400":
           description: Bad Request
           schema:
-            $ref: '#/definitions/api.ErrorResponse'
+            type: string
         "401":
           description: Unauthorized
           schema:
-            $ref: '#/definitions/api.ErrorResponse'
-        "404":
-          description: Job or tag does not exist
+            type: string
+        "403":
+          description: Forbidden
           schema:
-            $ref: '#/definitions/api.ErrorResponse'
+            type: string
+        "422":
+          description: 'Unprocessable Entity: creating user failed'
+          schema:
+            type: string
         "500":
           description: Internal Server Error
           schema:
-            $ref: '#/definitions/api.ErrorResponse'
+            type: string
       security:
       - ApiKeyAuth: []
-      summary: Removes all tags and job-relations for type:name tuple
+      summary: Adds a new user
       tags:
-      - Tag remove
+      - User
 securityDefinitions:
   ApiKeyAuth:
     in: header

cmd/cc-backend/cli.go

@@ -1,5 +1,5 @@
 // Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved. This file is part of cc-backend.
+// All rights reserved.
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 package main
@@ -7,13 +7,12 @@ package main
 import "flag"
 
 var (
-	flagReinitDB, flagInit, flagServer, flagSyncLDAP, flagGops, flagMigrateDB, flagRevertDB,
-	flagForceDB, flagDev, flagVersion, flagLogDateTime, flagApplyTags bool
-	flagNewUser, flagDelUser, flagGenJWT, flagConfigFile, flagImportJob, flagLogLevel string
+	flagReinitDB, flagInit, flagServer, flagSyncLDAP, flagGops, flagMigrateDB, flagRevertDB, flagForceDB, flagDev, flagVersion, flagLogDateTime bool
+	flagNewUser, flagDelUser, flagGenJWT, flagConfigFile, flagImportJob, flagLogLevel                                                           string
 )
 
 func cliInit() {
-	flag.BoolVar(&flagInit, "init", false, "Setup var directory, initialize sqlite database file, config.json and .env")
+	flag.BoolVar(&flagInit, "init", false, "Setup var directory, initialize swlite database file, config.json and .env")
 	flag.BoolVar(&flagReinitDB, "init-db", false, "Go through job-archive and re-initialize the 'job', 'tag', and 'jobtag' tables (all running jobs will be lost!)")
 	flag.BoolVar(&flagSyncLDAP, "sync-ldap", false, "Sync the 'hpc_user' table with ldap")
 	flag.BoolVar(&flagServer, "server", false, "Start a server, continues listening on port after initialization and argument handling")
@@ -22,14 +21,13 @@ func cliInit() {
 	flag.BoolVar(&flagVersion, "version", false, "Show version information and exit")
 	flag.BoolVar(&flagMigrateDB, "migrate-db", false, "Migrate database to supported version and exit")
 	flag.BoolVar(&flagRevertDB, "revert-db", false, "Migrate database to previous version and exit")
-	flag.BoolVar(&flagApplyTags, "apply-tags", false, "Run taggers on all completed jobs and exit")
 	flag.BoolVar(&flagForceDB, "force-db", false, "Force database version, clear dirty flag and exit")
 	flag.BoolVar(&flagLogDateTime, "logdate", false, "Set this flag to add date and time to log messages")
 	flag.StringVar(&flagConfigFile, "config", "./config.json", "Specify alternative path to `config.json`")
-	flag.StringVar(&flagNewUser, "add-user", "", "Add a new user. Argument format: <username>:[admin,support,manager,api,user]:<password>")
-	flag.StringVar(&flagDelUser, "del-user", "", "Remove a existing user. Argument format: <username>")
+	flag.StringVar(&flagNewUser, "add-user", "", "Add a new user. Argument format: `<username>:[admin,support,manager,api,user]:<password>`")
+	flag.StringVar(&flagDelUser, "del-user", "", "Remove user by `username`")
 	flag.StringVar(&flagGenJWT, "jwt", "", "Generate and print a JWT for the user specified by its `username`")
 	flag.StringVar(&flagImportJob, "import-job", "", "Import a job. Argument format: `<path-to-meta.json>:<path-to-data.json>,...`")
-	flag.StringVar(&flagLogLevel, "loglevel", "warn", "Sets the logging level: `[debug, info (default), warn, err, crit]`")
+	flag.StringVar(&flagLogLevel, "loglevel", "warn", "Sets the logging level: `[debug,info,warn (default),err,fatal,crit]`")
 	flag.Parse()
 }

cmd/cc-backend/init.go

@@ -1,15 +1,16 @@
 // Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved. This file is part of cc-backend.
+// All rights reserved.
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 package main
 
 import (
+	"fmt"
 	"os"
 
 	"github.com/ClusterCockpit/cc-backend/internal/repository"
-	cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
-	"github.com/ClusterCockpit/cc-lib/util"
+	"github.com/ClusterCockpit/cc-backend/internal/util"
+	"github.com/ClusterCockpit/cc-backend/pkg/log"
 )
 
 const envString = `
@@ -32,18 +33,6 @@ const configString = `
     "jwts": {
         "max-age": "2000h"
     },
-    "apiAllowedIPs": [
-      "*"
-    ],
-    "enable-resampling": {
-      "trigger": 30,
-      "resolutions": [
-        600,
-        300,
-        120,
-        60
-      ]
-    },
     "clusters": [
         {
             "name": "name",
@@ -73,23 +62,24 @@ const configString = `
 
 func initEnv() {
 	if util.CheckFileExists("var") {
-		cclog.Exit("Directory ./var already exists. Cautiously exiting application initialization.")
+		fmt.Print("Directory ./var already exists. Exiting!\n")
+		os.Exit(0)
 	}
 
 	if err := os.WriteFile("config.json", []byte(configString), 0o666); err != nil {
-		cclog.Abortf("Could not write default ./config.json with permissions '0o666'. Application initialization failed, exited.\nError: %s\n", err.Error())
+		log.Fatalf("Writing config.json failed: %s", err.Error())
 	}
 
 	if err := os.WriteFile(".env", []byte(envString), 0o666); err != nil {
-		cclog.Abortf("Could not write default ./.env file with permissions '0o666'. Application initialization failed, exited.\nError: %s\n", err.Error())
+		log.Fatalf("Writing .env failed: %s", err.Error())
 	}
 
 	if err := os.Mkdir("var", 0o777); err != nil {
-		cclog.Abortf("Could not create default ./var folder with permissions '0o777'. Application initialization failed, exited.\nError: %s\n", err.Error())
+		log.Fatalf("Mkdir var failed: %s", err.Error())
 	}
 
 	err := repository.MigrateDB("sqlite3", "./var/job.db")
 	if err != nil {
-		cclog.Abortf("Could not initialize default sqlite3 database as './var/job.db'. Application initialization failed, exited.\nError: %s\n", err.Error())
+		log.Fatalf("Initialize job.db failed: %s", err.Error())
 	}
 }

cmd/cc-backend/main.go

@@ -1,11 +1,10 @@
 // Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved. This file is part of cc-backend.
+// All rights reserved.
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 package main
 
 import (
-	"encoding/json"
 	"fmt"
 	"os"
 	"os/signal"
@@ -18,20 +17,14 @@ import (
 	"github.com/ClusterCockpit/cc-backend/internal/auth"
 	"github.com/ClusterCockpit/cc-backend/internal/config"
 	"github.com/ClusterCockpit/cc-backend/internal/importer"
-	"github.com/ClusterCockpit/cc-backend/internal/memorystore"
 	"github.com/ClusterCockpit/cc-backend/internal/metricdata"
 	"github.com/ClusterCockpit/cc-backend/internal/repository"
-	"github.com/ClusterCockpit/cc-backend/internal/tagger"
 	"github.com/ClusterCockpit/cc-backend/internal/taskManager"
 	"github.com/ClusterCockpit/cc-backend/pkg/archive"
-	"github.com/ClusterCockpit/cc-backend/web"
-	ccconf "github.com/ClusterCockpit/cc-lib/ccConfig"
-	cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
-	"github.com/ClusterCockpit/cc-lib/runtimeEnv"
-	"github.com/ClusterCockpit/cc-lib/schema"
-	"github.com/ClusterCockpit/cc-lib/util"
+	"github.com/ClusterCockpit/cc-backend/pkg/log"
+	"github.com/ClusterCockpit/cc-backend/pkg/runtimeEnv"
+	"github.com/ClusterCockpit/cc-backend/pkg/schema"
 	"github.com/google/gops/agent"
-	"github.com/joho/godotenv"
 
 	_ "github.com/go-sql-driver/mysql"
 	_ "github.com/mattn/go-sqlite3"
@@ -65,102 +58,87 @@ func main() {
 		os.Exit(0)
 	}
 
-	cclog.Init(flagLogLevel, flagLogDateTime)
-
-	// If init flag set, run tasks here before any file dependencies cause errors
-	if flagInit {
-		initEnv()
-		cclog.Exit("Successfully setup environment!\n" +
-			"Please review config.json and .env and adjust it to your needs.\n" +
-			"Add your job-archive at ./var/job-archive.")
-	}
+	// Apply config flags for pkg/log
+	log.Init(flagLogLevel, flagLogDateTime)
 
 	// See https://github.com/google/gops (Runtime overhead is almost zero)
 	if flagGops {
 		if err := agent.Listen(agent.Options{}); err != nil {
-			cclog.Abortf("Could not start gops agent with 'gops/agent.Listen(agent.Options{})'. Application startup failed, exited.\nError: %s\n", err.Error())
+			log.Fatalf("gops/agent.Listen failed: %s", err.Error())
 		}
 	}
 
-	err := godotenv.Load()
-	if err != nil {
-		cclog.Abortf("Could not parse existing .env file at location './.env'. Application startup failed, exited.\nError: %s\n", err.Error())
+	if err := runtimeEnv.LoadEnv("./.env"); err != nil && !os.IsNotExist(err) {
+		log.Fatalf("parsing './.env' file failed: %s", err.Error())
 	}
 
 	// Initialize sub-modules and handle command line flags.
 	// The order here is important!
-	ccconf.Init(flagConfigFile)
+	config.Init(flagConfigFile)
 
-	// Load and check main configuration
-	if cfg := ccconf.GetPackageConfig("main"); cfg != nil {
-		if clustercfg := ccconf.GetPackageConfig("clusters"); clustercfg != nil {
-			config.Init(cfg, clustercfg)
-		} else {
-			cclog.Abort("Cluster configuration must be present")
-		}
-	} else {
-		cclog.Abort("Main configuration must be present")
+	// As a special case for `db`, allow using an environment variable instead of the value
+	// stored in the config. This can be done for people having security concerns about storing
+	// the password for their mysql database in config.json.
+	if strings.HasPrefix(config.Keys.DB, "env:") {
+		envvar := strings.TrimPrefix(config.Keys.DB, "env:")
+		config.Keys.DB = os.Getenv(envvar)
 	}
 
 	if flagMigrateDB {
 		err := repository.MigrateDB(config.Keys.DBDriver, config.Keys.DB)
 		if err != nil {
-			cclog.Abortf("MigrateDB Failed: Could not migrate '%s' database at location '%s' to version %d.\nError: %s\n", config.Keys.DBDriver, config.Keys.DB, repository.Version, err.Error())
+			log.Fatal(err)
 		}
-		cclog.Exitf("MigrateDB Success: Migrated '%s' database at location '%s' to version %d.\n", config.Keys.DBDriver, config.Keys.DB, repository.Version)
+		os.Exit(0)
 	}
 
 	if flagRevertDB {
 		err := repository.RevertDB(config.Keys.DBDriver, config.Keys.DB)
 		if err != nil {
-			cclog.Abortf("RevertDB Failed: Could not revert '%s' database at location '%s' to version %d.\nError: %s\n", config.Keys.DBDriver, config.Keys.DB, (repository.Version - 1), err.Error())
+			log.Fatal(err)
 		}
-		cclog.Exitf("RevertDB Success: Reverted '%s' database at location '%s' to version %d.\n", config.Keys.DBDriver, config.Keys.DB, (repository.Version - 1))
+		os.Exit(0)
 	}
 
 	if flagForceDB {
 		err := repository.ForceDB(config.Keys.DBDriver, config.Keys.DB)
 		if err != nil {
-			cclog.Abortf("ForceDB Failed: Could not force '%s' database at location '%s' to version %d.\nError: %s\n", config.Keys.DBDriver, config.Keys.DB, repository.Version, err.Error())
+			log.Fatal(err)
 		}
-		cclog.Exitf("ForceDB Success: Forced '%s' database at location '%s' to version %d.\n", config.Keys.DBDriver, config.Keys.DB, repository.Version)
+		os.Exit(0)
 	}
 
 	repository.Connect(config.Keys.DBDriver, config.Keys.DB)
 
+	if flagInit {
+		initEnv()
+		fmt.Print("Successfully setup environment!\n")
+		fmt.Print("Please review config.json and .env and adjust it to your needs.\n")
+		fmt.Print("Add your job-archive at ./var/job-archive.\n")
+		os.Exit(0)
+	}
+
 	if !config.Keys.DisableAuthentication {
 
-		if cfg := ccconf.GetPackageConfig("auth"); cfg != nil {
-			auth.Init(&cfg)
-		} else {
-			cclog.Warn("Authentication disabled due to missing configuration")
-			auth.Init(nil)
-		}
+		auth.Init()
 
 		if flagNewUser != "" {
 			parts := strings.SplitN(flagNewUser, ":", 3)
 			if len(parts) != 3 || len(parts[0]) == 0 {
-				cclog.Abortf("Add User: Could not parse supplied argument format: No changes.\n"+
-					"Want: <username>:[admin,support,manager,api,user]:<password>\n"+
-					"Have: %s\n", flagNewUser)
+				log.Fatal("invalid argument format for user creation")
 			}
 
 			ur := repository.GetUserRepository()
 			if err := ur.AddUser(&schema.User{
 				Username: parts[0], Projects: make([]string, 0), Password: parts[2], Roles: strings.Split(parts[1], ","),
 			}); err != nil {
-				cclog.Abortf("Add User: Could not add new user authentication for '%s' and roles '%s'.\nError: %s\n", parts[0], parts[1], err.Error())
-			} else {
-				cclog.Printf("Add User: Added new user '%s' with roles '%s'.\n", parts[0], parts[1])
+				log.Fatalf("adding '%s' user authentication failed: %v", parts[0], err)
 			}
 		}
-
 		if flagDelUser != "" {
 			ur := repository.GetUserRepository()
 			if err := ur.DelUser(flagDelUser); err != nil {
-				cclog.Abortf("Delete User: Could not delete user '%s' from DB.\nError: %s\n", flagDelUser, err.Error())
-			} else {
-				cclog.Printf("Delete User: Deleted user '%s' from DB.\n", flagDelUser)
+				log.Fatalf("deleting user failed: %v", err)
 			}
 		}
 
@@ -168,101 +146,68 @@ func main() {
 
 		if flagSyncLDAP {
 			if authHandle.LdapAuth == nil {
-				cclog.Abort("Sync LDAP: LDAP authentication is not configured, could not synchronize. No changes, exited.")
+				log.Fatal("cannot sync: LDAP authentication is not configured")
 			}
 
 			if err := authHandle.LdapAuth.Sync(); err != nil {
-				cclog.Abortf("Sync LDAP: Could not synchronize, failed with error.\nError: %s\n", err.Error())
+				log.Fatalf("LDAP sync failed: %v", err)
 			}
-			cclog.Print("Sync LDAP: LDAP synchronization successfull.")
+			log.Info("LDAP sync successfull")
 		}
 
 		if flagGenJWT != "" {
 			ur := repository.GetUserRepository()
 			user, err := ur.GetUser(flagGenJWT)
 			if err != nil {
-				cclog.Abortf("JWT: Could not get supplied user '%s' from DB. No changes, exited.\nError: %s\n", flagGenJWT, err.Error())
+				log.Fatalf("could not get user from JWT: %v", err)
 			}
 
 			if !user.HasRole(schema.RoleApi) {
-				cclog.Warnf("JWT: User '%s' does not have the role 'api'. REST API endpoints will return error!\n", user.Username)
+				log.Warnf("user '%s' does not have the API role", user.Username)
 			}
 
 			jwt, err := authHandle.JwtAuth.ProvideJWT(user)
 			if err != nil {
-				cclog.Abortf("JWT: User '%s' found in DB, but failed to provide JWT.\nError: %s\n", user.Username, err.Error())
+				log.Fatalf("failed to provide JWT to user '%s': %v", user.Username, err)
 			}
 
-			cclog.Printf("JWT: Successfully generated JWT for user '%s': %s\n", user.Username, jwt)
+			fmt.Printf("MAIN > JWT for '%s': %s\n", user.Username, jwt)
 		}
 
 	} else if flagNewUser != "" || flagDelUser != "" {
-		cclog.Abort("Error: Arguments '--add-user' and '--del-user' can only be used if authentication is enabled. No changes, exited.")
+		log.Fatal("arguments --add-user and --del-user can only be used if authentication is enabled")
 	}
 
-	if archiveCfg := ccconf.GetPackageConfig("archive"); archiveCfg != nil {
-		err = archive.Init(archiveCfg, config.Keys.DisableArchive)
-	} else {
-		err = archive.Init(json.RawMessage("{\"kind\":\"file\",\"path\":\"./var/job-archive\"}"), config.Keys.DisableArchive)
-	}
-	if err != nil {
-		cclog.Abortf("Init: Failed to initialize archive.\nError: %s\n", err.Error())
+	if err := archive.Init(config.Keys.Archive, config.Keys.DisableArchive); err != nil {
+		log.Fatalf("failed to initialize archive: %s", err.Error())
 	}
 
 	if err := metricdata.Init(); err != nil {
-		cclog.Abortf("Init: Failed to initialize metricdata repository.\nError %s\n", err.Error())
+		log.Fatalf("failed to initialize metricdata repository: %s", err.Error())
 	}
 
 	if flagReinitDB {
 		if err := importer.InitDB(); err != nil {
-			cclog.Abortf("Init DB: Failed to re-initialize repository DB.\nError: %s\n", err.Error())
-		} else {
-			cclog.Print("Init DB: Sucessfully re-initialized repository DB.")
+			log.Fatalf("failed to re-initialize repository DB: %s", err.Error())
 		}
 	}
 
 	if flagImportJob != "" {
 		if err := importer.HandleImportFlag(flagImportJob); err != nil {
-			cclog.Abortf("Import Job: Job import failed.\nError: %s\n", err.Error())
-		} else {
-			cclog.Printf("Import Job: Imported Job '%s' into DB.\n", flagImportJob)
-		}
-	}
-
-	if config.Keys.EnableJobTaggers {
-		tagger.Init()
-	}
-
-	if flagApplyTags {
-		if err := tagger.RunTaggers(); err != nil {
-			cclog.Abortf("Running job taggers.\nError: %s\n", err.Error())
+			log.Fatalf("job import failed: %s", err.Error())
 		}
 	}
 
 	if !flagServer {
-		cclog.Exit("No errors, server flag not set. Exiting cc-backend.")
+		return
 	}
 
+	archiver.Start(repository.GetJobRepository())
+	taskManager.Start()
+	serverInit()
+
 	var wg sync.WaitGroup
 
-	// Metric Store starts after all flags have been processes
-	if memorystore.InternalCCMSFlag {
-		if mscfg := ccconf.GetPackageConfig("metric-store"); mscfg != nil {
-			memorystore.Init(mscfg, &wg)
-		} else {
-			cclog.Abort("Metric Store configuration must be present")
-		}
-	}
-	archiver.Start(repository.GetJobRepository())
-
-	taskManager.Start(ccconf.GetPackageConfig("cron"),
-		ccconf.GetPackageConfig("archive"))
-
-	cfg := ccconf.GetPackageConfig("ui")
-	web.Init(cfg)
-
-	serverInit()
-
 	wg.Add(1)
 	go func() {
 		defer wg.Done()
@@ -279,8 +224,6 @@ func main() {
 
 		serverShutdown()
 
-		util.FsWatcherShutdown()
-
 		taskManager.Shutdown()
 	}()
 
@@ -289,5 +232,5 @@ func main() {
 	}
 	runtimeEnv.SystemdNotifiy(true, "running")
 	wg.Wait()
-	cclog.Print("Graceful shutdown completed!")
+	log.Print("Graceful shutdown completed!")
 }

cmd/cc-backend/server.go

@@ -1,5 +1,5 @@
 // Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved. This file is part of cc-backend.
+// All rights reserved.
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 package main
@@ -18,7 +18,6 @@ import (
 	"time"
 
 	"github.com/99designs/gqlgen/graphql/handler"
-	"github.com/99designs/gqlgen/graphql/handler/transport"
 	"github.com/99designs/gqlgen/graphql/playground"
 	"github.com/ClusterCockpit/cc-backend/internal/api"
 	"github.com/ClusterCockpit/cc-backend/internal/archiver"
@@ -26,11 +25,10 @@ import (
 	"github.com/ClusterCockpit/cc-backend/internal/config"
 	"github.com/ClusterCockpit/cc-backend/internal/graph"
 	"github.com/ClusterCockpit/cc-backend/internal/graph/generated"
-	"github.com/ClusterCockpit/cc-backend/internal/memorystore"
 	"github.com/ClusterCockpit/cc-backend/internal/routerConfig"
+	"github.com/ClusterCockpit/cc-backend/pkg/log"
+	"github.com/ClusterCockpit/cc-backend/pkg/runtimeEnv"
 	"github.com/ClusterCockpit/cc-backend/web"
-	cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
-	"github.com/ClusterCockpit/cc-lib/runtimeEnv"
 	"github.com/gorilla/handlers"
 	"github.com/gorilla/mux"
 	httpSwagger "github.com/swaggo/http-swagger"
@@ -55,29 +53,18 @@ func serverInit() {
 	// Setup the http.Handler/Router used by the server
 	graph.Init()
 	resolver := graph.GetResolverInstance()
-	graphQLServer := handler.New(
+	graphQLEndpoint := handler.NewDefaultServer(
 		generated.NewExecutableSchema(generated.Config{Resolvers: resolver}))
 
-	// graphQLServer.AddTransport(transport.SSE{})
-	graphQLServer.AddTransport(transport.POST{})
-	// graphQLServer.AddTransport(transport.Websocket{
-	// 	KeepAlivePingInterval: 10 * time.Second,
-	// 	Upgrader: websocket.Upgrader{
-	// 		CheckOrigin: func(r *http.Request) bool {
-	// 			return true
-	// 		},
-	// 	},
-	// })
-
 	if os.Getenv("DEBUG") != "1" {
 		// Having this handler means that a error message is returned via GraphQL instead of the connection simply beeing closed.
 		// The problem with this is that then, no more stacktrace is printed to stderr.
-		graphQLServer.SetRecoverFunc(func(ctx context.Context, err any) error {
+		graphQLEndpoint.SetRecoverFunc(func(ctx context.Context, err interface{}) error {
 			switch e := err.(type) {
 			case string:
 				return fmt.Errorf("MAIN > Panic: %s", e)
 			case error:
-				return fmt.Errorf("MAIN > Panic caused by: %s", e.Error())
+				return fmt.Errorf("MAIN > Panic caused by: %w", e)
 			}
 
 			return errors.New("MAIN > Internal server error (panic)")
@@ -91,10 +78,10 @@ func serverInit() {
 	router = mux.NewRouter()
 	buildInfo := web.Build{Version: version, Hash: commit, Buildtime: date}
 
-	info := map[string]any{}
+	info := map[string]interface{}{}
 	info["hasOpenIDConnect"] = false
 
-	if auth.Keys.OpenIDConfig != nil {
+	if config.Keys.OpenIDConfig != nil {
 		openIDConnect := auth.NewOIDC(authHandle)
 		openIDConnect.RegisterEndpoints(router)
 		info["hasOpenIDConnect"] = true
@@ -102,7 +89,7 @@ func serverInit() {
 
 	router.HandleFunc("/login", func(rw http.ResponseWriter, r *http.Request) {
 		rw.Header().Add("Content-Type", "text/html; charset=utf-8")
-		cclog.Debugf("##%v##", info)
+		log.Debugf("##%v##", info)
 		web.RenderTemplate(rw, "login.tmpl", &web.Page{Title: "Login", Build: buildInfo, Infos: info})
 	}).Methods(http.MethodGet)
 	router.HandleFunc("/imprint", func(rw http.ResponseWriter, r *http.Request) {
@@ -119,7 +106,6 @@ func serverInit() {
 	userapi := router.PathPrefix("/userapi").Subrouter()
 	configapi := router.PathPrefix("/config").Subrouter()
 	frontendapi := router.PathPrefix("/frontend").Subrouter()
-	metricstoreapi := router.PathPrefix("/metricstore").Subrouter()
 
 	if !config.Keys.DisableAuthentication {
 		router.Handle("/login", authHandle.Login(
@@ -185,7 +171,7 @@ func serverInit() {
 		})
 
 		securedapi.Use(func(next http.Handler) http.Handler {
-			return authHandle.AuthAPI(
+			return authHandle.AuthApi(
 				// On success;
 				next,
 				// On failure: JSON Response
@@ -193,15 +179,7 @@ func serverInit() {
 		})
 
 		userapi.Use(func(next http.Handler) http.Handler {
-			return authHandle.AuthUserAPI(
-				// On success;
-				next,
-				// On failure: JSON Response
-				onFailureResponse)
-		})
-
-		metricstoreapi.Use(func(next http.Handler) http.Handler {
-			return authHandle.AuthMetricStoreAPI(
+			return authHandle.AuthUserApi(
 				// On success;
 				next,
 				// On failure: JSON Response
@@ -209,7 +187,7 @@ func serverInit() {
 		})
 
 		configapi.Use(func(next http.Handler) http.Handler {
-			return authHandle.AuthConfigAPI(
+			return authHandle.AuthConfigApi(
 				// On success;
 				next,
 				// On failure: JSON Response
@@ -217,7 +195,7 @@ func serverInit() {
 		})
 
 		frontendapi.Use(func(next http.Handler) http.Handler {
-			return authHandle.AuthFrontendAPI(
+			return authHandle.AuthFrontendApi(
 				// On success;
 				next,
 				// On failure: JSON Response
@@ -230,7 +208,7 @@ func serverInit() {
 		router.PathPrefix("/swagger/").Handler(httpSwagger.Handler(
 			httpSwagger.URL("http://" + config.Keys.Addr + "/swagger/doc.json"))).Methods(http.MethodGet)
 	}
-	secured.Handle("/query", graphQLServer)
+	secured.Handle("/query", graphQLEndpoint)
 
 	// Send a searchId and then reply with a redirect to a user, or directly send query to job table for jobid and project.
 	secured.HandleFunc("/search", func(rw http.ResponseWriter, r *http.Request) {
@@ -244,18 +222,14 @@ func serverInit() {
 	apiHandle.MountConfigApiRoutes(configapi)
 	apiHandle.MountFrontendApiRoutes(frontendapi)
 
-	if memorystore.InternalCCMSFlag {
-		apiHandle.MountMetricStoreApiRoutes(metricstoreapi)
-	}
-
 	if config.Keys.EmbedStaticFiles {
 		if i, err := os.Stat("./var/img"); err == nil {
 			if i.IsDir() {
-				cclog.Info("Use local directory for static images")
+				log.Info("Use local directory for static images")
 				router.PathPrefix("/img/").Handler(http.StripPrefix("/img/", http.FileServer(http.Dir("./var/img"))))
 			}
 		}
-		router.PathPrefix("/").Handler(http.StripPrefix("/", web.ServeFiles()))
+		router.PathPrefix("/").Handler(web.ServeFiles())
 	} else {
 		router.PathPrefix("/").Handler(http.FileServer(http.Dir(config.Keys.StaticFiles)))
 	}
@@ -267,46 +241,17 @@ func serverInit() {
 		handlers.AllowedHeaders([]string{"X-Requested-With", "Content-Type", "Authorization", "Origin"}),
 		handlers.AllowedMethods([]string{"GET", "POST", "HEAD", "OPTIONS"}),
 		handlers.AllowedOrigins([]string{"*"})))
-
-	// secured.NotFoundHandler = http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
-	// 	page := web.Page{
-	// 		Title: "ClusterCockpit - Not Found",
-	// 		Build: buildInfo,
-	// 	}
-	// 	rw.Header().Add("Content-Type", "text/html; charset=utf-8")
-	// 	web.RenderTemplate(rw, "404.tmpl", &page)
-	// })
-
-	// secured.NotFoundHandler = http.HandlerFunc(http.NotFound)
-	// router.NotFoundHandler = router.NewRoute().HandlerFunc(http.NotFound).GetHandler()
-
-	// printEndpoints(router)
 }
 
-// func printEndpoints(r *mux.Router) {
-// 	r.Walk(func(route *mux.Route, router *mux.Router, ancestors []*mux.Route) error {
-// 		path, err := route.GetPathTemplate()
-// 		if err != nil {
-// 			path = "nopath"
-// 		}
-// 		methods, err := route.GetMethods()
-// 		if err != nil {
-// 			methods = append(methods, "nomethod")
-// 		}
-// 		fmt.Printf("%v %s\n", methods, path)
-// 		return nil
-// 	})
-// }
-
 func serverStart() {
 	handler := handlers.CustomLoggingHandler(io.Discard, router, func(_ io.Writer, params handlers.LogFormatterParams) {
 		if strings.HasPrefix(params.Request.RequestURI, "/api/") {
-			cclog.Debugf("%s %s (%d, %.02fkb, %dms)",
+			log.Debugf("%s %s (%d, %.02fkb, %dms)",
 				params.Request.Method, params.URL.RequestURI(),
 				params.StatusCode, float32(params.Size)/1024,
 				time.Since(params.TimeStamp).Milliseconds())
 		} else {
-			cclog.Debugf("%s %s (%d, %.02fkb, %dms)",
+			log.Debugf("%s %s (%d, %.02fkb, %dms)",
 				params.Request.Method, params.URL.RequestURI(),
 				params.StatusCode, float32(params.Size)/1024,
 				time.Since(params.TimeStamp).Milliseconds())
@@ -323,20 +268,20 @@ func serverStart() {
 	// Start http or https server
 	listener, err := net.Listen("tcp", config.Keys.Addr)
 	if err != nil {
-		cclog.Abortf("Server Start: Starting http listener on '%s' failed.\nError: %s\n", config.Keys.Addr, err.Error())
+		log.Fatalf("starting http listener failed: %v", err)
 	}
 
-	if !strings.HasSuffix(config.Keys.Addr, ":80") && config.Keys.RedirectHTTPTo != "" {
+	if !strings.HasSuffix(config.Keys.Addr, ":80") && config.Keys.RedirectHttpTo != "" {
 		go func() {
-			http.ListenAndServe(":80", http.RedirectHandler(config.Keys.RedirectHTTPTo, http.StatusMovedPermanently))
+			http.ListenAndServe(":80", http.RedirectHandler(config.Keys.RedirectHttpTo, http.StatusMovedPermanently))
 		}()
 	}
 
-	if config.Keys.HTTPSCertFile != "" && config.Keys.HTTPSKeyFile != "" {
+	if config.Keys.HttpsCertFile != "" && config.Keys.HttpsKeyFile != "" {
 		cert, err := tls.LoadX509KeyPair(
-			config.Keys.HTTPSCertFile, config.Keys.HTTPSKeyFile)
+			config.Keys.HttpsCertFile, config.Keys.HttpsKeyFile)
 		if err != nil {
-			cclog.Abortf("Server Start: Loading X509 keypair failed. Check options 'https-cert-file' and 'https-key-file' in 'config.json'.\nError: %s\n", err.Error())
+			log.Fatalf("loading X509 keypair failed: %v", err)
 		}
 		listener = tls.NewListener(listener, &tls.Config{
 			Certificates: []tls.Certificate{cert},
@@ -347,20 +292,20 @@ func serverStart() {
 			MinVersion:               tls.VersionTLS12,
 			PreferServerCipherSuites: true,
 		})
-		cclog.Printf("HTTPS server listening at %s...\n", config.Keys.Addr)
+		fmt.Printf("HTTPS server listening at %s...", config.Keys.Addr)
 	} else {
-		cclog.Printf("HTTP server listening at %s...\n", config.Keys.Addr)
+		fmt.Printf("HTTP server listening at %s...", config.Keys.Addr)
 	}
 	//
 	// Because this program will want to bind to a privileged port (like 80), the listener must
 	// be established first, then the user can be changed, and after that,
 	// the actual http server can be started.
 	if err := runtimeEnv.DropPrivileges(config.Keys.Group, config.Keys.User); err != nil {
-		cclog.Abortf("Server Start: Error while preparing server start.\nError: %s\n", err.Error())
+		log.Fatalf("error while preparing server start: %s", err.Error())
 	}
 
 	if err = server.Serve(listener); err != nil && err != http.ErrServerClosed {
-		cclog.Abortf("Server Start: Starting server failed.\nError: %s\n", err.Error())
+		log.Fatalf("starting server failed: %v", err)
 	}
 }
 
@@ -368,11 +313,6 @@ func serverShutdown() {
 	// First shut down the server gracefully (waiting for all ongoing requests)
 	server.Shutdown(context.Background())
 
-	// Archive all the metric store data
-	if memorystore.InternalCCMSFlag {
-		memorystore.Shutdown()
-	}
-
 	// Then, wait for any async archivings still pending...
 	archiver.WaitForArchiving()
 }

configs/config-demo.json

@@ -1,42 +1,30 @@
 {
-  "main": {
-    "addr": "127.0.0.1:8080",
-    "short-running-jobs-duration": 300,
-    "resampling": {
-      "trigger": 30,
-      "resolutions": [
-        600,
-        300,
-        120,
-        60
-      ]
-    },
-    "apiAllowedIPs": [
-      "*"
-    ],
-    "emission-constant": 317
-  },
-  "cron": {
-    "commit-job-worker": "2m",
-    "duration-worker": "5m",
-    "footprint-worker": "10m"
-  },
+  "addr": "127.0.0.1:8080",
+  "short-running-jobs-duration": 300,
   "archive": {
     "kind": "file",
     "path": "./var/job-archive"
   },
-  "auth": {
-    "jwts": {
-      "max-age": "2000h"
-    }
+  "jwts": {
+    "max-age": "2000h"
   },
+  "enable-resampling": {
+    "trigger": 30,
+    "resolutions": [
+      600,
+      300,
+      120,
+      60
+    ]
+  },
+  "emission-constant": 317,
   "clusters": [
     {
       "name": "fritz",
       "metricDataRepository": {
-        "kind": "cc-metric-store-internal",
+        "kind": "cc-metric-store",
         "url": "http://localhost:8082",
-        "token": "eyJ0eXAiOiJKV1QiLCJhbGciOiJFZERTQSJ9.eyJ1c2VyIjoiYWRtaW4iLCJyb2xlcyI6WyJST0xFX0FETUlOIiwiUk9MRV9BTkFMWVNUIiwiUk9MRV9VU0VSIl19.d-3_3FZTsadPjDEdsWrrQ7nS0edMAR4zjl-eK7rJU3HziNBfI9PDHDIpJVHTNN5E5SlLGLFXctWyKAkwhXL-Dw"
+        "token": ""
       },
       "filterRanges": {
         "numNodes": {
@@ -56,9 +44,9 @@
     {
       "name": "alex",
       "metricDataRepository": {
-        "kind": "cc-metric-store-internal",
+        "kind": "cc-metric-store",
         "url": "http://localhost:8082",
-        "token": "eyJ0eXAiOiJKV1QiLCJhbGciOiJFZERTQSJ9.eyJ1c2VyIjoiYWRtaW4iLCJyb2xlcyI6WyJST0xFX0FETUlOIiwiUk9MRV9BTkFMWVNUIiwiUk9MRV9VU0VSIl19.d-3_3FZTsadPjDEdsWrrQ7nS0edMAR4zjl-eK7rJU3HziNBfI9PDHDIpJVHTNN5E5SlLGLFXctWyKAkwhXL-Dw"
+        "token": ""
       },
       "filterRanges": {
         "numNodes": {
@@ -75,19 +63,5 @@
         }
       }
     }
-  ],
-  "metric-store": {
-    "checkpoints": {
-      "file-format": "avro",
-      "interval": "1h",
-      "directory": "./var/checkpoints",
-      "restore": "48h"
-    },
-    "archive": {
-      "interval": "1h",
-      "directory": "./var/archive"
-    },
-    "retention-in-memory": "48h"
-  },
-  "ui-file": "./configs/uiConfig.json"
-}
+  ]
+}

configs/config-mariadb.json

@@ -12,7 +12,12 @@
   "db": "clustercockpit:demo@tcp(127.0.0.1:3306)/clustercockpit",
   "enable-resampling": {
     "trigger": 30,
-    "resolutions": [600, 300, 120, 60]
+    "resolutions": [
+      600,
+      300,
+      120,
+      60
+    ]
   },
   "emission-constant": 317,
   "clusters": [

configs/config.json

@@ -1,50 +1,50 @@
 {
-  "main": {
     "addr": "0.0.0.0:443",
+    "ldap": {
+        "url": "ldaps://test",
+        "user_base": "ou=people,ou=hpc,dc=test,dc=de",
+        "search_dn": "cn=hpcmonitoring,ou=roadm,ou=profile,ou=hpc,dc=test,dc=de",
+        "user_bind": "uid={username},ou=people,ou=hpc,dc=test,dc=de",
+        "user_filter": "(&(objectclass=posixAccount))"
+    },
     "https-cert-file": "/etc/letsencrypt/live/url/fullchain.pem",
     "https-key-file": "/etc/letsencrypt/live/url/privkey.pem",
     "user": "clustercockpit",
     "group": "clustercockpit",
-    "validate": false,
-    "apiAllowedIPs": ["*"],
-    "short-running-jobs-duration": 300,
-    "resampling": {
-      "trigger": 30,
-      "resolutions": [600, 300, 120, 60]
-    }
-  },
-  "cron": {
-    "commit-job-worker": "2m",
-    "duration-worker": "5m",
-    "footprint-worker": "10m"
-  },
-  "archive": {
-    "kind": "file",
-    "path": "./var/job-archive"
-  },
-  "clusters": [
-    {
-      "name": "test",
-      "metricDataRepository": {
-        "kind": "cc-metric-store",
-        "url": "http://localhost:8082",
-        "token": "eyJhbGciOiJF-E-pQBQ"
-      },
-      "filterRanges": {
-        "numNodes": {
-          "from": 1,
-          "to": 64
-        },
-        "duration": {
-          "from": 0,
-          "to": 86400
-        },
-        "startTime": {
-          "from": "2022-01-01T00:00:00Z",
-          "to": null
+    "archive": {
+        "kind": "file",
+        "path": "./var/job-archive"
+    },
+    "validate": true,
+    "clusters": [
+        {
+            "name": "test",
+            "metricDataRepository": {
+                "kind": "cc-metric-store",
+                "url": "http://localhost:8082",
+                "token": "eyJhbGciOiJF-E-pQBQ"
+            },
+            "filterRanges": {
+                "numNodes": {
+                    "from": 1,
+                    "to": 64
+                },
+                "duration": {
+                    "from": 0,
+                    "to": 86400
+                },
+                "startTime": {
+                    "from": "2022-01-01T00:00:00Z",
+                    "to": null
+                }
+            }
         }
-      }
-    }
-  ]
+    ],
+    "jwts": {
+        "cookieName": "",
+        "validateUser": false,
+        "max-age": "2000h",
+        "trustedIssuer": ""
+    },
+    "short-running-jobs-duration": 300
 }
-

configs/default_metrics.json

@@ -0,0 +1,12 @@
+{
+  "clusters": [
+    {
+      "name": "fritz",
+      "default_metrics": "cpu_load, flops_any, core_power, lustre_open, mem_used, mem_bw, net_bytes_in"
+    },
+    {
+      "name": "alex",
+      "default_metrics": "flops_any, mem_bw, mem_used, vectorization_ratio"
+    }
+  ]
+}

configs/uiConfig.json

@@ -1,45 +0,0 @@
-{
-  "jobList": {
-    "usePaging": false,
-    "showFootprint":false
-  },
-  "jobView": {
-    "showPolarPlot": true,
-    "showFootprint": true,
-    "showRoofline": true,
-    "showStatTable": true
-  },
-  "metricConfig": {
-    "jobListMetrics": ["mem_bw", "flops_dp"],
-    "jobViewPlotMetrics": ["mem_bw", "flops_dp"],
-    "jobViewTableMetrics": ["mem_bw", "flops_dp"],
-    "clusters": [
-      {
-        "name": "test",
-        "subClusters": [
-          {
-            "name": "one",
-            "jobListMetrics": ["mem_used", "flops_sp"]
-          }
-        ]
-      }
-    ]
-  },
-  "nodeList": {
-    "usePaging": true
-  },
-  "plotConfiguration": {
-    "plotsPerRow": 3,
-    "colorBackground": true,
-    "lineWidth": 3,
-    "colorScheme": [
-	"#00bfff",
-	"#0000ff",
-	"#ff00ff",
-	"#ff0000",
-	"#ff8000",
-	"#ffff00",
-	"#80ff00"
-    ]
-  }
-}

go.mod

@@ -1,40 +1,36 @@
 module github.com/ClusterCockpit/cc-backend
 
-go 1.24.0
-
-toolchain go1.24.1
+go 1.23.5
 
 require (
-	github.com/99designs/gqlgen v0.17.81
-	github.com/ClusterCockpit/cc-lib v0.10.1
+	github.com/99designs/gqlgen v0.17.63
+	github.com/ClusterCockpit/cc-lib v0.0.0-20250216162506-b737e48423c5
+	github.com/ClusterCockpit/cc-units v0.4.0
 	github.com/Masterminds/squirrel v1.5.4
-	github.com/coreos/go-oidc/v3 v3.16.0
-	github.com/expr-lang/expr v1.17.6
-	github.com/go-co-op/gocron/v2 v2.16.0
-	github.com/go-ldap/ldap/v3 v3.4.10
-	github.com/go-sql-driver/mysql v1.9.0
-	github.com/golang-jwt/jwt/v5 v5.2.2
-	github.com/golang-migrate/migrate/v4 v4.18.2
+	github.com/coreos/go-oidc/v3 v3.11.0
+	github.com/go-co-op/gocron/v2 v2.9.0
+	github.com/go-ldap/ldap/v3 v3.4.8
+	github.com/go-sql-driver/mysql v1.8.1
+	github.com/golang-jwt/jwt/v5 v5.2.1
+	github.com/golang-migrate/migrate/v4 v4.17.1
 	github.com/google/gops v0.3.28
 	github.com/gorilla/handlers v1.5.2
 	github.com/gorilla/mux v1.8.1
 	github.com/gorilla/sessions v1.4.0
-	github.com/influxdata/line-protocol/v2 v2.2.1
+	github.com/influxdata/influxdb-client-go/v2 v2.14.0
 	github.com/jmoiron/sqlx v1.4.0
-	github.com/joho/godotenv v1.5.1
-	github.com/linkedin/goavro/v2 v2.14.0
-	github.com/mattn/go-sqlite3 v1.14.24
-	github.com/nats-io/nats.go v1.47.0
-	github.com/prometheus/client_golang v1.23.2
-	github.com/prometheus/common v0.66.1
+	github.com/mattn/go-sqlite3 v1.14.22
+	github.com/prometheus/client_golang v1.20.5
+	github.com/prometheus/common v0.55.0
 	github.com/qustavo/sqlhooks/v2 v2.1.0
 	github.com/santhosh-tekuri/jsonschema/v5 v5.3.1
 	github.com/swaggo/http-swagger v1.3.4
-	github.com/swaggo/swag v1.16.6
-	github.com/vektah/gqlparser/v2 v2.5.30
-	golang.org/x/crypto v0.43.0
-	golang.org/x/oauth2 v0.32.0
-	golang.org/x/time v0.13.0
+	github.com/swaggo/swag v1.16.4
+	github.com/vektah/gqlparser/v2 v2.5.22
+	golang.org/x/crypto v0.33.0
+	golang.org/x/exp v0.0.0-20250215185904-eff6e970281f
+	golang.org/x/oauth2 v0.21.0
+	golang.org/x/time v0.10.0
 )
 
 require (
@@ -42,62 +38,61 @@ require (
 	github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect
 	github.com/KyleBanks/depth v1.2.1 // indirect
 	github.com/agnivade/levenshtein v1.2.1 // indirect
+	github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
-	github.com/cpuguy83/go-md2man/v2 v2.0.7 // indirect
+	github.com/cpuguy83/go-md2man/v2 v2.0.6 // indirect
+	github.com/expr-lang/expr v1.16.9 // indirect
 	github.com/felixge/httpsnoop v1.0.4 // indirect
-	github.com/fsnotify/fsnotify v1.9.0 // indirect
 	github.com/go-asn1-ber/asn1-ber v1.5.7 // indirect
-	github.com/go-jose/go-jose/v4 v4.1.3 // indirect
-	github.com/go-openapi/jsonpointer v0.22.1 // indirect
-	github.com/go-openapi/jsonreference v0.21.2 // indirect
-	github.com/go-openapi/spec v0.22.0 // indirect
-	github.com/go-openapi/swag/conv v0.25.1 // indirect
-	github.com/go-openapi/swag/jsonname v0.25.1 // indirect
-	github.com/go-openapi/swag/jsonutils v0.25.1 // indirect
-	github.com/go-openapi/swag/loading v0.25.1 // indirect
-	github.com/go-openapi/swag/stringutils v0.25.1 // indirect
-	github.com/go-openapi/swag/typeutils v0.25.1 // indirect
-	github.com/go-openapi/swag/yamlutils v0.25.1 // indirect
-	github.com/go-viper/mapstructure/v2 v2.4.0 // indirect
-	github.com/golang/snappy v0.0.4 // indirect
+	github.com/go-jose/go-jose/v4 v4.0.3 // indirect
+	github.com/go-openapi/jsonpointer v0.21.0 // indirect
+	github.com/go-openapi/jsonreference v0.21.0 // indirect
+	github.com/go-openapi/spec v0.21.0 // indirect
+	github.com/go-openapi/swag v0.23.0 // indirect
+	github.com/go-viper/mapstructure/v2 v2.2.1 // indirect
 	github.com/google/uuid v1.6.0 // indirect
 	github.com/gorilla/securecookie v1.1.2 // indirect
 	github.com/gorilla/websocket v1.5.3 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
 	github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
-	github.com/jonboulle/clockwork v0.5.0 // indirect
+	github.com/influxdata/line-protocol v0.0.0-20210922203350-b1ad95c89adf // indirect
+	github.com/influxdata/line-protocol/v2 v2.2.1 // indirect
+	github.com/jonboulle/clockwork v0.4.0 // indirect
+	github.com/josharian/intern v1.0.0 // indirect
 	github.com/jpillora/backoff v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
-	github.com/klauspost/compress v1.18.0 // indirect
+	github.com/klauspost/compress v1.17.9 // indirect
 	github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect
 	github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect
+	github.com/mailru/easyjson v0.9.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect
-	github.com/nats-io/nkeys v0.4.11 // indirect
+	github.com/nats-io/nats.go v1.39.0 // indirect
+	github.com/nats-io/nkeys v0.4.9 // indirect
 	github.com/nats-io/nuid v1.0.1 // indirect
-	github.com/prometheus/client_model v0.6.2 // indirect
-	github.com/prometheus/procfs v0.16.1 // indirect
+	github.com/oapi-codegen/runtime v1.1.1 // indirect
+	github.com/opencontainers/image-spec v1.1.0-rc2.0.20221005185240-3a7f492d3f1b // indirect
+	github.com/prometheus/client_model v0.6.1 // indirect
+	github.com/prometheus/procfs v0.15.1 // indirect
 	github.com/robfig/cron/v3 v3.0.1 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/sosodev/duration v1.3.1 // indirect
 	github.com/swaggo/files v1.0.1 // indirect
-	github.com/urfave/cli/v2 v2.27.7 // indirect
-	github.com/xrash/smetrics v0.0.0-20250705151800-55b8f293f342 // indirect
+	github.com/urfave/cli/v2 v2.27.5 // indirect
+	github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect
 	go.uber.org/atomic v1.11.0 // indirect
-	go.yaml.in/yaml/v2 v2.4.3 // indirect
-	go.yaml.in/yaml/v3 v3.0.4 // indirect
-	golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b // indirect
-	golang.org/x/mod v0.29.0 // indirect
-	golang.org/x/net v0.46.0 // indirect
-	golang.org/x/sync v0.17.0 // indirect
-	golang.org/x/sys v0.37.0 // indirect
-	golang.org/x/text v0.30.0 // indirect
-	golang.org/x/tools v0.38.0 // indirect
-	google.golang.org/protobuf v1.36.9 // indirect
+	golang.org/x/mod v0.23.0 // indirect
+	golang.org/x/net v0.35.0 // indirect
+	golang.org/x/sync v0.11.0 // indirect
+	golang.org/x/sys v0.30.0 // indirect
+	golang.org/x/text v0.22.0 // indirect
+	golang.org/x/tools v0.30.0 // indirect
+	google.golang.org/protobuf v1.36.1 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
-	sigs.k8s.io/yaml v1.6.0 // indirect
+	sigs.k8s.io/yaml v1.4.0 // indirect
 )

go.sum

@@ -1,124 +1,102 @@
 filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA=
 filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
-github.com/99designs/gqlgen v0.17.81 h1:kCkN/xVyRb5rEQpuwOHRTYq83i0IuTQg9vdIiwEerTs=
-github.com/99designs/gqlgen v0.17.81/go.mod h1:vgNcZlLwemsUhYim4dC1pvFP5FX0pr2Y+uYUoHFb1ig=
+github.com/99designs/gqlgen v0.17.63 h1:HCdaYDPd9HqUXRchEvmE3EFzELRwLlaJ8DBuyC8Cqto=
+github.com/99designs/gqlgen v0.17.63/go.mod h1:sVCM2iwIZisJjTI/DEC3fpH+HFgxY1496ZJ+jbT9IjA=
 github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0=
 github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
 github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 h1:mFRzDkZVAjdal+s7s0MwaRv9igoPqLRdzOLzw/8Xvq8=
 github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU=
-github.com/ClusterCockpit/cc-lib v0.10.1 h1:tjGEH8mFGgznYxO8BKLiiar0eZR1Oytk8x5iIQHZR5s=
-github.com/ClusterCockpit/cc-lib v0.10.1/go.mod h1:nvTZuxFCTwlos8I1rL5O1RPab7vRtkU8E/PGiaF6pQA=
+github.com/ClusterCockpit/cc-lib v0.0.0-20250216162506-b737e48423c5 h1:Suu9ivW5rz1dXAqHvA4V9dHI0jm4aIGzfpQGFvGzGEc=
+github.com/ClusterCockpit/cc-lib v0.0.0-20250216162506-b737e48423c5/go.mod h1:T2iDLmsbGjyMy1UtDkR+hofoQXfTMuwSdeFettTav9Q=
+github.com/ClusterCockpit/cc-units v0.4.0 h1:zP5DOu99GmErW0tCDf0gcLrlWt42RQ9dpoONEOh4cI0=
+github.com/ClusterCockpit/cc-units v0.4.0/go.mod h1:3S3PAhAayS3pbgcT4q9Vn9VJw22Op51X0YimtG77zBw=
 github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc=
 github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE=
 github.com/Masterminds/squirrel v1.5.4 h1:uUcX/aBc8O7Fg9kaISIUsHXdKuqehiXAMQTYX8afzqM=
 github.com/Masterminds/squirrel v1.5.4/go.mod h1:NNaOrjSoIDfDA40n7sr2tPNZRfjzjA400rg+riTZj10=
-github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
-github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
-github.com/NVIDIA/go-nvml v0.13.0-1 h1:OLX8Jq3dONuPOQPC7rndB6+iDmDakw0XTYgzMxObkEw=
-github.com/NVIDIA/go-nvml v0.13.0-1/go.mod h1:+KNA7c7gIBH7SKSJ1ntlwkfN80zdx8ovl4hrK3LmPt4=
-github.com/PuerkitoBio/goquery v1.10.3 h1:pFYcNSqHxBD06Fpj/KsbStFRsgRATgnf3LeXiUkhzPo=
-github.com/PuerkitoBio/goquery v1.10.3/go.mod h1:tMUX0zDMHXYlAQk6p35XxQMqMweEKB7iK7iLNd4RH4Y=
+github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow=
+github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM=
+github.com/PuerkitoBio/goquery v1.9.3 h1:mpJr/ikUA9/GNJB/DBZcGeFDXUtosHRyRrwh7KGdTG0=
+github.com/PuerkitoBio/goquery v1.9.3/go.mod h1:1ndLHPdTz+DyQPICCWYlYQMPl0oXZj0G6D4LCYA6u4U=
+github.com/RaveNoX/go-jsoncommentstrip v1.0.0/go.mod h1:78ihd09MekBnJnxpICcwzCMzGrKSKYe4AqU6PDYYpjk=
 github.com/agnivade/levenshtein v1.2.1 h1:EHBY3UOn1gwdy/VbFwgo4cxecRznFk7fKWN1KOX7eoM=
 github.com/agnivade/levenshtein v1.2.1/go.mod h1:QVVI16kDrtSuwcpd0p1+xMC6Z/VfhtCyDIjcwga4/DU=
 github.com/alexbrainman/sspi v0.0.0-20231016080023-1a75b4708caa h1:LHTHcTQiSGT7VVbI0o4wBRNQIgn917usHWOd6VAffYI=
 github.com/alexbrainman/sspi v0.0.0-20231016080023-1a75b4708caa/go.mod h1:cEWa1LVoE5KvSD9ONXsZrj0z6KqySlCCNKHlLzbqAt4=
 github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883 h1:bvNMNQO63//z+xNgfBlViaCIJKLlCJ6/fmUseuG0wVQ=
 github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8=
-github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kktS1LM=
-github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA=
+github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss=
+github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU=
 github.com/apapsch/go-jsonmerge/v2 v2.0.0 h1:axGnT1gRIfimI7gJifB699GoE/oq+F2MU7Dml6nw9rQ=
 github.com/apapsch/go-jsonmerge/v2 v2.0.0/go.mod h1:lvDnEdqiQrp0O42VQGgmlKpxL1AP2+08jFMw88y4klk=
 github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0 h1:jfIu9sQUG6Ig+0+Ap1h4unLjW6YQJpKZVmUzxsD4E/Q=
 github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0/go.mod h1:t2tdKJDJF9BV14lnkjHmOQgcvEKgtqs5a1N3LNdJhGE=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
+github.com/bmatcuk/doublestar v1.1.1/go.mod h1:UD6OnuiIn0yFxxA2le/rnRU1G4RaI4UvFv1sNto9p6w=
 github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
 github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
-github.com/coreos/go-oidc/v3 v3.16.0 h1:qRQUCFstKpXwmEjDQTIbyY/5jF00+asXzSkmkoa/mow=
-github.com/coreos/go-oidc/v3 v3.16.0/go.mod h1:wqPbKFrVnE90vty060SB40FCJ8fTHTxSwyXJqZH+sI8=
-github.com/cpuguy83/go-md2man/v2 v2.0.7 h1:zbFlGlXEAKlwXpmvle3d8Oe3YnkKIK4xSRTd3sHPnBo=
-github.com/cpuguy83/go-md2man/v2 v2.0.7/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
+github.com/coreos/go-oidc/v3 v3.11.0 h1:Ia3MxdwpSw702YW0xgfmP1GVCMA9aEFWu12XUZ3/OtI=
+github.com/coreos/go-oidc/v3 v3.11.0/go.mod h1:gE3LgjOgFoHi9a4ce4/tJczr0Ai2/BoDhf0r5lltWI0=
+github.com/cpuguy83/go-md2man/v2 v2.0.6 h1:XJtiaUW6dEEqVuZiMTn1ldk455QWwEIsMIJlo5vtkx0=
+github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/dgryski/trifles v0.0.0-20230903005119-f50d829f2e54 h1:SG7nF6SRlWhcT7cNTs5R6Hk4V2lcmLz2NsG2VnInyNo=
 github.com/dgryski/trifles v0.0.0-20230903005119-f50d829f2e54/go.mod h1:if7Fbed8SFyPtHLHbg49SI7NAdJiC5WIA09pe59rfAA=
-github.com/dhui/dktest v0.4.4 h1:+I4s6JRE1yGuqflzwqG+aIaMdgXIorCf5P98JnaAWa8=
-github.com/dhui/dktest v0.4.4/go.mod h1:4+22R4lgsdAXrDyaH4Nqx2JEz2hLp49MqQmm9HLCQhM=
-github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk=
-github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
-github.com/docker/docker v27.2.0+incompatible h1:Rk9nIVdfH3+Vz4cyI/uhbINhEZ/oLmc+CBXmH6fbNk4=
-github.com/docker/docker v27.2.0+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
-github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c=
-github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc=
+github.com/dhui/dktest v0.4.1 h1:/w+IWuDXVymg3IrRJCHHOkMK10m9aNVMOyD0X12YVTg=
+github.com/dhui/dktest v0.4.1/go.mod h1:DdOqcUpL7vgyP4GlF3X3w7HbSlz8cEQzwewPveYEQbA=
+github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m322EBzniBPB6ZIzuh8=
+github.com/docker/distribution v2.8.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
+github.com/docker/docker v24.0.9+incompatible h1:HPGzNmwfLZWdxHqK9/II92pyi1EpYKsAqcl4G0Of9v0=
+github.com/docker/docker v24.0.9+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
+github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ=
+github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
 github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
 github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
-github.com/expr-lang/expr v1.17.6 h1:1h6i8ONk9cexhDmowO/A64VPxHScu7qfSl2k8OlINec=
-github.com/expr-lang/expr v1.17.6/go.mod h1:8/vRC7+7HBzESEqt5kKpYXxrxkr31SaO8r40VO/1IT4=
+github.com/expr-lang/expr v1.16.9 h1:WUAzmR0JNI9JCiF0/ewwHB1gmcGw5wW7nWt8gc6PpCI=
+github.com/expr-lang/expr v1.16.9/go.mod h1:8/vRC7+7HBzESEqt5kKpYXxrxkr31SaO8r40VO/1IT4=
 github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
 github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
 github.com/frankban/quicktest v1.11.0/go.mod h1:K+q6oSqb0W0Ininfk863uOk1lMy69l/P6txr3mVT54s=
 github.com/frankban/quicktest v1.11.2/go.mod h1:K+q6oSqb0W0Ininfk863uOk1lMy69l/P6txr3mVT54s=
 github.com/frankban/quicktest v1.13.0 h1:yNZif1OkDfNoDfb9zZa9aXIpejNR4F23Wely0c+Qdqk=
 github.com/frankban/quicktest v1.13.0/go.mod h1:qLE0fzW0VuyUAJgPU19zByoIr0HtCHN/r/VLSOOIySU=
-github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k=
-github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
+github.com/go-asn1-ber/asn1-ber v1.5.5/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
 github.com/go-asn1-ber/asn1-ber v1.5.7 h1:DTX+lbVTWaTw1hQ+PbZPlnDZPEIs0SS/GCZAl535dDk=
 github.com/go-asn1-ber/asn1-ber v1.5.7/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
-github.com/go-co-op/gocron/v2 v2.16.0 h1:uqUF6WFZ4enRU45pWFNcn1xpDLc+jBOTKhPQI16Z1xs=
-github.com/go-co-op/gocron/v2 v2.16.0/go.mod h1:opexeOFy5BplhsKdA7bzY9zeYih8I8/WNJ4arTIFPVc=
-github.com/go-jose/go-jose/v4 v4.1.3 h1:CVLmWDhDVRa6Mi/IgCgaopNosCaHz7zrMeF9MlZRkrs=
-github.com/go-jose/go-jose/v4 v4.1.3/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08=
-github.com/go-ldap/ldap/v3 v3.4.10 h1:ot/iwPOhfpNVgB1o+AVXljizWZ9JTp7YF5oeyONmcJU=
-github.com/go-ldap/ldap/v3 v3.4.10/go.mod h1:JXh4Uxgi40P6E9rdsYqpUtbW46D9UTjJ9QSwGRznplY=
-github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=
-github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
-github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
-github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
-github.com/go-openapi/jsonpointer v0.22.1 h1:sHYI1He3b9NqJ4wXLoJDKmUmHkWy/L7rtEo92JUxBNk=
-github.com/go-openapi/jsonpointer v0.22.1/go.mod h1:pQT9OsLkfz1yWoMgYFy4x3U5GY5nUlsOn1qSBH5MkCM=
-github.com/go-openapi/jsonreference v0.21.2 h1:Wxjda4M/BBQllegefXrY/9aq1fxBA8sI5M/lFU6tSWU=
-github.com/go-openapi/jsonreference v0.21.2/go.mod h1:pp3PEjIsJ9CZDGCNOyXIQxsNuroxm8FAJ/+quA0yKzQ=
-github.com/go-openapi/spec v0.22.0 h1:xT/EsX4frL3U09QviRIZXvkh80yibxQmtoEvyqug0Tw=
-github.com/go-openapi/spec v0.22.0/go.mod h1:K0FhKxkez8YNS94XzF8YKEMULbFrRw4m15i2YUht4L0=
-github.com/go-openapi/swag v0.19.15 h1:D2NRCBzS9/pEY3gP9Nl8aDqGUcPFrwG2p+CNFrLyrCM=
-github.com/go-openapi/swag/conv v0.25.1 h1:+9o8YUg6QuqqBM5X6rYL/p1dpWeZRhoIt9x7CCP+he0=
-github.com/go-openapi/swag/conv v0.25.1/go.mod h1:Z1mFEGPfyIKPu0806khI3zF+/EUXde+fdeksUl2NiDs=
-github.com/go-openapi/swag/jsonname v0.25.1 h1:Sgx+qbwa4ej6AomWC6pEfXrA6uP2RkaNjA9BR8a1RJU=
-github.com/go-openapi/swag/jsonname v0.25.1/go.mod h1:71Tekow6UOLBD3wS7XhdT98g5J5GR13NOTQ9/6Q11Zo=
-github.com/go-openapi/swag/jsonutils v0.25.1 h1:AihLHaD0brrkJoMqEZOBNzTLnk81Kg9cWr+SPtxtgl8=
-github.com/go-openapi/swag/jsonutils v0.25.1/go.mod h1:JpEkAjxQXpiaHmRO04N1zE4qbUEg3b7Udll7AMGTNOo=
-github.com/go-openapi/swag/jsonutils/fixtures_test v0.25.1 h1:DSQGcdB6G0N9c/KhtpYc71PzzGEIc/fZ1no35x4/XBY=
-github.com/go-openapi/swag/jsonutils/fixtures_test v0.25.1/go.mod h1:kjmweouyPwRUEYMSrbAidoLMGeJ5p6zdHi9BgZiqmsg=
-github.com/go-openapi/swag/loading v0.25.1 h1:6OruqzjWoJyanZOim58iG2vj934TysYVptyaoXS24kw=
-github.com/go-openapi/swag/loading v0.25.1/go.mod h1:xoIe2EG32NOYYbqxvXgPzne989bWvSNoWoyQVWEZicc=
-github.com/go-openapi/swag/stringutils v0.25.1 h1:Xasqgjvk30eUe8VKdmyzKtjkVjeiXx1Iz0zDfMNpPbw=
-github.com/go-openapi/swag/stringutils v0.25.1/go.mod h1:JLdSAq5169HaiDUbTvArA2yQxmgn4D6h4A+4HqVvAYg=
-github.com/go-openapi/swag/typeutils v0.25.1 h1:rD/9HsEQieewNt6/k+JBwkxuAHktFtH3I3ysiFZqukA=
-github.com/go-openapi/swag/typeutils v0.25.1/go.mod h1:9McMC/oCdS4BKwk2shEB7x17P6HmMmA6dQRtAkSnNb8=
-github.com/go-openapi/swag/yamlutils v0.25.1 h1:mry5ez8joJwzvMbaTGLhw8pXUnhDK91oSJLDPF1bmGk=
-github.com/go-openapi/swag/yamlutils v0.25.1/go.mod h1:cm9ywbzncy3y6uPm/97ysW8+wZ09qsks+9RS8fLWKqg=
+github.com/go-co-op/gocron/v2 v2.9.0 h1:+0nTyI3mjc2FGIClBdDWpaLPCNrJ+62o9xbS0ZklEKQ=
+github.com/go-co-op/gocron/v2 v2.9.0/go.mod h1:xY7bJxGazKam1cz04EebrlP4S9q4iWdiAylMGP3jY9w=
+github.com/go-jose/go-jose/v4 v4.0.3 h1:o8aphO8Hv6RPmH+GfzVuyf7YXSBibp+8YyHdOoDESGo=
+github.com/go-jose/go-jose/v4 v4.0.3/go.mod h1:NKb5HO1EZccyMpiZNbdUw/14tiXNyUJh188dfnMCAfc=
+github.com/go-ldap/ldap/v3 v3.4.8 h1:loKJyspcRezt2Q3ZRMq2p/0v8iOurlmeXDPw6fikSvQ=
+github.com/go-ldap/ldap/v3 v3.4.8/go.mod h1:qS3Sjlu76eHfHGpUdWkAXQTw4beih+cHsco2jXlIXrk=
+github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ=
+github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY=
+github.com/go-openapi/jsonreference v0.21.0 h1:Rs+Y7hSXT83Jacb7kFyjn4ijOuVGSvOdF2+tg1TRrwQ=
+github.com/go-openapi/jsonreference v0.21.0/go.mod h1:LmZmgsrTkVg9LG4EaHeY8cBDslNPMo06cago5JNLkm4=
+github.com/go-openapi/spec v0.21.0 h1:LTVzPc3p/RzRnkQqLRndbAzjY0d0BCL72A6j3CdL9ZY=
+github.com/go-openapi/spec v0.21.0/go.mod h1:78u6VdPw81XU44qEWGhtr982gJ5BWg2c0I5XwVMotYk=
+github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE=
+github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ=
 github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
+github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y=
 github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg=
-github.com/go-sql-driver/mysql v1.9.0 h1:Y0zIbQXhQKmQgTp44Y1dp3wTXcn804QoTptLZT1vtvo=
-github.com/go-sql-driver/mysql v1.9.0/go.mod h1:pDetrLJeA3oMujJuvXc8RJoasr589B6A9fwzD3QMrqw=
-github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs=
-github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
+github.com/go-viper/mapstructure/v2 v2.2.1 h1:ZAaOCxANMuZx5RCeg0mBdEZk7DZasvvZIxtHqx8aGss=
+github.com/go-viper/mapstructure/v2 v2.2.1/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
-github.com/golang-jwt/jwt/v5 v5.2.2 h1:Rl4B7itRWVtYIHFrSNd7vhTiz9UpLdi6gZhZ3wEeDy8=
-github.com/golang-jwt/jwt/v5 v5.2.2/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
-github.com/golang-migrate/migrate/v4 v4.18.2 h1:2VSCMz7x7mjyTXx3m2zPokOY82LTRgxK1yQYKo6wWQ8=
-github.com/golang-migrate/migrate/v4 v4.18.2/go.mod h1:2CM6tJvn2kqPXwnXO/d3rAQYiyoIm180VsO8PRX6Rpk=
-github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
-github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
-github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk=
+github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
+github.com/golang-migrate/migrate/v4 v4.17.1 h1:4zQ6iqL6t6AiItphxJctQb3cFqWiSpMnX7wLTPnnYO4=
+github.com/golang-migrate/migrate/v4 v4.17.1/go.mod h1:m8hinFyWBn0SA4QKHuKh175Pm9wjmxj3S2Mia7dbXzM=
 github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
 github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
-github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
-github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=
 github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
@@ -173,16 +151,17 @@ github.com/jcmturner/rpc/v2 v2.0.3 h1:7FXXj8Ti1IaVFpSAziCZWNzbNuZmnvw/i6CqLNdWfZ
 github.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc=
 github.com/jmoiron/sqlx v1.4.0 h1:1PLqN7S1UYp5t4SrVVnt4nUVNemrDAtxlulVe+Qgm3o=
 github.com/jmoiron/sqlx v1.4.0/go.mod h1:ZrZ7UsYB/weZdl2Bxg6jCRO9c3YHl8r3ahlKmRT4JLY=
-github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
-github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
-github.com/jonboulle/clockwork v0.5.0 h1:Hyh9A8u51kptdkR+cqRpT1EebBwTn1oK9YfGYbdFz6I=
-github.com/jonboulle/clockwork v0.5.0/go.mod h1:3mZlmanh0g2NDKO5TWZVJAfofYk64M7XN3SzBPjZF60=
+github.com/jonboulle/clockwork v0.4.0 h1:p4Cf1aMWXnXAUh8lVfewRBx1zaTSYKrKMF2g3ST4RZ4=
+github.com/jonboulle/clockwork v0.4.0/go.mod h1:xgRqUGwRcjKCO1vbZUEtSLrqKoPSsUpK7fnezOII0kc=
+github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
+github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
 github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA=
 github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
 github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
 github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
-github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
-github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
+github.com/juju/gnuflag v0.0.0-20171113085948-2ce1bb71843d/go.mod h1:2PavIy+JPciBPrBUjwbNvtwB6RQlve+hkpll6QSNmOE=
+github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA=
+github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
 github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
 github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
@@ -190,6 +169,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
+github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
 github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 h1:SOEGU9fKiNWd/HOJuq6+3iTQz8KNCLtVX6idSoTLdUw=
 github.com/lann/builder v0.0.0-20180802200727-47ae307949d0/go.mod h1:dXGbAdH5GtBTC4WfIxhKZfyBF/HBFgRZSWwZ9g/He9o=
 github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 h1:P6pPBnrTSX3DEVR4fDembhRWSsG5rVo6hYhAB/ADZrk=
@@ -197,14 +178,11 @@ github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0/go.mod h1:vmVJ0l/dxyfGW6Fm
 github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
 github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
 github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
-github.com/linkedin/goavro/v2 v2.14.0 h1:aNO/js65U+Mwq4yB5f1h01c3wiM458qtRad1DN0CMUI=
-github.com/linkedin/goavro/v2 v2.14.0/go.mod h1:KXx+erlq+RPlGSPmLF7xGo6SAbh8sCQ53x064+ioxhk=
+github.com/mailru/easyjson v0.9.0 h1:PrnmzHw7262yW8sTBwxi1PdJA3Iw/EKBa8psRf7d9a4=
+github.com/mailru/easyjson v0.9.0/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUto+XFtTU=
 github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
+github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU=
 github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
-github.com/mattn/go-sqlite3 v1.14.24 h1:tpSp2G2KyMnnQu99ngJ47EIkWVmliIizyZBfPrBWDRM=
-github.com/mattn/go-sqlite3 v1.14.24/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
-github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0=
-github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo=
 github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0=
 github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
@@ -218,10 +196,10 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU=
 github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
-github.com/nats-io/nats.go v1.47.0 h1:YQdADw6J/UfGUd2Oy6tn4Hq6YHxCaJrVKayxxFqYrgM=
-github.com/nats-io/nats.go v1.47.0/go.mod h1:iRWIPokVIFbVijxuMQq4y9ttaBTMe0SFdlZfMDd+33g=
-github.com/nats-io/nkeys v0.4.11 h1:q44qGV008kYd9W1b1nEBkNzvnWxtRSQ7A8BoqRrcfa0=
-github.com/nats-io/nkeys v0.4.11/go.mod h1:szDimtgmfOi9n25JpfIdGw12tZFYXqhGxjhVxsatHVE=
+github.com/nats-io/nats.go v1.39.0 h1:2/yg2JQjiYYKLwDuBzV0FbB2sIV+eFNkEevlRi4n9lI=
+github.com/nats-io/nats.go v1.39.0/go.mod h1:MgRb8oOdigA6cYpEPhXJuRVH6UE/V4jblJ2jQ27IXYM=
+github.com/nats-io/nkeys v0.4.9 h1:qe9Faq2Gxwi6RZnZMXfmGMZkg3afLLOtrU+gDZJ35b0=
+github.com/nats-io/nkeys v0.4.9/go.mod h1:jcMqs+FLG+W5YO36OX6wFIFcmpdAns+w1Wm6D3I/evE=
 github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
 github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
@@ -229,27 +207,27 @@ github.com/oapi-codegen/runtime v1.1.1 h1:EXLHh0DXIJnWhdRPN2w4MXAzFyE4CskzhNLUmt
 github.com/oapi-codegen/runtime v1.1.1/go.mod h1:SK9X900oXmPWilYR5/WKPzt3Kqxn/uS/+lbpREv+eCg=
 github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
-github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug=
-github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM=
+github.com/opencontainers/image-spec v1.1.0-rc2.0.20221005185240-3a7f492d3f1b h1:YWuSjZCQAPM8UUBLkYUk1e+rZcvWHJmFb6i6rM44Xs8=
+github.com/opencontainers/image-spec v1.1.0-rc2.0.20221005185240-3a7f492d3f1b/go.mod h1:3OVijpioIKYWTqjiG0zfF6wvoJ4fAXGbjdZuI2NgsRQ=
 github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o=
-github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg=
-github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk=
-github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE=
-github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs=
-github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA=
-github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg=
-github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is=
+github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y=
+github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE=
+github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E=
+github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY=
+github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc=
+github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8=
+github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc=
+github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk=
 github.com/qustavo/sqlhooks/v2 v2.1.0 h1:54yBemHnGHp/7xgT+pxwmIlMSDNYKx5JW5dfRAiCZi0=
 github.com/qustavo/sqlhooks/v2 v2.1.0/go.mod h1:aMREyKo7fOKTwiLuWPsaHRXEmtqG4yREztO0idF83AU=
 github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
 github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
-github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8=
-github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4=
+github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M=
+github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA=
 github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 h1:lZUw3E0/J3roVtGQ+SCrUrg3ON6NgVqpn3+iol9aGu4=
@@ -258,6 +236,7 @@ github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8=
 github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I=
 github.com/sosodev/duration v1.3.1 h1:qtHBDMQ6lvMQsL15g4aopM4HEfOaYuhWBw3NPTtlqq4=
 github.com/sosodev/duration v1.3.1/go.mod h1:RQIBBX0+fMLc/D9+Jb/fwvVmo0eZvDDEERAikUR6SDg=
+github.com/spkg/bom v0.0.0-20160624110644-59b7046e48ad/go.mod h1:qLr4V1qq6nMqFKkMo8ZTx3f+BZEkzsRUY10Xsm2mwU0=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
@@ -265,58 +244,40 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.7.5/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
-github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
-github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
+github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
+github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/swaggo/files v1.0.1 h1:J1bVJ4XHZNq0I46UU90611i9/YzdrF7x92oX1ig5IdE=
 github.com/swaggo/files v1.0.1/go.mod h1:0qXmMNH6sXNf+73t65aKeB+ApmgxdnkQzVTAj2uaMUg=
 github.com/swaggo/http-swagger v1.3.4 h1:q7t/XLx0n15H1Q9/tk3Y9L4n210XzJF5WtnDX64a5ww=
 github.com/swaggo/http-swagger v1.3.4/go.mod h1:9dAh0unqMBAlbp1uE2Uc2mQTxNMU/ha4UbucIg1MFkQ=
-github.com/swaggo/swag v1.16.6 h1:qBNcx53ZaX+M5dxVyTrgQ0PJ/ACK+NzhwcbieTt+9yI=
-github.com/swaggo/swag v1.16.6/go.mod h1:ngP2etMK5a0P3QBizic5MEwpRmluJZPHjXcMoj4Xesg=
-github.com/urfave/cli/v2 v2.27.7 h1:bH59vdhbjLv3LAvIu6gd0usJHgoTTPhCFib8qqOwXYU=
-github.com/urfave/cli/v2 v2.27.7/go.mod h1:CyNAG/xg+iAOg0N4MPGZqVmv2rCoP267496AOXUZjA4=
-github.com/vektah/gqlparser/v2 v2.5.30 h1:EqLwGAFLIzt1wpx1IPpY67DwUujF1OfzgEyDsLrN6kE=
-github.com/vektah/gqlparser/v2 v2.5.30/go.mod h1:D1/VCZtV3LPnQrcPBeR/q5jkSQIPti0uYCP/RI0gIeo=
-github.com/xrash/smetrics v0.0.0-20250705151800-55b8f293f342 h1:FnBeRrxr7OU4VvAzt5X7s6266i6cSVkkFPS0TuXWbIg=
-github.com/xrash/smetrics v0.0.0-20250705151800-55b8f293f342/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM=
+github.com/swaggo/swag v1.16.4 h1:clWJtd9LStiG3VeijiCfOVODP6VpHtKdQy9ELFG3s1A=
+github.com/swaggo/swag v1.16.4/go.mod h1:VBsHJRsDvfYvqoiMKnsdwhNV9LEMHgEDZcyVYX0sxPg=
+github.com/urfave/cli/v2 v2.27.5 h1:WoHEJLdsXr6dDWoJgMq/CboDmyY/8HMMH1fTECbih+w=
+github.com/urfave/cli/v2 v2.27.5/go.mod h1:3Sevf16NykTbInEnD0yKkjDAeZDS0A6bzhBH5hrMvTQ=
+github.com/vektah/gqlparser/v2 v2.5.22 h1:yaaeJ0fu+nv1vUMW0Hl+aS1eiv1vMfapBNjpffAda1I=
+github.com/vektah/gqlparser/v2 v2.5.22/go.mod h1:xMl+ta8a5M1Yo1A1Iwt/k7gSpscwSnHZdw7tfhEGfTM=
+github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGCjxCBTO/36wtF6j2nSip77qHd4x4=
+github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
-go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 h1:TT4fX+nBOA/+LUkobKGW1ydGcn+G3vRw9+g5HwCphpk=
-go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0/go.mod h1:L7UH0GbB0p47T4Rri3uHjbpCFYrVrwc1I25QhNPiGK8=
-go.opentelemetry.io/otel v1.29.0 h1:PdomN/Al4q/lN6iBJEN3AwPvUiHPMlt93c8bqTG5Llw=
-go.opentelemetry.io/otel v1.29.0/go.mod h1:N/WtXPs1CNCUEx+Agz5uouwCba+i+bJGFicT8SR4NP8=
-go.opentelemetry.io/otel/metric v1.29.0 h1:vPf/HFWTNkPu1aYeIsc98l4ktOQaL6LeSoeV2g+8YLc=
-go.opentelemetry.io/otel/metric v1.29.0/go.mod h1:auu/QWieFVWx+DmQOUMgj0F8LHWdgalxXqvp7BII/W8=
-go.opentelemetry.io/otel/trace v1.29.0 h1:J/8ZNK4XgR7a21DZUAsbF8pZ5Jcw1VhACmnYt39JTi4=
-go.opentelemetry.io/otel/trace v1.29.0/go.mod h1:eHl3w0sp3paPkYstJOmAimxhiFXPg+MMTlEh3nsQgWQ=
 go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE=
 go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
 go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
 go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
-go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0=
-go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8=
-go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
-go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58=
-golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
 golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
-golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
-golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
-golang.org/x/crypto v0.43.0 h1:dduJYIi3A3KOfdGOHX8AVZ/jGiyPa3IbBozJ5kNuE04=
-golang.org/x/crypto v0.43.0/go.mod h1:BFbav4mRNlXJL4wNeejLpWxB7wMbc79PdRGhWKncxR0=
-golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b h1:M2rDM6z3Fhozi9O7NWsxAkg/yqS/lQJ6PmkyIV3YP+o=
-golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b/go.mod h1:3//PLf8L/X+8b4vuAfHzxeRUl04Adcb341+IGKfnqS8=
+golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
+golang.org/x/crypto v0.33.0 h1:IOBPskki6Lysi0lo9qQvbxiQ+FvsCC/YWOecCHAixus=
+golang.org/x/crypto v0.33.0/go.mod h1:bVdXmD7IV/4GdElGPozy6U7lWdRXA4qyRVGJV57uQ5M=
+golang.org/x/exp v0.0.0-20250215185904-eff6e970281f h1:oFMYAjX0867ZD2jcNiLBrI9BdpmEkvPyi5YrBGXbamg=
+golang.org/x/exp v0.0.0-20250215185904-eff6e970281f/go.mod h1:BHOTPb3L19zxehTsLoJXVaTktb06DFgmdW6Wb9s8jqk=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
 golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
-golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
-golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
-golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
-golang.org/x/mod v0.29.0 h1:HV8lRxZC4l2cr3Zq1LvtOsi/ThTgWnUk/y64QSs8GwA=
-golang.org/x/mod v0.29.0/go.mod h1:NyhrlYXJ2H4eJiRy/WDBO6HMqZQ6q9nk4JzS3NuCK+w=
+golang.org/x/mod v0.23.0 h1:Zb7khfcRGKk+kqfxFaP5tZqCnDZMjC5VtUBs87Hr6QM=
+golang.org/x/mod v0.23.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
@@ -324,23 +285,17 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug
 golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
-golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
 golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
-golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
-golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
-golang.org/x/net v0.46.0 h1:giFlY12I07fugqwPuWJi68oOnpfqFnJIJzaIIm2JVV4=
-golang.org/x/net v0.46.0/go.mod h1:Q9BGdFy1y4nkUwiLvT5qtyhAnEHgnQ/zd8PfU6nc210=
-golang.org/x/oauth2 v0.32.0 h1:jsCblLleRMDrxMN29H3z/k1KliIvpLgCkE6R8FXXNgY=
-golang.org/x/oauth2 v0.32.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA=
+golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
+golang.org/x/net v0.35.0 h1:T5GQRQb2y08kTAByq9L4/bz8cipCdA8FbRTXewonqY8=
+golang.org/x/net v0.35.0/go.mod h1:EglIi67kWsHKlRzzVMUD93VMSWGFOMSZgxFjparz1Qk=
+golang.org/x/oauth2 v0.21.0 h1:tsimM75w1tF/uws5rbeHzIWxEqElMehnc+iW793zsZs=
+golang.org/x/oauth2 v0.21.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
-golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
-golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
-golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
-golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug=
-golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
+golang.org/x/sync v0.11.0 h1:GGz8+XQP4FvTTrjZPzNKTMFtSXH80RAzG+5ghFPgK9w=
+golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -348,54 +303,46 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.37.0 h1:fdNQudmxPjkdUTPnLn5mdQv7Zwvbvpaxqs831goi9kQ=
-golang.org/x/sys v0.37.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
-golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
+golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc=
+golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
 golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
-golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU=
 golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
-golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
-golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
+golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
-golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
 golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
-golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
-golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
-golang.org/x/text v0.30.0 h1:yznKA/E9zq54KzlzBEAWn1NXSQ8DIp/NYMy88xJjl4k=
-golang.org/x/text v0.30.0/go.mod h1:yDdHFIX9t+tORqspjENWgzaCVXgk0yYnYuSZ8UzzBVM=
-golang.org/x/time v0.13.0 h1:eUlYslOIt32DgYD6utsuUeHs4d7AsEYLuIAdg7FlYgI=
-golang.org/x/time v0.13.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4=
+golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM=
+golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY=
+golang.org/x/time v0.10.0 h1:3usCWA8tQn0L8+hFJQNgzpWbd89begxN66o1Ojdn5L4=
+golang.org/x/time v0.10.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
 golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
-golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
-golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
-golang.org/x/tools v0.38.0 h1:Hx2Xv8hISq8Lm16jvBZ2VQf+RLmbd7wVUsALibYI/IQ=
-golang.org/x/tools v0.38.0/go.mod h1:yEsQ/d/YK8cjh0L6rZlY8tgtlKiBNTL14pGDJPJpYQs=
+golang.org/x/tools v0.30.0 h1:BgcpHewrV5AUp2G9MebG4XPFI1E2W41zU1SaqVA9vJY=
+golang.org/x/tools v0.30.0/go.mod h1:c347cR/OJfw5TI+GfX7RUPNMdDRRbjvYTS0jPyvsVtY=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-google.golang.org/protobuf v1.36.9 h1:w2gp2mA27hUeUzj9Ex9FBjsBm40zfaDtEWow293U7Iw=
-google.golang.org/protobuf v1.36.9/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU=
+google.golang.org/protobuf v1.36.1 h1:yBPeRvTftaleIgM3PZ/WBIZ7XM/eEYAaEyCwvyjq/gk=
+google.golang.org/protobuf v1.36.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs=
-sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4=
+sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E=
+sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY=

gqlgen.yml

@@ -30,9 +30,7 @@ resolver:
 # gqlgen will search for any type names in the schema in these go packages
 # if they match it will use them, otherwise it will generate them.
 autobind:
-  - "github.com/99designs/gqlgen/graphql/introspection"
   - "github.com/ClusterCockpit/cc-backend/internal/graph/model"
-  - "github.com/ClusterCockpit/cc-backend/internal/config"
 
 # This section declares type mapping between the GraphQL and go type systems
 #
@@ -52,51 +50,61 @@ models:
       - github.com/99designs/gqlgen/graphql.Int64
       - github.com/99designs/gqlgen/graphql.Int32
   Job:
-    model: "github.com/ClusterCockpit/cc-lib/schema.Job"
+    model: "github.com/ClusterCockpit/cc-backend/pkg/schema.Job"
     fields:
       tags:
         resolver: true
       metaData:
         resolver: true
   Cluster:
-    model: "github.com/ClusterCockpit/cc-lib/schema.Cluster"
+    model: "github.com/ClusterCockpit/cc-backend/pkg/schema.Cluster"
     fields:
       partitions:
         resolver: true
-  # Node:
-  #   model: "github.com/ClusterCockpit/cc-lib/schema.Node"
-  #   fields:
-  #     metaData:
-  #       resolver: true
-  NullableFloat: { model: "github.com/ClusterCockpit/cc-lib/schema.Float" }
-  MetricScope: { model: "github.com/ClusterCockpit/cc-lib/schema.MetricScope" }
-  MetricValue: { model: "github.com/ClusterCockpit/cc-lib/schema.MetricValue" }
+  NullableFloat:
+    { model: "github.com/ClusterCockpit/cc-backend/pkg/schema.Float" }
+  MetricScope:
+    { model: "github.com/ClusterCockpit/cc-backend/pkg/schema.MetricScope" }
+  MetricValue:
+    { model: "github.com/ClusterCockpit/cc-backend/pkg/schema.MetricValue" }
   JobStatistics:
-    { model: "github.com/ClusterCockpit/cc-lib/schema.JobStatistics" }
+    { model: "github.com/ClusterCockpit/cc-backend/pkg/schema.JobStatistics" }
   GlobalMetricListItem:
-    { model: "github.com/ClusterCockpit/cc-lib/schema.GlobalMetricListItem" }
+    {
+      model: "github.com/ClusterCockpit/cc-backend/pkg/schema.GlobalMetricListItem",
+    }
   ClusterSupport:
-    { model: "github.com/ClusterCockpit/cc-lib/schema.ClusterSupport" }
-  Tag: { model: "github.com/ClusterCockpit/cc-lib/schema.Tag" }
-  Resource: { model: "github.com/ClusterCockpit/cc-lib/schema.Resource" }
-  JobState: { model: "github.com/ClusterCockpit/cc-lib/schema.JobState" }
-  Node: { model: "github.com/ClusterCockpit/cc-lib/schema.Node" }
-  SchedulerState:
-    { model: "github.com/ClusterCockpit/cc-lib/schema.SchedulerState" }
-  HealthState:
-    { model: "github.com/ClusterCockpit/cc-lib/schema.MonitoringState" }
-  JobMetric: { model: "github.com/ClusterCockpit/cc-lib/schema.JobMetric" }
-  Series: { model: "github.com/ClusterCockpit/cc-lib/schema.Series" }
+    { model: "github.com/ClusterCockpit/cc-backend/pkg/schema.ClusterSupport" }
+  Tag: { model: "github.com/ClusterCockpit/cc-backend/pkg/schema.Tag" }
+  Resource:
+    { model: "github.com/ClusterCockpit/cc-backend/pkg/schema.Resource" }
+  JobState:
+    { model: "github.com/ClusterCockpit/cc-backend/pkg/schema.JobState" }
+  TimeRange:
+    { model: "github.com/ClusterCockpit/cc-backend/pkg/schema.TimeRange" }
+  IntRange:
+    { model: "github.com/ClusterCockpit/cc-backend/pkg/schema.IntRange" }
+  JobMetric:
+    { model: "github.com/ClusterCockpit/cc-backend/pkg/schema.JobMetric" }
+  Series: { model: "github.com/ClusterCockpit/cc-backend/pkg/schema.Series" }
   MetricStatistics:
-    { model: "github.com/ClusterCockpit/cc-lib/schema.MetricStatistics" }
+    {
+      model: "github.com/ClusterCockpit/cc-backend/pkg/schema.MetricStatistics",
+    }
   MetricConfig:
-    { model: "github.com/ClusterCockpit/cc-lib/schema.MetricConfig" }
+    { model: "github.com/ClusterCockpit/cc-backend/pkg/schema.MetricConfig" }
   SubClusterConfig:
-    { model: "github.com/ClusterCockpit/cc-lib/schema.SubClusterConfig" }
-  Accelerator: { model: "github.com/ClusterCockpit/cc-lib/schema.Accelerator" }
-  Topology: { model: "github.com/ClusterCockpit/cc-lib/schema.Topology" }
+    {
+      model: "github.com/ClusterCockpit/cc-backend/pkg/schema.SubClusterConfig",
+    }
+  Accelerator:
+    { model: "github.com/ClusterCockpit/cc-backend/pkg/schema.Accelerator" }
+  Topology:
+    { model: "github.com/ClusterCockpit/cc-backend/pkg/schema.Topology" }
   FilterRanges:
-    { model: "github.com/ClusterCockpit/cc-lib/schema.FilterRanges" }
-  SubCluster: { model: "github.com/ClusterCockpit/cc-lib/schema.SubCluster" }
-  StatsSeries: { model: "github.com/ClusterCockpit/cc-lib/schema.StatsSeries" }
-  Unit: { model: "github.com/ClusterCockpit/cc-lib/schema.Unit" }
+    { model: "github.com/ClusterCockpit/cc-backend/pkg/schema.FilterRanges" }
+  SubCluster:
+    { model: "github.com/ClusterCockpit/cc-backend/pkg/schema.SubCluster" }
+  StatsSeries:
+    { model: "github.com/ClusterCockpit/cc-backend/pkg/schema.StatsSeries" }
+  Unit: { model: "github.com/ClusterCockpit/cc-backend/pkg/schema.Unit" }

internal/api/api_test.go

@@ -1,5 +1,5 @@
 // Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved. This file is part of cc-backend.
+// All rights reserved.
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 package api_test
@@ -27,9 +27,8 @@ import (
 	"github.com/ClusterCockpit/cc-backend/internal/metricdata"
 	"github.com/ClusterCockpit/cc-backend/internal/repository"
 	"github.com/ClusterCockpit/cc-backend/pkg/archive"
-	ccconf "github.com/ClusterCockpit/cc-lib/ccConfig"
-	cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
-	"github.com/ClusterCockpit/cc-lib/schema"
+	"github.com/ClusterCockpit/cc-backend/pkg/log"
+	"github.com/ClusterCockpit/cc-backend/pkg/schema"
 	"github.com/gorilla/mux"
 
 	_ "github.com/mattn/go-sqlite3"
@@ -37,22 +36,15 @@ import (
 
 func setup(t *testing.T) *api.RestApi {
 	const testconfig = `{
-		"main": {
 	"addr":            "0.0.0.0:8080",
 	"validate": false,
-  "apiAllowedIPs": [
-    "*"
-  ]
-	},
 	"archive": {
 		"kind": "file",
 		"path": "./var/job-archive"
 	},
-	"auth": {
-  "jwts": {
-      "max-age": "2m"
-  }
-	},
+    "jwts": {
+        "max-age": "2m"
+    },
 	"clusters": [
 	{
 	   "name": "testcluster",
@@ -121,22 +113,22 @@ func setup(t *testing.T) *api.RestApi {
 		]
 	}`
 
-	cclog.Init("info", true)
+	log.Init("info", true)
 	tmpdir := t.TempDir()
 	jobarchive := filepath.Join(tmpdir, "job-archive")
-	if err := os.Mkdir(jobarchive, 0o777); err != nil {
+	if err := os.Mkdir(jobarchive, 0777); err != nil {
 		t.Fatal(err)
 	}
 
-	if err := os.WriteFile(filepath.Join(jobarchive, "version.txt"), fmt.Appendf(nil, "%d", 2), 0o666); err != nil {
+	if err := os.WriteFile(filepath.Join(jobarchive, "version.txt"), []byte(fmt.Sprintf("%d", 2)), 0666); err != nil {
 		t.Fatal(err)
 	}
 
-	if err := os.Mkdir(filepath.Join(jobarchive, "testcluster"), 0o777); err != nil {
+	if err := os.Mkdir(filepath.Join(jobarchive, "testcluster"), 0777); err != nil {
 		t.Fatal(err)
 	}
 
-	if err := os.WriteFile(filepath.Join(jobarchive, "testcluster", "cluster.json"), []byte(testclusterJson), 0o666); err != nil {
+	if err := os.WriteFile(filepath.Join(jobarchive, "testcluster", "cluster.json"), []byte(testclusterJson), 0666); err != nil {
 		t.Fatal(err)
 	}
 
@@ -147,22 +139,11 @@ func setup(t *testing.T) *api.RestApi {
 	}
 
 	cfgFilePath := filepath.Join(tmpdir, "config.json")
-	if err := os.WriteFile(cfgFilePath, []byte(testconfig), 0o666); err != nil {
+	if err := os.WriteFile(cfgFilePath, []byte(testconfig), 0666); err != nil {
 		t.Fatal(err)
 	}
 
-	ccconf.Init(cfgFilePath)
-
-	// Load and check main configuration
-	if cfg := ccconf.GetPackageConfig("main"); cfg != nil {
-		if clustercfg := ccconf.GetPackageConfig("clusters"); clustercfg != nil {
-			config.Init(cfg, clustercfg)
-		} else {
-			cclog.Abort("Cluster configuration must be present")
-		}
-	} else {
-		cclog.Abort("Main configuration must be present")
-	}
+	config.Init(cfgFilePath)
 	archiveCfg := fmt.Sprintf("{\"kind\": \"file\",\"path\": \"%s\"}", jobarchive)
 
 	repository.Connect("sqlite3", dbfilepath)
@@ -176,14 +157,7 @@ func setup(t *testing.T) *api.RestApi {
 	}
 
 	archiver.Start(repository.GetJobRepository())
-
-	if cfg := ccconf.GetPackageConfig("auth"); cfg != nil {
-		auth.Init(&cfg)
-	} else {
-		cclog.Warn("Authentication disabled due to missing configuration")
-		auth.Init(nil)
-	}
-
+	auth.Init()
 	graph.Init()
 
 	return api.New()
@@ -227,11 +201,11 @@ func TestRestApi(t *testing.T) {
 	restapi.MountApiRoutes(r)
 
 	var TestJobId int64 = 123
-	TestClusterName := "testcluster"
+	var TestClusterName string = "testcluster"
 	var TestStartTime int64 = 123456789
 
 	const startJobBody string = `{
-    "jobId":            123,
+        "jobId":            123,
 		"user":             "testuser",
 		"project":          "testproj",
 		"cluster":          "testcluster",
@@ -241,9 +215,10 @@ func TestRestApi(t *testing.T) {
 		"numNodes":         1,
 		"numHwthreads":     8,
 		"numAcc":           0,
-		"shared":           "none",
+		"exclusive":        1,
 		"monitoringStatus": 1,
 		"smt":              1,
+		"tags":             [{ "type": "testTagType", "name": "testTagName", "scope": "testuser" }],
 		"resources": [
 			{
 				"hostname": "host123",
@@ -274,17 +249,16 @@ func TestRestApi(t *testing.T) {
 		if response.StatusCode != http.StatusCreated {
 			t.Fatal(response.Status, recorder.Body.String())
 		}
-		// resolver := graph.GetResolverInstance()
-		restapi.JobRepository.SyncJobs()
+		resolver := graph.GetResolverInstance()
 		job, err := restapi.JobRepository.Find(&TestJobId, &TestClusterName, &TestStartTime)
 		if err != nil {
 			t.Fatal(err)
 		}
 
-		// job.Tags, err = resolver.Job().Tags(ctx, job)
-		// if err != nil {
-		// 	t.Fatal(err)
-		// }
+		job.Tags, err = resolver.Job().Tags(ctx, job)
+		if err != nil {
+			t.Fatal(err)
+		}
 
 		if job.JobID != 123 ||
 			job.User != "testuser" ||
@@ -293,20 +267,21 @@ func TestRestApi(t *testing.T) {
 			job.SubCluster != "sc1" ||
 			job.Partition != "default" ||
 			job.Walltime != 3600 ||
-			job.ArrayJobID != 0 ||
+			job.ArrayJobId != 0 ||
 			job.NumNodes != 1 ||
 			job.NumHWThreads != 8 ||
 			job.NumAcc != 0 ||
+			job.Exclusive != 1 ||
 			job.MonitoringStatus != 1 ||
 			job.SMT != 1 ||
 			!reflect.DeepEqual(job.Resources, []*schema.Resource{{Hostname: "host123", HWThreads: []int{0, 1, 2, 3, 4, 5, 6, 7}}}) ||
-			job.StartTime != 123456789 {
+			job.StartTime.Unix() != 123456789 {
 			t.Fatalf("unexpected job properties: %#v", job)
 		}
 
-		// if len(job.Tags) != 1 || job.Tags[0].Type != "testTagType" || job.Tags[0].Name != "testTagName" || job.Tags[0].Scope != "testuser" {
-		// 	t.Fatalf("unexpected tags: %#v", job.Tags)
-		// }
+		if len(job.Tags) != 1 || job.Tags[0].Type != "testTagType" || job.Tags[0].Name != "testTagName" || job.Tags[0].Scope != "testuser" {
+			t.Fatalf("unexpected tags: %#v", job.Tags)
+		}
 	}); !ok {
 		return
 	}
@@ -374,7 +349,7 @@ func TestRestApi(t *testing.T) {
 
 	t.Run("CheckDoubleStart", func(t *testing.T) {
 		// Starting a job with the same jobId and cluster should only be allowed if the startTime is far appart!
-		body := strings.ReplaceAll(startJobBody, `"startTime": 123456789`, `"startTime": 123456790`)
+		body := strings.Replace(startJobBody, `"startTime": 123456789`, `"startTime": 123456790`, -1)
 
 		req := httptest.NewRequest(http.MethodPost, "/jobs/start_job/", bytes.NewBuffer([]byte(body)))
 		recorder := httptest.NewRecorder()
@@ -396,7 +371,7 @@ func TestRestApi(t *testing.T) {
 		"partition":        "default",
 		"walltime":         3600,
 		"numNodes":         1,
-		"shared":        	"none",
+		"exclusive":        1,
 		"monitoringStatus": 1,
 		"smt":              1,
 		"resources": [
@@ -424,7 +399,6 @@ func TestRestApi(t *testing.T) {
 	}
 
 	time.Sleep(1 * time.Second)
-	restapi.JobRepository.SyncJobs()
 
 	const stopJobBodyFailed string = `{
     "jobId":     12345,

internal/api/cluster.go

@@ -1,71 +0,0 @@
-// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved. This file is part of cc-backend.
-// Use of this source code is governed by a MIT-style
-// license that can be found in the LICENSE file.
-
-package api
-
-import (
-	"bufio"
-	"encoding/json"
-	"fmt"
-	"net/http"
-
-	"github.com/ClusterCockpit/cc-backend/internal/repository"
-	"github.com/ClusterCockpit/cc-backend/pkg/archive"
-	"github.com/ClusterCockpit/cc-lib/schema"
-)
-
-// GetClustersAPIResponse model
-type GetClustersAPIResponse struct {
-	Clusters []*schema.Cluster `json:"clusters"` // Array of clusters
-}
-
-// getClusters godoc
-// @summary     Lists all cluster configs
-// @tags Cluster query
-// @description Get a list of all cluster configs. Specific cluster can be requested using query parameter.
-// @produce     json
-// @param       cluster        query    string            false "Job Cluster"
-// @success     200            {object} api.GetClustersApiResponse  "Array of clusters"
-// @failure     400            {object} api.ErrorResponse       "Bad Request"
-// @failure     401            {object} api.ErrorResponse       "Unauthorized"
-// @failure     403            {object} api.ErrorResponse       "Forbidden"
-// @failure     500            {object} api.ErrorResponse       "Internal Server Error"
-// @security    ApiKeyAuth
-// @router      /api/clusters/ [get]
-func (api *RestApi) getClusters(rw http.ResponseWriter, r *http.Request) {
-	if user := repository.GetUserFromContext(r.Context()); user != nil &&
-		!user.HasRole(schema.RoleApi) {
-
-		handleError(fmt.Errorf("missing role: %v", schema.GetRoleString(schema.RoleApi)), http.StatusForbidden, rw)
-		return
-	}
-
-	rw.Header().Add("Content-Type", "application/json")
-	bw := bufio.NewWriter(rw)
-	defer bw.Flush()
-
-	var clusters []*schema.Cluster
-
-	if r.URL.Query().Has("cluster") {
-		name := r.URL.Query().Get("cluster")
-		cluster := archive.GetCluster(name)
-		if cluster == nil {
-			handleError(fmt.Errorf("unknown cluster: %s", name), http.StatusBadRequest, rw)
-			return
-		}
-		clusters = append(clusters, cluster)
-	} else {
-		clusters = archive.Clusters
-	}
-
-	payload := GetClustersAPIResponse{
-		Clusters: clusters,
-	}
-
-	if err := json.NewEncoder(bw).Encode(payload); err != nil {
-		handleError(err, http.StatusInternalServerError, rw)
-		return
-	}
-}

internal/api/docs.go

@@ -23,7 +23,7 @@ const docTemplate = `{
     "host": "{{.Host}}",
     "basePath": "{{.BasePath}}",
     "paths": {
-        "/api/clusters/": {
+        "/clusters/": {
             "get": {
                 "security": [
                     {
@@ -80,7 +80,7 @@ const docTemplate = `{
                 }
             }
         },
-        "/api/jobs/": {
+        "/jobs/": {
             "get": {
                 "security": [
                     {
@@ -175,7 +175,7 @@ const docTemplate = `{
                 }
             }
         },
-        "/api/jobs/delete_job/": {
+        "/jobs/delete_job/": {
             "delete": {
                 "security": [
                     {
@@ -208,7 +208,7 @@ const docTemplate = `{
                     "200": {
                         "description": "Success message",
                         "schema": {
-                            "$ref": "#/definitions/api.DefaultApiResponse"
+                            "$ref": "#/definitions/api.DeleteJobApiResponse"
                         }
                     },
                     "400": {
@@ -250,7 +250,7 @@ const docTemplate = `{
                 }
             }
         },
-        "/api/jobs/delete_job/{id}": {
+        "/jobs/delete_job/{id}": {
             "delete": {
                 "security": [
                     {
@@ -278,7 +278,7 @@ const docTemplate = `{
                     "200": {
                         "description": "Success message",
                         "schema": {
-                            "$ref": "#/definitions/api.DefaultApiResponse"
+                            "$ref": "#/definitions/api.DeleteJobApiResponse"
                         }
                     },
                     "400": {
@@ -320,7 +320,7 @@ const docTemplate = `{
                 }
             }
         },
-        "/api/jobs/delete_job_before/{ts}": {
+        "/jobs/delete_job_before/{ts}": {
             "delete": {
                 "security": [
                     {
@@ -348,7 +348,7 @@ const docTemplate = `{
                     "200": {
                         "description": "Success message",
                         "schema": {
-                            "$ref": "#/definitions/api.DefaultApiResponse"
+                            "$ref": "#/definitions/api.DeleteJobApiResponse"
                         }
                     },
                     "400": {
@@ -390,7 +390,7 @@ const docTemplate = `{
                 }
             }
         },
-        "/api/jobs/edit_meta/{id}": {
+        "/jobs/edit_meta/{id}": {
             "post": {
                 "security": [
                     {
@@ -460,14 +460,14 @@ const docTemplate = `{
                 }
             }
         },
-        "/api/jobs/start_job/": {
+        "/jobs/start_job/": {
             "post": {
                 "security": [
                     {
                         "ApiKeyAuth": []
                     }
                 ],
-                "description": "Job specified in request body will be saved to database as \"running\" with new DB ID.\nJob specifications follow the 'Job' scheme, API will fail to execute if requirements are not met.",
+                "description": "Job specified in request body will be saved to database as \"running\" with new DB ID.\nJob specifications follow the 'JobMeta' scheme, API will fail to execute if requirements are not met.",
                 "consumes": [
                     "application/json"
                 ],
@@ -485,7 +485,7 @@ const docTemplate = `{
                         "in": "body",
                         "required": true,
                         "schema": {
-                            "$ref": "#/definitions/schema.Job"
+                            "$ref": "#/definitions/schema.JobMeta"
                         }
                     }
                 ],
@@ -493,7 +493,7 @@ const docTemplate = `{
                     "201": {
                         "description": "Job added successfully",
                         "schema": {
-                            "$ref": "#/definitions/api.DefaultApiResponse"
+                            "$ref": "#/definitions/api.StartJobApiResponse"
                         }
                     },
                     "400": {
@@ -529,14 +529,14 @@ const docTemplate = `{
                 }
             }
         },
-        "/api/jobs/stop_job/": {
+        "/jobs/stop_job/": {
             "post": {
                 "security": [
                     {
                         "ApiKeyAuth": []
                     }
                 ],
-                "description": "Job to stop is specified by request body. All fields are required in this case.\nReturns full job resource information according to 'Job' scheme.",
+                "description": "Job to stop is specified by request body. All fields are required in this case.\nReturns full job resource information according to 'JobMeta' scheme.",
                 "produces": [
                     "application/json"
                 ],
@@ -559,7 +559,7 @@ const docTemplate = `{
                     "200": {
                         "description": "Success message",
                         "schema": {
-                            "$ref": "#/definitions/schema.Job"
+                            "$ref": "#/definitions/schema.JobMeta"
                         }
                     },
                     "400": {
@@ -587,7 +587,7 @@ const docTemplate = `{
                         }
                     },
                     "422": {
-                        "description": "Unprocessable Entity: job has already been stopped",
+                        "description": "Unprocessable Entity: finding job failed: sql: no rows in result set",
                         "schema": {
                             "$ref": "#/definitions/api.ErrorResponse"
                         }
@@ -601,7 +601,7 @@ const docTemplate = `{
                 }
             }
         },
-        "/api/jobs/tag_job/{id}": {
+        "/jobs/tag_job/{id}": {
             "post": {
                 "security": [
                     {
@@ -674,14 +674,14 @@ const docTemplate = `{
                 }
             }
         },
-        "/api/jobs/{id}": {
+        "/jobs/{id}": {
             "get": {
                 "security": [
                     {
                         "ApiKeyAuth": []
                     }
                 ],
-                "description": "Job to get is specified by database ID\nReturns full job resource information according to 'Job' scheme and all metrics according to 'JobData'.",
+                "description": "Job to get is specified by database ID\nReturns full job resource information according to 'JobMeta' scheme and all metrics according to 'JobData'.",
                 "produces": [
                     "application/json"
                 ],
@@ -755,7 +755,7 @@ const docTemplate = `{
                         "ApiKeyAuth": []
                     }
                 ],
-                "description": "Job to get is specified by database ID\nReturns full job resource information according to 'Job' scheme and all metrics according to 'JobData'.",
+                "description": "Job to get is specified by database ID\nReturns full job resource information according to 'JobMeta' scheme and all metrics according to 'JobData'.",
                 "consumes": [
                     "application/json"
                 ],
@@ -833,74 +833,119 @@ const docTemplate = `{
                 }
             }
         },
-        "/api/nodestats/": {
+        "/user/{id}": {
             "post": {
                 "security": [
                     {
                         "ApiKeyAuth": []
                     }
                 ],
-                "description": "Returns a JSON-encoded list of users.\nRequired query-parameter defines if all users or only users with additional special roles are returned.",
+                "description": "Modifies user defined by username (id) in one of four possible ways.\nIf more than one formValue is set then only the highest priority field is used.\nOnly accessible from IPs registered with apiAllowedIPs configuration option.",
+                "consumes": [
+                    "multipart/form-data"
+                ],
                 "produces": [
-                    "application/json"
+                    "text/plain"
                 ],
                 "tags": [
-                    "Nodestates"
+                    "User"
                 ],
-                "summary": "Deliver updated Slurm node states",
+                "summary": "Updates an existing user",
                 "parameters": [
                     {
-                        "description": "Request body containing nodes and their states",
-                        "name": "request",
-                        "in": "body",
-                        "required": true,
-                        "schema": {
-                            "$ref": "#/definitions/api.UpdateNodeStatesRequest"
-                        }
+                        "type": "string",
+                        "description": "Database ID of User",
+                        "name": "id",
+                        "in": "path",
+                        "required": true
+                    },
+                    {
+                        "enum": [
+                            "admin",
+                            "support",
+                            "manager",
+                            "user",
+                            "api"
+                        ],
+                        "type": "string",
+                        "description": "Priority 1: Role to add",
+                        "name": "add-role",
+                        "in": "formData"
+                    },
+                    {
+                        "enum": [
+                            "admin",
+                            "support",
+                            "manager",
+                            "user",
+                            "api"
+                        ],
+                        "type": "string",
+                        "description": "Priority 2: Role to remove",
+                        "name": "remove-role",
+                        "in": "formData"
+                    },
+                    {
+                        "type": "string",
+                        "description": "Priority 3: Project to add",
+                        "name": "add-project",
+                        "in": "formData"
+                    },
+                    {
+                        "type": "string",
+                        "description": "Priority 4: Project to remove",
+                        "name": "remove-project",
+                        "in": "formData"
                     }
                 ],
                 "responses": {
                     "200": {
-                        "description": "Success message",
+                        "description": "Success Response Message",
                         "schema": {
-                            "$ref": "#/definitions/api.DefaultApiResponse"
+                            "type": "string"
                         }
                     },
                     "400": {
                         "description": "Bad Request",
                         "schema": {
-                            "$ref": "#/definitions/api.ErrorResponse"
+                            "type": "string"
                         }
                     },
                     "401": {
                         "description": "Unauthorized",
                         "schema": {
-                            "$ref": "#/definitions/api.ErrorResponse"
+                            "type": "string"
                         }
                     },
                     "403": {
                         "description": "Forbidden",
                         "schema": {
-                            "$ref": "#/definitions/api.ErrorResponse"
+                            "type": "string"
+                        }
+                    },
+                    "422": {
+                        "description": "Unprocessable Entity: The user could not be updated",
+                        "schema": {
+                            "type": "string"
                         }
                     },
                     "500": {
                         "description": "Internal Server Error",
                         "schema": {
-                            "$ref": "#/definitions/api.ErrorResponse"
+                            "type": "string"
                         }
                     }
                 }
             }
         },
-        "/api/users/": {
+        "/users/": {
             "get": {
                 "security": [
                     {
                         "ApiKeyAuth": []
                     }
                 ],
-                "description": "Returns a JSON-encoded list of users.\nRequired query-parameter defines if all users or only users with additional special roles are returned.",
+                "description": "Returns a JSON-encoded list of users.\nRequired query-parameter defines if all users or only users with additional special roles are returned.\nOnly accessible from IPs registered with apiAllowedIPs configuration option.",
                 "produces": [
                     "application/json"
                 ],
@@ -952,111 +997,70 @@ const docTemplate = `{
                         }
                     }
                 }
-            }
-        },
-        "/jobs/tag_job/{id}": {
-            "delete": {
+            },
+            "post": {
                 "security": [
                     {
                         "ApiKeyAuth": []
                     }
                 ],
-                "description": "Removes tag(s) from a job specified by DB ID. Name and Type of Tag(s) must match.\nTag Scope is required for matching, options: \"global\", \"admin\". Private tags can not be deleted via API.\nIf tagged job is already finished: Tag will be removed from respective archive files.",
+                "description": "User specified in form data will be saved to database.\nOnly accessible from IPs registered with apiAllowedIPs configuration option.",
                 "consumes": [
-                    "application/json"
-                ],
-                "produces": [
-                    "application/json"
-                ],
-                "tags": [
-                    "Job add and modify"
-                ],
-                "summary": "Removes one or more tags from a job",
-                "parameters": [
-                    {
-                        "type": "integer",
-                        "description": "Job Database ID",
-                        "name": "id",
-                        "in": "path",
-                        "required": true
-                    },
-                    {
-                        "description": "Array of tag-objects to remove",
-                        "name": "request",
-                        "in": "body",
-                        "required": true,
-                        "schema": {
-                            "type": "array",
-                            "items": {
-                                "$ref": "#/definitions/api.ApiTag"
-                            }
-                        }
-                    }
-                ],
-                "responses": {
-                    "200": {
-                        "description": "Updated job resource",
-                        "schema": {
-                            "$ref": "#/definitions/schema.Job"
-                        }
-                    },
-                    "400": {
-                        "description": "Bad Request",
-                        "schema": {
-                            "$ref": "#/definitions/api.ErrorResponse"
-                        }
-                    },
-                    "401": {
-                        "description": "Unauthorized",
-                        "schema": {
-                            "$ref": "#/definitions/api.ErrorResponse"
-                        }
-                    },
-                    "404": {
-                        "description": "Job or tag does not exist",
-                        "schema": {
-                            "$ref": "#/definitions/api.ErrorResponse"
-                        }
-                    },
-                    "500": {
-                        "description": "Internal Server Error",
-                        "schema": {
-                            "$ref": "#/definitions/api.ErrorResponse"
-                        }
-                    }
-                }
-            }
-        },
-        "/tags/": {
-            "delete": {
-                "security": [
-                    {
-                        "ApiKeyAuth": []
-                    }
-                ],
-                "description": "Removes tags by type and name. Name and Type of Tag(s) must match.\nTag Scope is required for matching, options: \"global\", \"admin\". Private tags can not be deleted via API.\nTag wills be removed from respective archive files.",
-                "consumes": [
-                    "application/json"
+                    "multipart/form-data"
                 ],
                 "produces": [
                     "text/plain"
                 ],
                 "tags": [
-                    "Tag remove"
+                    "User"
                 ],
-                "summary": "Removes all tags and job-relations for type:name tuple",
+                "summary": "Adds a new user",
                 "parameters": [
                     {
-                        "description": "Array of tag-objects to remove",
-                        "name": "request",
-                        "in": "body",
-                        "required": true,
-                        "schema": {
-                            "type": "array",
-                            "items": {
-                                "$ref": "#/definitions/api.ApiTag"
-                            }
-                        }
+                        "type": "string",
+                        "description": "Unique user ID",
+                        "name": "username",
+                        "in": "formData",
+                        "required": true
+                    },
+                    {
+                        "type": "string",
+                        "description": "User password",
+                        "name": "password",
+                        "in": "formData",
+                        "required": true
+                    },
+                    {
+                        "enum": [
+                            "admin",
+                            "support",
+                            "manager",
+                            "user",
+                            "api"
+                        ],
+                        "type": "string",
+                        "description": "User role",
+                        "name": "role",
+                        "in": "formData",
+                        "required": true
+                    },
+                    {
+                        "type": "string",
+                        "description": "Managed project, required for new manager role user",
+                        "name": "project",
+                        "in": "formData"
+                    },
+                    {
+                        "type": "string",
+                        "description": "Users name",
+                        "name": "name",
+                        "in": "formData"
+                    },
+                    {
+                        "type": "string",
+                        "description": "Users email",
+                        "name": "email",
+                        "in": "formData"
                     }
                 ],
                 "responses": {
@@ -1069,25 +1073,93 @@ const docTemplate = `{
                     "400": {
                         "description": "Bad Request",
                         "schema": {
-                            "$ref": "#/definitions/api.ErrorResponse"
+                            "type": "string"
                         }
                     },
                     "401": {
                         "description": "Unauthorized",
                         "schema": {
-                            "$ref": "#/definitions/api.ErrorResponse"
+                            "type": "string"
                         }
                     },
-                    "404": {
-                        "description": "Job or tag does not exist",
+                    "403": {
+                        "description": "Forbidden",
                         "schema": {
-                            "$ref": "#/definitions/api.ErrorResponse"
+                            "type": "string"
+                        }
+                    },
+                    "422": {
+                        "description": "Unprocessable Entity: creating user failed",
+                        "schema": {
+                            "type": "string"
                         }
                     },
                     "500": {
                         "description": "Internal Server Error",
                         "schema": {
-                            "$ref": "#/definitions/api.ErrorResponse"
+                            "type": "string"
+                        }
+                    }
+                }
+            },
+            "delete": {
+                "security": [
+                    {
+                        "ApiKeyAuth": []
+                    }
+                ],
+                "description": "User defined by username in form data will be deleted from database.\nOnly accessible from IPs registered with apiAllowedIPs configuration option.",
+                "consumes": [
+                    "multipart/form-data"
+                ],
+                "produces": [
+                    "text/plain"
+                ],
+                "tags": [
+                    "User"
+                ],
+                "summary": "Deletes a user",
+                "parameters": [
+                    {
+                        "type": "string",
+                        "description": "User ID to delete",
+                        "name": "username",
+                        "in": "formData",
+                        "required": true
+                    }
+                ],
+                "responses": {
+                    "200": {
+                        "description": "User deleted successfully"
+                    },
+                    "400": {
+                        "description": "Bad Request",
+                        "schema": {
+                            "type": "string"
+                        }
+                    },
+                    "401": {
+                        "description": "Unauthorized",
+                        "schema": {
+                            "type": "string"
+                        }
+                    },
+                    "403": {
+                        "description": "Forbidden",
+                        "schema": {
+                            "type": "string"
+                        }
+                    },
+                    "422": {
+                        "description": "Unprocessable Entity: deleting user failed",
+                        "schema": {
+                            "type": "string"
+                        }
+                    },
+                    "500": {
+                        "description": "Internal Server Error",
+                        "schema": {
+                            "type": "string"
                         }
                     }
                 }
@@ -1141,14 +1213,6 @@ const docTemplate = `{
                 }
             }
         },
-        "api.DefaultApiResponse": {
-            "type": "object",
-            "properties": {
-                "msg": {
-                    "type": "string"
-                }
-            }
-        },
         "api.DeleteJobApiRequest": {
             "type": "object",
             "required": [
@@ -1172,6 +1236,14 @@ const docTemplate = `{
                 }
             }
         },
+        "api.DeleteJobApiResponse": {
+            "type": "object",
+            "properties": {
+                "msg": {
+                    "type": "string"
+                }
+            }
+        },
         "api.EditMetaRequest": {
             "type": "object",
             "properties": {
@@ -1235,7 +1307,7 @@ const docTemplate = `{
                     "description": "Array of jobs",
                     "type": "array",
                     "items": {
-                        "$ref": "#/definitions/schema.Job"
+                        "$ref": "#/definitions/schema.JobMeta"
                     }
                 },
                 "page": {
@@ -1258,35 +1330,11 @@ const docTemplate = `{
                 }
             }
         },
-        "api.Node": {
+        "api.StartJobApiResponse": {
             "type": "object",
             "properties": {
-                "cpusAllocated": {
-                    "type": "integer"
-                },
-                "cpusTotal": {
-                    "type": "integer"
-                },
-                "gpusAllocated": {
-                    "type": "integer"
-                },
-                "gpusTotal": {
-                    "type": "integer"
-                },
-                "hostname": {
+                "msg": {
                     "type": "string"
-                },
-                "memoryAllocated": {
-                    "type": "integer"
-                },
-                "memoryTotal": {
-                    "type": "integer"
-                },
-                "states": {
-                    "type": "array",
-                    "items": {
-                        "type": "string"
-                    }
                 }
             }
         },
@@ -1323,21 +1371,6 @@ const docTemplate = `{
                 }
             }
         },
-        "api.UpdateNodeStatesRequest": {
-            "type": "object",
-            "properties": {
-                "cluster": {
-                    "type": "string",
-                    "example": "fritz"
-                },
-                "nodes": {
-                    "type": "array",
-                    "items": {
-                        "$ref": "#/definitions/api.Node"
-                    }
-                }
-            }
-        },
         "schema.Accelerator": {
             "type": "object",
             "properties": {
@@ -1373,6 +1406,7 @@ const docTemplate = `{
             }
         },
         "schema.Job": {
+            "description": "Information of a HPC job.",
             "type": "object",
             "properties": {
                 "arrayJobId": {
@@ -1397,15 +1431,19 @@ const docTemplate = `{
                 "energyFootprint": {
                     "type": "object",
                     "additionalProperties": {
-                        "type": "number",
-                        "format": "float64"
+                        "type": "number"
                     }
                 },
+                "exclusive": {
+                    "type": "integer",
+                    "maximum": 2,
+                    "minimum": 0,
+                    "example": 1
+                },
                 "footprint": {
                     "type": "object",
                     "additionalProperties": {
-                        "type": "number",
-                        "format": "float64"
+                        "type": "number"
                     }
                 },
                 "id": {
@@ -1417,18 +1455,12 @@ const docTemplate = `{
                 },
                 "jobState": {
                     "enum": [
-                        "boot_fail",
-                        "cancelled",
                         "completed",
-                        "deadline",
                         "failed",
-                        "node_fail",
-                        "out-of-memory",
-                        "pending",
-                        "preempted",
-                        "running",
-                        "suspended",
-                        "timeout"
+                        "cancelled",
+                        "stopped",
+                        "timeout",
+                        "out_of_memory"
                     ],
                     "allOf": [
                         {
@@ -1472,48 +1504,23 @@ const docTemplate = `{
                     "type": "string",
                     "example": "abcd200"
                 },
-                "requestedMemory": {
-                    "description": "in MB",
-                    "type": "integer",
-                    "minimum": 1,
-                    "example": 128000
-                },
                 "resources": {
                     "type": "array",
                     "items": {
                         "$ref": "#/definitions/schema.Resource"
                     }
                 },
-                "shared": {
-                    "type": "string",
-                    "enum": [
-                        "none",
-                        "single_user",
-                        "multi_user"
-                    ]
-                },
                 "smt": {
                     "type": "integer",
                     "example": 4
                 },
                 "startTime": {
-                    "type": "integer",
-                    "example": 1649723812
-                },
-                "statistics": {
-                    "type": "object",
-                    "additionalProperties": {
-                        "$ref": "#/definitions/schema.JobStatistics"
-                    }
+                    "type": "string"
                 },
                 "subCluster": {
                     "type": "string",
                     "example": "main"
                 },
-                "submitTime": {
-                    "type": "integer",
-                    "example": 1649723812
-                },
                 "tags": {
                     "type": "array",
                     "items": {
@@ -1556,6 +1563,147 @@ const docTemplate = `{
                 }
             }
         },
+        "schema.JobMeta": {
+            "description": "Meta data information of a HPC job.",
+            "type": "object",
+            "properties": {
+                "arrayJobId": {
+                    "type": "integer",
+                    "example": 123000
+                },
+                "cluster": {
+                    "type": "string",
+                    "example": "fritz"
+                },
+                "concurrentJobs": {
+                    "$ref": "#/definitions/schema.JobLinkResultList"
+                },
+                "duration": {
+                    "type": "integer",
+                    "minimum": 1,
+                    "example": 43200
+                },
+                "energy": {
+                    "type": "number"
+                },
+                "energyFootprint": {
+                    "type": "object",
+                    "additionalProperties": {
+                        "type": "number"
+                    }
+                },
+                "exclusive": {
+                    "type": "integer",
+                    "maximum": 2,
+                    "minimum": 0,
+                    "example": 1
+                },
+                "footprint": {
+                    "type": "object",
+                    "additionalProperties": {
+                        "type": "number"
+                    }
+                },
+                "id": {
+                    "type": "integer"
+                },
+                "jobId": {
+                    "type": "integer",
+                    "example": 123000
+                },
+                "jobState": {
+                    "enum": [
+                        "completed",
+                        "failed",
+                        "cancelled",
+                        "stopped",
+                        "timeout",
+                        "out_of_memory"
+                    ],
+                    "allOf": [
+                        {
+                            "$ref": "#/definitions/schema.JobState"
+                        }
+                    ],
+                    "example": "completed"
+                },
+                "metaData": {
+                    "type": "object",
+                    "additionalProperties": {
+                        "type": "string"
+                    }
+                },
+                "monitoringStatus": {
+                    "type": "integer",
+                    "maximum": 3,
+                    "minimum": 0,
+                    "example": 1
+                },
+                "numAcc": {
+                    "type": "integer",
+                    "minimum": 1,
+                    "example": 2
+                },
+                "numHwthreads": {
+                    "type": "integer",
+                    "minimum": 1,
+                    "example": 20
+                },
+                "numNodes": {
+                    "type": "integer",
+                    "minimum": 1,
+                    "example": 2
+                },
+                "partition": {
+                    "type": "string",
+                    "example": "main"
+                },
+                "project": {
+                    "type": "string",
+                    "example": "abcd200"
+                },
+                "resources": {
+                    "type": "array",
+                    "items": {
+                        "$ref": "#/definitions/schema.Resource"
+                    }
+                },
+                "smt": {
+                    "type": "integer",
+                    "example": 4
+                },
+                "startTime": {
+                    "type": "integer",
+                    "minimum": 1,
+                    "example": 1649723812
+                },
+                "statistics": {
+                    "type": "object",
+                    "additionalProperties": {
+                        "$ref": "#/definitions/schema.JobStatistics"
+                    }
+                },
+                "subCluster": {
+                    "type": "string",
+                    "example": "main"
+                },
+                "tags": {
+                    "type": "array",
+                    "items": {
+                        "$ref": "#/definitions/schema.Tag"
+                    }
+                },
+                "user": {
+                    "type": "string",
+                    "example": "abcd100h"
+                },
+                "walltime": {
+                    "type": "integer",
+                    "minimum": 1,
+                    "example": 86400
+                }
+            }
+        },
         "schema.JobMetric": {
             "type": "object",
             "properties": {
@@ -1579,32 +1727,24 @@ const docTemplate = `{
         "schema.JobState": {
             "type": "string",
             "enum": [
-                "boot_fail",
-                "cancelled",
-                "completed",
-                "deadline",
-                "failed",
-                "node_fail",
-                "out_of_memory",
-                "pending",
-                "preempted",
                 "running",
-                "suspended",
-                "timeout"
+                "completed",
+                "failed",
+                "cancelled",
+                "stopped",
+                "timeout",
+                "preempted",
+                "out_of_memory"
             ],
             "x-enum-varnames": [
-                "JobStateBootFail",
-                "JobStateCancelled",
-                "JobStateCompleted",
-                "JobStateDeadline",
-                "JobStateFailed",
-                "JobStateNodeFail",
-                "JobStateOutOfMemory",
-                "JobStatePending",
-                "JobStatePreempted",
                 "JobStateRunning",
-                "JobStateSuspended",
-                "JobStateTimeout"
+                "JobStateCompleted",
+                "JobStateFailed",
+                "JobStateCancelled",
+                "JobStateStopped",
+                "JobStateTimeout",
+                "JobStatePreempted",
+                "JobStateOutOfMemory"
             ]
         },
         "schema.JobStatistics": {
@@ -1803,8 +1943,7 @@ const docTemplate = `{
                     "additionalProperties": {
                         "type": "array",
                         "items": {
-                            "type": "number",
-                            "format": "float64"
+                            "type": "number"
                         }
                     }
                 }
@@ -1892,9 +2031,6 @@ const docTemplate = `{
                 },
                 "remove": {
                     "type": "boolean"
-                },
-                "unit": {
-                    "$ref": "#/definitions/schema.Unit"
                 }
             }
         },
@@ -1997,7 +2133,7 @@ const docTemplate = `{
 var SwaggerInfo = &swag.Spec{
 	Version:          "1.0.0",
 	Host:             "localhost:8080",
-	BasePath:         "",
+	BasePath:         "/api",
 	Schemes:          []string{},
 	Title:            "ClusterCockpit REST API",
 	Description:      "API for batch job control.",

internal/api/job.go

@@ -1,984 +0,0 @@
-// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved. This file is part of cc-backend.
-// Use of this source code is governed by a MIT-style
-// license that can be found in the LICENSE file.
-package api
-
-import (
-	"bufio"
-	"database/sql"
-	"encoding/json"
-	"errors"
-	"fmt"
-	"net/http"
-	"strconv"
-	"strings"
-	"sync"
-	"time"
-
-	"github.com/ClusterCockpit/cc-backend/internal/archiver"
-	"github.com/ClusterCockpit/cc-backend/internal/config"
-	"github.com/ClusterCockpit/cc-backend/internal/graph"
-	"github.com/ClusterCockpit/cc-backend/internal/graph/model"
-	"github.com/ClusterCockpit/cc-backend/internal/importer"
-	"github.com/ClusterCockpit/cc-backend/internal/metricDataDispatcher"
-	"github.com/ClusterCockpit/cc-backend/internal/repository"
-	"github.com/ClusterCockpit/cc-backend/pkg/archive"
-	cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
-	"github.com/ClusterCockpit/cc-lib/schema"
-	"github.com/gorilla/mux"
-)
-
-// StopJobApiRequest model
-type StopJobApiRequest struct {
-	JobId     *int64          `json:"jobId" example:"123000"`
-	Cluster   *string         `json:"cluster" example:"fritz"`
-	StartTime *int64          `json:"startTime" example:"1649723812"`
-	State     schema.JobState `json:"jobState" validate:"required" example:"completed"`
-	StopTime  int64           `json:"stopTime" validate:"required" example:"1649763839"`
-}
-
-// DeleteJobApiRequest model
-type DeleteJobApiRequest struct {
-	JobId     *int64  `json:"jobId" validate:"required" example:"123000"` // Cluster Job ID of job
-	Cluster   *string `json:"cluster" example:"fritz"`                    // Cluster of job
-	StartTime *int64  `json:"startTime" example:"1649723812"`             // Start Time of job as epoch
-}
-
-// GetJobsApiResponse model
-type GetJobsApiResponse struct {
-	Jobs  []*schema.Job `json:"jobs"`  // Array of jobs
-	Items int           `json:"items"` // Number of jobs returned
-	Page  int           `json:"page"`  // Page id returned
-}
-
-// ApiTag model
-type ApiTag struct {
-	// Tag Type
-	Type  string `json:"type" example:"Debug"`
-	Name  string `json:"name" example:"Testjob"` // Tag Name
-	Scope string `json:"scope" example:"global"` // Tag Scope for Frontend Display
-}
-
-// ApiMeta model
-type EditMetaRequest struct {
-	Key   string `json:"key" example:"jobScript"`
-	Value string `json:"value" example:"bash script"`
-}
-
-type TagJobApiRequest []*ApiTag
-
-type GetJobApiRequest []string
-
-type GetJobApiResponse struct {
-	Meta *schema.Job
-	Data []*JobMetricWithName
-}
-
-type GetCompleteJobApiResponse struct {
-	Meta *schema.Job
-	Data schema.JobData
-}
-
-type JobMetricWithName struct {
-	Metric *schema.JobMetric  `json:"metric"`
-	Name   string             `json:"name"`
-	Scope  schema.MetricScope `json:"scope"`
-}
-
-// getJobs godoc
-// @summary     Lists all jobs
-// @tags Job query
-// @description Get a list of all jobs. Filters can be applied using query parameters.
-// @description Number of results can be limited by page. Results are sorted by descending startTime.
-// @produce     json
-// @param       state          query    string            false "Job State" Enums(running, completed, failed, cancelled, stopped, timeout)
-// @param       cluster        query    string            false "Job Cluster"
-// @param       start-time     query    string            false "Syntax: '$from-$to', as unix epoch timestamps in seconds"
-// @param       items-per-page query    int               false "Items per page (Default: 25)"
-// @param       page           query    int               false "Page Number (Default: 1)"
-// @param       with-metadata  query    bool              false "Include metadata (e.g. jobScript) in response"
-// @success     200            {object} api.GetJobsApiResponse  "Job array and page info"
-// @failure     400            {object} api.ErrorResponse       "Bad Request"
-// @failure     401   		   {object} api.ErrorResponse       "Unauthorized"
-// @failure     403            {object} api.ErrorResponse       "Forbidden"
-// @failure     500            {object} api.ErrorResponse       "Internal Server Error"
-// @security    ApiKeyAuth
-// @router      /api/jobs/ [get]
-func (api *RestApi) getJobs(rw http.ResponseWriter, r *http.Request) {
-	withMetadata := false
-	filter := &model.JobFilter{}
-	page := &model.PageRequest{ItemsPerPage: 25, Page: 1}
-	order := &model.OrderByInput{Field: "startTime", Type: "col", Order: model.SortDirectionEnumDesc}
-
-	for key, vals := range r.URL.Query() {
-		switch key {
-		// TODO: add project filter
-		case "state":
-			for _, s := range vals {
-				state := schema.JobState(s)
-				if !state.Valid() {
-					handleError(fmt.Errorf("invalid query parameter value: state"),
-						http.StatusBadRequest, rw)
-					return
-				}
-				filter.State = append(filter.State, state)
-			}
-		case "cluster":
-			filter.Cluster = &model.StringInput{Eq: &vals[0]}
-		case "start-time": // ?startTime=1753707480-1754053139
-			st := strings.Split(vals[0], "-")
-			if len(st) != 2 {
-				handleError(fmt.Errorf("invalid query parameter value: startTime"),
-					http.StatusBadRequest, rw)
-				return
-			}
-			from, err := strconv.ParseInt(st[0], 10, 64)
-			if err != nil {
-				handleError(err, http.StatusBadRequest, rw)
-				return
-			}
-			to, err := strconv.ParseInt(st[1], 10, 64)
-			if err != nil {
-				handleError(err, http.StatusBadRequest, rw)
-				return
-			}
-			ufrom, uto := time.Unix(from, 0), time.Unix(to, 0)
-			filter.StartTime = &config.TimeRange{From: &ufrom, To: &uto}
-		case "page":
-			x, err := strconv.Atoi(vals[0])
-			if err != nil {
-				handleError(err, http.StatusBadRequest, rw)
-				return
-			}
-			page.Page = x
-		case "items-per-page":
-			x, err := strconv.Atoi(vals[0])
-			if err != nil {
-				handleError(err, http.StatusBadRequest, rw)
-				return
-			}
-			page.ItemsPerPage = x
-		case "with-metadata":
-			withMetadata = true
-		default:
-			handleError(fmt.Errorf("invalid query parameter: %s", key),
-				http.StatusBadRequest, rw)
-			return
-		}
-	}
-
-	jobs, err := api.JobRepository.QueryJobs(r.Context(), []*model.JobFilter{filter}, page, order)
-	if err != nil {
-		handleError(err, http.StatusInternalServerError, rw)
-		return
-	}
-
-	results := make([]*schema.Job, 0, len(jobs))
-	for _, job := range jobs {
-		if withMetadata {
-			if _, err = api.JobRepository.FetchMetadata(job); err != nil {
-				handleError(err, http.StatusInternalServerError, rw)
-				return
-			}
-		}
-
-		job.Tags, err = api.JobRepository.GetTags(repository.GetUserFromContext(r.Context()), job.ID)
-		if err != nil {
-			handleError(err, http.StatusInternalServerError, rw)
-			return
-		}
-
-		if job.MonitoringStatus == schema.MonitoringStatusArchivingSuccessful {
-			job.Statistics, err = archive.GetStatistics(job)
-			if err != nil {
-				handleError(err, http.StatusInternalServerError, rw)
-				return
-			}
-		}
-
-		results = append(results, job)
-	}
-
-	cclog.Debugf("/api/jobs: %d jobs returned", len(results))
-	rw.Header().Add("Content-Type", "application/json")
-	bw := bufio.NewWriter(rw)
-	defer bw.Flush()
-
-	payload := GetJobsApiResponse{
-		Jobs:  results,
-		Items: page.ItemsPerPage,
-		Page:  page.Page,
-	}
-
-	if err := json.NewEncoder(bw).Encode(payload); err != nil {
-		handleError(err, http.StatusInternalServerError, rw)
-		return
-	}
-}
-
-// getCompleteJobById godoc
-// @summary   Get job meta and optional all metric data
-// @tags Job query
-// @description Job to get is specified by database ID
-// @description Returns full job resource information according to 'Job' scheme and all metrics according to 'JobData'.
-// @produce     json
-// @param       id          path     int                  true "Database ID of Job"
-// @param       all-metrics query    bool                 false "Include all available metrics"
-// @success     200     {object} api.GetJobApiResponse      "Job resource"
-// @failure     400     {object} api.ErrorResponse          "Bad Request"
-// @failure     401     {object} api.ErrorResponse          "Unauthorized"
-// @failure     403     {object} api.ErrorResponse          "Forbidden"
-// @failure     404     {object} api.ErrorResponse          "Resource not found"
-// @failure     422     {object} api.ErrorResponse          "Unprocessable Entity: finding job failed: sql: no rows in result set"
-// @failure     500     {object} api.ErrorResponse          "Internal Server Error"
-// @security    ApiKeyAuth
-// @router      /api/jobs/{id} [get]
-func (api *RestApi) getCompleteJobById(rw http.ResponseWriter, r *http.Request) {
-	// Fetch job from db
-	id, ok := mux.Vars(r)["id"]
-	var job *schema.Job
-	var err error
-	if ok {
-		id, e := strconv.ParseInt(id, 10, 64)
-		if e != nil {
-			handleError(fmt.Errorf("integer expected in path for id: %w", e), http.StatusBadRequest, rw)
-			return
-		}
-
-		job, err = api.JobRepository.FindById(r.Context(), id) // Get Job from Repo by ID
-	} else {
-		handleError(fmt.Errorf("the parameter 'id' is required"), http.StatusBadRequest, rw)
-		return
-	}
-	if err != nil {
-		handleError(fmt.Errorf("finding job with db id %s failed: %w", id, err), http.StatusUnprocessableEntity, rw)
-		return
-	}
-
-	job.Tags, err = api.JobRepository.GetTags(repository.GetUserFromContext(r.Context()), job.ID)
-	if err != nil {
-		handleError(err, http.StatusInternalServerError, rw)
-		return
-
-	}
-	if _, err = api.JobRepository.FetchMetadata(job); err != nil {
-
-		handleError(err, http.StatusInternalServerError, rw)
-		return
-	}
-
-	var scopes []schema.MetricScope
-
-	if job.NumNodes == 1 {
-		scopes = []schema.MetricScope{"core"}
-	} else {
-		scopes = []schema.MetricScope{"node"}
-	}
-
-	var data schema.JobData
-
-	metricConfigs := archive.GetCluster(job.Cluster).MetricConfig
-	resolution := 0
-
-	for _, mc := range metricConfigs {
-		resolution = max(resolution, mc.Timestep)
-	}
-
-	if r.URL.Query().Get("all-metrics") == "true" {
-		data, err = metricDataDispatcher.LoadData(job, nil, scopes, r.Context(), resolution)
-		if err != nil {
-			cclog.Warnf("REST: error while loading all-metrics job data for JobID %d on %s", job.JobID, job.Cluster)
-			return
-		}
-	}
-
-	cclog.Debugf("/api/job/%s: get job %d", id, job.JobID)
-	rw.Header().Add("Content-Type", "application/json")
-	bw := bufio.NewWriter(rw)
-	defer bw.Flush()
-
-	payload := GetCompleteJobApiResponse{
-		Meta: job,
-		Data: data,
-	}
-
-	if err := json.NewEncoder(bw).Encode(payload); err != nil {
-		handleError(err, http.StatusInternalServerError, rw)
-		return
-	}
-}
-
-// getJobById godoc
-// @summary   Get job meta and configurable metric data
-// @tags Job query
-// @description Job to get is specified by database ID
-// @description Returns full job resource information according to 'Job' scheme and all metrics according to 'JobData'.
-// @accept      json
-// @produce     json
-// @param       id          path     int                  true "Database ID of Job"
-// @param       request     body     api.GetJobApiRequest true  "Array of metric names"
-// @success     200     {object} api.GetJobApiResponse      "Job resource"
-// @failure     400     {object} api.ErrorResponse          "Bad Request"
-// @failure     401     {object} api.ErrorResponse          "Unauthorized"
-// @failure     403     {object} api.ErrorResponse          "Forbidden"
-// @failure     404     {object} api.ErrorResponse          "Resource not found"
-// @failure     422     {object} api.ErrorResponse          "Unprocessable Entity: finding job failed: sql: no rows in result set"
-// @failure     500     {object} api.ErrorResponse          "Internal Server Error"
-// @security    ApiKeyAuth
-// @router      /api/jobs/{id} [post]
-func (api *RestApi) getJobById(rw http.ResponseWriter, r *http.Request) {
-	// Fetch job from db
-	id, ok := mux.Vars(r)["id"]
-	var job *schema.Job
-	var err error
-	if ok {
-		id, e := strconv.ParseInt(id, 10, 64)
-		if e != nil {
-			handleError(fmt.Errorf("integer expected in path for id: %w", e), http.StatusBadRequest, rw)
-			return
-		}
-
-		job, err = api.JobRepository.FindById(r.Context(), id)
-	} else {
-		handleError(errors.New("the parameter 'id' is required"), http.StatusBadRequest, rw)
-		return
-	}
-	if err != nil {
-		handleError(fmt.Errorf("finding job with db id %s failed: %w", id, err), http.StatusUnprocessableEntity, rw)
-		return
-	}
-
-	job.Tags, err = api.JobRepository.GetTags(repository.GetUserFromContext(r.Context()), job.ID)
-	if err != nil {
-		handleError(err, http.StatusInternalServerError, rw)
-		return
-
-	}
-	if _, err = api.JobRepository.FetchMetadata(job); err != nil {
-
-		handleError(err, http.StatusInternalServerError, rw)
-		return
-	}
-
-	var metrics GetJobApiRequest
-	if err = decode(r.Body, &metrics); err != nil {
-		http.Error(rw, err.Error(), http.StatusBadRequest)
-		return
-	}
-
-	var scopes []schema.MetricScope
-
-	if job.NumNodes == 1 {
-		scopes = []schema.MetricScope{"core"}
-	} else {
-		scopes = []schema.MetricScope{"node"}
-	}
-
-	metricConfigs := archive.GetCluster(job.Cluster).MetricConfig
-	resolution := 0
-
-	for _, mc := range metricConfigs {
-		resolution = max(resolution, mc.Timestep)
-	}
-
-	data, err := metricDataDispatcher.LoadData(job, metrics, scopes, r.Context(), resolution)
-	if err != nil {
-		cclog.Warnf("REST: error while loading job data for JobID %d on %s", job.JobID, job.Cluster)
-		return
-	}
-
-	res := []*JobMetricWithName{}
-	for name, md := range data {
-		for scope, metric := range md {
-			res = append(res, &JobMetricWithName{
-				Name:   name,
-				Scope:  scope,
-				Metric: metric,
-			})
-		}
-	}
-
-	cclog.Debugf("/api/job/%s: get job %d", id, job.JobID)
-	rw.Header().Add("Content-Type", "application/json")
-	bw := bufio.NewWriter(rw)
-	defer bw.Flush()
-
-	payload := GetJobApiResponse{
-		Meta: job,
-		Data: res,
-	}
-
-	if err := json.NewEncoder(bw).Encode(payload); err != nil {
-		handleError(err, http.StatusInternalServerError, rw)
-		return
-	}
-}
-
-// editMeta godoc
-// @summary    Edit meta-data json
-// @tags Job add and modify
-// @description Edit key value pairs in job metadata json
-// @description If a key already exists its content will be overwritten
-// @accept      json
-// @produce     json
-// @param       id      path     int                  true "Job Database ID"
-// @param       request body     api.EditMetaRequest  true "Kay value pair to add"
-// @success     200     {object} schema.Job                "Updated job resource"
-// @failure     400     {object} api.ErrorResponse         "Bad Request"
-// @failure     401     {object} api.ErrorResponse         "Unauthorized"
-// @failure     404     {object} api.ErrorResponse         "Job does not exist"
-// @failure     500     {object} api.ErrorResponse         "Internal Server Error"
-// @security    ApiKeyAuth
-// @router      /api/jobs/edit_meta/{id} [post]
-func (api *RestApi) editMeta(rw http.ResponseWriter, r *http.Request) {
-	id, err := strconv.ParseInt(mux.Vars(r)["id"], 10, 64)
-	if err != nil {
-		http.Error(rw, err.Error(), http.StatusBadRequest)
-		return
-	}
-
-	job, err := api.JobRepository.FindById(r.Context(), id)
-	if err != nil {
-		http.Error(rw, err.Error(), http.StatusNotFound)
-		return
-	}
-
-	var req EditMetaRequest
-	if err := decode(r.Body, &req); err != nil {
-		http.Error(rw, err.Error(), http.StatusBadRequest)
-		return
-	}
-
-	if err := api.JobRepository.UpdateMetadata(job, req.Key, req.Value); err != nil {
-		http.Error(rw, err.Error(), http.StatusInternalServerError)
-		return
-	}
-
-	rw.Header().Add("Content-Type", "application/json")
-	rw.WriteHeader(http.StatusOK)
-	json.NewEncoder(rw).Encode(job)
-}
-
-// tagJob godoc
-// @summary     Adds one or more tags to a job
-// @tags Job add and modify
-// @description Adds tag(s) to a job specified by DB ID. Name and Type of Tag(s) can be chosen freely.
-// @description Tag Scope for frontend visibility will default to "global" if none entered, other options: "admin" or specific username.
-// @description If tagged job is already finished: Tag will be written directly to respective archive files.
-// @accept      json
-// @produce     json
-// @param       id      path     int                  true "Job Database ID"
-// @param       request body     api.TagJobApiRequest true "Array of tag-objects to add"
-// @success     200     {object} schema.Job                "Updated job resource"
-// @failure     400     {object} api.ErrorResponse         "Bad Request"
-// @failure     401     {object} api.ErrorResponse         "Unauthorized"
-// @failure     404     {object} api.ErrorResponse         "Job or tag does not exist"
-// @failure     500     {object} api.ErrorResponse         "Internal Server Error"
-// @security    ApiKeyAuth
-// @router      /api/jobs/tag_job/{id} [post]
-func (api *RestApi) tagJob(rw http.ResponseWriter, r *http.Request) {
-	id, err := strconv.ParseInt(mux.Vars(r)["id"], 10, 64)
-	if err != nil {
-		http.Error(rw, err.Error(), http.StatusBadRequest)
-		return
-	}
-
-	job, err := api.JobRepository.FindById(r.Context(), id)
-	if err != nil {
-		http.Error(rw, err.Error(), http.StatusNotFound)
-		return
-	}
-
-	job.Tags, err = api.JobRepository.GetTags(repository.GetUserFromContext(r.Context()), job.ID)
-	if err != nil {
-		http.Error(rw, err.Error(), http.StatusInternalServerError)
-		return
-	}
-
-	var req TagJobApiRequest
-	if err := decode(r.Body, &req); err != nil {
-		http.Error(rw, err.Error(), http.StatusBadRequest)
-		return
-	}
-
-	for _, tag := range req {
-		tagId, err := api.JobRepository.AddTagOrCreate(repository.GetUserFromContext(r.Context()), *job.ID, tag.Type, tag.Name, tag.Scope)
-		if err != nil {
-			http.Error(rw, err.Error(), http.StatusInternalServerError)
-			return
-		}
-
-		job.Tags = append(job.Tags, &schema.Tag{
-			ID:    tagId,
-			Type:  tag.Type,
-			Name:  tag.Name,
-			Scope: tag.Scope,
-		})
-	}
-
-	rw.Header().Add("Content-Type", "application/json")
-	rw.WriteHeader(http.StatusOK)
-	json.NewEncoder(rw).Encode(job)
-}
-
-// removeTagJob godoc
-// @summary     Removes one or more tags from a job
-// @tags Job add and modify
-// @description Removes tag(s) from a job specified by DB ID. Name and Type of Tag(s) must match.
-// @description Tag Scope is required for matching, options: "global", "admin". Private tags can not be deleted via API.
-// @description If tagged job is already finished: Tag will be removed from respective archive files.
-// @accept      json
-// @produce     json
-// @param       id      path     int                  true "Job Database ID"
-// @param       request body     api.TagJobApiRequest true "Array of tag-objects to remove"
-// @success     200     {object} schema.Job                "Updated job resource"
-// @failure     400     {object} api.ErrorResponse         "Bad Request"
-// @failure     401     {object} api.ErrorResponse         "Unauthorized"
-// @failure     404     {object} api.ErrorResponse         "Job or tag does not exist"
-// @failure     500     {object} api.ErrorResponse         "Internal Server Error"
-// @security    ApiKeyAuth
-// @router      /jobs/tag_job/{id} [delete]
-func (api *RestApi) removeTagJob(rw http.ResponseWriter, r *http.Request) {
-	id, err := strconv.ParseInt(mux.Vars(r)["id"], 10, 64)
-	if err != nil {
-		http.Error(rw, err.Error(), http.StatusBadRequest)
-		return
-	}
-
-	job, err := api.JobRepository.FindById(r.Context(), id)
-	if err != nil {
-		http.Error(rw, err.Error(), http.StatusNotFound)
-		return
-	}
-
-	job.Tags, err = api.JobRepository.GetTags(repository.GetUserFromContext(r.Context()), job.ID)
-	if err != nil {
-		http.Error(rw, err.Error(), http.StatusInternalServerError)
-		return
-	}
-
-	var req TagJobApiRequest
-	if err := decode(r.Body, &req); err != nil {
-		http.Error(rw, err.Error(), http.StatusBadRequest)
-		return
-	}
-
-	for _, rtag := range req {
-		// Only Global and Admin Tags
-		if rtag.Scope != "global" && rtag.Scope != "admin" {
-			cclog.Warnf("Cannot delete private tag for job %d: Skip", job.JobID)
-			continue
-		}
-
-		remainingTags, err := api.JobRepository.RemoveJobTagByRequest(repository.GetUserFromContext(r.Context()), *job.ID, rtag.Type, rtag.Name, rtag.Scope)
-		if err != nil {
-			http.Error(rw, err.Error(), http.StatusInternalServerError)
-			return
-		}
-
-		job.Tags = remainingTags
-	}
-
-	rw.Header().Add("Content-Type", "application/json")
-	rw.WriteHeader(http.StatusOK)
-	json.NewEncoder(rw).Encode(job)
-}
-
-// removeTags godoc
-// @summary     Removes all tags and job-relations for type:name tuple
-// @tags Tag remove
-// @description Removes tags by type and name. Name and Type of Tag(s) must match.
-// @description Tag Scope is required for matching, options: "global", "admin". Private tags can not be deleted via API.
-// @description Tag wills be removed from respective archive files.
-// @accept      json
-// @produce     plain
-// @param       request body     api.TagJobApiRequest true "Array of tag-objects to remove"
-// @success     200     {string} string                    "Success Response"
-// @failure     400     {object} api.ErrorResponse         "Bad Request"
-// @failure     401     {object} api.ErrorResponse         "Unauthorized"
-// @failure     404     {object} api.ErrorResponse         "Job or tag does not exist"
-// @failure     500     {object} api.ErrorResponse         "Internal Server Error"
-// @security    ApiKeyAuth
-// @router      /tags/ [delete]
-func (api *RestApi) removeTags(rw http.ResponseWriter, r *http.Request) {
-	var req TagJobApiRequest
-	if err := decode(r.Body, &req); err != nil {
-		http.Error(rw, err.Error(), http.StatusBadRequest)
-		return
-	}
-
-	targetCount := len(req)
-	currentCount := 0
-	for _, rtag := range req {
-		// Only Global and Admin Tags
-		if rtag.Scope != "global" && rtag.Scope != "admin" {
-			cclog.Warn("Cannot delete private tag: Skip")
-			continue
-		}
-
-		err := api.JobRepository.RemoveTagByRequest(rtag.Type, rtag.Name, rtag.Scope)
-		if err != nil {
-			http.Error(rw, err.Error(), http.StatusInternalServerError)
-			return
-		} else {
-			currentCount++
-		}
-	}
-
-	rw.WriteHeader(http.StatusOK)
-	fmt.Fprintf(rw, "Deleted Tags from DB: %d successfull of %d requested\n", currentCount, targetCount)
-}
-
-// startJob godoc
-// @summary     Adds a new job as "running"
-// @tags Job add and modify
-// @description Job specified in request body will be saved to database as "running" with new DB ID.
-// @description Job specifications follow the 'Job' scheme, API will fail to execute if requirements are not met.
-// @accept      json
-// @produce     json
-// @param       request body     schema.Job true "Job to add"
-// @success     201     {object} api.DefaultApiResponse    "Job added successfully"
-// @failure     400     {object} api.ErrorResponse            "Bad Request"
-// @failure     401     {object} api.ErrorResponse            "Unauthorized"
-// @failure     403     {object} api.ErrorResponse            "Forbidden"
-// @failure     422     {object} api.ErrorResponse            "Unprocessable Entity: The combination of jobId, clusterId and startTime does already exist"
-// @failure     500     {object} api.ErrorResponse            "Internal Server Error"
-// @security    ApiKeyAuth
-// @router      /api/jobs/start_job/ [post]
-func (api *RestApi) startJob(rw http.ResponseWriter, r *http.Request) {
-	req := schema.Job{
-		Shared:           "none",
-		MonitoringStatus: schema.MonitoringStatusRunningOrArchiving,
-	}
-	if err := decode(r.Body, &req); err != nil {
-		handleError(fmt.Errorf("parsing request body failed: %w", err), http.StatusBadRequest, rw)
-		return
-	}
-
-	cclog.Printf("REST: %s\n", req.GoString())
-	req.State = schema.JobStateRunning
-
-	if err := importer.SanityChecks(&req); err != nil {
-		handleError(err, http.StatusBadRequest, rw)
-		return
-	}
-
-	// aquire lock to avoid race condition between API calls
-	var unlockOnce sync.Once
-	api.RepositoryMutex.Lock()
-	defer unlockOnce.Do(api.RepositoryMutex.Unlock)
-
-	// Check if combination of (job_id, cluster_id, start_time) already exists:
-	jobs, err := api.JobRepository.FindAll(&req.JobID, &req.Cluster, nil)
-	if err != nil && err != sql.ErrNoRows {
-		handleError(fmt.Errorf("checking for duplicate failed: %w", err), http.StatusInternalServerError, rw)
-		return
-	} else if err == nil {
-		for _, job := range jobs {
-			if (req.StartTime - job.StartTime) < 86400 {
-				handleError(fmt.Errorf("a job with that jobId, cluster and startTime already exists: dbid: %d, jobid: %d", job.ID, job.JobID), http.StatusUnprocessableEntity, rw)
-				return
-			}
-		}
-	}
-
-	id, err := api.JobRepository.Start(&req)
-	if err != nil {
-		handleError(fmt.Errorf("insert into database failed: %w", err), http.StatusInternalServerError, rw)
-		return
-	}
-	// unlock here, adding Tags can be async
-	unlockOnce.Do(api.RepositoryMutex.Unlock)
-
-	for _, tag := range req.Tags {
-		if _, err := api.JobRepository.AddTagOrCreate(repository.GetUserFromContext(r.Context()), id, tag.Type, tag.Name, tag.Scope); err != nil {
-			http.Error(rw, err.Error(), http.StatusInternalServerError)
-			handleError(fmt.Errorf("adding tag to new job %d failed: %w", id, err), http.StatusInternalServerError, rw)
-			return
-		}
-	}
-
-	cclog.Printf("new job (id: %d): cluster=%s, jobId=%d, user=%s, startTime=%d", id, req.Cluster, req.JobID, req.User, req.StartTime)
-	rw.Header().Add("Content-Type", "application/json")
-	rw.WriteHeader(http.StatusCreated)
-	json.NewEncoder(rw).Encode(DefaultApiResponse{
-		Message: "success",
-	})
-}
-
-// stopJobByRequest godoc
-// @summary     Marks job as completed and triggers archiving
-// @tags Job add and modify
-// @description Job to stop is specified by request body. All fields are required in this case.
-// @description Returns full job resource information according to 'Job' scheme.
-// @produce     json
-// @param       request body     api.StopJobApiRequest true "All fields required"
-// @success     200     {object} schema.Job                 "Success message"
-// @failure     400     {object} api.ErrorResponse          "Bad Request"
-// @failure     401     {object} api.ErrorResponse          "Unauthorized"
-// @failure     403     {object} api.ErrorResponse          "Forbidden"
-// @failure     404     {object} api.ErrorResponse          "Resource not found"
-// @failure     422     {object} api.ErrorResponse          "Unprocessable Entity: job has already been stopped"
-// @failure     500     {object} api.ErrorResponse          "Internal Server Error"
-// @security    ApiKeyAuth
-// @router      /api/jobs/stop_job/ [post]
-func (api *RestApi) stopJobByRequest(rw http.ResponseWriter, r *http.Request) {
-	// Parse request body
-	req := StopJobApiRequest{}
-	if err := decode(r.Body, &req); err != nil {
-		handleError(fmt.Errorf("parsing request body failed: %w", err), http.StatusBadRequest, rw)
-		return
-	}
-
-	// Fetch job (that will be stopped) from db
-	var job *schema.Job
-	var err error
-	if req.JobId == nil {
-		handleError(errors.New("the field 'jobId' is required"), http.StatusBadRequest, rw)
-		return
-	}
-
-	// cclog.Printf("loading db job for stopJobByRequest... : stopJobApiRequest=%v", req)
-	job, err = api.JobRepository.Find(req.JobId, req.Cluster, req.StartTime)
-	if err != nil {
-		job, err = api.JobRepository.FindCached(req.JobId, req.Cluster, req.StartTime)
-		// FIXME: Previous error is hidden
-		if err != nil {
-			handleError(fmt.Errorf("finding job failed: %w", err), http.StatusUnprocessableEntity, rw)
-			return
-		}
-	}
-
-	api.checkAndHandleStopJob(rw, job, req)
-}
-
-// deleteJobById godoc
-// @summary     Remove a job from the sql database
-// @tags Job remove
-// @description Job to remove is specified by database ID. This will not remove the job from the job archive.
-// @produce     json
-// @param       id      path     int                   true "Database ID of Job"
-// @success     200     {object} api.DefaultApiResponse  "Success message"
-// @failure     400     {object} api.ErrorResponse          "Bad Request"
-// @failure     401     {object} api.ErrorResponse          "Unauthorized"
-// @failure     403     {object} api.ErrorResponse          "Forbidden"
-// @failure     404     {object} api.ErrorResponse          "Resource not found"
-// @failure     422     {object} api.ErrorResponse          "Unprocessable Entity: finding job failed: sql: no rows in result set"
-// @failure     500     {object} api.ErrorResponse          "Internal Server Error"
-// @security    ApiKeyAuth
-// @router      /api/jobs/delete_job/{id} [delete]
-func (api *RestApi) deleteJobById(rw http.ResponseWriter, r *http.Request) {
-	// Fetch job (that will be stopped) from db
-	id, ok := mux.Vars(r)["id"]
-	var err error
-	if ok {
-		id, e := strconv.ParseInt(id, 10, 64)
-		if e != nil {
-			handleError(fmt.Errorf("integer expected in path for id: %w", e), http.StatusBadRequest, rw)
-			return
-		}
-
-		err = api.JobRepository.DeleteJobById(id)
-	} else {
-		handleError(errors.New("the parameter 'id' is required"), http.StatusBadRequest, rw)
-		return
-	}
-	if err != nil {
-		handleError(fmt.Errorf("deleting job failed: %w", err), http.StatusUnprocessableEntity, rw)
-		return
-	}
-	rw.Header().Add("Content-Type", "application/json")
-	rw.WriteHeader(http.StatusOK)
-	json.NewEncoder(rw).Encode(DefaultApiResponse{
-		Message: fmt.Sprintf("Successfully deleted job %s", id),
-	})
-}
-
-// deleteJobByRequest godoc
-// @summary     Remove a job from the sql database
-// @tags Job remove
-// @description Job to delete is specified by request body. All fields are required in this case.
-// @accept      json
-// @produce     json
-// @param       request body     api.DeleteJobApiRequest true "All fields required"
-// @success     200     {object} api.DefaultApiResponse  "Success message"
-// @failure     400     {object} api.ErrorResponse          "Bad Request"
-// @failure     401     {object} api.ErrorResponse          "Unauthorized"
-// @failure     403     {object} api.ErrorResponse          "Forbidden"
-// @failure     404     {object} api.ErrorResponse          "Resource not found"
-// @failure     422     {object} api.ErrorResponse          "Unprocessable Entity: finding job failed: sql: no rows in result set"
-// @failure     500     {object} api.ErrorResponse          "Internal Server Error"
-// @security    ApiKeyAuth
-// @router      /api/jobs/delete_job/ [delete]
-func (api *RestApi) deleteJobByRequest(rw http.ResponseWriter, r *http.Request) {
-	// Parse request body
-	req := DeleteJobApiRequest{}
-	if err := decode(r.Body, &req); err != nil {
-		handleError(fmt.Errorf("parsing request body failed: %w", err), http.StatusBadRequest, rw)
-		return
-	}
-
-	// Fetch job (that will be deleted) from db
-	var job *schema.Job
-	var err error
-	if req.JobId == nil {
-		handleError(errors.New("the field 'jobId' is required"), http.StatusBadRequest, rw)
-		return
-	}
-
-	job, err = api.JobRepository.Find(req.JobId, req.Cluster, req.StartTime)
-	if err != nil {
-		handleError(fmt.Errorf("finding job failed: %w", err), http.StatusUnprocessableEntity, rw)
-		return
-	}
-
-	err = api.JobRepository.DeleteJobById(*job.ID)
-	if err != nil {
-		handleError(fmt.Errorf("deleting job failed: %w", err), http.StatusUnprocessableEntity, rw)
-		return
-	}
-
-	rw.Header().Add("Content-Type", "application/json")
-	rw.WriteHeader(http.StatusOK)
-	json.NewEncoder(rw).Encode(DefaultApiResponse{
-		Message: fmt.Sprintf("Successfully deleted job %d", job.ID),
-	})
-}
-
-// deleteJobBefore godoc
-// @summary     Remove a job from the sql database
-// @tags Job remove
-// @description Remove all jobs with start time before timestamp. The jobs will not be removed from the job archive.
-// @produce     json
-// @param       ts      path     int                   true "Unix epoch timestamp"
-// @success     200     {object} api.DefaultApiResponse  "Success message"
-// @failure     400     {object} api.ErrorResponse          "Bad Request"
-// @failure     401     {object} api.ErrorResponse          "Unauthorized"
-// @failure     403     {object} api.ErrorResponse          "Forbidden"
-// @failure     404     {object} api.ErrorResponse          "Resource not found"
-// @failure     422     {object} api.ErrorResponse          "Unprocessable Entity: finding job failed: sql: no rows in result set"
-// @failure     500     {object} api.ErrorResponse          "Internal Server Error"
-// @security    ApiKeyAuth
-// @router      /api/jobs/delete_job_before/{ts} [delete]
-func (api *RestApi) deleteJobBefore(rw http.ResponseWriter, r *http.Request) {
-	var cnt int
-	// Fetch job (that will be stopped) from db
-	id, ok := mux.Vars(r)["ts"]
-	var err error
-	if ok {
-		ts, e := strconv.ParseInt(id, 10, 64)
-		if e != nil {
-			handleError(fmt.Errorf("integer expected in path for ts: %w", e), http.StatusBadRequest, rw)
-			return
-		}
-
-		cnt, err = api.JobRepository.DeleteJobsBefore(ts)
-	} else {
-		handleError(errors.New("the parameter 'ts' is required"), http.StatusBadRequest, rw)
-		return
-	}
-	if err != nil {
-		handleError(fmt.Errorf("deleting jobs failed: %w", err), http.StatusUnprocessableEntity, rw)
-		return
-	}
-
-	rw.Header().Add("Content-Type", "application/json")
-	rw.WriteHeader(http.StatusOK)
-	json.NewEncoder(rw).Encode(DefaultApiResponse{
-		Message: fmt.Sprintf("Successfully deleted %d jobs", cnt),
-	})
-}
-
-func (api *RestApi) checkAndHandleStopJob(rw http.ResponseWriter, job *schema.Job, req StopJobApiRequest) {
-	// Sanity checks
-	if job.State != schema.JobStateRunning {
-		handleError(fmt.Errorf("jobId %d (id %d) on %s : job has already been stopped (state is: %s)", job.JobID, job.ID, job.Cluster, job.State), http.StatusUnprocessableEntity, rw)
-		return
-	}
-
-	if job == nil || job.StartTime > req.StopTime {
-		handleError(fmt.Errorf("jobId %d (id %d) on %s : stopTime %d must be larger/equal than startTime %d", job.JobID, job.ID, job.Cluster, req.StopTime, job.StartTime), http.StatusBadRequest, rw)
-		return
-	}
-
-	if req.State != "" && !req.State.Valid() {
-		handleError(fmt.Errorf("jobId %d (id %d) on %s : invalid requested job state: %#v", job.JobID, job.ID, job.Cluster, req.State), http.StatusBadRequest, rw)
-		return
-	} else if req.State == "" {
-		req.State = schema.JobStateCompleted
-	}
-
-	// Mark job as stopped in the database (update state and duration)
-	job.Duration = int32(req.StopTime - job.StartTime)
-	job.State = req.State
-	api.JobRepository.Mutex.Lock()
-	if err := api.JobRepository.Stop(*job.ID, job.Duration, job.State, job.MonitoringStatus); err != nil {
-		if err := api.JobRepository.StopCached(*job.ID, job.Duration, job.State, job.MonitoringStatus); err != nil {
-			api.JobRepository.Mutex.Unlock()
-			handleError(fmt.Errorf("jobId %d (id %d) on %s : marking job as '%s' (duration: %d) in DB failed: %w", job.JobID, job.ID, job.Cluster, job.State, job.Duration, err), http.StatusInternalServerError, rw)
-			return
-		}
-	}
-	api.JobRepository.Mutex.Unlock()
-
-	cclog.Printf("archiving job... (dbid: %d): cluster=%s, jobId=%d, user=%s, startTime=%d, duration=%d, state=%s", job.ID, job.Cluster, job.JobID, job.User, job.StartTime, job.Duration, job.State)
-
-	// Send a response (with status OK). This means that errors that happen from here on forward
-	// can *NOT* be communicated to the client. If reading from a MetricDataRepository or
-	// writing to the filesystem fails, the client will not know.
-	rw.Header().Add("Content-Type", "application/json")
-	rw.WriteHeader(http.StatusOK)
-	json.NewEncoder(rw).Encode(job)
-
-	// Monitoring is disabled...
-	if job.MonitoringStatus == schema.MonitoringStatusDisabled {
-		return
-	}
-
-	// Trigger async archiving
-	archiver.TriggerArchiving(job)
-}
-
-func (api *RestApi) getJobMetrics(rw http.ResponseWriter, r *http.Request) {
-	id := mux.Vars(r)["id"]
-	metrics := r.URL.Query()["metric"]
-	var scopes []schema.MetricScope
-	for _, scope := range r.URL.Query()["scope"] {
-		var s schema.MetricScope
-		if err := s.UnmarshalGQL(scope); err != nil {
-			http.Error(rw, err.Error(), http.StatusBadRequest)
-			return
-		}
-		scopes = append(scopes, s)
-	}
-
-	rw.Header().Add("Content-Type", "application/json")
-	rw.WriteHeader(http.StatusOK)
-
-	type Respone struct {
-		Data *struct {
-			JobMetrics []*model.JobMetricWithName `json:"jobMetrics"`
-		} `json:"data"`
-		Error *struct {
-			Message string `json:"message"`
-		} `json:"error"`
-	}
-
-	resolver := graph.GetResolverInstance()
-	data, err := resolver.Query().JobMetrics(r.Context(), id, metrics, scopes, nil)
-	if err != nil {
-		json.NewEncoder(rw).Encode(Respone{
-			Error: &struct {
-				Message string "json:\"message\""
-			}{Message: err.Error()},
-		})
-		return
-	}
-
-	json.NewEncoder(rw).Encode(Respone{
-		Data: &struct {
-			JobMetrics []*model.JobMetricWithName "json:\"jobMetrics\""
-		}{JobMetrics: data},
-	})
-}

internal/api/memorystore.go

@@ -1,177 +0,0 @@
-// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved.
-// Use of this source code is governed by a MIT-style
-// license that can be found in the LICENSE file.
-
-package api
-
-import (
-	"bufio"
-	"encoding/json"
-	"errors"
-	"fmt"
-	"io"
-	"net/http"
-	"strconv"
-	"strings"
-
-	"github.com/ClusterCockpit/cc-backend/internal/memorystore"
-	cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
-
-	"github.com/influxdata/line-protocol/v2/lineprotocol"
-)
-
-// handleFree godoc
-// @summary
-// @tags free
-// @description This endpoint allows the users to free the Buffers from the
-// metric store. This endpoint offers the users to remove then systematically
-// and also allows then to prune the data under node, if they do not want to
-// remove the whole node.
-// @produce     json
-// @param       to        query    string        false  "up to timestamp"
-// @success     200            {string} string  "ok"
-// @failure     400            {object} api.ErrorResponse       "Bad Request"
-// @failure     401            {object} api.ErrorResponse       "Unauthorized"
-// @failure     403            {object} api.ErrorResponse       "Forbidden"
-// @failure     500            {object} api.ErrorResponse       "Internal Server Error"
-// @security    ApiKeyAuth
-// @router      /free/ [post]
-func freeMetrics(rw http.ResponseWriter, r *http.Request) {
-	rawTo := r.URL.Query().Get("to")
-	if rawTo == "" {
-		handleError(errors.New("'to' is a required query parameter"), http.StatusBadRequest, rw)
-		return
-	}
-
-	to, err := strconv.ParseInt(rawTo, 10, 64)
-	if err != nil {
-		handleError(err, http.StatusInternalServerError, rw)
-		return
-	}
-
-	// // TODO: lastCheckpoint might be modified by different go-routines.
-	// // Load it using the sync/atomic package?
-	// freeUpTo := lastCheckpoint.Unix()
-	// if to < freeUpTo {
-	// 	freeUpTo = to
-	// }
-
-	bodyDec := json.NewDecoder(r.Body)
-	var selectors [][]string
-	err = bodyDec.Decode(&selectors)
-	if err != nil {
-		http.Error(rw, err.Error(), http.StatusBadRequest)
-		return
-	}
-
-	ms := memorystore.GetMemoryStore()
-	n := 0
-	for _, sel := range selectors {
-		bn, err := ms.Free(sel, to)
-		if err != nil {
-			handleError(err, http.StatusInternalServerError, rw)
-			return
-		}
-
-		n += bn
-	}
-
-	rw.WriteHeader(http.StatusOK)
-	fmt.Fprintf(rw, "buffers freed: %d\n", n)
-}
-
-// handleWrite godoc
-// @summary Receive metrics in InfluxDB line-protocol
-// @tags write
-// @description Write data to the in-memory store in the InfluxDB line-protocol using [this format](https://github.com/ClusterCockpit/cc-specifications/blob/master/metrics/lineprotocol_alternative.md)
-
-// @accept      plain
-// @produce     json
-// @param       cluster        query string false "If the lines in the body do not have a cluster tag, use this value instead."
-// @success     200            {string} string  "ok"
-// @failure     400            {object} api.ErrorResponse       "Bad Request"
-// @failure     401            {object} api.ErrorResponse       "Unauthorized"
-// @failure     403            {object} api.ErrorResponse       "Forbidden"
-// @failure     500            {object} api.ErrorResponse       "Internal Server Error"
-// @security    ApiKeyAuth
-// @router      /write/ [post]
-func writeMetrics(rw http.ResponseWriter, r *http.Request) {
-	bytes, err := io.ReadAll(r.Body)
-	rw.Header().Add("Content-Type", "application/json")
-	if err != nil {
-		handleError(err, http.StatusInternalServerError, rw)
-		return
-	}
-
-	ms := memorystore.GetMemoryStore()
-	dec := lineprotocol.NewDecoderWithBytes(bytes)
-	if err := memorystore.DecodeLine(dec, ms, r.URL.Query().Get("cluster")); err != nil {
-		cclog.Errorf("/api/write error: %s", err.Error())
-		handleError(err, http.StatusBadRequest, rw)
-		return
-	}
-	rw.WriteHeader(http.StatusOK)
-}
-
-// handleDebug godoc
-// @summary Debug endpoint
-// @tags debug
-// @description This endpoint allows the users to print the content of
-// nodes/clusters/metrics to review the state of the data.
-// @produce     json
-// @param       selector        query    string            false "Selector"
-// @success     200            {string} string  "Debug dump"
-// @failure     400            {object} api.ErrorResponse       "Bad Request"
-// @failure     401            {object} api.ErrorResponse       "Unauthorized"
-// @failure     403            {object} api.ErrorResponse       "Forbidden"
-// @failure     500            {object} api.ErrorResponse       "Internal Server Error"
-// @security    ApiKeyAuth
-// @router      /debug/ [post]
-func debugMetrics(rw http.ResponseWriter, r *http.Request) {
-	raw := r.URL.Query().Get("selector")
-	rw.Header().Add("Content-Type", "application/json")
-	selector := []string{}
-	if len(raw) != 0 {
-		selector = strings.Split(raw, ":")
-	}
-
-	ms := memorystore.GetMemoryStore()
-	if err := ms.DebugDump(bufio.NewWriter(rw), selector); err != nil {
-		handleError(err, http.StatusBadRequest, rw)
-		return
-	}
-}
-
-// handleHealthCheck godoc
-// @summary HealthCheck endpoint
-// @tags healthcheck
-// @description This endpoint allows the users to check if a node is healthy
-// @produce     json
-// @param       selector        query    string            false "Selector"
-// @success     200            {string} string  "Debug dump"
-// @failure     400            {object} api.ErrorResponse       "Bad Request"
-// @failure     401            {object} api.ErrorResponse       "Unauthorized"
-// @failure     403            {object} api.ErrorResponse       "Forbidden"
-// @failure     500            {object} api.ErrorResponse       "Internal Server Error"
-// @security    ApiKeyAuth
-// @router      /healthcheck/ [get]
-func metricsHealth(rw http.ResponseWriter, r *http.Request) {
-	rawCluster := r.URL.Query().Get("cluster")
-	rawNode := r.URL.Query().Get("node")
-
-	if rawCluster == "" || rawNode == "" {
-		handleError(errors.New("'cluster' and 'node' are required query parameter"), http.StatusBadRequest, rw)
-		return
-	}
-
-	rw.Header().Add("Content-Type", "application/json")
-
-	selector := []string{rawCluster, rawNode}
-
-	ms := memorystore.GetMemoryStore()
-	if err := ms.HealthCheck(bufio.NewWriter(rw), selector); err != nil {
-		handleError(err, http.StatusBadRequest, rw)
-		return
-	}
-}

internal/api/nats.go

@@ -0,0 +1,57 @@
+// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
+// All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+package api
+
+import (
+	"sync"
+
+	"github.com/ClusterCockpit/cc-backend/internal/config"
+	"github.com/ClusterCockpit/cc-backend/pkg/log"
+	"github.com/ClusterCockpit/cc-backend/pkg/schema"
+	lp "github.com/ClusterCockpit/cc-lib/ccMessage"
+	"github.com/ClusterCockpit/cc-lib/sinks"
+)
+
+type NatsClient struct {
+	SinkManager sinks.SinkManager
+	SinkChannel chan lp.CCMessage
+}
+
+var (
+	initOnce sync.Once
+	ni       *NatsClient
+)
+
+func Init(wg *sync.WaitGroup) {
+	initOnce.Do(func() {
+		ni = &NatsClient{}
+		var err error
+
+		if len(config.Keys.SinkConfigFile) == 0 {
+			log.Error("Sink configuration file must be set")
+			return
+		}
+		ni.SinkManager, err = sinks.New(wg, config.Keys.SinkConfigFile)
+		if err != nil {
+			log.Error(err.Error())
+			return
+		}
+
+		ni.SinkChannel = make(chan lp.CCMessage, 200)
+		ni.SinkManager.AddInput(ni.SinkChannel)
+		ni.SinkManager.Start()
+	})
+}
+
+func Shutdown() {
+	if ni.SinkManager != nil {
+		log.Debug("Shutdown SinkManager...")
+		ni.SinkManager.Close()
+	}
+}
+
+func forwardJob(job schema.BaseJob) {
+  payload := lp.NewEvent("start_job", nil , meta map[string]string, event string, tm time.Time)
+}

internal/api/node.go

@@ -1,80 +0,0 @@
-// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved. This file is part of cc-backend.
-// Use of this source code is governed by a MIT-style
-// license that can be found in the LICENSE file.
-
-package api
-
-import (
-	"fmt"
-	"net/http"
-	"strings"
-	"time"
-
-	"github.com/ClusterCockpit/cc-backend/internal/repository"
-	"github.com/ClusterCockpit/cc-lib/schema"
-)
-
-type UpdateNodeStatesRequest struct {
-	Nodes   []schema.NodePayload `json:"nodes"`
-	Cluster string               `json:"cluster" example:"fritz"`
-}
-
-// this routine assumes that only one of them exists per node
-func determineState(states []string) schema.SchedulerState {
-	for _, state := range states {
-		switch strings.ToLower(state) {
-		case "allocated":
-			return schema.NodeStateAllocated
-		case "reserved":
-			return schema.NodeStateReserved
-		case "idle":
-			return schema.NodeStateIdle
-		case "down":
-			return schema.NodeStateDown
-		case "mixed":
-			return schema.NodeStateMixed
-		}
-	}
-
-	return schema.NodeStateUnknown
-}
-
-// updateNodeStates godoc
-// @summary     Deliver updated Slurm node states
-// @tags Nodestates
-// @description Returns a JSON-encoded list of users.
-// @description Required query-parameter defines if all users or only users with additional special roles are returned.
-// @produce     json
-// @param       request body UpdateNodeStatesRequest true "Request body containing nodes and their states"
-// @success     200     {object} api.DefaultApiResponse "Success message"
-// @failure     400     {object} api.ErrorResponse      "Bad Request"
-// @failure     401     {object} api.ErrorResponse      "Unauthorized"
-// @failure     403     {object} api.ErrorResponse      "Forbidden"
-// @failure     500     {object} api.ErrorResponse      "Internal Server Error"
-// @security    ApiKeyAuth
-// @router      /api/nodestats/ [post]
-func (api *RestApi) updateNodeStates(rw http.ResponseWriter, r *http.Request) {
-	// Parse request body
-	req := UpdateNodeStatesRequest{}
-	if err := decode(r.Body, &req); err != nil {
-		handleError(fmt.Errorf("parsing request body failed: %w", err),
-			http.StatusBadRequest, rw)
-		return
-	}
-	repo := repository.GetNodeRepository()
-
-	for _, node := range req.Nodes {
-		state := determineState(node.States)
-		nodeState := schema.NodeStateDB{
-			TimeStamp: time.Now().Unix(), NodeState: state,
-			CpusAllocated:   node.CpusAllocated,
-			MemoryAllocated: node.MemoryAllocated,
-			GpusAllocated:   node.GpusAllocated,
-			HealthState:     schema.MonitoringStateFull,
-			JobsRunning:     node.JobsRunning,
-		}
-
-		repo.UpdateNodeState(node.Hostname, req.Cluster, &nodeState)
-	}
-}

internal/api/user.go

@@ -1,159 +0,0 @@
-// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved. This file is part of cc-backend.
-// Use of this source code is governed by a MIT-style
-// license that can be found in the LICENSE file.
-package api
-
-import (
-	"encoding/json"
-	"fmt"
-	"net/http"
-
-	"github.com/ClusterCockpit/cc-backend/internal/repository"
-	"github.com/ClusterCockpit/cc-lib/schema"
-	"github.com/gorilla/mux"
-)
-
-type ApiReturnedUser struct {
-	Username string   `json:"username"`
-	Name     string   `json:"name"`
-	Roles    []string `json:"roles"`
-	Email    string   `json:"email"`
-	Projects []string `json:"projects"`
-}
-
-// getUsers godoc
-// @summary     Returns a list of users
-// @tags User
-// @description Returns a JSON-encoded list of users.
-// @description Required query-parameter defines if all users or only users with additional special roles are returned.
-// @produce     json
-// @param       not-just-user query bool true "If returned list should contain all users or only users with additional special roles"
-// @success     200     {array} api.ApiReturnedUser "List of users returned successfully"
-// @failure     400     {string} string             "Bad Request"
-// @failure     401     {string} string             "Unauthorized"
-// @failure     403     {string} string             "Forbidden"
-// @failure     500     {string} string             "Internal Server Error"
-// @security    ApiKeyAuth
-// @router      /api/users/ [get]
-func (api *RestApi) getUsers(rw http.ResponseWriter, r *http.Request) {
-	// SecuredCheck() only worked with TokenAuth: Removed
-
-	if user := repository.GetUserFromContext(r.Context()); !user.HasRole(schema.RoleAdmin) {
-		http.Error(rw, "Only admins are allowed to fetch a list of users", http.StatusForbidden)
-		return
-	}
-
-	users, err := repository.GetUserRepository().ListUsers(r.URL.Query().Get("not-just-user") == "true")
-	if err != nil {
-		http.Error(rw, err.Error(), http.StatusInternalServerError)
-		return
-	}
-
-	json.NewEncoder(rw).Encode(users)
-}
-
-func (api *RestApi) updateUser(rw http.ResponseWriter, r *http.Request) {
-	// SecuredCheck() only worked with TokenAuth: Removed
-
-	if user := repository.GetUserFromContext(r.Context()); !user.HasRole(schema.RoleAdmin) {
-		http.Error(rw, "Only admins are allowed to update a user", http.StatusForbidden)
-		return
-	}
-
-	// Get Values
-	newrole := r.FormValue("add-role")
-	delrole := r.FormValue("remove-role")
-	newproj := r.FormValue("add-project")
-	delproj := r.FormValue("remove-project")
-
-	// TODO: Handle anything but roles...
-	if newrole != "" {
-		if err := repository.GetUserRepository().AddRole(r.Context(), mux.Vars(r)["id"], newrole); err != nil {
-			http.Error(rw, err.Error(), http.StatusUnprocessableEntity)
-			return
-		}
-		rw.Write([]byte("Add Role Success"))
-	} else if delrole != "" {
-		if err := repository.GetUserRepository().RemoveRole(r.Context(), mux.Vars(r)["id"], delrole); err != nil {
-			http.Error(rw, err.Error(), http.StatusUnprocessableEntity)
-			return
-		}
-		rw.Write([]byte("Remove Role Success"))
-	} else if newproj != "" {
-		if err := repository.GetUserRepository().AddProject(r.Context(), mux.Vars(r)["id"], newproj); err != nil {
-			http.Error(rw, err.Error(), http.StatusUnprocessableEntity)
-			return
-		}
-		rw.Write([]byte("Add Project Success"))
-	} else if delproj != "" {
-		if err := repository.GetUserRepository().RemoveProject(r.Context(), mux.Vars(r)["id"], delproj); err != nil {
-			http.Error(rw, err.Error(), http.StatusUnprocessableEntity)
-			return
-		}
-		rw.Write([]byte("Remove Project Success"))
-	} else {
-		http.Error(rw, "Not Add or Del [role|project]?", http.StatusInternalServerError)
-	}
-}
-
-func (api *RestApi) createUser(rw http.ResponseWriter, r *http.Request) {
-	// SecuredCheck() only worked with TokenAuth: Removed
-
-	rw.Header().Set("Content-Type", "text/plain")
-	me := repository.GetUserFromContext(r.Context())
-	if !me.HasRole(schema.RoleAdmin) {
-		http.Error(rw, "Only admins are allowed to create new users", http.StatusForbidden)
-		return
-	}
-
-	username, password, role, name, email, project := r.FormValue("username"),
-		r.FormValue("password"), r.FormValue("role"), r.FormValue("name"),
-		r.FormValue("email"), r.FormValue("project")
-
-	if len(password) == 0 && role != schema.GetRoleString(schema.RoleApi) {
-		http.Error(rw, "Only API users are allowed to have a blank password (login will be impossible)", http.StatusBadRequest)
-		return
-	}
-
-	if len(project) != 0 && role != schema.GetRoleString(schema.RoleManager) {
-		http.Error(rw, "only managers require a project (can be changed later)",
-			http.StatusBadRequest)
-		return
-	} else if len(project) == 0 && role == schema.GetRoleString(schema.RoleManager) {
-		http.Error(rw, "managers require a project to manage (can be changed later)",
-			http.StatusBadRequest)
-		return
-	}
-
-	if err := repository.GetUserRepository().AddUser(&schema.User{
-		Username: username,
-		Name:     name,
-		Password: password,
-		Email:    email,
-		Projects: []string{project},
-		Roles:    []string{role},
-	}); err != nil {
-		http.Error(rw, err.Error(), http.StatusUnprocessableEntity)
-		return
-	}
-
-	fmt.Fprintf(rw, "User %v successfully created!\n", username)
-}
-
-func (api *RestApi) deleteUser(rw http.ResponseWriter, r *http.Request) {
-	// SecuredCheck() only worked with TokenAuth: Removed
-
-	if user := repository.GetUserFromContext(r.Context()); !user.HasRole(schema.RoleAdmin) {
-		http.Error(rw, "Only admins are allowed to delete a user", http.StatusForbidden)
-		return
-	}
-
-	username := r.FormValue("username")
-	if err := repository.GetUserRepository().DelUser(username); err != nil {
-		http.Error(rw, err.Error(), http.StatusUnprocessableEntity)
-		return
-	}
-
-	rw.WriteHeader(http.StatusOK)
-}

internal/archiver/archiveWorker.go

@@ -1,5 +1,5 @@
 // Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved. This file is part of cc-backend.
+// All rights reserved.
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 package archiver
@@ -10,8 +10,8 @@ import (
 	"time"
 
 	"github.com/ClusterCockpit/cc-backend/internal/repository"
-	cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
-	"github.com/ClusterCockpit/cc-lib/schema"
+	"github.com/ClusterCockpit/cc-backend/pkg/log"
+	"github.com/ClusterCockpit/cc-backend/pkg/schema"
 	sq "github.com/Masterminds/squirrel"
 )
 
@@ -40,8 +40,8 @@ func archivingWorker() {
 			// not using meta data, called to load JobMeta into Cache?
 			// will fail if job meta not in repository
 			if _, err := jobRepo.FetchMetadata(job); err != nil {
-				cclog.Errorf("archiving job (dbid: %d) failed at check metadata step: %s", job.ID, err.Error())
-				jobRepo.UpdateMonitoringStatus(*job.ID, schema.MonitoringStatusArchivingFailed)
+				log.Errorf("archiving job (dbid: %d) failed at check metadata step: %s", job.ID, err.Error())
+				jobRepo.UpdateMonitoringStatus(job.ID, schema.MonitoringStatusArchivingFailed)
 				continue
 			}
 
@@ -49,34 +49,30 @@ func archivingWorker() {
 			// TODO: Maybe use context with cancel/timeout here
 			jobMeta, err := ArchiveJob(job, context.Background())
 			if err != nil {
-				cclog.Errorf("archiving job (dbid: %d) failed at archiving job step: %s", job.ID, err.Error())
-				jobRepo.UpdateMonitoringStatus(*job.ID, schema.MonitoringStatusArchivingFailed)
+				log.Errorf("archiving job (dbid: %d) failed at archiving job step: %s", job.ID, err.Error())
+				jobRepo.UpdateMonitoringStatus(job.ID, schema.MonitoringStatusArchivingFailed)
 				continue
 			}
 
 			stmt := sq.Update("job").Where("job.id = ?", job.ID)
 
 			if stmt, err = jobRepo.UpdateFootprint(stmt, jobMeta); err != nil {
-				cclog.Errorf("archiving job (dbid: %d) failed at update Footprint step: %s", job.ID, err.Error())
+				log.Errorf("archiving job (dbid: %d) failed at update Footprint step: %s", job.ID, err.Error())
 				continue
 			}
 			if stmt, err = jobRepo.UpdateEnergy(stmt, jobMeta); err != nil {
-				cclog.Errorf("archiving job (dbid: %d) failed at update Energy step: %s", job.ID, err.Error())
+				log.Errorf("archiving job (dbid: %d) failed at update Energy step: %s", job.ID, err.Error())
 				continue
 			}
 			// Update the jobs database entry one last time:
 			stmt = jobRepo.MarkArchived(stmt, schema.MonitoringStatusArchivingSuccessful)
 			if err := jobRepo.Execute(stmt); err != nil {
-				cclog.Errorf("archiving job (dbid: %d) failed at db execute: %s", job.ID, err.Error())
+				log.Errorf("archiving job (dbid: %d) failed at db execute: %s", job.ID, err.Error())
 				continue
 			}
-			cclog.Debugf("archiving job %d took %s", job.JobID, time.Since(start))
-			cclog.Printf("archiving job (dbid: %d) successful", job.ID)
-
-			repository.CallJobStopHooks(job)
+			log.Debugf("archiving job %d took %s", job.JobID, time.Since(start))
+			log.Printf("archiving job (dbid: %d) successful", job.ID)
 			archivePending.Done()
-		default:
-			continue
 		}
 	}
 }
@@ -84,7 +80,7 @@ func archivingWorker() {
 // Trigger async archiving
 func TriggerArchiving(job *schema.Job) {
 	if archiveChannel == nil {
-		cclog.Fatal("Cannot archive without archiving channel. Did you Start the archiver?")
+		log.Fatal("Cannot archive without archiving channel. Did you Start the archiver?")
 	}
 
 	archivePending.Add(1)

internal/archiver/archiver.go

@@ -1,5 +1,5 @@
 // Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved. This file is part of cc-backend.
+// All rights reserved.
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 package archiver
@@ -11,12 +11,12 @@ import (
 	"github.com/ClusterCockpit/cc-backend/internal/config"
 	"github.com/ClusterCockpit/cc-backend/internal/metricDataDispatcher"
 	"github.com/ClusterCockpit/cc-backend/pkg/archive"
-	cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
-	"github.com/ClusterCockpit/cc-lib/schema"
+	"github.com/ClusterCockpit/cc-backend/pkg/log"
+	"github.com/ClusterCockpit/cc-backend/pkg/schema"
 )
 
 // Writes a running job to the job-archive
-func ArchiveJob(job *schema.Job, ctx context.Context) (*schema.Job, error) {
+func ArchiveJob(job *schema.Job, ctx context.Context) (*schema.JobMeta, error) {
 	allMetrics := make([]string, 0)
 	metricConfigs := archive.GetCluster(job.Cluster).MetricConfig
 	for _, mc := range metricConfigs {
@@ -36,11 +36,15 @@ func ArchiveJob(job *schema.Job, ctx context.Context) (*schema.Job, error) {
 
 	jobData, err := metricDataDispatcher.LoadData(job, allMetrics, scopes, ctx, 0) // 0 Resulotion-Value retrieves highest res (60s)
 	if err != nil {
-		cclog.Error("Error wile loading job data for archiving")
+		log.Error("Error wile loading job data for archiving")
 		return nil, err
 	}
 
-	job.Statistics = make(map[string]schema.JobStatistics)
+	jobMeta := &schema.JobMeta{
+		BaseJob:    job.BaseJob,
+		StartTime:  job.StartTime.Unix(),
+		Statistics: make(map[string]schema.JobStatistics),
+	}
 
 	for metric, data := range jobData {
 		avg, min, max := 0.0, math.MaxFloat32, -math.MaxFloat32
@@ -56,13 +60,12 @@ func ArchiveJob(job *schema.Job, ctx context.Context) (*schema.Job, error) {
 			max = math.Max(max, series.Statistics.Max)
 		}
 
-		// Round AVG Result to 2 Digits
-		job.Statistics[metric] = schema.JobStatistics{
+		jobMeta.Statistics[metric] = schema.JobStatistics{
 			Unit: schema.Unit{
 				Prefix: archive.GetMetricConfig(job.Cluster, metric).Unit.Prefix,
 				Base:   archive.GetMetricConfig(job.Cluster, metric).Unit.Base,
 			},
-			Avg: (math.Round((avg/float64(job.NumNodes))*100) / 100),
+			Avg: avg / float64(job.NumNodes),
 			Min: min,
 			Max: max,
 		}
@@ -72,8 +75,8 @@ func ArchiveJob(job *schema.Job, ctx context.Context) (*schema.Job, error) {
 	// only return the JobMeta structure as the
 	// statistics in there are needed.
 	if config.Keys.DisableArchive {
-		return job, nil
+		return jobMeta, nil
 	}
 
-	return job, archive.GetHandle().ImportJob(job, &jobData)
+	return jobMeta, archive.GetHandle().ImportJob(jobMeta, &jobData)
 }

internal/auth/auth.go

@@ -1,24 +1,18 @@
 // Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved. This file is part of cc-backend.
+// All rights reserved.
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
-
-// Package auth implements various authentication methods
 package auth
 
 import (
-	"bytes"
 	"context"
 	"crypto/rand"
 	"database/sql"
 	"encoding/base64"
-	"encoding/json"
 	"errors"
-	"fmt"
 	"net"
 	"net/http"
 	"os"
-	"strings"
 	"sync"
 	"time"
 
@@ -26,9 +20,8 @@ import (
 
 	"github.com/ClusterCockpit/cc-backend/internal/config"
 	"github.com/ClusterCockpit/cc-backend/internal/repository"
-	cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
-	"github.com/ClusterCockpit/cc-lib/schema"
-	"github.com/ClusterCockpit/cc-lib/util"
+	"github.com/ClusterCockpit/cc-backend/pkg/log"
+	"github.com/ClusterCockpit/cc-backend/pkg/schema"
 	"github.com/gorilla/sessions"
 )
 
@@ -55,14 +48,6 @@ func getIPUserLimiter(ip, username string) *rate.Limiter {
 	return limiter.(*rate.Limiter)
 }
 
-type AuthConfig struct {
-	LdapConfig   *LdapConfig    `json:"ldap"`
-	JwtConfig    *JWTAuthConfig `json:"jwts"`
-	OpenIDConfig *OpenIDConfig  `json:"oidc"`
-}
-
-var Keys AuthConfig
-
 type Authentication struct {
 	sessionStore   *sessions.CookieStore
 	LdapAuth       *LdapAuthenticator
@@ -78,7 +63,7 @@ func (auth *Authentication) AuthViaSession(
 ) (*schema.User, error) {
 	session, err := auth.sessionStore.Get(r, "session")
 	if err != nil {
-		cclog.Error("Error while getting session store")
+		log.Error("Error while getting session store")
 		return nil, err
 	}
 
@@ -99,22 +84,22 @@ func (auth *Authentication) AuthViaSession(
 	}, nil
 }
 
-func Init(authCfg *json.RawMessage) {
+func Init() {
 	initOnce.Do(func() {
 		authInstance = &Authentication{}
 
 		sessKey := os.Getenv("SESSION_KEY")
 		if sessKey == "" {
-			cclog.Warn("environment variable 'SESSION_KEY' not set (will use non-persistent random key)")
+			log.Warn("environment variable 'SESSION_KEY' not set (will use non-persistent random key)")
 			bytes := make([]byte, 32)
 			if _, err := rand.Read(bytes); err != nil {
-				cclog.Fatal("Error while initializing authentication -> failed to generate random bytes for session key")
+				log.Fatal("Error while initializing authentication -> failed to generate random bytes for session key")
 			}
 			authInstance.sessionStore = sessions.NewCookieStore(bytes)
 		} else {
 			bytes, err := base64.StdEncoding.DecodeString(sessKey)
 			if err != nil {
-				cclog.Fatal("Error while initializing authentication -> decoding session key failed")
+				log.Fatal("Error while initializing authentication -> decoding session key failed")
 			}
 			authInstance.sessionStore = sessions.NewCookieStore(bytes)
 		}
@@ -123,55 +108,44 @@ func Init(authCfg *json.RawMessage) {
 			authInstance.SessionMaxAge = d
 		}
 
-		if authCfg == nil {
-			return
-		}
-
-		config.Validate(configSchema, *authCfg)
-		dec := json.NewDecoder(bytes.NewReader(*authCfg))
-		dec.DisallowUnknownFields()
-		if err := dec.Decode(&Keys); err != nil {
-			cclog.Errorf("error while decoding ldap config: %v", err)
-		}
-
-		if Keys.LdapConfig != nil {
+		if config.Keys.LdapConfig != nil {
 			ldapAuth := &LdapAuthenticator{}
 			if err := ldapAuth.Init(); err != nil {
-				cclog.Warn("Error while initializing authentication -> ldapAuth init failed")
+				log.Warn("Error while initializing authentication -> ldapAuth init failed")
 			} else {
 				authInstance.LdapAuth = ldapAuth
 				authInstance.authenticators = append(authInstance.authenticators, authInstance.LdapAuth)
 			}
 		} else {
-			cclog.Info("Missing LDAP configuration: No LDAP support!")
+			log.Info("Missing LDAP configuration: No LDAP support!")
 		}
 
-		if Keys.JwtConfig != nil {
+		if config.Keys.JwtConfig != nil {
 			authInstance.JwtAuth = &JWTAuthenticator{}
 			if err := authInstance.JwtAuth.Init(); err != nil {
-				cclog.Fatal("Error while initializing authentication -> jwtAuth init failed")
+				log.Fatal("Error while initializing authentication -> jwtAuth init failed")
 			}
 
 			jwtSessionAuth := &JWTSessionAuthenticator{}
 			if err := jwtSessionAuth.Init(); err != nil {
-				cclog.Info("jwtSessionAuth init failed: No JWT login support!")
+				log.Info("jwtSessionAuth init failed: No JWT login support!")
 			} else {
 				authInstance.authenticators = append(authInstance.authenticators, jwtSessionAuth)
 			}
 
 			jwtCookieSessionAuth := &JWTCookieSessionAuthenticator{}
 			if err := jwtCookieSessionAuth.Init(); err != nil {
-				cclog.Info("jwtCookieSessionAuth init failed: No JWT cookie login support!")
+				log.Info("jwtCookieSessionAuth init failed: No JWT cookie login support!")
 			} else {
 				authInstance.authenticators = append(authInstance.authenticators, jwtCookieSessionAuth)
 			}
 		} else {
-			cclog.Info("Missing JWT configuration: No JWT token support!")
+			log.Info("Missing JWT configuration: No JWT token support!")
 		}
 
 		authInstance.LocalAuth = &LocalAuthenticator{}
 		if err := authInstance.LocalAuth.Init(); err != nil {
-			cclog.Fatal("Error while initializing authentication -> localAuth init failed")
+			log.Fatal("Error while initializing authentication -> localAuth init failed")
 		}
 		authInstance.authenticators = append(authInstance.authenticators, authInstance.LocalAuth)
 	})
@@ -179,7 +153,7 @@ func Init(authCfg *json.RawMessage) {
 
 func GetAuthInstance() *Authentication {
 	if authInstance == nil {
-		cclog.Fatal("Authentication module not initialized!")
+		log.Fatal("Authentication module not initialized!")
 	}
 
 	return authInstance
@@ -190,14 +164,14 @@ func handleTokenUser(tokenUser *schema.User) {
 	dbUser, err := r.GetUser(tokenUser.Username)
 
 	if err != nil && err != sql.ErrNoRows {
-		cclog.Errorf("Error while loading user '%s': %v", tokenUser.Username, err)
-	} else if err == sql.ErrNoRows && Keys.JwtConfig.SyncUserOnLogin { // Adds New User
+		log.Errorf("Error while loading user '%s': %v", tokenUser.Username, err)
+	} else if err == sql.ErrNoRows && config.Keys.JwtConfig.SyncUserOnLogin { // Adds New User
 		if err := r.AddUser(tokenUser); err != nil {
-			cclog.Errorf("Error while adding user '%s' to DB: %v", tokenUser.Username, err)
+			log.Errorf("Error while adding user '%s' to DB: %v", tokenUser.Username, err)
 		}
-	} else if err == nil && Keys.JwtConfig.UpdateUserOnLogin { // Update Existing User
+	} else if err == nil && config.Keys.JwtConfig.UpdateUserOnLogin { // Update Existing User
 		if err := r.UpdateUser(dbUser, tokenUser); err != nil {
-			cclog.Errorf("Error while updating user '%s' to DB: %v", dbUser.Username, err)
+			log.Errorf("Error while updating user '%s' to DB: %v", dbUser.Username, err)
 		}
 	}
 }
@@ -207,14 +181,14 @@ func handleOIDCUser(OIDCUser *schema.User) {
 	dbUser, err := r.GetUser(OIDCUser.Username)
 
 	if err != nil && err != sql.ErrNoRows {
-		cclog.Errorf("Error while loading user '%s': %v", OIDCUser.Username, err)
-	} else if err == sql.ErrNoRows && Keys.OpenIDConfig.SyncUserOnLogin { // Adds New User
+		log.Errorf("Error while loading user '%s': %v", OIDCUser.Username, err)
+	} else if err == sql.ErrNoRows && config.Keys.OpenIDConfig.SyncUserOnLogin { // Adds New User
 		if err := r.AddUser(OIDCUser); err != nil {
-			cclog.Errorf("Error while adding user '%s' to DB: %v", OIDCUser.Username, err)
+			log.Errorf("Error while adding user '%s' to DB: %v", OIDCUser.Username, err)
 		}
-	} else if err == nil && Keys.OpenIDConfig.UpdateUserOnLogin { // Update Existing User
+	} else if err == nil && config.Keys.OpenIDConfig.UpdateUserOnLogin { // Update Existing User
 		if err := r.UpdateUser(dbUser, OIDCUser); err != nil {
-			cclog.Errorf("Error while updating user '%s' to DB: %v", dbUser.Username, err)
+			log.Errorf("Error while updating user '%s' to DB: %v", dbUser.Username, err)
 		}
 	}
 }
@@ -222,7 +196,7 @@ func handleOIDCUser(OIDCUser *schema.User) {
 func (auth *Authentication) SaveSession(rw http.ResponseWriter, r *http.Request, user *schema.User) error {
 	session, err := auth.sessionStore.New(r, "session")
 	if err != nil {
-		cclog.Errorf("session creation failed: %s", err.Error())
+		log.Errorf("session creation failed: %s", err.Error())
 		http.Error(rw, err.Error(), http.StatusInternalServerError)
 		return err
 	}
@@ -230,7 +204,7 @@ func (auth *Authentication) SaveSession(rw http.ResponseWriter, r *http.Request,
 	if auth.SessionMaxAge != 0 {
 		session.Options.MaxAge = int(auth.SessionMaxAge.Seconds())
 	}
-	if config.Keys.HTTPSCertFile == "" && config.Keys.HTTPSKeyFile == "" {
+	if config.Keys.HttpsCertFile == "" && config.Keys.HttpsKeyFile == "" {
 		session.Options.Secure = false
 	}
 	session.Options.SameSite = http.SameSiteStrictMode
@@ -238,7 +212,7 @@ func (auth *Authentication) SaveSession(rw http.ResponseWriter, r *http.Request,
 	session.Values["projects"] = user.Projects
 	session.Values["roles"] = user.Roles
 	if err := auth.sessionStore.Save(r, rw, session); err != nil {
-		cclog.Warnf("session save failed: %s", err.Error())
+		log.Warnf("session save failed: %s", err.Error())
 		http.Error(rw, err.Error(), http.StatusInternalServerError)
 		return err
 	}
@@ -259,9 +233,9 @@ func (auth *Authentication) Login(
 
 		limiter := getIPUserLimiter(ip, username)
 		if !limiter.Allow() {
-			cclog.Warnf("AUTH/RATE > Too many login attempts for combination IP: %s, Username: %s", ip, username)
-			onfailure(rw, r, errors.New("too many login attempts, try again in a few minutes"))
-			return
+				log.Warnf("AUTH/RATE > Too many login attempts for combination IP: %s, Username: %s", ip, username)
+				onfailure(rw, r, errors.New("Too many login attempts, try again in a few minutes."))
+				return
 		}
 
 		var dbUser *schema.User
@@ -269,7 +243,7 @@ func (auth *Authentication) Login(
 			var err error
 			dbUser, err = repository.GetUserRepository().GetUser(username)
 			if err != nil && err != sql.ErrNoRows {
-				cclog.Errorf("Error while loading user '%v'", username)
+				log.Errorf("Error while loading user '%v'", username)
 			}
 		}
 
@@ -279,12 +253,12 @@ func (auth *Authentication) Login(
 			if user, ok = authenticator.CanLogin(dbUser, username, rw, r); !ok {
 				continue
 			} else {
-				cclog.Debugf("Can login with user %v", user)
+				log.Debugf("Can login with user %v", user)
 			}
 
 			user, err := authenticator.Login(user, rw, r)
 			if err != nil {
-				cclog.Warnf("user login failed: %s", err.Error())
+				log.Warnf("user login failed: %s", err.Error())
 				onfailure(rw, r, err)
 				return
 			}
@@ -293,7 +267,7 @@ func (auth *Authentication) Login(
 				return
 			}
 
-			cclog.Infof("login successfull: user: %#v (roles: %v, projects: %v)", user.Username, user.Roles, user.Projects)
+			log.Infof("login successfull: user: %#v (roles: %v, projects: %v)", user.Username, user.Roles, user.Projects)
 			ctx := context.WithValue(r.Context(), repository.ContextUserKey, user)
 
 			if r.FormValue("redirect") != "" {
@@ -305,7 +279,7 @@ func (auth *Authentication) Login(
 			return
 		}
 
-		cclog.Debugf("login failed: no authenticator applied")
+		log.Debugf("login failed: no authenticator applied")
 		onfailure(rw, r, errors.New("no authenticator applied"))
 	})
 }
@@ -317,14 +291,14 @@ func (auth *Authentication) Auth(
 	return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
 		user, err := auth.JwtAuth.AuthViaJWT(rw, r)
 		if err != nil {
-			cclog.Infof("auth -> authentication failed: %s", err.Error())
+			log.Infof("auth -> authentication failed: %s", err.Error())
 			http.Error(rw, err.Error(), http.StatusUnauthorized)
 			return
 		}
 		if user == nil {
 			user, err = auth.AuthViaSession(rw, r)
 			if err != nil {
-				cclog.Infof("auth -> authentication failed: %s", err.Error())
+				log.Infof("auth -> authentication failed: %s", err.Error())
 				http.Error(rw, err.Error(), http.StatusUnauthorized)
 				return
 			}
@@ -335,30 +309,22 @@ func (auth *Authentication) Auth(
 			return
 		}
 
-		cclog.Info("auth -> authentication failed")
+		log.Info("auth -> authentication failed")
 		onfailure(rw, r, errors.New("unauthorized (please login first)"))
 	})
 }
 
-func (auth *Authentication) AuthAPI(
+func (auth *Authentication) AuthApi(
 	onsuccess http.Handler,
 	onfailure func(rw http.ResponseWriter, r *http.Request, authErr error),
 ) http.Handler {
 	return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
 		user, err := auth.JwtAuth.AuthViaJWT(rw, r)
 		if err != nil {
-			cclog.Infof("auth api -> authentication failed: %s", err.Error())
+			log.Infof("auth api -> authentication failed: %s", err.Error())
 			onfailure(rw, r, err)
 			return
 		}
-
-		ipErr := securedCheck(user, r)
-		if ipErr != nil {
-			cclog.Infof("auth api -> secured check failed: %s", ipErr.Error())
-			onfailure(rw, r, ipErr)
-			return
-		}
-
 		if user != nil {
 			switch {
 			case len(user.Roles) == 1:
@@ -374,63 +340,26 @@ func (auth *Authentication) AuthAPI(
 					return
 				}
 			default:
-				cclog.Info("auth api -> authentication failed: missing role")
+				log.Info("auth api -> authentication failed: missing role")
 				onfailure(rw, r, errors.New("unauthorized"))
 			}
 		}
-		cclog.Info("auth api -> authentication failed: no auth")
+		log.Info("auth api -> authentication failed: no auth")
 		onfailure(rw, r, errors.New("unauthorized"))
 	})
 }
 
-func (auth *Authentication) AuthUserAPI(
+func (auth *Authentication) AuthUserApi(
 	onsuccess http.Handler,
 	onfailure func(rw http.ResponseWriter, r *http.Request, authErr error),
 ) http.Handler {
 	return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
 		user, err := auth.JwtAuth.AuthViaJWT(rw, r)
 		if err != nil {
-			cclog.Infof("auth user api -> authentication failed: %s", err.Error())
+			log.Infof("auth user api -> authentication failed: %s", err.Error())
 			onfailure(rw, r, err)
 			return
 		}
-
-		if user != nil {
-			switch {
-			case len(user.Roles) == 1:
-				if user.HasRole(schema.RoleApi) {
-					ctx := context.WithValue(r.Context(), repository.ContextUserKey, user)
-					onsuccess.ServeHTTP(rw, r.WithContext(ctx))
-					return
-				}
-			case len(user.Roles) >= 2:
-				if user.HasRole(schema.RoleApi) && user.HasAnyRole([]schema.Role{schema.RoleUser, schema.RoleManager, schema.RoleSupport, schema.RoleAdmin}) {
-					ctx := context.WithValue(r.Context(), repository.ContextUserKey, user)
-					onsuccess.ServeHTTP(rw, r.WithContext(ctx))
-					return
-				}
-			default:
-				cclog.Info("auth user api -> authentication failed: missing role")
-				onfailure(rw, r, errors.New("unauthorized"))
-			}
-		}
-		cclog.Info("auth user api -> authentication failed: no auth")
-		onfailure(rw, r, errors.New("unauthorized"))
-	})
-}
-
-func (auth *Authentication) AuthMetricStoreAPI(
-	onsuccess http.Handler,
-	onfailure func(rw http.ResponseWriter, r *http.Request, authErr error),
-) http.Handler {
-	return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
-		user, err := auth.JwtAuth.AuthViaJWT(rw, r)
-		if err != nil {
-			cclog.Infof("auth metricstore api -> authentication failed: %s", err.Error())
-			onfailure(rw, r, err)
-			return
-		}
-
 		if user != nil {
 			switch {
 			case len(user.Roles) == 1:
@@ -446,23 +375,23 @@ func (auth *Authentication) AuthMetricStoreAPI(
 					return
 				}
 			default:
-				cclog.Info("auth metricstore api -> authentication failed: missing role")
+				log.Info("auth user api -> authentication failed: missing role")
 				onfailure(rw, r, errors.New("unauthorized"))
 			}
 		}
-		cclog.Info("auth metricstore api -> authentication failed: no auth")
+		log.Info("auth user api -> authentication failed: no auth")
 		onfailure(rw, r, errors.New("unauthorized"))
 	})
 }
 
-func (auth *Authentication) AuthConfigAPI(
+func (auth *Authentication) AuthConfigApi(
 	onsuccess http.Handler,
 	onfailure func(rw http.ResponseWriter, r *http.Request, authErr error),
 ) http.Handler {
 	return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
 		user, err := auth.AuthViaSession(rw, r)
 		if err != nil {
-			cclog.Infof("auth config api -> authentication failed: %s", err.Error())
+			log.Infof("auth config api -> authentication failed: %s", err.Error())
 			onfailure(rw, r, err)
 			return
 		}
@@ -471,19 +400,19 @@ func (auth *Authentication) AuthConfigAPI(
 			onsuccess.ServeHTTP(rw, r.WithContext(ctx))
 			return
 		}
-		cclog.Info("auth config api -> authentication failed: no auth")
+		log.Info("auth config api -> authentication failed: no auth")
 		onfailure(rw, r, errors.New("unauthorized"))
 	})
 }
 
-func (auth *Authentication) AuthFrontendAPI(
+func (auth *Authentication) AuthFrontendApi(
 	onsuccess http.Handler,
 	onfailure func(rw http.ResponseWriter, r *http.Request, authErr error),
 ) http.Handler {
 	return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
 		user, err := auth.AuthViaSession(rw, r)
 		if err != nil {
-			cclog.Infof("auth frontend api -> authentication failed: %s", err.Error())
+			log.Infof("auth frontend api -> authentication failed: %s", err.Error())
 			onfailure(rw, r, err)
 			return
 		}
@@ -492,7 +421,7 @@ func (auth *Authentication) AuthFrontendAPI(
 			onsuccess.ServeHTTP(rw, r.WithContext(ctx))
 			return
 		}
-		cclog.Info("auth frontend api -> authentication failed: no auth")
+		log.Info("auth frontend api -> authentication failed: no auth")
 		onfailure(rw, r, errors.New("unauthorized"))
 	})
 }
@@ -516,39 +445,3 @@ func (auth *Authentication) Logout(onsuccess http.Handler) http.Handler {
 		onsuccess.ServeHTTP(rw, r)
 	})
 }
-
-// Helper Moved To MiddleWare Auth Handlers
-func securedCheck(user *schema.User, r *http.Request) error {
-	if user == nil {
-		return fmt.Errorf("no user for secured check")
-	}
-
-	// extract IP address for checking
-	IPAddress := r.Header.Get("X-Real-Ip")
-	if IPAddress == "" {
-		IPAddress = r.Header.Get("X-Forwarded-For")
-	}
-	if IPAddress == "" {
-		IPAddress = r.RemoteAddr
-	}
-
-	// FIXME: IPV6 not handled
-	if strings.Contains(IPAddress, ":") {
-		IPAddress = strings.Split(IPAddress, ":")[0]
-	}
-
-	// If nothing declared in config: deny all request to this api endpoint
-	if len(config.Keys.APIAllowedIPs) == 0 {
-		return fmt.Errorf("missing configuration key ApiAllowedIPs")
-	}
-	// If wildcard declared in config: Continue
-	if config.Keys.APIAllowedIPs[0] == "*" {
-		return nil
-	}
-	// check if IP is allowed
-	if !util.Contains(config.Keys.APIAllowedIPs, IPAddress) {
-		return fmt.Errorf("unknown ip: %v", IPAddress)
-	}
-
-	return nil
-}

internal/auth/jwt.go

@@ -1,8 +1,7 @@
 // Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved. This file is part of cc-backend.
+// All rights reserved.
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
-
 package auth
 
 import (
@@ -14,34 +13,13 @@ import (
 	"strings"
 	"time"
 
+	"github.com/ClusterCockpit/cc-backend/internal/config"
 	"github.com/ClusterCockpit/cc-backend/internal/repository"
-	cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
-	"github.com/ClusterCockpit/cc-lib/schema"
+	"github.com/ClusterCockpit/cc-backend/pkg/log"
+	"github.com/ClusterCockpit/cc-backend/pkg/schema"
 	"github.com/golang-jwt/jwt/v5"
 )
 
-type JWTAuthConfig struct {
-	// Specifies for how long a JWT token shall be valid
-	// as a string parsable by time.ParseDuration().
-	MaxAge string `json:"max-age"`
-
-	// Specifies which cookie should be checked for a JWT token (if no authorization header is present)
-	CookieName string `json:"cookieName"`
-
-	// Deny login for users not in database (but defined in JWT).
-	// Ignore user roles defined in JWTs ('roles' claim), get them from db.
-	ValidateUser bool `json:"validateUser"`
-
-	// Specifies which issuer should be accepted when validating external JWTs ('iss' claim)
-	TrustedIssuer string `json:"trustedIssuer"`
-
-	// Should an non-existent user be added to the DB based on the information in the token
-	SyncUserOnLogin bool `json:"syncUserOnLogin"`
-
-	// Should an existent user be updated in the DB based on the information in the token
-	UpdateUserOnLogin bool `json:"updateUserOnLogin"`
-}
-
 type JWTAuthenticator struct {
 	publicKey  ed25519.PublicKey
 	privateKey ed25519.PrivateKey
@@ -50,17 +28,17 @@ type JWTAuthenticator struct {
 func (ja *JWTAuthenticator) Init() error {
 	pubKey, privKey := os.Getenv("JWT_PUBLIC_KEY"), os.Getenv("JWT_PRIVATE_KEY")
 	if pubKey == "" || privKey == "" {
-		cclog.Warn("environment variables 'JWT_PUBLIC_KEY' or 'JWT_PRIVATE_KEY' not set (token based authentication will not work)")
+		log.Warn("environment variables 'JWT_PUBLIC_KEY' or 'JWT_PRIVATE_KEY' not set (token based authentication will not work)")
 	} else {
 		bytes, err := base64.StdEncoding.DecodeString(pubKey)
 		if err != nil {
-			cclog.Warn("Could not decode JWT public key")
+			log.Warn("Could not decode JWT public key")
 			return err
 		}
 		ja.publicKey = ed25519.PublicKey(bytes)
 		bytes, err = base64.StdEncoding.DecodeString(privKey)
 		if err != nil {
-			cclog.Warn("Could not decode JWT private key")
+			log.Warn("Could not decode JWT private key")
 			return err
 		}
 		ja.privateKey = ed25519.PrivateKey(bytes)
@@ -84,7 +62,7 @@ func (ja *JWTAuthenticator) AuthViaJWT(
 		return nil, nil
 	}
 
-	token, err := jwt.Parse(rawtoken, func(t *jwt.Token) (any, error) {
+	token, err := jwt.Parse(rawtoken, func(t *jwt.Token) (interface{}, error) {
 		if t.Method != jwt.SigningMethodEdDSA {
 			return nil, errors.New("only Ed25519/EdDSA supported")
 		}
@@ -92,11 +70,11 @@ func (ja *JWTAuthenticator) AuthViaJWT(
 		return ja.publicKey, nil
 	})
 	if err != nil {
-		cclog.Warn("Error while parsing JWT token")
+		log.Warn("Error while parsing JWT token")
 		return nil, err
 	}
 	if !token.Valid {
-		cclog.Warn("jwt token claims are not valid")
+		log.Warn("jwt token claims are not valid")
 		return nil, errors.New("jwt token claims are not valid")
 	}
 
@@ -107,19 +85,19 @@ func (ja *JWTAuthenticator) AuthViaJWT(
 	var roles []string
 
 	// Validate user + roles from JWT against database?
-	if Keys.JwtConfig.ValidateUser {
+	if config.Keys.JwtConfig.ValidateUser {
 		ur := repository.GetUserRepository()
 		user, err := ur.GetUser(sub)
 		// Deny any logins for unknown usernames
 		if err != nil {
-			cclog.Warn("Could not find user from JWT in internal database.")
+			log.Warn("Could not find user from JWT in internal database.")
 			return nil, errors.New("unknown user")
 		}
 		// Take user roles from database instead of trusting the JWT
 		roles = user.Roles
 	} else {
 		// Extract roles from JWT (if present)
-		if rawroles, ok := claims["roles"].([]any); ok {
+		if rawroles, ok := claims["roles"].([]interface{}); ok {
 			for _, rr := range rawroles {
 				if r, ok := rr.(string); ok {
 					roles = append(roles, r)
@@ -136,7 +114,7 @@ func (ja *JWTAuthenticator) AuthViaJWT(
 	}, nil
 }
 
-// ProvideJWT generates a new JWT that can be used for authentication
+// Generate a new JWT that can be used for authentication
 func (ja *JWTAuthenticator) ProvideJWT(user *schema.User) (string, error) {
 	if ja.privateKey == nil {
 		return "", errors.New("environment variable 'JWT_PRIVATE_KEY' not set")
@@ -148,8 +126,8 @@ func (ja *JWTAuthenticator) ProvideJWT(user *schema.User) (string, error) {
 		"roles": user.Roles,
 		"iat":   now.Unix(),
 	}
-	if Keys.JwtConfig.MaxAge != "" {
-		d, err := time.ParseDuration(Keys.JwtConfig.MaxAge)
+	if config.Keys.JwtConfig.MaxAge != "" {
+		d, err := time.ParseDuration(config.Keys.JwtConfig.MaxAge)
 		if err != nil {
 			return "", errors.New("cannot parse max-age config key")
 		}

internal/auth/jwtCookieSession.go

@@ -1,8 +1,7 @@
 // Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved. This file is part of cc-backend.
+// All rights reserved.
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
-
 package auth
 
 import (
@@ -14,9 +13,10 @@ import (
 	"net/http"
 	"os"
 
+	"github.com/ClusterCockpit/cc-backend/internal/config"
 	"github.com/ClusterCockpit/cc-backend/internal/repository"
-	cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
-	"github.com/ClusterCockpit/cc-lib/schema"
+	"github.com/ClusterCockpit/cc-backend/pkg/log"
+	"github.com/ClusterCockpit/cc-backend/pkg/schema"
 	"github.com/golang-jwt/jwt/v5"
 )
 
@@ -31,18 +31,18 @@ var _ Authenticator = (*JWTCookieSessionAuthenticator)(nil)
 func (ja *JWTCookieSessionAuthenticator) Init() error {
 	pubKey, privKey := os.Getenv("JWT_PUBLIC_KEY"), os.Getenv("JWT_PRIVATE_KEY")
 	if pubKey == "" || privKey == "" {
-		cclog.Warn("environment variables 'JWT_PUBLIC_KEY' or 'JWT_PRIVATE_KEY' not set (token based authentication will not work)")
+		log.Warn("environment variables 'JWT_PUBLIC_KEY' or 'JWT_PRIVATE_KEY' not set (token based authentication will not work)")
 		return errors.New("environment variables 'JWT_PUBLIC_KEY' or 'JWT_PRIVATE_KEY' not set (token based authentication will not work)")
 	} else {
 		bytes, err := base64.StdEncoding.DecodeString(pubKey)
 		if err != nil {
-			cclog.Warn("Could not decode JWT public key")
+			log.Warn("Could not decode JWT public key")
 			return err
 		}
 		ja.publicKey = ed25519.PublicKey(bytes)
 		bytes, err = base64.StdEncoding.DecodeString(privKey)
 		if err != nil {
-			cclog.Warn("Could not decode JWT private key")
+			log.Warn("Could not decode JWT private key")
 			return err
 		}
 		ja.privateKey = ed25519.PrivateKey(bytes)
@@ -53,35 +53,36 @@ func (ja *JWTCookieSessionAuthenticator) Init() error {
 	if keyFound && pubKeyCrossLogin != "" {
 		bytes, err := base64.StdEncoding.DecodeString(pubKeyCrossLogin)
 		if err != nil {
-			cclog.Warn("Could not decode cross login JWT public key")
+			log.Warn("Could not decode cross login JWT public key")
 			return err
 		}
 		ja.publicKeyCrossLogin = ed25519.PublicKey(bytes)
 	} else {
 		ja.publicKeyCrossLogin = nil
-		cclog.Debug("environment variable 'CROSS_LOGIN_JWT_PUBLIC_KEY' not set (cross login token based authentication will not work)")
+		log.Debug("environment variable 'CROSS_LOGIN_JWT_PUBLIC_KEY' not set (cross login token based authentication will not work)")
 		return errors.New("environment variable 'CROSS_LOGIN_JWT_PUBLIC_KEY' not set (cross login token based authentication will not work)")
 	}
 
+	jc := config.Keys.JwtConfig
 	// Warn if other necessary settings are not configured
-	if Keys.JwtConfig != nil {
-		if Keys.JwtConfig.CookieName == "" {
-			cclog.Info("cookieName for JWTs not configured (cross login via JWT cookie will fail)")
+	if jc != nil {
+		if jc.CookieName == "" {
+			log.Info("cookieName for JWTs not configured (cross login via JWT cookie will fail)")
 			return errors.New("cookieName for JWTs not configured (cross login via JWT cookie will fail)")
 		}
-		if !Keys.JwtConfig.ValidateUser {
-			cclog.Info("forceJWTValidationViaDatabase not set to true: CC will accept users and roles defined in JWTs regardless of its own database!")
+		if !jc.ValidateUser {
+			log.Info("forceJWTValidationViaDatabase not set to true: CC will accept users and roles defined in JWTs regardless of its own database!")
 		}
-		if Keys.JwtConfig.TrustedIssuer == "" {
-			cclog.Info("trustedExternalIssuer for JWTs not configured (cross login via JWT cookie will fail)")
+		if jc.TrustedIssuer == "" {
+			log.Info("trustedExternalIssuer for JWTs not configured (cross login via JWT cookie will fail)")
 			return errors.New("trustedExternalIssuer for JWTs not configured (cross login via JWT cookie will fail)")
 		}
 	} else {
-		cclog.Warn("config for JWTs not configured (cross login via JWT cookie will fail)")
+		log.Warn("config for JWTs not configured (cross login via JWT cookie will fail)")
 		return errors.New("config for JWTs not configured (cross login via JWT cookie will fail)")
 	}
 
-	cclog.Info("JWT Cookie Session authenticator successfully registered")
+	log.Info("JWT Cookie Session authenticator successfully registered")
 	return nil
 }
 
@@ -91,7 +92,7 @@ func (ja *JWTCookieSessionAuthenticator) CanLogin(
 	rw http.ResponseWriter,
 	r *http.Request,
 ) (*schema.User, bool) {
-	jc := Keys.JwtConfig
+	jc := config.Keys.JwtConfig
 	cookieName := ""
 	if jc.CookieName != "" {
 		cookieName = jc.CookieName
@@ -114,7 +115,7 @@ func (ja *JWTCookieSessionAuthenticator) Login(
 	rw http.ResponseWriter,
 	r *http.Request,
 ) (*schema.User, error) {
-	jc := Keys.JwtConfig
+	jc := config.Keys.JwtConfig
 	jwtCookie, err := r.Cookie(jc.CookieName)
 	var rawtoken string
 
@@ -122,7 +123,7 @@ func (ja *JWTCookieSessionAuthenticator) Login(
 		rawtoken = jwtCookie.Value
 	}
 
-	token, err := jwt.Parse(rawtoken, func(t *jwt.Token) (any, error) {
+	token, err := jwt.Parse(rawtoken, func(t *jwt.Token) (interface{}, error) {
 		if t.Method != jwt.SigningMethodEdDSA {
 			return nil, errors.New("only Ed25519/EdDSA supported")
 		}
@@ -139,12 +140,12 @@ func (ja *JWTCookieSessionAuthenticator) Login(
 		return ja.publicKey, nil
 	})
 	if err != nil {
-		cclog.Warn("JWT cookie session: error while parsing token")
+		log.Warn("JWT cookie session: error while parsing token")
 		return nil, err
 	}
 
 	if !token.Valid {
-		cclog.Warn("jwt token claims are not valid")
+		log.Warn("jwt token claims are not valid")
 		return nil, errors.New("jwt token claims are not valid")
 	}
 
@@ -158,18 +159,18 @@ func (ja *JWTCookieSessionAuthenticator) Login(
 		var err error
 		user, err = repository.GetUserRepository().GetUser(sub)
 		if err != nil && err != sql.ErrNoRows {
-			cclog.Errorf("Error while loading user '%v'", sub)
+			log.Errorf("Error while loading user '%v'", sub)
 		}
 
 		// Deny any logins for unknown usernames
 		if user == nil {
-			cclog.Warn("Could not find user from JWT in internal database.")
+			log.Warn("Could not find user from JWT in internal database.")
 			return nil, errors.New("unknown user")
 		}
 	} else {
 		var name string
-		if wrap, ok := claims["name"].(map[string]any); ok {
-			if vals, ok := wrap["values"].([]any); ok {
+		if wrap, ok := claims["name"].(map[string]interface{}); ok {
+			if vals, ok := wrap["values"].([]interface{}); ok {
 				if len(vals) != 0 {
 					name = fmt.Sprintf("%v", vals[0])
 
@@ -181,7 +182,7 @@ func (ja *JWTCookieSessionAuthenticator) Login(
 		}
 
 		// Extract roles from JWT (if present)
-		if rawroles, ok := claims["roles"].([]any); ok {
+		if rawroles, ok := claims["roles"].([]interface{}); ok {
 			for _, rr := range rawroles {
 				if r, ok := rr.(string); ok {
 					roles = append(roles, r)

internal/auth/jwtSession.go

@@ -1,8 +1,7 @@
 // Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved. This file is part of cc-backend.
+// All rights reserved.
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
-
 package auth
 
 import (
@@ -14,9 +13,10 @@ import (
 	"os"
 	"strings"
 
+	"github.com/ClusterCockpit/cc-backend/internal/config"
 	"github.com/ClusterCockpit/cc-backend/internal/repository"
-	cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
-	"github.com/ClusterCockpit/cc-lib/schema"
+	"github.com/ClusterCockpit/cc-backend/pkg/log"
+	"github.com/ClusterCockpit/cc-backend/pkg/schema"
 	"github.com/golang-jwt/jwt/v5"
 )
 
@@ -30,13 +30,13 @@ func (ja *JWTSessionAuthenticator) Init() error {
 	if pubKey := os.Getenv("CROSS_LOGIN_JWT_HS512_KEY"); pubKey != "" {
 		bytes, err := base64.StdEncoding.DecodeString(pubKey)
 		if err != nil {
-			cclog.Warn("Could not decode cross login JWT HS512 key")
+			log.Warn("Could not decode cross login JWT HS512 key")
 			return err
 		}
 		ja.loginTokenKey = bytes
 	}
 
-	cclog.Info("JWT Session authenticator successfully registered")
+	log.Info("JWT Session authenticator successfully registered")
 	return nil
 }
 
@@ -60,19 +60,19 @@ func (ja *JWTSessionAuthenticator) Login(
 		rawtoken = r.URL.Query().Get("login-token")
 	}
 
-	token, err := jwt.Parse(rawtoken, func(t *jwt.Token) (any, error) {
+	token, err := jwt.Parse(rawtoken, func(t *jwt.Token) (interface{}, error) {
 		if t.Method == jwt.SigningMethodHS256 || t.Method == jwt.SigningMethodHS512 {
 			return ja.loginTokenKey, nil
 		}
 		return nil, fmt.Errorf("unkown signing method for login token: %s (known: HS256, HS512, EdDSA)", t.Method.Alg())
 	})
 	if err != nil {
-		cclog.Warn("Error while parsing jwt token")
+		log.Warn("Error while parsing jwt token")
 		return nil, err
 	}
 
 	if !token.Valid {
-		cclog.Warn("jwt token claims are not valid")
+		log.Warn("jwt token claims are not valid")
 		return nil, errors.New("jwt token claims are not valid")
 	}
 
@@ -82,22 +82,22 @@ func (ja *JWTSessionAuthenticator) Login(
 	var roles []string
 	projects := make([]string, 0)
 
-	if Keys.JwtConfig.ValidateUser {
+	if config.Keys.JwtConfig.ValidateUser {
 		var err error
 		user, err = repository.GetUserRepository().GetUser(sub)
 		if err != nil && err != sql.ErrNoRows {
-			cclog.Errorf("Error while loading user '%v'", sub)
+			log.Errorf("Error while loading user '%v'", sub)
 		}
 
 		// Deny any logins for unknown usernames
 		if user == nil {
-			cclog.Warn("Could not find user from JWT in internal database.")
+			log.Warn("Could not find user from JWT in internal database.")
 			return nil, errors.New("unknown user")
 		}
 	} else {
 		var name string
-		if wrap, ok := claims["name"].(map[string]any); ok {
-			if vals, ok := wrap["values"].([]any); ok {
+		if wrap, ok := claims["name"].(map[string]interface{}); ok {
+			if vals, ok := wrap["values"].([]interface{}); ok {
 				if len(vals) != 0 {
 					name = fmt.Sprintf("%v", vals[0])
 
@@ -109,7 +109,7 @@ func (ja *JWTSessionAuthenticator) Login(
 		}
 
 		// Extract roles from JWT (if present)
-		if rawroles, ok := claims["roles"].([]any); ok {
+		if rawroles, ok := claims["roles"].([]interface{}); ok {
 			for _, rr := range rawroles {
 				if r, ok := rr.(string); ok {
 					if schema.IsValidRole(r) {
@@ -119,7 +119,7 @@ func (ja *JWTSessionAuthenticator) Login(
 			}
 		}
 
-		if rawprojs, ok := claims["projects"].([]any); ok {
+		if rawprojs, ok := claims["projects"].([]interface{}); ok {
 			for _, pp := range rawprojs {
 				if p, ok := pp.(string); ok {
 					projects = append(projects, p)
@@ -138,7 +138,7 @@ func (ja *JWTSessionAuthenticator) Login(
 			AuthSource: schema.AuthViaToken,
 		}
 
-		if Keys.JwtConfig.SyncUserOnLogin || Keys.JwtConfig.UpdateUserOnLogin {
+		if config.Keys.JwtConfig.SyncUserOnLogin || config.Keys.JwtConfig.UpdateUserOnLogin {
 			handleTokenUser(user)
 		}
 	}

internal/auth/ldap.go

@@ -1,8 +1,7 @@
 // Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved. This file is part of cc-backend.
+// All rights reserved.
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
-
 package auth
 
 import (
@@ -12,26 +11,13 @@ import (
 	"os"
 	"strings"
 
+	"github.com/ClusterCockpit/cc-backend/internal/config"
 	"github.com/ClusterCockpit/cc-backend/internal/repository"
-	cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
-	"github.com/ClusterCockpit/cc-lib/schema"
+	"github.com/ClusterCockpit/cc-backend/pkg/log"
+	"github.com/ClusterCockpit/cc-backend/pkg/schema"
 	"github.com/go-ldap/ldap/v3"
 )
 
-type LdapConfig struct {
-	URL             string `json:"url"`
-	UserBase        string `json:"user_base"`
-	SearchDN        string `json:"search_dn"`
-	UserBind        string `json:"user_bind"`
-	UserFilter      string `json:"user_filter"`
-	UserAttr        string `json:"username_attr"`
-	SyncInterval    string `json:"sync_interval"` // Parsed using time.ParseDuration.
-	SyncDelOldUsers bool   `json:"sync_del_old_users"`
-
-	// Should an non-existent user be added to the DB if user exists in ldap directory
-	SyncUserOnLogin bool `json:"syncUserOnLogin"`
-}
-
 type LdapAuthenticator struct {
 	syncPassword string
 	UserAttr     string
@@ -42,11 +28,13 @@ var _ Authenticator = (*LdapAuthenticator)(nil)
 func (la *LdapAuthenticator) Init() error {
 	la.syncPassword = os.Getenv("LDAP_ADMIN_PASSWORD")
 	if la.syncPassword == "" {
-		cclog.Warn("environment variable 'LDAP_ADMIN_PASSWORD' not set (ldap sync will not work)")
+		log.Warn("environment variable 'LDAP_ADMIN_PASSWORD' not set (ldap sync will not work)")
 	}
 
-	if Keys.LdapConfig.UserAttr != "" {
-		la.UserAttr = Keys.LdapConfig.UserAttr
+	lc := config.Keys.LdapConfig
+
+	if lc.UserAttr != "" {
+		la.UserAttr = lc.UserAttr
 	} else {
 		la.UserAttr = "gecos"
 	}
@@ -60,7 +48,7 @@ func (la *LdapAuthenticator) CanLogin(
 	rw http.ResponseWriter,
 	r *http.Request,
 ) (*schema.User, bool) {
-	lc := Keys.LdapConfig
+	lc := config.Keys.LdapConfig
 
 	if user != nil {
 		if user.AuthSource == schema.AuthViaLDAP {
@@ -70,7 +58,7 @@ func (la *LdapAuthenticator) CanLogin(
 		if lc.SyncUserOnLogin {
 			l, err := la.getLdapConnection(true)
 			if err != nil {
-				cclog.Error("LDAP connection error")
+				log.Error("LDAP connection error")
 			}
 			defer l.Close()
 
@@ -83,12 +71,12 @@ func (la *LdapAuthenticator) CanLogin(
 
 			sr, err := l.Search(searchRequest)
 			if err != nil {
-				cclog.Warn(err)
+				log.Warn(err)
 				return nil, false
 			}
 
 			if len(sr.Entries) != 1 {
-				cclog.Warn("LDAP: User does not exist or too many entries returned")
+				log.Warn("LDAP: User does not exist or too many entries returned")
 				return nil, false
 			}
 
@@ -108,7 +96,7 @@ func (la *LdapAuthenticator) CanLogin(
 			}
 
 			if err := repository.GetUserRepository().AddUser(user); err != nil {
-				cclog.Errorf("User '%s' LDAP: Insert into DB failed", username)
+				log.Errorf("User '%s' LDAP: Insert into DB failed", username)
 				return nil, false
 			}
 
@@ -126,14 +114,14 @@ func (la *LdapAuthenticator) Login(
 ) (*schema.User, error) {
 	l, err := la.getLdapConnection(false)
 	if err != nil {
-		cclog.Warn("Error while getting ldap connection")
+		log.Warn("Error while getting ldap connection")
 		return nil, err
 	}
 	defer l.Close()
 
-	userDn := strings.ReplaceAll(Keys.LdapConfig.UserBind, "{username}", user.Username)
+	userDn := strings.Replace(config.Keys.LdapConfig.UserBind, "{username}", user.Username, -1)
 	if err := l.Bind(userDn, r.FormValue("password")); err != nil {
-		cclog.Errorf("AUTH/LDAP > Authentication for user %s failed: %v",
+		log.Errorf("AUTH/LDAP > Authentication for user %s failed: %v",
 			user.Username, err)
 		return nil, fmt.Errorf("Authentication failed")
 	}
@@ -142,11 +130,11 @@ func (la *LdapAuthenticator) Login(
 }
 
 func (la *LdapAuthenticator) Sync() error {
-	const InDB int = 1
-	const InLdap int = 2
-	const InBoth int = 3
+	const IN_DB int = 1
+	const IN_LDAP int = 2
+	const IN_BOTH int = 3
 	ur := repository.GetUserRepository()
-	lc := Keys.LdapConfig
+	lc := config.Keys.LdapConfig
 
 	users := map[string]int{}
 	usernames, err := ur.GetLdapUsernames()
@@ -155,12 +143,12 @@ func (la *LdapAuthenticator) Sync() error {
 	}
 
 	for _, username := range usernames {
-		users[username] = InDB
+		users[username] = IN_DB
 	}
 
 	l, err := la.getLdapConnection(true)
 	if err != nil {
-		cclog.Error("LDAP connection error")
+		log.Error("LDAP connection error")
 		return err
 	}
 	defer l.Close()
@@ -171,7 +159,7 @@ func (la *LdapAuthenticator) Sync() error {
 		lc.UserFilter,
 		[]string{"dn", "uid", la.UserAttr}, nil))
 	if err != nil {
-		cclog.Warn("LDAP search error")
+		log.Warn("LDAP search error")
 		return err
 	}
 
@@ -184,18 +172,18 @@ func (la *LdapAuthenticator) Sync() error {
 
 		_, ok := users[username]
 		if !ok {
-			users[username] = InLdap
+			users[username] = IN_LDAP
 			newnames[username] = entry.GetAttributeValue(la.UserAttr)
 		} else {
-			users[username] = InBoth
+			users[username] = IN_BOTH
 		}
 	}
 
 	for username, where := range users {
-		if where == InDB && lc.SyncDelOldUsers {
+		if where == IN_DB && lc.SyncDelOldUsers {
 			ur.DelUser(username)
-			cclog.Debugf("sync: remove %v (does not show up in LDAP anymore)", username)
-		} else if where == InLdap {
+			log.Debugf("sync: remove %v (does not show up in LDAP anymore)", username)
+		} else if where == IN_LDAP {
 			name := newnames[username]
 
 			var roles []string
@@ -210,9 +198,9 @@ func (la *LdapAuthenticator) Sync() error {
 				AuthSource: schema.AuthViaLDAP,
 			}
 
-			cclog.Debugf("sync: add %v (name: %v, roles: [user], ldap: true)", username, name)
+			log.Debugf("sync: add %v (name: %v, roles: [user], ldap: true)", username, name)
 			if err := ur.AddUser(user); err != nil {
-				cclog.Errorf("User '%s' LDAP: Insert into DB failed", username)
+				log.Errorf("User '%s' LDAP: Insert into DB failed", username)
 				return err
 			}
 		}
@@ -222,17 +210,17 @@ func (la *LdapAuthenticator) Sync() error {
 }
 
 func (la *LdapAuthenticator) getLdapConnection(admin bool) (*ldap.Conn, error) {
-	lc := Keys.LdapConfig
-	conn, err := ldap.DialURL(lc.URL)
+	lc := config.Keys.LdapConfig
+	conn, err := ldap.DialURL(lc.Url)
 	if err != nil {
-		cclog.Warn("LDAP URL dial failed")
+		log.Warn("LDAP URL dial failed")
 		return nil, err
 	}
 
 	if admin {
 		if err := conn.Bind(lc.SearchDN, la.syncPassword); err != nil {
 			conn.Close()
-			cclog.Warn("LDAP connection bind failed")
+			log.Warn("LDAP connection bind failed")
 			return nil, err
 		}
 	}

internal/auth/local.go

@@ -1,16 +1,15 @@
 // Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved. This file is part of cc-backend.
+// All rights reserved.
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
-
 package auth
 
 import (
 	"fmt"
 	"net/http"
 
-	cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
-	"github.com/ClusterCockpit/cc-lib/schema"
+	"github.com/ClusterCockpit/cc-backend/pkg/log"
+	"github.com/ClusterCockpit/cc-backend/pkg/schema"
 	"golang.org/x/crypto/bcrypt"
 )
 
@@ -28,19 +27,19 @@ func (la *LocalAuthenticator) CanLogin(
 	user *schema.User,
 	username string,
 	rw http.ResponseWriter,
-	r *http.Request,
-) (*schema.User, bool) {
+	r *http.Request) (*schema.User, bool) {
+
 	return user, user != nil && user.AuthSource == schema.AuthViaLocalPassword
 }
 
 func (la *LocalAuthenticator) Login(
 	user *schema.User,
 	rw http.ResponseWriter,
-	r *http.Request,
-) (*schema.User, error) {
+	r *http.Request) (*schema.User, error) {
+
 	if e := bcrypt.CompareHashAndPassword([]byte(user.Password),
 		[]byte(r.FormValue("password"))); e != nil {
-		cclog.Errorf("AUTH/LOCAL > Authentication for user %s failed!", user.Username)
+		log.Errorf("AUTH/LOCAL > Authentication for user %s failed!", user.Username)
 		return nil, fmt.Errorf("Authentication failed")
 	}
 

internal/auth/oidc.go

@@ -1,8 +1,7 @@
 // Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved. This file is part of cc-backend.
+// All rights reserved.
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
-
 package auth
 
 import (
@@ -14,20 +13,15 @@ import (
 	"os"
 	"time"
 
+	"github.com/ClusterCockpit/cc-backend/internal/config"
 	"github.com/ClusterCockpit/cc-backend/internal/repository"
-	cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
-	"github.com/ClusterCockpit/cc-lib/schema"
+	"github.com/ClusterCockpit/cc-backend/pkg/log"
+	"github.com/ClusterCockpit/cc-backend/pkg/schema"
 	"github.com/coreos/go-oidc/v3/oidc"
 	"github.com/gorilla/mux"
 	"golang.org/x/oauth2"
 )
 
-type OpenIDConfig struct {
-	Provider          string `json:"provider"`
-	SyncUserOnLogin   bool   `json:"syncUserOnLogin"`
-	UpdateUserOnLogin bool   `json:"updateUserOnLogin"`
-}
-
 type OIDC struct {
 	client         *oauth2.Config
 	provider       *oidc.Provider
@@ -55,17 +49,17 @@ func setCallbackCookie(w http.ResponseWriter, r *http.Request, name, value strin
 }
 
 func NewOIDC(a *Authentication) *OIDC {
-	provider, err := oidc.NewProvider(context.Background(), Keys.OpenIDConfig.Provider)
+	provider, err := oidc.NewProvider(context.Background(), config.Keys.OpenIDConfig.Provider)
 	if err != nil {
-		cclog.Fatal(err)
+		log.Fatal(err)
 	}
 	clientID := os.Getenv("OID_CLIENT_ID")
 	if clientID == "" {
-		cclog.Warn("environment variable 'OID_CLIENT_ID' not set (Open ID connect auth will not work)")
+		log.Warn("environment variable 'OID_CLIENT_ID' not set (Open ID connect auth will not work)")
 	}
 	clientSecret := os.Getenv("OID_CLIENT_SECRET")
 	if clientSecret == "" {
-		cclog.Warn("environment variable 'OID_CLIENT_SECRET' not set (Open ID connect auth will not work)")
+		log.Warn("environment variable 'OID_CLIENT_SECRET' not set (Open ID connect auth will not work)")
 	}
 
 	client := &oauth2.Config{
@@ -174,12 +168,12 @@ func (oa *OIDC) OAuth2Callback(rw http.ResponseWriter, r *http.Request) {
 		AuthSource: schema.AuthViaOIDC,
 	}
 
-	if Keys.OpenIDConfig.SyncUserOnLogin || Keys.OpenIDConfig.UpdateUserOnLogin {
+	if config.Keys.OpenIDConfig.SyncUserOnLogin || config.Keys.OpenIDConfig.UpdateUserOnLogin {
 		handleOIDCUser(user)
 	}
 
 	oa.authentication.SaveSession(rw, r, user)
-	cclog.Infof("login successfull: user: %#v (roles: %v, projects: %v)", user.Username, user.Roles, user.Projects)
+	log.Infof("login successfull: user: %#v (roles: %v, projects: %v)", user.Username, user.Roles, user.Projects)
 	ctx := context.WithValue(r.Context(), repository.ContextUserKey, user)
 	http.RedirectHandler("/", http.StatusTemporaryRedirect).ServeHTTP(rw, r.WithContext(ctx))
 }

internal/auth/schema.go

@@ -1,96 +0,0 @@
-// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved. This file is part of cc-backend.
-// Use of this source code is governed by a MIT-style
-// license that can be found in the LICENSE file.
-
-package auth
-
-var configSchema = `
-	{
-    "jwts": {
-      "description": "For JWT token authentication.",
-      "type": "object",
-      "properties": {
-        "max-age": {
-          "description": "Configure how long a token is valid. As string parsable by time.ParseDuration()",
-          "type": "string"
-        },
-        "cookieName": {
-          "description": "Cookie that should be checked for a JWT token.",
-          "type": "string"
-        },
-        "validateUser": {
-          "description": "Deny login for users not in database (but defined in JWT). Overwrite roles in JWT with database roles.",
-          "type": "boolean"
-        },
-        "trustedIssuer": {
-          "description": "Issuer that should be accepted when validating external JWTs ",
-          "type": "string"
-        },
-        "syncUserOnLogin": {
-          "description": "Add non-existent user to DB at login attempt with values provided in JWT.",
-          "type": "boolean"
-        }
-      },
-      "required": ["max-age"]
-    },
-    "oidc": {
-      "provider": {
-        "description": "",
-        "type": "string"
-      },
-      "syncUserOnLogin": {
-        "description": "",
-        "type": "boolean"
-      },
-      "updateUserOnLogin": {
-        "description": "",
-        "type": "boolean"
-      },
-      "required": ["provider"]
-    },
-    "ldap": {
-      "description": "For LDAP Authentication and user synchronisation.",
-      "type": "object",
-      "properties": {
-        "url": {
-          "description": "URL of LDAP directory server.",
-          "type": "string"
-        },
-        "user_base": {
-          "description": "Base DN of user tree root.",
-          "type": "string"
-        },
-        "search_dn": {
-          "description": "DN for authenticating LDAP admin account with general read rights.",
-          "type": "string"
-        },
-        "user_bind": {
-          "description": "Expression used to authenticate users via LDAP bind. Must contain uid={username}.",
-          "type": "string"
-        },
-        "user_filter": {
-          "description": "Filter to extract users for syncing.",
-          "type": "string"
-        },
-        "username_attr": {
-          "description": "Attribute with full username. Default: gecos",
-          "type": "string"
-        },
-        "sync_interval": {
-          "description": "Interval used for syncing local user table with LDAP directory. Parsed using time.ParseDuration.",
-          "type": "string"
-        },
-        "sync_del_old_users": {
-          "description": "Delete obsolete users in database.",
-          "type": "boolean"
-        },
-        "syncUserOnLogin": {
-          "description": "Add non-existent user to DB at login attempt if user exists in Ldap directory",
-          "type": "boolean"
-        }
-      },
-      "required": ["url", "user_base", "search_dn", "user_bind", "user_filter"]
-    },
-  "required": ["jwts"]
-	}`

internal/config/config.go

@@ -1,143 +1,72 @@
 // Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved. This file is part of cc-backend.
+// All rights reserved.
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
-
-// Package config implements the program configuration data structures, validation and parsing
 package config
 
 import (
 	"bytes"
 	"encoding/json"
-	"time"
+	"log"
+	"os"
 
-	cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
+	"github.com/ClusterCockpit/cc-backend/pkg/schema"
 )
 
-type ProgramConfig struct {
-	// Address where the http (or https) server will listen on (for example: 'localhost:80').
-	Addr string `json:"addr"`
-
-	// Addresses from which secured admin API endpoints can be reached, can be wildcard "*"
-	APIAllowedIPs []string `json:"apiAllowedIPs"`
-
-	// Drop root permissions once .env was read and the port was taken.
-	User  string `json:"user"`
-	Group string `json:"group"`
-
-	// Disable authentication (for everything: API, Web-UI, ...)
-	DisableAuthentication bool `json:"disable-authentication"`
-
-	// If `embed-static-files` is true (default), the frontend files are directly
-	// embeded into the go binary and expected to be in web/frontend. Only if
-	// it is false the files in `static-files` are served instead.
-	EmbedStaticFiles bool   `json:"embed-static-files"`
-	StaticFiles      string `json:"static-files"`
-
-	// 'sqlite3' or 'mysql' (mysql will work for mariadb as well)
-	DBDriver string `json:"db-driver"`
-
-	// For sqlite3 a filename, for mysql a DSN in this format: https://github.com/go-sql-driver/mysql#dsn-data-source-name (Without query parameters!).
-	DB string `json:"db"`
-
-	// Keep all metric data in the metric data repositories,
-	// do not write to the job-archive.
-	DisableArchive bool `json:"disable-archive"`
-
-	EnableJobTaggers bool `json:"enable-job-taggers"`
-
-	// Validate json input against schema
-	Validate bool `json:"validate"`
-
-	// If 0 or empty, the session does not expire!
-	SessionMaxAge string `json:"session-max-age"`
-
-	// If both those options are not empty, use HTTPS using those certificates.
-	HTTPSCertFile string `json:"https-cert-file"`
-	HTTPSKeyFile  string `json:"https-key-file"`
-
-	// If not the empty string and `addr` does not end in ":80",
-	// redirect every request incoming at port 80 to that url.
-	RedirectHTTPTo string `json:"redirect-http-to"`
-
-	// Where to store MachineState files
-	MachineStateDir string `json:"machine-state-dir"`
-
-	// If not zero, automatically mark jobs as stopped running X seconds longer than their walltime.
-	StopJobsExceedingWalltime int `json:"stop-jobs-exceeding-walltime"`
-
-	// Defines time X in seconds in which jobs are considered to be "short" and will be filtered in specific views.
-	ShortRunningJobsDuration int `json:"short-running-jobs-duration"`
-
-	// Energy Mix CO2 Emission Constant [g/kWh]
-	// If entered, displays estimated CO2 emission for job based on jobs totalEnergy
-	EmissionConstant int `json:"emission-constant"`
-
-	// If exists, will enable dynamic zoom in frontend metric plots using the configured values
-	EnableResampling *ResampleConfig `json:"resampling"`
-}
-
-type ResampleConfig struct {
-	// Array of resampling target resolutions, in seconds; Example: [600,300,60]
-	Resolutions []int `json:"resolutions"`
-	// Trigger next zoom level at less than this many visible datapoints
-	Trigger int `json:"trigger"`
-}
-
-type IntRange struct {
-	From int `json:"from"`
-	To   int `json:"to"`
-}
-
-type TimeRange struct {
-	From  *time.Time `json:"from"`
-	To    *time.Time `json:"to"`
-	Range string     `json:"range,omitempty"`
-}
-
-type FilterRanges struct {
-	Duration  *IntRange  `json:"duration"`
-	NumNodes  *IntRange  `json:"numNodes"`
-	StartTime *TimeRange `json:"startTime"`
-}
-
-type ClusterConfig struct {
-	Name                 string          `json:"name"`
-	FilterRanges         *FilterRanges   `json:"filterRanges"`
-	MetricDataRepository json.RawMessage `json:"metricDataRepository"`
-}
-
-var Clusters []*ClusterConfig
-
-var Keys ProgramConfig = ProgramConfig{
+var Keys schema.ProgramConfig = schema.ProgramConfig{
 	Addr:                      "localhost:8080",
 	DisableAuthentication:     false,
 	EmbedStaticFiles:          true,
 	DBDriver:                  "sqlite3",
 	DB:                        "./var/job.db",
+	Archive:                   json.RawMessage(`{\"kind\":\"file\",\"path\":\"./var/job-archive\"}`),
 	DisableArchive:            false,
 	Validate:                  false,
 	SessionMaxAge:             "168h",
 	StopJobsExceedingWalltime: 0,
 	ShortRunningJobsDuration:  5 * 60,
+	UiDefaults: map[string]interface{}{
+		"analysis_view_histogramMetrics":         []string{"flops_any", "mem_bw", "mem_used"},
+		"analysis_view_scatterPlotMetrics":       [][]string{{"flops_any", "mem_bw"}, {"flops_any", "cpu_load"}, {"cpu_load", "mem_bw"}},
+		"job_view_nodestats_selectedMetrics":     []string{"flops_any", "mem_bw", "mem_used"},
+		"job_view_selectedMetrics":               []string{"flops_any", "mem_bw", "mem_used"},
+		"job_view_showFootprint":                 true,
+		"job_list_usePaging":                     false,
+		"plot_general_colorBackground":           true,
+		"plot_general_colorscheme":               []string{"#00bfff", "#0000ff", "#ff00ff", "#ff0000", "#ff8000", "#ffff00", "#80ff00"},
+		"plot_general_lineWidth":                 3,
+		"plot_list_jobsPerPage":                  50,
+		"plot_list_selectedMetrics":              []string{"cpu_load", "mem_used", "flops_any", "mem_bw"},
+		"plot_view_plotsPerRow":                  3,
+		"plot_view_showPolarplot":                true,
+		"plot_view_showRoofline":                 true,
+		"plot_view_showStatTable":                true,
+		"system_view_selectedMetric":             "cpu_load",
+		"analysis_view_selectedTopEntity":        "user",
+		"analysis_view_selectedTopCategory":      "totalWalltime",
+		"status_view_selectedTopUserCategory":    "totalJobs",
+		"status_view_selectedTopProjectCategory": "totalJobs",
+	},
 }
 
-func Init(mainConfig json.RawMessage, clusterConfig json.RawMessage) {
-	Validate(configSchema, mainConfig)
-	dec := json.NewDecoder(bytes.NewReader(mainConfig))
-	dec.DisallowUnknownFields()
-	if err := dec.Decode(&Keys); err != nil {
-		cclog.Abortf("Config Init: Could not decode config file '%s'.\nError: %s\n", mainConfig, err.Error())
-	}
+func Init(flagConfigFile string) {
+	raw, err := os.ReadFile(flagConfigFile)
+	if err != nil {
+		if !os.IsNotExist(err) {
+			log.Fatalf("CONFIG ERROR: %v", err)
+		}
+	} else {
+		if err := schema.Validate(schema.Config, bytes.NewReader(raw)); err != nil {
+			log.Fatalf("Validate config: %v\n", err)
+		}
+		dec := json.NewDecoder(bytes.NewReader(raw))
+		dec.DisallowUnknownFields()
+		if err := dec.Decode(&Keys); err != nil {
+			log.Fatalf("could not decode: %v", err)
+		}
 
-	Validate(clustersSchema, clusterConfig)
-	dec = json.NewDecoder(bytes.NewReader(clusterConfig))
-	dec.DisallowUnknownFields()
-	if err := dec.Decode(&Clusters); err != nil {
-		cclog.Abortf("Config Init: Could not decode config file '%s'.\nError: %s\n", mainConfig, err.Error())
-	}
-
-	if len(Clusters) < 1 {
-		cclog.Abort("Config Init: At least one cluster required in config. Exited with error.")
+		if Keys.Clusters == nil || len(Keys.Clusters) < 1 {
+			log.Fatal("At least one cluster required in config!")
+		}
 	}
 }

internal/config/config_test.go

@@ -1,30 +1,16 @@
 // Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved. This file is part of cc-backend.
+// All rights reserved.
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
-
 package config
 
 import (
 	"testing"
-
-	ccconf "github.com/ClusterCockpit/cc-lib/ccConfig"
-	cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
 )
 
 func TestInit(t *testing.T) {
 	fp := "../../configs/config.json"
-	ccconf.Init(fp)
-	if cfg := ccconf.GetPackageConfig("main"); cfg != nil {
-		if clustercfg := ccconf.GetPackageConfig("clusters"); clustercfg != nil {
-			Init(cfg, clustercfg)
-		} else {
-			cclog.Abort("Cluster configuration must be present")
-		}
-	} else {
-		cclog.Abort("Main configuration must be present")
-	}
-
+	Init(fp)
 	if Keys.Addr != "0.0.0.0:443" {
 		t.Errorf("wrong addr\ngot: %s \nwant: 0.0.0.0:443", Keys.Addr)
 	}
@@ -32,17 +18,7 @@ func TestInit(t *testing.T) {
 
 func TestInitMinimal(t *testing.T) {
 	fp := "../../configs/config-demo.json"
-	ccconf.Init(fp)
-	if cfg := ccconf.GetPackageConfig("main"); cfg != nil {
-		if clustercfg := ccconf.GetPackageConfig("clusters"); clustercfg != nil {
-			Init(cfg, clustercfg)
-		} else {
-			cclog.Abort("Cluster configuration must be present")
-		}
-	} else {
-		cclog.Abort("Main configuration must be present")
-	}
-
+	Init(fp)
 	if Keys.Addr != "127.0.0.1:8080" {
 		t.Errorf("wrong addr\ngot: %s \nwant: 127.0.0.1:8080", Keys.Addr)
 	}

internal/config/default_metrics.go

@@ -1,8 +1,3 @@
-// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved. This file is part of cc-backend.
-// Use of this source code is governed by a MIT-style
-// license that can be found in the LICENSE file.
-
 package config
 
 import (
@@ -11,8 +6,6 @@ import (
 	"strings"
 )
 
-// DEPRECATED: SUPERSEDED BY NEW USER CONFIG - userConfig.go / web.go
-
 type DefaultMetricsCluster struct {
 	Name           string `json:"name"`
 	DefaultMetrics string `json:"default_metrics"`
@@ -23,7 +16,7 @@ type DefaultMetricsConfig struct {
 }
 
 func LoadDefaultMetricsConfig() (*DefaultMetricsConfig, error) {
-	filePath := "default_metrics.json"
+	filePath := "configs/default_metrics.json"
 	if _, err := os.Stat(filePath); os.IsNotExist(err) {
 		return nil, nil
 	}

internal/config/schema.go

@@ -1,201 +0,0 @@
-// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved. This file is part of cc-backend.
-// Use of this source code is governed by a MIT-style
-// license that can be found in the LICENSE file.
-
-package config
-
-var configSchema = `
-	{
-  "type": "object",
-  "properties": {
-    "addr": {
-      "description": "Address where the http (or https) server will listen on (for example: 'localhost:80').",
-      "type": "string"
-    },
-    "apiAllowedIPs": {
-      "description": "Addresses from which secured API endpoints can be reached",
-      "type": "array",
-      "items": {
-        "type": "string"
-      }
-    },
-    "user": {
-      "description": "Drop root permissions once .env was read and the port was taken. Only applicable if using privileged port.",
-      "type": "string"
-    },
-    "group": {
-      "description": "Drop root permissions once .env was read and the port was taken. Only applicable if using privileged port.",
-      "type": "string"
-    },
-    "disable-authentication": {
-      "description": "Disable authentication (for everything: API, Web-UI, ...).",
-      "type": "boolean"
-    },
-    "embed-static-files": {
-      "description": "If all files in web/frontend/public should be served from within the binary itself (they are embedded) or not.",
-      "type": "boolean"
-    },
-    "static-files": {
-      "description": "Folder where static assets can be found, if embed-static-files is false.",
-      "type": "string"
-    },
-    "db": {
-      "description": "For sqlite3 a filename, for mysql a DSN in this format: https://github.com/go-sql-driver/mysql#dsn-data-source-name (Without query parameters!).",
-      "type": "string"
-    },
-    "disable-archive": {
-      "description": "Keep all metric data in the metric data repositories, do not write to the job-archive.",
-      "type": "boolean"
-    },
-    "enable-job-taggers": {
-      "description": "Turn on automatic application and jobclass taggers",
-      "type": "boolean"
-    },
-    "validate": {
-      "description": "Validate all input json documents against json schema.",
-      "type": "boolean"
-    },
-    "session-max-age": {
-      "description": "Specifies for how long a session shall be valid  as a string parsable by time.ParseDuration(). If 0 or empty, the session/token does not expire!",
-      "type": "string"
-    },
-    "https-cert-file": {
-      "description": "Filepath to SSL certificate. If also https-key-file is set use HTTPS using those certificates.",
-      "type": "string"
-    },
-    "https-key-file": {
-      "description": "Filepath to SSL key file. If also https-cert-file is set use HTTPS using those certificates.",
-      "type": "string"
-    },
-    "redirect-http-to": {
-      "description": "If not the empty string and addr does not end in :80, redirect every request incoming at port 80 to that url.",
-      "type": "string"
-    },
-    "stop-jobs-exceeding-walltime": {
-      "description": "If not zero, automatically mark jobs as stopped running X seconds longer than their walltime. Only applies if walltime is set for job.",
-      "type": "integer"
-    },
-    "short-running-jobs-duration": {
-      "description": "Do not show running jobs shorter than X seconds.",
-      "type": "integer"
-    },
-    "emission-constant": {
-      "description": ".",
-      "type": "integer"
-    },
-    "cron-frequency": {
-      "description": "Frequency of cron job workers.",
-      "type": "object",
-      "properties": {
-        "duration-worker": {
-          "description": "Duration Update Worker [Defaults to '5m']",
-          "type": "string"
-        },
-        "footprint-worker": {
-          "description": "Metric-Footprint Update Worker [Defaults to '10m']",
-          "type": "string"
-        }
-      }
-    },
-    "enable-resampling": {
-      "description": "Enable dynamic zoom in frontend metric plots.",
-      "type": "object",
-      "properties": {
-        "trigger": {
-          "description": "Trigger next zoom level at less than this many visible datapoints.",
-          "type": "integer"
-        },
-        "resolutions": {
-          "description": "Array of resampling target resolutions, in seconds.",
-          "type": "array",
-          "items": {
-            "type": "integer"
-          }
-        }
-      },
-      "required": ["trigger", "resolutions"]
-    }
-	},
-  "required": ["apiAllowedIPs"]
-	}`
-
-var clustersSchema = `
-  {
-    "type": "array",
-    "items": {
-      "type": "object",
-      "properties": {
-        "name": {
-          "description": "The name of the cluster.",
-          "type": "string"
-        },
-        "metricDataRepository": {
-          "description": "Type of the metric data repository for this cluster",
-          "type": "object",
-          "properties": {
-            "kind": {
-              "type": "string",
-                "enum": ["influxdb", "prometheus", "cc-metric-store", "cc-metric-store-internal", "test"]
-            },
-            "url": {
-              "type": "string"
-            },
-            "token": {
-              "type": "string"
-            }
-          },
-          "required": ["kind"]
-        },
-        "filterRanges": {
-          "description": "This option controls the slider ranges for the UI controls of numNodes, duration, and startTime.",
-          "type": "object",
-          "properties": {
-            "numNodes": {
-              "description": "UI slider range for number of nodes",
-              "type": "object",
-              "properties": {
-                "from": {
-                  "type": "integer"
-                },
-                "to": {
-                  "type": "integer"
-                }
-              },
-              "required": ["from", "to"]
-            },
-            "duration": {
-              "description": "UI slider range for duration",
-              "type": "object",
-              "properties": {
-                "from": {
-                  "type": "integer"
-                },
-                "to": {
-                  "type": "integer"
-                }
-              },
-              "required": ["from", "to"]
-            },
-            "startTime": {
-              "description": "UI slider range for start time",
-              "type": "object",
-              "properties": {
-                "from": {
-                  "type": "string",
-                  "format": "date-time"
-                },
-                "to": {
-                  "type": "null"
-                }
-              },
-              "required": ["from", "to"]
-            }
-          },
-          "required": ["numNodes", "duration", "startTime"]
-        }
-      },
-      "required": ["name", "metricDataRepository", "filterRanges"],
-      "minItems": 1
-    }
-  }`

internal/config/validate.go

@@ -1,29 +0,0 @@
-// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved. This file is part of cc-backend.
-// Use of this source code is governed by a MIT-style
-// license that can be found in the LICENSE file.
-
-package config
-
-import (
-	"encoding/json"
-
-	cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
-	"github.com/santhosh-tekuri/jsonschema/v5"
-)
-
-func Validate(schema string, instance json.RawMessage) {
-	sch, err := jsonschema.CompileString("schema.json", schema)
-	if err != nil {
-		cclog.Fatalf("%#v", err)
-	}
-
-	var v any
-	if err := json.Unmarshal([]byte(instance), &v); err != nil {
-		cclog.Fatal(err)
-	}
-
-	if err = sch.Validate(v); err != nil {
-		cclog.Fatalf("%#v", err)
-	}
-}

internal/graph/model/models.go

@@ -1,6 +1,5 @@
 // Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved. This file is part of cc-backend.
+// All rights reserved.
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
-
 package model

internal/graph/model/models_gen.go

@@ -3,14 +3,12 @@
 package model
 
 import (
-	"bytes"
 	"fmt"
 	"io"
 	"strconv"
 	"time"
 
-	"github.com/ClusterCockpit/cc-backend/internal/config"
-	"github.com/ClusterCockpit/cc-lib/schema"
+	"github.com/ClusterCockpit/cc-backend/pkg/schema"
 )
 
 type Count struct {
@@ -52,7 +50,6 @@ type IntRangeOutput struct {
 
 type JobFilter struct {
 	Tags            []string          `json:"tags,omitempty"`
-	DbID            []string          `json:"dbId,omitempty"`
 	JobID           *StringInput      `json:"jobId,omitempty"`
 	ArrayJobID      *int              `json:"arrayJobId,omitempty"`
 	User            *StringInput      `json:"user,omitempty"`
@@ -60,16 +57,16 @@ type JobFilter struct {
 	JobName         *StringInput      `json:"jobName,omitempty"`
 	Cluster         *StringInput      `json:"cluster,omitempty"`
 	Partition       *StringInput      `json:"partition,omitempty"`
-	Duration        *config.IntRange  `json:"duration,omitempty"`
+	Duration        *schema.IntRange  `json:"duration,omitempty"`
 	Energy          *FloatRange       `json:"energy,omitempty"`
 	MinRunningFor   *int              `json:"minRunningFor,omitempty"`
-	NumNodes        *config.IntRange  `json:"numNodes,omitempty"`
-	NumAccelerators *config.IntRange  `json:"numAccelerators,omitempty"`
-	NumHWThreads    *config.IntRange  `json:"numHWThreads,omitempty"`
-	StartTime       *config.TimeRange `json:"startTime,omitempty"`
+	NumNodes        *schema.IntRange  `json:"numNodes,omitempty"`
+	NumAccelerators *schema.IntRange  `json:"numAccelerators,omitempty"`
+	NumHWThreads    *schema.IntRange  `json:"numHWThreads,omitempty"`
+	StartTime       *schema.TimeRange `json:"startTime,omitempty"`
 	State           []schema.JobState `json:"state,omitempty"`
 	MetricStats     []*MetricStatItem `json:"metricStats,omitempty"`
-	Shared          *string           `json:"shared,omitempty"`
+	Exclusive       *int              `json:"exclusive,omitempty"`
 	Node            *StringInput      `json:"node,omitempty"`
 }
 
@@ -98,23 +95,9 @@ type JobResultList struct {
 	HasNextPage *bool         `json:"hasNextPage,omitempty"`
 }
 
-type JobStats struct {
-	ID              int           `json:"id"`
-	JobID           string        `json:"jobId"`
-	StartTime       int           `json:"startTime"`
-	Duration        int           `json:"duration"`
-	Cluster         string        `json:"cluster"`
-	SubCluster      string        `json:"subCluster"`
-	NumNodes        int           `json:"numNodes"`
-	NumHWThreads    *int          `json:"numHWThreads,omitempty"`
-	NumAccelerators *int          `json:"numAccelerators,omitempty"`
-	Stats           []*NamedStats `json:"stats"`
-}
-
 type JobsStatistics struct {
 	ID             string               `json:"id"`
 	Name           string               `json:"name"`
-	TotalUsers     int                  `json:"totalUsers"`
 	TotalJobs      int                  `json:"totalJobs"`
 	RunningJobs    int                  `json:"runningJobs"`
 	ShortJobs      int                  `json:"shortJobs"`
@@ -159,49 +142,12 @@ type MetricStatItem struct {
 type Mutation struct {
 }
 
-type NamedStats struct {
-	Name string                   `json:"name"`
-	Data *schema.MetricStatistics `json:"data"`
-}
-
-type NamedStatsWithScope struct {
-	Name  string             `json:"name"`
-	Scope schema.MetricScope `json:"scope"`
-	Stats []*ScopedStats     `json:"stats"`
-}
-
-type NodeFilter struct {
-	Hostname       *StringInput           `json:"hostname,omitempty"`
-	Cluster        *StringInput           `json:"cluster,omitempty"`
-	Subcluster     *StringInput           `json:"subcluster,omitempty"`
-	SchedulerState *schema.SchedulerState `json:"schedulerState,omitempty"`
-	HealthState    *string                `json:"healthState,omitempty"`
-	TimeStart      *int                   `json:"timeStart,omitempty"`
-}
-
 type NodeMetrics struct {
 	Host       string               `json:"host"`
 	SubCluster string               `json:"subCluster"`
 	Metrics    []*JobMetricWithName `json:"metrics"`
 }
 
-type NodeStateResultList struct {
-	Items []*schema.Node `json:"items"`
-	Count *int           `json:"count,omitempty"`
-}
-
-type NodeStates struct {
-	State string `json:"state"`
-	Count int    `json:"count"`
-}
-
-type NodeStatesTimed struct {
-	State string `json:"state"`
-	Type  string `json:"type"`
-	Count int    `json:"count"`
-	Time  int    `json:"time"`
-}
-
 type NodesResultList struct {
 	Items       []*NodeMetrics `json:"items"`
 	Offset      *int           `json:"offset,omitempty"`
@@ -222,10 +168,7 @@ type PageRequest struct {
 	Page         int `json:"page"`
 }
 
-type ScopedStats struct {
-	Hostname string                   `json:"hostname"`
-	ID       *string                  `json:"id,omitempty"`
-	Data     *schema.MetricStatistics `json:"data"`
+type Query struct {
 }
 
 type StringInput struct {
@@ -258,22 +201,20 @@ type User struct {
 type Aggregate string
 
 const (
-	AggregateUser       Aggregate = "USER"
-	AggregateProject    Aggregate = "PROJECT"
-	AggregateCluster    Aggregate = "CLUSTER"
-	AggregateSubcluster Aggregate = "SUBCLUSTER"
+	AggregateUser    Aggregate = "USER"
+	AggregateProject Aggregate = "PROJECT"
+	AggregateCluster Aggregate = "CLUSTER"
 )
 
 var AllAggregate = []Aggregate{
 	AggregateUser,
 	AggregateProject,
 	AggregateCluster,
-	AggregateSubcluster,
 }
 
 func (e Aggregate) IsValid() bool {
 	switch e {
-	case AggregateUser, AggregateProject, AggregateCluster, AggregateSubcluster:
+	case AggregateUser, AggregateProject, AggregateCluster:
 		return true
 	}
 	return false
@@ -283,7 +224,7 @@ func (e Aggregate) String() string {
 	return string(e)
 }
 
-func (e *Aggregate) UnmarshalGQL(v any) error {
+func (e *Aggregate) UnmarshalGQL(v interface{}) error {
 	str, ok := v.(string)
 	if !ok {
 		return fmt.Errorf("enums must be strings")
@@ -300,26 +241,11 @@ func (e Aggregate) MarshalGQL(w io.Writer) {
 	fmt.Fprint(w, strconv.Quote(e.String()))
 }
 
-func (e *Aggregate) UnmarshalJSON(b []byte) error {
-	s, err := strconv.Unquote(string(b))
-	if err != nil {
-		return err
-	}
-	return e.UnmarshalGQL(s)
-}
-
-func (e Aggregate) MarshalJSON() ([]byte, error) {
-	var buf bytes.Buffer
-	e.MarshalGQL(&buf)
-	return buf.Bytes(), nil
-}
-
 type SortByAggregate string
 
 const (
 	SortByAggregateTotalwalltime  SortByAggregate = "TOTALWALLTIME"
 	SortByAggregateTotaljobs      SortByAggregate = "TOTALJOBS"
-	SortByAggregateTotalusers     SortByAggregate = "TOTALUSERS"
 	SortByAggregateTotalnodes     SortByAggregate = "TOTALNODES"
 	SortByAggregateTotalnodehours SortByAggregate = "TOTALNODEHOURS"
 	SortByAggregateTotalcores     SortByAggregate = "TOTALCORES"
@@ -331,7 +257,6 @@ const (
 var AllSortByAggregate = []SortByAggregate{
 	SortByAggregateTotalwalltime,
 	SortByAggregateTotaljobs,
-	SortByAggregateTotalusers,
 	SortByAggregateTotalnodes,
 	SortByAggregateTotalnodehours,
 	SortByAggregateTotalcores,
@@ -342,7 +267,7 @@ var AllSortByAggregate = []SortByAggregate{
 
 func (e SortByAggregate) IsValid() bool {
 	switch e {
-	case SortByAggregateTotalwalltime, SortByAggregateTotaljobs, SortByAggregateTotalusers, SortByAggregateTotalnodes, SortByAggregateTotalnodehours, SortByAggregateTotalcores, SortByAggregateTotalcorehours, SortByAggregateTotalaccs, SortByAggregateTotalacchours:
+	case SortByAggregateTotalwalltime, SortByAggregateTotaljobs, SortByAggregateTotalnodes, SortByAggregateTotalnodehours, SortByAggregateTotalcores, SortByAggregateTotalcorehours, SortByAggregateTotalaccs, SortByAggregateTotalacchours:
 		return true
 	}
 	return false
@@ -352,7 +277,7 @@ func (e SortByAggregate) String() string {
 	return string(e)
 }
 
-func (e *SortByAggregate) UnmarshalGQL(v any) error {
+func (e *SortByAggregate) UnmarshalGQL(v interface{}) error {
 	str, ok := v.(string)
 	if !ok {
 		return fmt.Errorf("enums must be strings")
@@ -369,20 +294,6 @@ func (e SortByAggregate) MarshalGQL(w io.Writer) {
 	fmt.Fprint(w, strconv.Quote(e.String()))
 }
 
-func (e *SortByAggregate) UnmarshalJSON(b []byte) error {
-	s, err := strconv.Unquote(string(b))
-	if err != nil {
-		return err
-	}
-	return e.UnmarshalGQL(s)
-}
-
-func (e SortByAggregate) MarshalJSON() ([]byte, error) {
-	var buf bytes.Buffer
-	e.MarshalGQL(&buf)
-	return buf.Bytes(), nil
-}
-
 type SortDirectionEnum string
 
 const (
@@ -407,7 +318,7 @@ func (e SortDirectionEnum) String() string {
 	return string(e)
 }
 
-func (e *SortDirectionEnum) UnmarshalGQL(v any) error {
+func (e *SortDirectionEnum) UnmarshalGQL(v interface{}) error {
 	str, ok := v.(string)
 	if !ok {
 		return fmt.Errorf("enums must be strings")
@@ -423,17 +334,3 @@ func (e *SortDirectionEnum) UnmarshalGQL(v any) error {
 func (e SortDirectionEnum) MarshalGQL(w io.Writer) {
 	fmt.Fprint(w, strconv.Quote(e.String()))
 }
-
-func (e *SortDirectionEnum) UnmarshalJSON(b []byte) error {
-	s, err := strconv.Unquote(string(b))
-	if err != nil {
-		return err
-	}
-	return e.UnmarshalGQL(s)
-}
-
-func (e SortDirectionEnum) MarshalJSON() ([]byte, error) {
-	var buf bytes.Buffer
-	e.MarshalGQL(&buf)
-	return buf.Bytes(), nil
-}

internal/graph/resolver.go

@@ -4,7 +4,7 @@ import (
 	"sync"
 
 	"github.com/ClusterCockpit/cc-backend/internal/repository"
-	cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
+	"github.com/ClusterCockpit/cc-backend/pkg/log"
 	"github.com/jmoiron/sqlx"
 )
 
@@ -32,7 +32,7 @@ func Init() {
 
 func GetResolverInstance() *Resolver {
 	if resolverInstance == nil {
-		cclog.Fatal("Authentication module not initialized!")
+		log.Fatal("Authentication module not initialized!")
 	}
 
 	return resolverInstance

internal/graph/schema.resolvers.go

@@ -2,7 +2,7 @@ package graph
 
 // This file will be automatically regenerated based on the schema, any resolver implementations
 // will be copied through when generating and any unknown code will be moved to the end.
-// Code generated by github.com/99designs/gqlgen version v0.17.81
+// Code generated by github.com/99designs/gqlgen version v0.17.57
 
 import (
 	"context"
@@ -20,8 +20,8 @@ import (
 	"github.com/ClusterCockpit/cc-backend/internal/metricDataDispatcher"
 	"github.com/ClusterCockpit/cc-backend/internal/repository"
 	"github.com/ClusterCockpit/cc-backend/pkg/archive"
-	cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
-	"github.com/ClusterCockpit/cc-lib/schema"
+	"github.com/ClusterCockpit/cc-backend/pkg/log"
+	"github.com/ClusterCockpit/cc-backend/pkg/schema"
 )
 
 // Partitions is the resolver for the partitions field.
@@ -29,21 +29,15 @@ func (r *clusterResolver) Partitions(ctx context.Context, obj *schema.Cluster) (
 	return r.Repo.Partitions(obj.Name)
 }
 
-// StartTime is the resolver for the startTime field.
-func (r *jobResolver) StartTime(ctx context.Context, obj *schema.Job) (*time.Time, error) {
-	timestamp := time.Unix(obj.StartTime, 0)
-	return &timestamp, nil
-}
-
 // Tags is the resolver for the tags field.
 func (r *jobResolver) Tags(ctx context.Context, obj *schema.Job) ([]*schema.Tag, error) {
-	return r.Repo.GetTags(repository.GetUserFromContext(ctx), obj.ID)
+	return r.Repo.GetTags(repository.GetUserFromContext(ctx), &obj.ID)
 }
 
 // ConcurrentJobs is the resolver for the concurrentJobs field.
 func (r *jobResolver) ConcurrentJobs(ctx context.Context, obj *schema.Job) (*model.JobLinkResultList, error) {
 	// FIXME: Make the hardcoded duration configurable
-	if obj.Shared != "none" && obj.Duration > 600 {
+	if obj.Exclusive != 1 && obj.Duration > 600 {
 		return r.Repo.FindConcurrentJobs(ctx, obj)
 	}
 
@@ -54,7 +48,7 @@ func (r *jobResolver) ConcurrentJobs(ctx context.Context, obj *schema.Job) (*mod
 func (r *jobResolver) Footprint(ctx context.Context, obj *schema.Job) ([]*model.FootprintValue, error) {
 	rawFootprint, err := r.Repo.FetchFootprint(obj)
 	if err != nil {
-		cclog.Warn("Error while fetching job footprint data")
+		log.Warn("Error while fetching job footprint data")
 		return nil, err
 	}
 
@@ -79,7 +73,7 @@ func (r *jobResolver) Footprint(ctx context.Context, obj *schema.Job) ([]*model.
 func (r *jobResolver) EnergyFootprint(ctx context.Context, obj *schema.Job) ([]*model.EnergyFootprintValue, error) {
 	rawEnergyFootprint, err := r.Repo.FetchEnergyFootprint(obj)
 	if err != nil {
-		cclog.Warn("Error while fetching job energy footprint data")
+		log.Warn("Error while fetching job energy footprint data")
 		return nil, err
 	}
 
@@ -131,75 +125,40 @@ func (r *metricValueResolver) Name(ctx context.Context, obj *schema.MetricValue)
 
 // CreateTag is the resolver for the createTag field.
 func (r *mutationResolver) CreateTag(ctx context.Context, typeArg string, name string, scope string) (*schema.Tag, error) {
-	user := repository.GetUserFromContext(ctx)
-	if user == nil {
-		return nil, fmt.Errorf("no user in context")
+	id, err := r.Repo.CreateTag(typeArg, name, scope)
+	if err != nil {
+		log.Warn("Error while creating tag")
+		return nil, err
 	}
 
-	// Test Access: Admins && Admin Tag OR Support/Admin and Global Tag OR Everyone && Private Tag
-	if user.HasRole(schema.RoleAdmin) && scope == "admin" ||
-		user.HasAnyRole([]schema.Role{schema.RoleAdmin, schema.RoleSupport}) && scope == "global" ||
-		user.Username == scope {
-		// Create in DB
-		id, err := r.Repo.CreateTag(typeArg, name, scope)
-		if err != nil {
-			cclog.Warn("Error while creating tag")
-			return nil, err
-		}
-		return &schema.Tag{ID: id, Type: typeArg, Name: name, Scope: scope}, nil
-	} else {
-		cclog.Warnf("Not authorized to create tag with scope: %s", scope)
-		return nil, fmt.Errorf("not authorized to create tag with scope: %s", scope)
-	}
+	return &schema.Tag{ID: id, Type: typeArg, Name: name, Scope: scope}, nil
 }
 
 // DeleteTag is the resolver for the deleteTag field.
 func (r *mutationResolver) DeleteTag(ctx context.Context, id string) (string, error) {
-	// This Uses ID string <-> ID string, removeTagFromList uses []string <-> []int
 	panic(fmt.Errorf("not implemented: DeleteTag - deleteTag"))
 }
 
 // AddTagsToJob is the resolver for the addTagsToJob field.
 func (r *mutationResolver) AddTagsToJob(ctx context.Context, job string, tagIds []string) ([]*schema.Tag, error) {
-	user := repository.GetUserFromContext(ctx)
-	if user == nil {
-		return nil, fmt.Errorf("no user in context")
-	}
-
+	// Selectable Tags Pre-Filtered by Scope in Frontend: No backend check required
 	jid, err := strconv.ParseInt(job, 10, 64)
 	if err != nil {
-		cclog.Warn("Error while adding tag to job")
+		log.Warn("Error while adding tag to job")
 		return nil, err
 	}
 
 	tags := []*schema.Tag{}
 	for _, tagId := range tagIds {
-		// Get ID
 		tid, err := strconv.ParseInt(tagId, 10, 64)
 		if err != nil {
-			cclog.Warn("Error while parsing tag id")
+			log.Warn("Error while parsing tag id")
 			return nil, err
 		}
 
-		// Test Exists
-		_, _, tscope, exists := r.Repo.TagInfo(tid)
-		if !exists {
-			cclog.Warnf("Tag does not exist (ID): %d", tid)
-			return nil, fmt.Errorf("tag does not exist (ID): %d", tid)
-		}
-
-		// Test Access: Admins && Admin Tag OR Support/Admin and Global Tag OR Everyone && Private Tag
-		if user.HasRole(schema.RoleAdmin) && tscope == "admin" ||
-			user.HasAnyRole([]schema.Role{schema.RoleAdmin, schema.RoleSupport}) && tscope == "global" ||
-			user.Username == tscope {
-			// Add to Job
-			if tags, err = r.Repo.AddTag(user, jid, tid); err != nil {
-				cclog.Warn("Error while adding tag")
-				return nil, err
-			}
-		} else {
-			cclog.Warnf("Not authorized to add tag: %d", tid)
-			return nil, fmt.Errorf("not authorized to add tag: %d", tid)
+		if tags, err = r.Repo.AddTag(repository.GetUserFromContext(ctx), jid, tid); err != nil {
+			log.Warn("Error while adding tag")
+			return nil, err
 		}
 	}
 
@@ -208,123 +167,40 @@ func (r *mutationResolver) AddTagsToJob(ctx context.Context, job string, tagIds
 
 // RemoveTagsFromJob is the resolver for the removeTagsFromJob field.
 func (r *mutationResolver) RemoveTagsFromJob(ctx context.Context, job string, tagIds []string) ([]*schema.Tag, error) {
-	user := repository.GetUserFromContext(ctx)
-	if user == nil {
-		return nil, fmt.Errorf("no user in context")
-	}
-
+	// Removable Tags Pre-Filtered by Scope in Frontend: No backend check required
 	jid, err := strconv.ParseInt(job, 10, 64)
 	if err != nil {
-		cclog.Warn("Error while parsing job id")
+		log.Warn("Error while parsing job id")
 		return nil, err
 	}
 
 	tags := []*schema.Tag{}
 	for _, tagId := range tagIds {
-		// Get ID
 		tid, err := strconv.ParseInt(tagId, 10, 64)
 		if err != nil {
-			cclog.Warn("Error while parsing tag id")
+			log.Warn("Error while parsing tag id")
 			return nil, err
 		}
 
-		// Test Exists
-		_, _, tscope, exists := r.Repo.TagInfo(tid)
-		if !exists {
-			cclog.Warnf("Tag does not exist (ID): %d", tid)
-			return nil, fmt.Errorf("tag does not exist (ID): %d", tid)
-		}
-
-		// Test Access: Admins && Admin Tag OR Support/Admin and Global Tag OR Everyone && Private Tag
-		if user.HasRole(schema.RoleAdmin) && tscope == "admin" ||
-			user.HasAnyRole([]schema.Role{schema.RoleAdmin, schema.RoleSupport}) && tscope == "global" ||
-			user.Username == tscope {
-			// Remove from Job
-			if tags, err = r.Repo.RemoveTag(user, jid, tid); err != nil {
-				cclog.Warn("Error while removing tag")
-				return nil, err
-			}
-		} else {
-			cclog.Warnf("Not authorized to remove tag: %d", tid)
-			return nil, fmt.Errorf("not authorized to remove tag: %d", tid)
-		}
-
-	}
-
-	return tags, nil
-}
-
-// RemoveTagFromList is the resolver for the removeTagFromList field.
-func (r *mutationResolver) RemoveTagFromList(ctx context.Context, tagIds []string) ([]int, error) {
-	// Needs Contextuser
-	user := repository.GetUserFromContext(ctx)
-	if user == nil {
-		return nil, fmt.Errorf("no user in context")
-	}
-
-	tags := []int{}
-	for _, tagId := range tagIds {
-		// Get ID
-		tid, err := strconv.ParseInt(tagId, 10, 64)
-		if err != nil {
-			cclog.Warn("Error while parsing tag id for removal")
+		if tags, err = r.Repo.RemoveTag(repository.GetUserFromContext(ctx), jid, tid); err != nil {
+			log.Warn("Error while removing tag")
 			return nil, err
 		}
-
-		// Test Exists
-		_, _, tscope, exists := r.Repo.TagInfo(tid)
-		if !exists {
-			cclog.Warnf("Tag does not exist (ID): %d", tid)
-			return nil, fmt.Errorf("tag does not exist (ID): %d", tid)
-		}
-
-		// Test Access: Admins && Admin Tag OR Everyone && Private Tag
-		if user.HasRole(schema.RoleAdmin) && (tscope == "global" || tscope == "admin") || user.Username == tscope {
-			// Remove from DB
-			if err = r.Repo.RemoveTagById(tid); err != nil {
-				cclog.Warn("Error while removing tag")
-				return nil, err
-			} else {
-				tags = append(tags, int(tid))
-			}
-		} else {
-			cclog.Warnf("Not authorized to remove tag: %d", tid)
-			return nil, fmt.Errorf("not authorized to remove tag: %d", tid)
-		}
 	}
+
 	return tags, nil
 }
 
 // UpdateConfiguration is the resolver for the updateConfiguration field.
 func (r *mutationResolver) UpdateConfiguration(ctx context.Context, name string, value string) (*string, error) {
 	if err := repository.GetUserCfgRepo().UpdateConfig(name, value, repository.GetUserFromContext(ctx)); err != nil {
-		cclog.Warn("Error while updating user config")
+		log.Warn("Error while updating user config")
 		return nil, err
 	}
 
 	return nil, nil
 }
 
-// ID is the resolver for the id field.
-func (r *nodeResolver) ID(ctx context.Context, obj *schema.Node) (string, error) {
-	panic(fmt.Errorf("not implemented: ID - id"))
-}
-
-// SchedulerState is the resolver for the schedulerState field.
-func (r *nodeResolver) SchedulerState(ctx context.Context, obj *schema.Node) (schema.SchedulerState, error) {
-	panic(fmt.Errorf("not implemented: SchedulerState - schedulerState"))
-}
-
-// HealthState is the resolver for the healthState field.
-func (r *nodeResolver) HealthState(ctx context.Context, obj *schema.Node) (string, error) {
-	panic(fmt.Errorf("not implemented: HealthState - healthState"))
-}
-
-// MetaData is the resolver for the metaData field.
-func (r *nodeResolver) MetaData(ctx context.Context, obj *schema.Node) (any, error) {
-	panic(fmt.Errorf("not implemented: MetaData - metaData"))
-}
-
 // Clusters is the resolver for the clusters field.
 func (r *queryResolver) Clusters(ctx context.Context) ([]*schema.Cluster, error) {
 	return archive.Clusters, nil
@@ -349,7 +225,7 @@ func (r *queryResolver) User(ctx context.Context, username string) (*model.User,
 func (r *queryResolver) AllocatedNodes(ctx context.Context, cluster string) ([]*model.Count, error) {
 	data, err := r.Repo.AllocatedNodes(cluster)
 	if err != nil {
-		cclog.Warn("Error while fetching allocated nodes")
+		log.Warn("Error while fetching allocated nodes")
 		return nil, err
 	}
 
@@ -364,81 +240,17 @@ func (r *queryResolver) AllocatedNodes(ctx context.Context, cluster string) ([]*
 	return counts, nil
 }
 
-// Node is the resolver for the node field.
-func (r *queryResolver) Node(ctx context.Context, id string) (*schema.Node, error) {
-	repo := repository.GetNodeRepository()
-	numericId, err := strconv.ParseInt(id, 10, 64)
-	if err != nil {
-		cclog.Warn("Error while parsing job id")
-		return nil, err
-	}
-	return repo.GetNodeById(numericId, false)
-}
-
-// Nodes is the resolver for the nodes field.
-func (r *queryResolver) Nodes(ctx context.Context, filter []*model.NodeFilter, order *model.OrderByInput) (*model.NodeStateResultList, error) {
-	repo := repository.GetNodeRepository()
-	nodes, err := repo.QueryNodes(ctx, filter, order)
-	count := len(nodes)
-	return &model.NodeStateResultList{Items: nodes, Count: &count}, err
-}
-
-// NodeStates is the resolver for the nodeStates field.
-func (r *queryResolver) NodeStates(ctx context.Context, filter []*model.NodeFilter) ([]*model.NodeStates, error) {
-	repo := repository.GetNodeRepository()
-
-	stateCounts, serr := repo.CountNodeStates(ctx, filter)
-	if serr != nil {
-		cclog.Warnf("Error while counting nodeStates: %s", serr.Error())
-		return nil, serr
-	}
-
-	healthCounts, herr := repo.CountHealthStates(ctx, filter)
-	if herr != nil {
-		cclog.Warnf("Error while counting healthStates: %s", herr.Error())
-		return nil, herr
-	}
-
-	allCounts := make([]*model.NodeStates, 0)
-	allCounts = append(stateCounts, healthCounts...)
-
-	return allCounts, nil
-}
-
-// NodeStatesTimed is the resolver for the nodeStatesTimed field.
-func (r *queryResolver) NodeStatesTimed(ctx context.Context, filter []*model.NodeFilter) ([]*model.NodeStatesTimed, error) {
-	panic(fmt.Errorf("not implemented: NodeStatesTimed - NodeStatesTimed"))
-	// repo := repository.GetNodeRepository()
-
-	// stateCounts, serr := repo.CountNodeStates(ctx, filter)
-	// if serr != nil {
-	// 	cclog.Warnf("Error while counting nodeStates: %s", serr.Error())
-	// 	return nil, serr
-	// }
-
-	// healthCounts, herr := repo.CountHealthStates(ctx, filter)
-	// if herr != nil {
-	// 	cclog.Warnf("Error while counting healthStates: %s", herr.Error())
-	// 	return nil, herr
-	// }
-
-	// allCounts := make([]*model.NodeStates, 0)
-	// allCounts = append(stateCounts, healthCounts...)
-
-	// return allCounts, nil
-}
-
 // Job is the resolver for the job field.
 func (r *queryResolver) Job(ctx context.Context, id string) (*schema.Job, error) {
 	numericId, err := strconv.ParseInt(id, 10, 64)
 	if err != nil {
-		cclog.Warn("Error while parsing job id")
+		log.Warn("Error while parsing job id")
 		return nil, err
 	}
 
 	job, err := r.Repo.FindById(ctx, numericId)
 	if err != nil {
-		cclog.Warn("Error while finding job by id")
+		log.Warn("Error while finding job by id")
 		return nil, err
 	}
 
@@ -465,13 +277,13 @@ func (r *queryResolver) JobMetrics(ctx context.Context, id string, metrics []str
 
 	job, err := r.Query().Job(ctx, id)
 	if err != nil {
-		cclog.Warn("Error while querying job for metrics")
+		log.Warn("Error while querying job for metrics")
 		return nil, err
 	}
 
 	data, err := metricDataDispatcher.LoadData(job, metrics, scopes, ctx, *resolution)
 	if err != nil {
-		cclog.Warn("Error while loading job data")
+		log.Warn("Error while loading job data")
 		return nil, err
 	}
 
@@ -489,67 +301,9 @@ func (r *queryResolver) JobMetrics(ctx context.Context, id string, metrics []str
 	return res, err
 }
 
-// JobStats is the resolver for the jobStats field.
-func (r *queryResolver) JobStats(ctx context.Context, id string, metrics []string) ([]*model.NamedStats, error) {
-	job, err := r.Query().Job(ctx, id)
-	if err != nil {
-		cclog.Warnf("Error while querying job %s for metadata", id)
-		return nil, err
-	}
-
-	data, err := metricDataDispatcher.LoadJobStats(job, metrics, ctx)
-	if err != nil {
-		cclog.Warnf("Error while loading jobStats data for job id %s", id)
-		return nil, err
-	}
-
-	res := []*model.NamedStats{}
-	for name, md := range data {
-		res = append(res, &model.NamedStats{
-			Name: name,
-			Data: &md,
-		})
-	}
-
-	return res, err
-}
-
-// ScopedJobStats is the resolver for the scopedJobStats field.
-func (r *queryResolver) ScopedJobStats(ctx context.Context, id string, metrics []string, scopes []schema.MetricScope) ([]*model.NamedStatsWithScope, error) {
-	job, err := r.Query().Job(ctx, id)
-	if err != nil {
-		cclog.Warnf("Error while querying job %s for metadata", id)
-		return nil, err
-	}
-
-	data, err := metricDataDispatcher.LoadScopedJobStats(job, metrics, scopes, ctx)
-	if err != nil {
-		cclog.Warnf("Error while loading scopedJobStats data for job id %s", id)
-		return nil, err
-	}
-
-	res := make([]*model.NamedStatsWithScope, 0)
-	for name, scoped := range data {
-		for scope, stats := range scoped {
-
-			mdlStats := make([]*model.ScopedStats, 0)
-			for _, stat := range stats {
-				mdlStats = append(mdlStats, &model.ScopedStats{
-					Hostname: stat.Hostname,
-					ID:       stat.Id,
-					Data:     stat.Data,
-				})
-			}
-
-			res = append(res, &model.NamedStatsWithScope{
-				Name:  name,
-				Scope: scope,
-				Stats: mdlStats,
-			})
-		}
-	}
-
-	return res, nil
+// JobsFootprints is the resolver for the jobsFootprints field.
+func (r *queryResolver) JobsFootprints(ctx context.Context, filter []*model.JobFilter, metrics []string) (*model.Footprints, error) {
+	return r.jobsFootprints(ctx, filter, metrics)
 }
 
 // Jobs is the resolver for the jobs field.
@@ -563,35 +317,40 @@ func (r *queryResolver) Jobs(ctx context.Context, filter []*model.JobFilter, pag
 
 	jobs, err := r.Repo.QueryJobs(ctx, filter, page, order)
 	if err != nil {
-		cclog.Warn("Error while querying jobs")
+		log.Warn("Error while querying jobs")
 		return nil, err
 	}
 
 	count, err := r.Repo.CountJobs(ctx, filter)
 	if err != nil {
-		cclog.Warn("Error while counting jobs")
+		log.Warn("Error while counting jobs")
 		return nil, err
 	}
 
-	// Note: Even if App-Default 'config.Keys.UiDefaults["job_list_usePaging"]' is set, always return hasNextPage boolean.
-	// Users can decide in frontend to use continuous scroll, even if app-default is paging!
-	/*
-	  Example Page 4 @ 10 IpP : Does item 41 exist?
-	  Minimal Page 41 @ 1 IpP : If len(result) is 1, Page 5 @ 10 IpP exists.
-	*/
-	nextPage := &model.PageRequest{
-		ItemsPerPage: 1,
-		Page:         ((page.Page * page.ItemsPerPage) + 1),
-	}
-	nextJobs, err := r.Repo.QueryJobs(ctx, filter, nextPage, order)
-	if err != nil {
-		cclog.Warn("Error while querying next jobs")
-		return nil, err
-	}
+	if !config.Keys.UiDefaults["job_list_usePaging"].(bool) {
+		hasNextPage := false
+		// page.Page += 1 		   : Simple, but expensive
+		// Example Page 4 @ 10 IpP : Does item 41 exist?
+		// Minimal Page 41 @ 1 IpP : If len(result) is 1, Page 5 @ 10 IpP exists.
+		nextPage := &model.PageRequest{
+			ItemsPerPage: 1,
+			Page:         ((page.Page * page.ItemsPerPage) + 1),
+		}
 
-	hasNextPage := len(nextJobs) == 1
+		nextJobs, err := r.Repo.QueryJobs(ctx, filter, nextPage, order)
+		if err != nil {
+			log.Warn("Error while querying next jobs")
+			return nil, err
+		}
 
-	return &model.JobResultList{Items: jobs, Count: &count, HasNextPage: &hasNextPage}, nil
+		if len(nextJobs) == 1 {
+			hasNextPage = true
+		}
+
+		return &model.JobResultList{Items: jobs, Count: &count, HasNextPage: &hasNextPage}, nil
+	} else {
+		return &model.JobResultList{Items: jobs, Count: &count}, nil
+	}
 }
 
 // JobsStatistics is the resolver for the jobsStatistics field.
@@ -600,10 +359,10 @@ func (r *queryResolver) JobsStatistics(ctx context.Context, filter []*model.JobF
 	var stats []*model.JobsStatistics
 
 	// Top Level Defaults
-	defaultDurationBins := "1h"
-	defaultMetricBins := 10
+	var defaultDurationBins string = "1h"
+	var defaultMetricBins int = 10
 
-	if requireField(ctx, "totalJobs") || requireField(ctx, "totalUsers") || requireField(ctx, "totalWalltime") || requireField(ctx, "totalNodes") || requireField(ctx, "totalCores") ||
+	if requireField(ctx, "totalJobs") || requireField(ctx, "totalWalltime") || requireField(ctx, "totalNodes") || requireField(ctx, "totalCores") ||
 		requireField(ctx, "totalAccs") || requireField(ctx, "totalNodeHours") || requireField(ctx, "totalCoreHours") || requireField(ctx, "totalAccHours") {
 		if groupBy == nil {
 			stats, err = r.Repo.JobsStats(ctx, filter)
@@ -670,62 +429,6 @@ func (r *queryResolver) JobsStatistics(ctx context.Context, filter []*model.JobF
 	return stats, nil
 }
 
-// JobsMetricStats is the resolver for the jobsMetricStats field.
-func (r *queryResolver) JobsMetricStats(ctx context.Context, filter []*model.JobFilter, metrics []string) ([]*model.JobStats, error) {
-	// No Paging, Fixed Order by StartTime ASC
-	order := &model.OrderByInput{
-		Field: "startTime",
-		Type:  "col",
-		Order: "ASC",
-	}
-
-	jobs, err := r.Repo.QueryJobs(ctx, filter, nil, order)
-	if err != nil {
-		cclog.Warn("Error while querying jobs for comparison")
-		return nil, err
-	}
-
-	res := []*model.JobStats{}
-	for _, job := range jobs {
-		data, err := metricDataDispatcher.LoadJobStats(job, metrics, ctx)
-		if err != nil {
-			cclog.Warnf("Error while loading comparison jobStats data for job id %d", job.JobID)
-			continue
-			// return nil, err
-		}
-
-		sres := []*model.NamedStats{}
-		for name, md := range data {
-			sres = append(sres, &model.NamedStats{
-				Name: name,
-				Data: &md,
-			})
-		}
-
-		numThreadsInt := int(job.NumHWThreads)
-		numAccsInt := int(job.NumAcc)
-		res = append(res, &model.JobStats{
-			ID:              int(*job.ID),
-			JobID:           strconv.Itoa(int(job.JobID)),
-			StartTime:       int(job.StartTime),
-			Duration:        int(job.Duration),
-			Cluster:         job.Cluster,
-			SubCluster:      job.SubCluster,
-			NumNodes:        int(job.NumNodes),
-			NumHWThreads:    &numThreadsInt,
-			NumAccelerators: &numAccsInt,
-			Stats:           sres,
-		})
-	}
-	return res, err
-}
-
-// JobsFootprints is the resolver for the jobsFootprints field.
-func (r *queryResolver) JobsFootprints(ctx context.Context, filter []*model.JobFilter, metrics []string) (*model.Footprints, error) {
-	// NOTE: Legacy Naming! This resolver is for normalized histograms in analysis view only - *Not* related to DB "footprint" column!
-	return r.jobsFootprints(ctx, filter, metrics)
-}
-
 // RooflineHeatmap is the resolver for the rooflineHeatmap field.
 func (r *queryResolver) RooflineHeatmap(ctx context.Context, filter []*model.JobFilter, rows int, cols int, minX float64, minY float64, maxX float64, maxY float64) ([][]float64, error) {
 	return r.rooflineHeatmap(ctx, filter, rows, cols, minX, minY, maxX, maxY)
@@ -746,7 +449,7 @@ func (r *queryResolver) NodeMetrics(ctx context.Context, cluster string, nodes [
 
 	data, err := metricDataDispatcher.LoadNodeData(cluster, metrics, nodes, scopes, from, to, ctx)
 	if err != nil {
-		cclog.Warn("error while loading node data")
+		log.Warn("error while loading node data")
 		return nil, err
 	}
 
@@ -758,7 +461,7 @@ func (r *queryResolver) NodeMetrics(ctx context.Context, cluster string, nodes [
 		}
 		host.SubCluster, err = archive.GetSubClusterByNode(cluster, hostname)
 		if err != nil {
-			cclog.Warnf("error in nodeMetrics resolver: %s", err)
+			log.Warnf("error in nodeMetrics resolver: %s", err)
 		}
 
 		for metric, scopedMetrics := range metrics {
@@ -802,7 +505,7 @@ func (r *queryResolver) NodeMetricsList(ctx context.Context, cluster string, sub
 
 	data, totalNodes, hasNextPage, err := metricDataDispatcher.LoadNodeListData(cluster, subCluster, nodeFilter, metrics, scopes, *resolution, from, to, page, ctx)
 	if err != nil {
-		cclog.Warn("error while loading node data")
+		log.Warn("error while loading node data")
 		return nil, err
 	}
 
@@ -814,7 +517,7 @@ func (r *queryResolver) NodeMetricsList(ctx context.Context, cluster string, sub
 		}
 		host.SubCluster, err = archive.GetSubClusterByNode(cluster, hostname)
 		if err != nil {
-			cclog.Warnf("error in nodeMetrics resolver: %s", err)
+			log.Warnf("error in nodeMetrics resolver: %s", err)
 		}
 
 		for metric, scopedMetrics := range metrics {
@@ -860,9 +563,6 @@ func (r *Resolver) MetricValue() generated.MetricValueResolver { return &metricV
 // Mutation returns generated.MutationResolver implementation.
 func (r *Resolver) Mutation() generated.MutationResolver { return &mutationResolver{r} }
 
-// Node returns generated.NodeResolver implementation.
-func (r *Resolver) Node() generated.NodeResolver { return &nodeResolver{r} }
-
 // Query returns generated.QueryResolver implementation.
 func (r *Resolver) Query() generated.QueryResolver { return &queryResolver{r} }
 
@@ -873,6 +573,5 @@ type clusterResolver struct{ *Resolver }
 type jobResolver struct{ *Resolver }
 type metricValueResolver struct{ *Resolver }
 type mutationResolver struct{ *Resolver }
-type nodeResolver struct{ *Resolver }
 type queryResolver struct{ *Resolver }
 type subClusterResolver struct{ *Resolver }

internal/graph/util.go

@@ -1,5 +1,5 @@
 // Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved. This file is part of cc-backend.
+// All rights reserved.
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 package graph
@@ -12,8 +12,9 @@ import (
 	"github.com/99designs/gqlgen/graphql"
 	"github.com/ClusterCockpit/cc-backend/internal/graph/model"
 	"github.com/ClusterCockpit/cc-backend/internal/metricDataDispatcher"
-	cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
-	"github.com/ClusterCockpit/cc-lib/schema"
+	"github.com/ClusterCockpit/cc-backend/pkg/log"
+	"github.com/ClusterCockpit/cc-backend/pkg/schema"
+	// "github.com/ClusterCockpit/cc-backend/pkg/archive"
 )
 
 const MAX_JOBS_FOR_ANALYSIS = 500
@@ -27,7 +28,7 @@ func (r *queryResolver) rooflineHeatmap(
 ) ([][]float64, error) {
 	jobs, err := r.Repo.QueryJobs(ctx, filter, &model.PageRequest{Page: 1, ItemsPerPage: MAX_JOBS_FOR_ANALYSIS + 1}, nil)
 	if err != nil {
-		cclog.Error("Error while querying jobs for roofline")
+		log.Error("Error while querying jobs for roofline")
 		return nil, err
 	}
 	if len(jobs) > MAX_JOBS_FOR_ANALYSIS {
@@ -55,13 +56,13 @@ func (r *queryResolver) rooflineHeatmap(
 
 		jobdata, err := metricDataDispatcher.LoadData(job, []string{"flops_any", "mem_bw"}, []schema.MetricScope{schema.MetricScopeNode}, ctx, 0)
 		if err != nil {
-			cclog.Errorf("Error while loading roofline metrics for job %d", job.ID)
+			log.Errorf("Error while loading roofline metrics for job %d", job.ID)
 			return nil, err
 		}
 
 		flops_, membw_ := jobdata["flops_any"], jobdata["mem_bw"]
 		if flops_ == nil && membw_ == nil {
-			cclog.Infof("rooflineHeatmap(): 'flops_any' or 'mem_bw' missing for job %d", job.ID)
+			log.Infof("rooflineHeatmap(): 'flops_any' or 'mem_bw' missing for job %d", job.ID)
 			continue
 			// return nil, fmt.Errorf("GRAPH/UTIL > 'flops_any' or 'mem_bw' missing for job %d", job.ID)
 		}
@@ -69,7 +70,7 @@ func (r *queryResolver) rooflineHeatmap(
 		flops, ok1 := flops_["node"]
 		membw, ok2 := membw_["node"]
 		if !ok1 || !ok2 {
-			cclog.Info("rooflineHeatmap() query not implemented for where flops_any or mem_bw not available at 'node' level")
+			log.Info("rooflineHeatmap() query not implemented for where flops_any or mem_bw not available at 'node' level")
 			continue
 			// TODO/FIXME:
 			// return nil, errors.New("GRAPH/UTIL > todo: rooflineHeatmap() query not implemented for where flops_any or mem_bw not available at 'node' level")
@@ -104,7 +105,7 @@ func (r *queryResolver) rooflineHeatmap(
 func (r *queryResolver) jobsFootprints(ctx context.Context, filter []*model.JobFilter, metrics []string) (*model.Footprints, error) {
 	jobs, err := r.Repo.QueryJobs(ctx, filter, &model.PageRequest{Page: 1, ItemsPerPage: MAX_JOBS_FOR_ANALYSIS + 1}, nil)
 	if err != nil {
-		cclog.Error("Error while querying jobs for footprint")
+		log.Error("Error while querying jobs for footprint")
 		return nil, err
 	}
 	if len(jobs) > MAX_JOBS_FOR_ANALYSIS {
@@ -127,7 +128,7 @@ func (r *queryResolver) jobsFootprints(ctx context.Context, filter []*model.JobF
 		}
 
 		if err := metricDataDispatcher.LoadAverages(job, metrics, avgs, ctx); err != nil {
-			cclog.Error("Error while loading averages for footprint")
+			log.Error("Error while loading averages for footprint")
 			return nil, err
 		}
 

internal/importer/handleImport.go

@@ -1,5 +1,5 @@
 // Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved. This file is part of cc-backend.
+// All rights reserved.
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 package importer
@@ -15,8 +15,8 @@ import (
 	"github.com/ClusterCockpit/cc-backend/internal/config"
 	"github.com/ClusterCockpit/cc-backend/internal/repository"
 	"github.com/ClusterCockpit/cc-backend/pkg/archive"
-	cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
-	"github.com/ClusterCockpit/cc-lib/schema"
+	"github.com/ClusterCockpit/cc-backend/pkg/log"
+	"github.com/ClusterCockpit/cc-backend/pkg/schema"
 )
 
 // Import all jobs specified as `<path-to-meta.json>:<path-to-data.json>,...`
@@ -31,7 +31,7 @@ func HandleImportFlag(flag string) error {
 
 		raw, err := os.ReadFile(files[0])
 		if err != nil {
-			cclog.Warn("Error while reading metadata file for import")
+			log.Warn("Error while reading metadata file for import")
 			return err
 		}
 
@@ -42,18 +42,15 @@ func HandleImportFlag(flag string) error {
 		}
 		dec := json.NewDecoder(bytes.NewReader(raw))
 		dec.DisallowUnknownFields()
-		job := schema.Job{
-			Shared:           "none",
-			MonitoringStatus: schema.MonitoringStatusRunningOrArchiving,
-		}
+		job := schema.JobMeta{BaseJob: schema.JobDefaults}
 		if err = dec.Decode(&job); err != nil {
-			cclog.Warn("Error while decoding raw json metadata for import")
+			log.Warn("Error while decoding raw json metadata for import")
 			return err
 		}
 
 		raw, err = os.ReadFile(files[1])
 		if err != nil {
-			cclog.Warn("Error while reading jobdata file for import")
+			log.Warn("Error while reading jobdata file for import")
 			return err
 		}
 
@@ -66,7 +63,7 @@ func HandleImportFlag(flag string) error {
 		dec.DisallowUnknownFields()
 		jobData := schema.JobData{}
 		if err = dec.Decode(&jobData); err != nil {
-			cclog.Warn("Error while decoding raw json jobdata for import")
+			log.Warn("Error while decoding raw json jobdata for import")
 			return err
 		}
 
@@ -74,7 +71,7 @@ func HandleImportFlag(flag string) error {
 
 		sc, err := archive.GetSubCluster(job.Cluster, job.SubCluster)
 		if err != nil {
-			cclog.Errorf("cannot get subcluster: %s", err.Error())
+			log.Errorf("cannot get subcluster: %s", err.Error())
 			return err
 		}
 
@@ -94,80 +91,72 @@ func HandleImportFlag(flag string) error {
 
 		job.RawFootprint, err = json.Marshal(job.Footprint)
 		if err != nil {
-			cclog.Warn("Error while marshaling job footprint")
+			log.Warn("Error while marshaling job footprint")
 			return err
 		}
 
 		job.EnergyFootprint = make(map[string]float64)
+		var totalEnergy float64
+		var energy float64
 
-		// Total Job Energy Outside Loop
-		totalEnergy := 0.0
 		for _, fp := range sc.EnergyFootprint {
-			// Always Init Metric Energy Inside Loop
-			metricEnergy := 0.0
 			if i, err := archive.MetricIndex(sc.MetricConfig, fp); err == nil {
 				// Note: For DB data, calculate and save as kWh
+				// Energy: Power (in Watts) * Time (in Seconds)
 				if sc.MetricConfig[i].Energy == "energy" { // this metric has energy as unit (Joules)
-					cclog.Warnf("Update EnergyFootprint for Job %d and Metric %s on cluster %s: Set to 'energy' in cluster.json: Not implemented, will return 0.0", job.JobID, job.Cluster, fp)
-					// FIXME: Needs sum as stats type
 				} else if sc.MetricConfig[i].Energy == "power" { // this metric has power as unit (Watt)
-					// Energy: Power (in Watts) * Time (in Seconds)
-					// Unit: (W * (s / 3600)) / 1000 = kWh
-					// Round 2 Digits: round(Energy * 100) / 100
-					// Here: (All-Node Metric Average * Number of Nodes) * (Job Duration in Seconds / 3600) / 1000
-					// Note: Shared Jobs handled correctly since "Node Average" is based on partial resources, while "numNodes" factor is 1
-					rawEnergy := ((repository.LoadJobStat(&job, fp, "avg") * float64(job.NumNodes)) * (float64(job.Duration) / 3600.0)) / 1000.0
-					metricEnergy = math.Round(rawEnergy*100.0) / 100.0
+					// Unit: ( W * s ) / 3600 / 1000 = kWh ; Rounded to 2 nearest digits
+					energy = math.Round(((repository.LoadJobStat(&job, fp, "avg")*float64(job.Duration))/3600/1000)*100) / 100
 				}
 			} else {
-				cclog.Warnf("Error while collecting energy metric %s for job, DB ID '%v', return '0.0'", fp, job.ID)
+				log.Warnf("Error while collecting energy metric %s for job, DB ID '%v', return '0.0'", fp, job.ID)
 			}
 
-			job.EnergyFootprint[fp] = metricEnergy
-			totalEnergy += metricEnergy
+			job.EnergyFootprint[fp] = energy
+			totalEnergy += energy
 		}
 
-		job.Energy = (math.Round(totalEnergy*100.0) / 100.0)
+		job.Energy = (math.Round(totalEnergy*100) / 100)
 		if job.RawEnergyFootprint, err = json.Marshal(job.EnergyFootprint); err != nil {
-			cclog.Warnf("Error while marshaling energy footprint for job INTO BYTES, DB ID '%v'", job.ID)
+			log.Warnf("Error while marshaling energy footprint for job INTO BYTES, DB ID '%v'", job.ID)
 			return err
 		}
 
 		job.RawResources, err = json.Marshal(job.Resources)
 		if err != nil {
-			cclog.Warn("Error while marshaling job resources")
+			log.Warn("Error while marshaling job resources")
 			return err
 		}
 		job.RawMetaData, err = json.Marshal(job.MetaData)
 		if err != nil {
-			cclog.Warn("Error while marshaling job metadata")
+			log.Warn("Error while marshaling job metadata")
 			return err
 		}
 
-		if err = SanityChecks(&job); err != nil {
-			cclog.Warn("BaseJob SanityChecks failed")
+		if err = SanityChecks(&job.BaseJob); err != nil {
+			log.Warn("BaseJob SanityChecks failed")
 			return err
 		}
 
 		if err = archive.GetHandle().ImportJob(&job, &jobData); err != nil {
-			cclog.Error("Error while importing job")
+			log.Error("Error while importing job")
 			return err
 		}
 
 		id, err := r.InsertJob(&job)
 		if err != nil {
-			cclog.Warn("Error while job db insert")
+			log.Warn("Error while job db insert")
 			return err
 		}
 
 		for _, tag := range job.Tags {
 			if err := r.ImportTag(id, tag.Type, tag.Name, tag.Scope); err != nil {
-				cclog.Error("Error while adding or creating tag on import")
+				log.Error("Error while adding or creating tag on import")
 				return err
 			}
 		}
 
-		cclog.Infof("successfully imported a new job (jobId: %d, cluster: %s, dbid: %d)", job.JobID, job.Cluster, id)
+		log.Infof("successfully imported a new job (jobId: %d, cluster: %s, dbid: %d)", job.JobID, job.Cluster, id)
 	}
 	return nil
 }

internal/importer/importer_test.go

@@ -1,5 +1,5 @@
 // Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved. This file is part of cc-backend.
+// All rights reserved.
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 package importer_test
@@ -16,8 +16,7 @@ import (
 	"github.com/ClusterCockpit/cc-backend/internal/importer"
 	"github.com/ClusterCockpit/cc-backend/internal/repository"
 	"github.com/ClusterCockpit/cc-backend/pkg/archive"
-	ccconf "github.com/ClusterCockpit/cc-lib/ccConfig"
-	cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
+	"github.com/ClusterCockpit/cc-backend/pkg/log"
 )
 
 func copyFile(s string, d string) error {
@@ -37,16 +36,15 @@ func copyFile(s string, d string) error {
 
 func setup(t *testing.T) *repository.JobRepository {
 	const testconfig = `{
-		"main": {
 	"addr":            "0.0.0.0:8080",
 	"validate": false,
-  "apiAllowedIPs": [
-    "*"
-  ]},
 	"archive": {
 		"kind": "file",
 		"path": "./var/job-archive"
 	},
+    "jwts": {
+        "max-age": "2m"
+    },
 	"clusters": [
 	{
 	   "name": "testcluster",
@@ -77,7 +75,7 @@ func setup(t *testing.T) *repository.JobRepository {
 	 }
 	]}`
 
-	cclog.Init("info", true)
+	log.Init("info", true)
 	tmpdir := t.TempDir()
 
 	jobarchive := filepath.Join(tmpdir, "job-archive")
@@ -107,19 +105,7 @@ func setup(t *testing.T) *repository.JobRepository {
 		t.Fatal(err)
 	}
 
-	ccconf.Init(cfgFilePath)
-
-	// Load and check main configuration
-	if cfg := ccconf.GetPackageConfig("main"); cfg != nil {
-		if clustercfg := ccconf.GetPackageConfig("clusters"); clustercfg != nil {
-			config.Init(cfg, clustercfg)
-		} else {
-			t.Fatal("Cluster configuration must be present")
-		}
-	} else {
-		t.Fatal("Main configuration must be present")
-	}
-
+	config.Init(cfgFilePath)
 	archiveCfg := fmt.Sprintf("{\"kind\": \"file\",\"path\": \"%s\"}", jobarchive)
 
 	if err := archive.Init(json.RawMessage(archiveCfg), config.Keys.DisableArchive); err != nil {
@@ -177,7 +163,7 @@ func TestHandleImportFlag(t *testing.T) {
 			}
 
 			result := readResult(t, testname)
-			job, err := r.FindCached(&result.JobId, &result.Cluster, &result.StartTime)
+			job, err := r.Find(&result.JobId, &result.Cluster, &result.StartTime)
 			if err != nil {
 				t.Fatal(err)
 			}

internal/importer/initDB.go

@@ -1,5 +1,5 @@
 // Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved. This file is part of cc-backend.
+// All rights reserved.
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 package importer
@@ -13,8 +13,8 @@ import (
 
 	"github.com/ClusterCockpit/cc-backend/internal/repository"
 	"github.com/ClusterCockpit/cc-backend/pkg/archive"
-	cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
-	"github.com/ClusterCockpit/cc-lib/schema"
+	"github.com/ClusterCockpit/cc-backend/pkg/log"
+	"github.com/ClusterCockpit/cc-backend/pkg/schema"
 )
 
 const (
@@ -27,20 +27,20 @@ const (
 func InitDB() error {
 	r := repository.GetJobRepository()
 	if err := r.Flush(); err != nil {
-		cclog.Errorf("repository initDB(): %v", err)
+		log.Errorf("repository initDB(): %v", err)
 		return err
 	}
 	starttime := time.Now()
-	cclog.Print("Building job table...")
+	log.Print("Building job table...")
 
 	t, err := r.TransactionInit()
 	if err != nil {
-		cclog.Warn("Error while initializing SQL transactions")
+		log.Warn("Error while initializing SQL transactions")
 		return err
 	}
 	tags := make(map[string]int64)
 
-	// Not using cclog.Print because we want the line to end with `\r` and
+	// Not using log.Print because we want the line to end with `\r` and
 	// this function is only ever called when a special command line flag
 	// is passed anyways.
 	fmt.Printf("%d jobs inserted...\r", 0)
@@ -60,14 +60,19 @@ func InitDB() error {
 		}
 
 		jobMeta.MonitoringStatus = schema.MonitoringStatusArchivingSuccessful
+		job := schema.Job{
+			BaseJob:       jobMeta.BaseJob,
+			StartTime:     time.Unix(jobMeta.StartTime, 0),
+			StartTimeUnix: jobMeta.StartTime,
+		}
 
 		sc, err := archive.GetSubCluster(jobMeta.Cluster, jobMeta.SubCluster)
 		if err != nil {
-			cclog.Errorf("cannot get subcluster: %s", err.Error())
+			log.Errorf("cannot get subcluster: %s", err.Error())
 			return err
 		}
 
-		jobMeta.Footprint = make(map[string]float64)
+		job.Footprint = make(map[string]float64)
 
 		for _, fp := range sc.Footprint {
 			statType := "avg"
@@ -78,79 +83,71 @@ func InitDB() error {
 
 			name := fmt.Sprintf("%s_%s", fp, statType)
 
-			jobMeta.Footprint[name] = repository.LoadJobStat(jobMeta, fp, statType)
+			job.Footprint[name] = repository.LoadJobStat(jobMeta, fp, statType)
 		}
 
-		jobMeta.RawFootprint, err = json.Marshal(jobMeta.Footprint)
+		job.RawFootprint, err = json.Marshal(job.Footprint)
 		if err != nil {
-			cclog.Warn("Error while marshaling job footprint")
+			log.Warn("Error while marshaling job footprint")
 			return err
 		}
 
-		jobMeta.EnergyFootprint = make(map[string]float64)
+		job.EnergyFootprint = make(map[string]float64)
+		var totalEnergy float64
+		var energy float64
 
-		// Total Job Energy Outside Loop
-		totalEnergy := 0.0
 		for _, fp := range sc.EnergyFootprint {
-			// Always Init Metric Energy Inside Loop
-			metricEnergy := 0.0
 			if i, err := archive.MetricIndex(sc.MetricConfig, fp); err == nil {
 				// Note: For DB data, calculate and save as kWh
+				// Energy: Power (in Watts) * Time (in Seconds)
 				if sc.MetricConfig[i].Energy == "energy" { // this metric has energy as unit (Joules)
-					cclog.Warnf("Update EnergyFootprint for Job %d and Metric %s on cluster %s: Set to 'energy' in cluster.json: Not implemented, will return 0.0", jobMeta.JobID, jobMeta.Cluster, fp)
-					// FIXME: Needs sum as stats type
 				} else if sc.MetricConfig[i].Energy == "power" { // this metric has power as unit (Watt)
-					// Energy: Power (in Watts) * Time (in Seconds)
-					// Unit: (W * (s / 3600)) / 1000 = kWh
-					// Round 2 Digits: round(Energy * 100) / 100
-					// Here: (All-Node Metric Average * Number of Nodes) * (Job Duration in Seconds / 3600) / 1000
-					// Note: Shared Jobs handled correctly since "Node Average" is based on partial resources, while "numNodes" factor is 1
-					rawEnergy := ((repository.LoadJobStat(jobMeta, fp, "avg") * float64(jobMeta.NumNodes)) * (float64(jobMeta.Duration) / 3600.0)) / 1000.0
-					metricEnergy = math.Round(rawEnergy*100.0) / 100.0
+					// Unit: ( W * s ) / 3600 / 1000 = kWh ; Rounded to 2 nearest digits
+					energy = math.Round(((repository.LoadJobStat(jobMeta, fp, "avg")*float64(jobMeta.Duration))/3600/1000)*100) / 100
 				}
 			} else {
-				cclog.Warnf("Error while collecting energy metric %s for job, DB ID '%v', return '0.0'", fp, jobMeta.ID)
+				log.Warnf("Error while collecting energy metric %s for job, DB ID '%v', return '0.0'", fp, jobMeta.ID)
 			}
 
-			jobMeta.EnergyFootprint[fp] = metricEnergy
-			totalEnergy += metricEnergy
+			job.EnergyFootprint[fp] = energy
+			totalEnergy += energy
 		}
 
-		jobMeta.Energy = (math.Round(totalEnergy*100.0) / 100.0)
-		if jobMeta.RawEnergyFootprint, err = json.Marshal(jobMeta.EnergyFootprint); err != nil {
-			cclog.Warnf("Error while marshaling energy footprint for job INTO BYTES, DB ID '%v'", jobMeta.ID)
+		job.Energy = (math.Round(totalEnergy*100) / 100)
+		if job.RawEnergyFootprint, err = json.Marshal(job.EnergyFootprint); err != nil {
+			log.Warnf("Error while marshaling energy footprint for job INTO BYTES, DB ID '%v'", jobMeta.ID)
 			return err
 		}
 
-		jobMeta.RawResources, err = json.Marshal(jobMeta.Resources)
+		job.RawResources, err = json.Marshal(job.Resources)
 		if err != nil {
-			cclog.Errorf("repository initDB(): %v", err)
+			log.Errorf("repository initDB(): %v", err)
 			errorOccured++
 			continue
 		}
 
-		jobMeta.RawMetaData, err = json.Marshal(jobMeta.MetaData)
+		job.RawMetaData, err = json.Marshal(job.MetaData)
 		if err != nil {
-			cclog.Errorf("repository initDB(): %v", err)
+			log.Errorf("repository initDB(): %v", err)
 			errorOccured++
 			continue
 		}
 
-		if err := SanityChecks(jobMeta); err != nil {
-			cclog.Errorf("repository initDB(): %v", err)
+		if err := SanityChecks(&job.BaseJob); err != nil {
+			log.Errorf("repository initDB(): %v", err)
 			errorOccured++
 			continue
 		}
 
 		id, err := r.TransactionAddNamed(t,
-			repository.NamedJobInsert, jobMeta)
+			repository.NamedJobInsert, job)
 		if err != nil {
-			cclog.Errorf("repository initDB(): %v", err)
+			log.Errorf("repository initDB(): %v", err)
 			errorOccured++
 			continue
 		}
 
-		for _, tag := range jobMeta.Tags {
+		for _, tag := range job.Tags {
 			tagstr := tag.Name + ":" + tag.Type
 			tagId, ok := tags[tagstr]
 			if !ok {
@@ -158,7 +155,7 @@ func InitDB() error {
 					addTagQuery,
 					tag.Name, tag.Type)
 				if err != nil {
-					cclog.Errorf("Error adding tag: %v", err)
+					log.Errorf("Error adding tag: %v", err)
 					errorOccured++
 					continue
 				}
@@ -176,21 +173,21 @@ func InitDB() error {
 	}
 
 	if errorOccured > 0 {
-		cclog.Warnf("Error in import of %d jobs!", errorOccured)
+		log.Warnf("Error in import of %d jobs!", errorOccured)
 	}
 
 	r.TransactionEnd(t)
-	cclog.Printf("A total of %d jobs have been registered in %.3f seconds.\n", i, time.Since(starttime).Seconds())
+	log.Printf("A total of %d jobs have been registered in %.3f seconds.\n", i, time.Since(starttime).Seconds())
 	return nil
 }
 
 // This function also sets the subcluster if necessary!
-func SanityChecks(job *schema.Job) error {
+func SanityChecks(job *schema.BaseJob) error {
 	if c := archive.GetCluster(job.Cluster); c == nil {
 		return fmt.Errorf("no such cluster: %v", job.Cluster)
 	}
 	if err := archive.AssignSubCluster(job); err != nil {
-		cclog.Warn("Error while assigning subcluster to job")
+		log.Warn("Error while assigning subcluster to job")
 		return err
 	}
 	if !job.State.Valid() {

internal/importer/normalize.go

@@ -1,5 +1,5 @@
 // Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved. This file is part of cc-backend.
+// All rights reserved.
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 package importer
@@ -7,7 +7,7 @@ package importer
 import (
 	"math"
 
-	ccunits "github.com/ClusterCockpit/cc-lib/ccUnits"
+	ccunits "github.com/ClusterCockpit/cc-units"
 )
 
 func getNormalizationFactor(v float64) (float64, int) {

internal/importer/normalize_test.go

@@ -1,5 +1,5 @@
 // Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved. This file is part of cc-backend.
+// All rights reserved.
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 package importer
@@ -8,7 +8,7 @@ import (
 	"fmt"
 	"testing"
 
-	ccunits "github.com/ClusterCockpit/cc-lib/ccUnits"
+	ccunits "github.com/ClusterCockpit/cc-units"
 )
 
 func TestNormalizeFactor(t *testing.T) {

internal/importer/testdata/meta-fritzError.input

@@ -1 +1 @@
-{"jobId":398955,"user":"k106eb10","project":"k106eb","cluster":"fritz","subCluster":"main","partition":"singlenode","arrayJobId":0,"numNodes":1,"numHwthreads":72,"numAcc":0,"shared":"none","monitoringStatus":1,"smt":0,"jobState":"completed","duration":260,"walltime":86340,"resources":[{"hostname":"f0720"}],"metaData":{"jobName":"ams_pipeline","jobScript":"#!/bin/bash  -l\n#SBATCH --job-name=ams_pipeline\n#SBATCH --time=23:59:00\n#SBATCH --partition=singlenode\n#SBATCH --ntasks=72\n#SBATCH --hint=multithread\n#SBATCH --chdir=/home/atuin/k106eb/k106eb10/ACE/Ni-Al/DFT/VASP_PBE_500_0.125_0.1_NM/AlNi/binaries/bulk/base-hcp/occ-shaken/hcp16.occ.4.shake.0/cfg/NiAl3NiAl11\n#SBATCH --export=NONE\nunset SLURM_EXPORT_ENV\nuss=$(whoami)\nfind /dev/shm/ -user $uss -type f -mmin +30 -delete\ncd \"/home/atuin/k106eb/k106eb10/ACE/Ni-Al/DFT/VASP_PBE_500_0.125_0.1_NM/AlNi/binaries/bulk/base-hcp/occ-shaken/hcp16.occ.4.shake.0/cfg/NiAl3NiAl11\"\nams_pipeline pipeline.json \u003e \"/home/atuin/k106eb/k106eb10/ACE/Ni-Al/DFT/VASP_PBE_500_0.125_0.1_NM/AlNi/binaries/bulk/base-hcp/occ-shaken/hcp16.occ.4.shake.0/cfg/NiAl3NiAl11/ams_pipeline_job.sh.out\" 2\u003e \"/home/atuin/k106eb/k106eb10/ACE/Ni-Al/DFT/VASP_PBE_500_0.125_0.1_NM/AlNi/binaries/bulk/base-hcp/occ-shaken/hcp16.occ.4.shake.0/cfg/NiAl3NiAl11/ams_pipeline_job.sh.err\"\n","slurmInfo":"\nJobId=398955 JobName=ams_pipeline\n   UserId=k106eb10(210387) GroupId=80111\n   Account=k106eb QOS=normal \n   Requeue=False Restarts=0 BatchFlag=True \n   TimeLimit=1439\n   SubmitTime=2023-02-09T14:11:22\n   Partition=singlenode \n   NodeList=f0720\n   NumNodes=1 NumCPUs=72 NumTasks=72 CPUs/Task=1\n   NTasksPerNode:Socket:Core=0:None:None\n   TRES_req=cpu=72,mem=250000M,node=1,billing=72\n   TRES_alloc=cpu=72,node=1,billing=72\n   Command=/home/atuin/k106eb/k106eb10/ACE/Ni-Al/DFT/VASP_PBE_500_0.125_0.1_NM/AlNi/binaries/bulk/base-hcp/occ-shaken/hcp16.occ.4.shake.0/cfg/NiAl3NiAl11/ams_pipeline_job.sh\n   WorkDir=/home/atuin/k106eb/k106eb10/ACE/Ni-Al/DFT/VASP_PBE_500_0.125_0.1_NM/AlNi/binaries/bulk/base-hcp/occ-shaken/hcp16.occ.4.shake.0/cfg/NiAl3NiAl11\n   StdErr=\n   StdOut=ams_pipeline.o%j\n"},"startTime":1675956725,"statistics":{"clock":{"unit":{"base":"Hz","prefix":"M"},"avg":2335.254,"min":800.418,"max":2734.922},"cpu_load":{"unit":{"base":""},"avg":52.72,"min":34.46,"max":71.91},"cpu_power":{"unit":{"base":"W"},"avg":407.767,"min":93.932,"max":497.636},"cpu_user":{"unit":{"base":""},"avg":63.678,"min":19.872,"max":96.633},"flops_any":{"unit":{"base":"F/s","prefix":"G"},"avg":635.672,"min":0,"max":1332.874},"flops_dp":{"unit":{"base":"F/s","prefix":"G"},"avg":261.006,"min":0,"max":382.294},"flops_sp":{"unit":{"base":"F/s","prefix":"G"},"avg":113.659,"min":0,"max":568.286},"ib_recv":{"unit":{"base":"B/s"},"avg":27981.111,"min":69.4,"max":48084.589},"ib_recv_pkts":{"unit":{"base":"packets/s"},"avg":398.939,"min":0.5,"max":693.817},"ib_xmit":{"unit":{"base":"B/s"},"avg":188.513,"min":39.597,"max":724.568},"ib_xmit_pkts":{"unit":{"base":"packets/s"},"avg":0.867,"min":0.2,"max":2.933},"ipc":{"unit":{"base":"IPC"},"avg":0.944,"min":0.564,"max":1.291},"mem_bw":{"unit":{"base":"B/s","prefix":"G"},"avg":79.565,"min":0.021,"max":116.02},"mem_power":{"unit":{"base":"W"},"avg":24.692,"min":7.883,"max":31.318},"mem_used":{"unit":{"base":"B","prefix":"G"},"avg":22.566,"min":8.225,"max":27.613},"nfs4_read":{"unit":{"base":"B/s","prefix":"M"},"avg":647,"min":0,"max":1946},"nfs4_total":{"unit":{"base":"B/s","prefix":"M"},"avg":6181.6,"min":1270,"max":11411},"nfs4_write":{"unit":{"base":"B/s","prefix":"M"},"avg":22.4,"min":11,"max":29},"vectorization_ratio":{"unit":{"base":"%"},"avg":77.351,"min":0,"max":98.837}}}
+{"jobId":398955,"user":"k106eb10","project":"k106eb","cluster":"fritz","subCluster":"main","partition":"singlenode","arrayJobId":0,"numNodes":1,"numHwthreads":72,"numAcc":0,"exclusive":1,"monitoringStatus":1,"smt":0,"jobState":"completed","duration":260,"walltime":86340,"resources":[{"hostname":"f0720"}],"metaData":{"jobName":"ams_pipeline","jobScript":"#!/bin/bash  -l\n#SBATCH --job-name=ams_pipeline\n#SBATCH --time=23:59:00\n#SBATCH --partition=singlenode\n#SBATCH --ntasks=72\n#SBATCH --hint=multithread\n#SBATCH --chdir=/home/atuin/k106eb/k106eb10/ACE/Ni-Al/DFT/VASP_PBE_500_0.125_0.1_NM/AlNi/binaries/bulk/base-hcp/occ-shaken/hcp16.occ.4.shake.0/cfg/NiAl3NiAl11\n#SBATCH --export=NONE\nunset SLURM_EXPORT_ENV\nuss=$(whoami)\nfind /dev/shm/ -user $uss -type f -mmin +30 -delete\ncd \"/home/atuin/k106eb/k106eb10/ACE/Ni-Al/DFT/VASP_PBE_500_0.125_0.1_NM/AlNi/binaries/bulk/base-hcp/occ-shaken/hcp16.occ.4.shake.0/cfg/NiAl3NiAl11\"\nams_pipeline pipeline.json \u003e \"/home/atuin/k106eb/k106eb10/ACE/Ni-Al/DFT/VASP_PBE_500_0.125_0.1_NM/AlNi/binaries/bulk/base-hcp/occ-shaken/hcp16.occ.4.shake.0/cfg/NiAl3NiAl11/ams_pipeline_job.sh.out\" 2\u003e \"/home/atuin/k106eb/k106eb10/ACE/Ni-Al/DFT/VASP_PBE_500_0.125_0.1_NM/AlNi/binaries/bulk/base-hcp/occ-shaken/hcp16.occ.4.shake.0/cfg/NiAl3NiAl11/ams_pipeline_job.sh.err\"\n","slurmInfo":"\nJobId=398955 JobName=ams_pipeline\n   UserId=k106eb10(210387) GroupId=80111\n   Account=k106eb QOS=normal \n   Requeue=False Restarts=0 BatchFlag=True \n   TimeLimit=1439\n   SubmitTime=2023-02-09T14:11:22\n   Partition=singlenode \n   NodeList=f0720\n   NumNodes=1 NumCPUs=72 NumTasks=72 CPUs/Task=1\n   NTasksPerNode:Socket:Core=0:None:None\n   TRES_req=cpu=72,mem=250000M,node=1,billing=72\n   TRES_alloc=cpu=72,node=1,billing=72\n   Command=/home/atuin/k106eb/k106eb10/ACE/Ni-Al/DFT/VASP_PBE_500_0.125_0.1_NM/AlNi/binaries/bulk/base-hcp/occ-shaken/hcp16.occ.4.shake.0/cfg/NiAl3NiAl11/ams_pipeline_job.sh\n   WorkDir=/home/atuin/k106eb/k106eb10/ACE/Ni-Al/DFT/VASP_PBE_500_0.125_0.1_NM/AlNi/binaries/bulk/base-hcp/occ-shaken/hcp16.occ.4.shake.0/cfg/NiAl3NiAl11\n   StdErr=\n   StdOut=ams_pipeline.o%j\n"},"startTime":1675956725,"statistics":{"clock":{"unit":{"base":"Hz","prefix":"M"},"avg":2335.254,"min":800.418,"max":2734.922},"cpu_load":{"unit":{"base":""},"avg":52.72,"min":34.46,"max":71.91},"cpu_power":{"unit":{"base":"W"},"avg":407.767,"min":93.932,"max":497.636},"cpu_user":{"unit":{"base":""},"avg":63.678,"min":19.872,"max":96.633},"flops_any":{"unit":{"base":"F/s","prefix":"G"},"avg":635.672,"min":0,"max":1332.874},"flops_dp":{"unit":{"base":"F/s","prefix":"G"},"avg":261.006,"min":0,"max":382.294},"flops_sp":{"unit":{"base":"F/s","prefix":"G"},"avg":113.659,"min":0,"max":568.286},"ib_recv":{"unit":{"base":"B/s"},"avg":27981.111,"min":69.4,"max":48084.589},"ib_recv_pkts":{"unit":{"base":"packets/s"},"avg":398.939,"min":0.5,"max":693.817},"ib_xmit":{"unit":{"base":"B/s"},"avg":188.513,"min":39.597,"max":724.568},"ib_xmit_pkts":{"unit":{"base":"packets/s"},"avg":0.867,"min":0.2,"max":2.933},"ipc":{"unit":{"base":"IPC"},"avg":0.944,"min":0.564,"max":1.291},"mem_bw":{"unit":{"base":"B/s","prefix":"G"},"avg":79.565,"min":0.021,"max":116.02},"mem_power":{"unit":{"base":"W"},"avg":24.692,"min":7.883,"max":31.318},"mem_used":{"unit":{"base":"B","prefix":"G"},"avg":22.566,"min":8.225,"max":27.613},"nfs4_read":{"unit":{"base":"B/s","prefix":"M"},"avg":647,"min":0,"max":1946},"nfs4_total":{"unit":{"base":"B/s","prefix":"M"},"avg":6181.6,"min":1270,"max":11411},"nfs4_write":{"unit":{"base":"B/s","prefix":"M"},"avg":22.4,"min":11,"max":29},"vectorization_ratio":{"unit":{"base":"%"},"avg":77.351,"min":0,"max":98.837}}}

internal/importer/testdata/meta-fritzMinimal.input

@@ -1 +1 @@
-{"jobId":398764,"user":"k106eb10","project":"k106eb","cluster":"fritz","subCluster":"main","numNodes":1,"shared":"none","jobState":"completed","duration":177,"resources":[{"hostname":"f0649"}],"startTime":1675954353,"statistics":{"clock":{"unit":{"base":"Hz","prefix":"M"},"avg":1336.519,"min":801.564,"max":2348.215},"cpu_load":{"unit":{"base":""},"avg":31.64,"min":17.36,"max":45.54},"cpu_power":{"unit":{"base":"W"},"avg":150.018,"min":93.672,"max":261.592},"cpu_user":{"unit":{"base":""},"avg":28.518,"min":0.09,"max":57.343},"flops_any":{"unit":{"base":"F/s","prefix":"G"},"avg":45.012,"min":0,"max":135.037},"flops_dp":{"unit":{"base":"F/s","prefix":"G"},"avg":22.496,"min":0,"max":67.488},"flops_sp":{"unit":{"base":"F/s","prefix":"G"},"avg":0.02,"min":0,"max":0.061},"ib_recv":{"unit":{"base":"B/s"},"avg":14442.82,"min":219.998,"max":42581.368},"ib_recv_pkts":{"unit":{"base":"packets/s"},"avg":201.532,"min":1.25,"max":601.345},"ib_xmit":{"unit":{"base":"B/s"},"avg":282.098,"min":56.2,"max":569.363},"ib_xmit_pkts":{"unit":{"base":"packets/s"},"avg":1.228,"min":0.433,"max":2},"ipc":{"unit":{"base":"IPC"},"avg":0.77,"min":0.564,"max":0.906},"mem_bw":{"unit":{"base":"B/s","prefix":"G"},"avg":4.872,"min":0.025,"max":14.552},"mem_power":{"unit":{"base":"W"},"avg":7.725,"min":6.286,"max":10.556},"mem_used":{"unit":{"base":"B","prefix":"G"},"avg":6.162,"min":6.103,"max":6.226},"nfs4_read":{"unit":{"base":"B/s","prefix":"M"},"avg":1045.333,"min":311,"max":1525},"nfs4_total":{"unit":{"base":"B/s","prefix":"M"},"avg":6430,"min":2796,"max":11518},"nfs4_write":{"unit":{"base":"B/s","prefix":"M"},"avg":24.333,"min":0,"max":38},"vectorization_ratio":{"unit":{"base":"%"},"avg":25.528,"min":0,"max":76.585}}}
+{"jobId":398764,"user":"k106eb10","project":"k106eb","cluster":"fritz","subCluster":"main","numNodes":1,"exclusive":1,"jobState":"completed","duration":177,"resources":[{"hostname":"f0649"}],"startTime":1675954353,"statistics":{"clock":{"unit":{"base":"Hz","prefix":"M"},"avg":1336.519,"min":801.564,"max":2348.215},"cpu_load":{"unit":{"base":""},"avg":31.64,"min":17.36,"max":45.54},"cpu_power":{"unit":{"base":"W"},"avg":150.018,"min":93.672,"max":261.592},"cpu_user":{"unit":{"base":""},"avg":28.518,"min":0.09,"max":57.343},"flops_any":{"unit":{"base":"F/s","prefix":"G"},"avg":45.012,"min":0,"max":135.037},"flops_dp":{"unit":{"base":"F/s","prefix":"G"},"avg":22.496,"min":0,"max":67.488},"flops_sp":{"unit":{"base":"F/s","prefix":"G"},"avg":0.02,"min":0,"max":0.061},"ib_recv":{"unit":{"base":"B/s"},"avg":14442.82,"min":219.998,"max":42581.368},"ib_recv_pkts":{"unit":{"base":"packets/s"},"avg":201.532,"min":1.25,"max":601.345},"ib_xmit":{"unit":{"base":"B/s"},"avg":282.098,"min":56.2,"max":569.363},"ib_xmit_pkts":{"unit":{"base":"packets/s"},"avg":1.228,"min":0.433,"max":2},"ipc":{"unit":{"base":"IPC"},"avg":0.77,"min":0.564,"max":0.906},"mem_bw":{"unit":{"base":"B/s","prefix":"G"},"avg":4.872,"min":0.025,"max":14.552},"mem_power":{"unit":{"base":"W"},"avg":7.725,"min":6.286,"max":10.556},"mem_used":{"unit":{"base":"B","prefix":"G"},"avg":6.162,"min":6.103,"max":6.226},"nfs4_read":{"unit":{"base":"B/s","prefix":"M"},"avg":1045.333,"min":311,"max":1525},"nfs4_total":{"unit":{"base":"B/s","prefix":"M"},"avg":6430,"min":2796,"max":11518},"nfs4_write":{"unit":{"base":"B/s","prefix":"M"},"avg":24.333,"min":0,"max":38},"vectorization_ratio":{"unit":{"base":"%"},"avg":25.528,"min":0,"max":76.585}}}

internal/memorystore/api.go

@@ -1,217 +0,0 @@
-// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved. This file is part of cc-backend.
-// Use of this source code is governed by a MIT-style
-// license that can be found in the LICENSE file.
-
-package memorystore
-
-import (
-	"math"
-
-	"github.com/ClusterCockpit/cc-lib/schema"
-	"github.com/ClusterCockpit/cc-lib/util"
-)
-
-type APIMetricData struct {
-	Error      *string           `json:"error,omitempty"`
-	Data       schema.FloatArray `json:"data,omitempty"`
-	From       int64             `json:"from"`
-	To         int64             `json:"to"`
-	Resolution int64             `json:"resolution"`
-	Avg        schema.Float      `json:"avg"`
-	Min        schema.Float      `json:"min"`
-	Max        schema.Float      `json:"max"`
-}
-
-type APIQueryRequest struct {
-	Cluster     string     `json:"cluster"`
-	Queries     []APIQuery `json:"queries"`
-	ForAllNodes []string   `json:"for-all-nodes"`
-	From        int64      `json:"from"`
-	To          int64      `json:"to"`
-	WithStats   bool       `json:"with-stats"`
-	WithData    bool       `json:"with-data"`
-	WithPadding bool       `json:"with-padding"`
-}
-
-type APIQueryResponse struct {
-	Queries []APIQuery        `json:"queries,omitempty"`
-	Results [][]APIMetricData `json:"results"`
-}
-
-type APIQuery struct {
-	Type        *string      `json:"type,omitempty"`
-	SubType     *string      `json:"subtype,omitempty"`
-	Metric      string       `json:"metric"`
-	Hostname    string       `json:"host"`
-	Resolution  int64        `json:"resolution"`
-	TypeIds     []string     `json:"type-ids,omitempty"`
-	SubTypeIds  []string     `json:"subtype-ids,omitempty"`
-	ScaleFactor schema.Float `json:"scale-by,omitempty"`
-	Aggregate   bool         `json:"aggreg"`
-}
-
-// TODO: Optimize this, just like the stats endpoint!
-func (data *APIMetricData) AddStats() {
-	n := 0
-	sum, min, max := 0.0, math.MaxFloat64, -math.MaxFloat64
-	for _, x := range data.Data {
-		if x.IsNaN() {
-			continue
-		}
-
-		n += 1
-		sum += float64(x)
-		min = math.Min(min, float64(x))
-		max = math.Max(max, float64(x))
-	}
-
-	if n > 0 {
-		avg := sum / float64(n)
-		data.Avg = schema.Float(avg)
-		data.Min = schema.Float(min)
-		data.Max = schema.Float(max)
-	} else {
-		data.Avg, data.Min, data.Max = schema.NaN, schema.NaN, schema.NaN
-	}
-}
-
-func (data *APIMetricData) ScaleBy(f schema.Float) {
-	if f == 0 || f == 1 {
-		return
-	}
-
-	data.Avg *= f
-	data.Min *= f
-	data.Max *= f
-	for i := 0; i < len(data.Data); i++ {
-		data.Data[i] *= f
-	}
-}
-
-func (data *APIMetricData) PadDataWithNull(ms *MemoryStore, from, to int64, metric string) {
-	minfo, ok := ms.Metrics[metric]
-	if !ok {
-		return
-	}
-
-	if (data.From / minfo.Frequency) > (from / minfo.Frequency) {
-		padfront := int((data.From / minfo.Frequency) - (from / minfo.Frequency))
-		ndata := make([]schema.Float, 0, padfront+len(data.Data))
-		for range padfront {
-			ndata = append(ndata, schema.NaN)
-		}
-		for j := 0; j < len(data.Data); j++ {
-			ndata = append(ndata, data.Data[j])
-		}
-		data.Data = ndata
-	}
-}
-
-func FetchData(req APIQueryRequest) (*APIQueryResponse, error) {
-	req.WithData = true
-	req.WithData = true
-	req.WithData = true
-
-	ms := GetMemoryStore()
-
-	response := APIQueryResponse{
-		Results: make([][]APIMetricData, 0, len(req.Queries)),
-	}
-	if req.ForAllNodes != nil {
-		nodes := ms.ListChildren([]string{req.Cluster})
-		for _, node := range nodes {
-			for _, metric := range req.ForAllNodes {
-				q := APIQuery{
-					Metric:   metric,
-					Hostname: node,
-				}
-				req.Queries = append(req.Queries, q)
-				response.Queries = append(response.Queries, q)
-			}
-		}
-	}
-
-	for _, query := range req.Queries {
-		sels := make([]util.Selector, 0, 1)
-		if query.Aggregate || query.Type == nil {
-			sel := util.Selector{{String: req.Cluster}, {String: query.Hostname}}
-			if query.Type != nil {
-				if len(query.TypeIds) == 1 {
-					sel = append(sel, util.SelectorElement{String: *query.Type + query.TypeIds[0]})
-				} else {
-					ids := make([]string, len(query.TypeIds))
-					for i, id := range query.TypeIds {
-						ids[i] = *query.Type + id
-					}
-					sel = append(sel, util.SelectorElement{Group: ids})
-				}
-
-				if query.SubType != nil {
-					if len(query.SubTypeIds) == 1 {
-						sel = append(sel, util.SelectorElement{String: *query.SubType + query.SubTypeIds[0]})
-					} else {
-						ids := make([]string, len(query.SubTypeIds))
-						for i, id := range query.SubTypeIds {
-							ids[i] = *query.SubType + id
-						}
-						sel = append(sel, util.SelectorElement{Group: ids})
-					}
-				}
-			}
-			sels = append(sels, sel)
-		} else {
-			for _, typeID := range query.TypeIds {
-				if query.SubType != nil {
-					for _, subTypeID := range query.SubTypeIds {
-						sels = append(sels, util.Selector{
-							{String: req.Cluster},
-							{String: query.Hostname},
-							{String: *query.Type + typeID},
-							{String: *query.SubType + subTypeID},
-						})
-					}
-				} else {
-					sels = append(sels, util.Selector{
-						{String: req.Cluster},
-						{String: query.Hostname},
-						{String: *query.Type + typeID},
-					})
-				}
-			}
-		}
-
-		// log.Printf("query: %#v\n", query)
-		// log.Printf("sels: %#v\n", sels)
-		var err error
-		res := make([]APIMetricData, 0, len(sels))
-		for _, sel := range sels {
-			data := APIMetricData{}
-
-			data.Data, data.From, data.To, data.Resolution, err = ms.Read(sel, query.Metric, req.From, req.To, query.Resolution)
-			if err != nil {
-				msg := err.Error()
-				data.Error = &msg
-				res = append(res, data)
-				continue
-			}
-
-			if req.WithStats {
-				data.AddStats()
-			}
-			if query.ScaleFactor != 0 {
-				data.ScaleBy(query.ScaleFactor)
-			}
-			if req.WithPadding {
-				data.PadDataWithNull(ms, req.From, req.To, query.Metric)
-			}
-			if !req.WithData {
-				data.Data = nil
-			}
-			res = append(res, data)
-		}
-		response.Results = append(response.Results, res)
-	}
-
-	return &response, nil
-}

internal/memorystore/archive.go

@@ -1,191 +0,0 @@
-// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved. This file is part of cc-backend.
-// Use of this source code is governed by a MIT-style
-// license that can be found in the LICENSE file.
-
-package memorystore
-
-import (
-	"archive/zip"
-	"bufio"
-	"context"
-	"errors"
-	"fmt"
-	"io"
-	"os"
-	"path/filepath"
-	"sync"
-	"sync/atomic"
-	"time"
-
-	cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
-)
-
-func Archiving(wg *sync.WaitGroup, ctx context.Context) {
-	go func() {
-		defer wg.Done()
-		d, err := time.ParseDuration(Keys.Archive.Interval)
-		if err != nil {
-			cclog.Fatalf("[METRICSTORE]> error parsing archive interval duration: %v\n", err)
-		}
-		if d <= 0 {
-			return
-		}
-
-		ticks := func() <-chan time.Time {
-			if d <= 0 {
-				return nil
-			}
-			return time.NewTicker(d).C
-		}()
-		for {
-			select {
-			case <-ctx.Done():
-				return
-			case <-ticks:
-				t := time.Now().Add(-d)
-				cclog.Printf("[METRICSTORE]> start archiving checkpoints (older than %s)...\n", t.Format(time.RFC3339))
-				n, err := ArchiveCheckpoints(Keys.Checkpoints.RootDir,
-					Keys.Archive.RootDir, t.Unix(), Keys.Archive.DeleteInstead)
-
-				if err != nil {
-					cclog.Printf("[METRICSTORE]> archiving failed: %s\n", err.Error())
-				} else {
-					cclog.Printf("[METRICSTORE]> done: %d files zipped and moved to archive\n", n)
-				}
-			}
-		}
-	}()
-}
-
-var ErrNoNewArchiveData error = errors.New("all data already archived")
-
-// ZIP all checkpoint files older than `from` together and write them to the `archiveDir`,
-// deleting them from the `checkpointsDir`.
-func ArchiveCheckpoints(checkpointsDir, archiveDir string, from int64, deleteInstead bool) (int, error) {
-	entries1, err := os.ReadDir(checkpointsDir)
-	if err != nil {
-		return 0, err
-	}
-
-	type workItem struct {
-		cdir, adir    string
-		cluster, host string
-	}
-
-	var wg sync.WaitGroup
-	n, errs := int32(0), int32(0)
-	work := make(chan workItem, NumWorkers)
-
-	wg.Add(NumWorkers)
-	for worker := 0; worker < NumWorkers; worker++ {
-		go func() {
-			defer wg.Done()
-			for workItem := range work {
-				m, err := archiveCheckpoints(workItem.cdir, workItem.adir, from, deleteInstead)
-				if err != nil {
-					cclog.Errorf("error while archiving %s/%s: %s", workItem.cluster, workItem.host, err.Error())
-					atomic.AddInt32(&errs, 1)
-				}
-				atomic.AddInt32(&n, int32(m))
-			}
-		}()
-	}
-
-	for _, de1 := range entries1 {
-		entries2, e := os.ReadDir(filepath.Join(checkpointsDir, de1.Name()))
-		if e != nil {
-			err = e
-		}
-
-		for _, de2 := range entries2 {
-			cdir := filepath.Join(checkpointsDir, de1.Name(), de2.Name())
-			adir := filepath.Join(archiveDir, de1.Name(), de2.Name())
-			work <- workItem{
-				adir: adir, cdir: cdir,
-				cluster: de1.Name(), host: de2.Name(),
-			}
-		}
-	}
-
-	close(work)
-	wg.Wait()
-
-	if err != nil {
-		return int(n), err
-	}
-
-	if errs > 0 {
-		return int(n), fmt.Errorf("%d errors happend while archiving (%d successes)", errs, n)
-	}
-	return int(n), nil
-}
-
-// Helper function for `ArchiveCheckpoints`.
-func archiveCheckpoints(dir string, archiveDir string, from int64, deleteInstead bool) (int, error) {
-	entries, err := os.ReadDir(dir)
-	if err != nil {
-		return 0, err
-	}
-
-	extension := Keys.Checkpoints.FileFormat
-	files, err := findFiles(entries, from, extension, false)
-	if err != nil {
-		return 0, err
-	}
-
-	if deleteInstead {
-		n := 0
-		for _, checkpoint := range files {
-			filename := filepath.Join(dir, checkpoint)
-			if err = os.Remove(filename); err != nil {
-				return n, err
-			}
-			n += 1
-		}
-		return n, nil
-	}
-
-	filename := filepath.Join(archiveDir, fmt.Sprintf("%d.zip", from))
-	f, err := os.OpenFile(filename, os.O_CREATE|os.O_WRONLY, 0o644)
-	if err != nil && os.IsNotExist(err) {
-		err = os.MkdirAll(archiveDir, 0o755)
-		if err == nil {
-			f, err = os.OpenFile(filename, os.O_CREATE|os.O_WRONLY, 0o644)
-		}
-	}
-	if err != nil {
-		return 0, err
-	}
-	defer f.Close()
-	bw := bufio.NewWriter(f)
-	defer bw.Flush()
-	zw := zip.NewWriter(bw)
-	defer zw.Close()
-
-	n := 0
-	for _, checkpoint := range files {
-		filename := filepath.Join(dir, checkpoint)
-		r, err := os.Open(filename)
-		if err != nil {
-			return n, err
-		}
-		defer r.Close()
-
-		w, err := zw.Create(checkpoint)
-		if err != nil {
-			return n, err
-		}
-
-		if _, err = io.Copy(w, r); err != nil {
-			return n, err
-		}
-
-		if err = os.Remove(filename); err != nil {
-			return n, err
-		}
-		n += 1
-	}
-
-	return n, nil
-}

internal/memorystore/avroCheckpoint.go

@@ -1,482 +0,0 @@
-// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved. This file is part of cc-backend.
-// Use of this source code is governed by a MIT-style
-// license that can be found in the LICENSE file.
-
-package memorystore
-
-import (
-	"bufio"
-	"encoding/json"
-	"errors"
-	"fmt"
-	"os"
-	"path"
-	"sort"
-	"strconv"
-	"strings"
-	"sync"
-	"sync/atomic"
-	"time"
-
-	cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
-	"github.com/ClusterCockpit/cc-lib/schema"
-	"github.com/linkedin/goavro/v2"
-)
-
-var NumAvroWorkers int = 4
-var startUp bool = true
-var ErrNoNewData error = errors.New("no data in the pool")
-
-func (as *AvroStore) ToCheckpoint(dir string, dumpAll bool) (int, error) {
-	levels := make([]*AvroLevel, 0)
-	selectors := make([][]string, 0)
-	as.root.lock.RLock()
-	// Cluster
-	for sel1, l1 := range as.root.children {
-		l1.lock.RLock()
-		// Node
-		for sel2, l2 := range l1.children {
-			l2.lock.RLock()
-			// Frequency
-			for sel3, l3 := range l2.children {
-				levels = append(levels, l3)
-				selectors = append(selectors, []string{sel1, sel2, sel3})
-			}
-			l2.lock.RUnlock()
-		}
-		l1.lock.RUnlock()
-	}
-	as.root.lock.RUnlock()
-
-	type workItem struct {
-		level    *AvroLevel
-		dir      string
-		selector []string
-	}
-
-	n, errs := int32(0), int32(0)
-
-	var wg sync.WaitGroup
-	wg.Add(NumAvroWorkers)
-	work := make(chan workItem, NumAvroWorkers*2)
-	for range NumAvroWorkers {
-		go func() {
-			defer wg.Done()
-
-			for workItem := range work {
-				from := getTimestamp(workItem.dir)
-
-				if err := workItem.level.toCheckpoint(workItem.dir, from, dumpAll); err != nil {
-					if err == ErrNoNewArchiveData {
-						continue
-					}
-
-					cclog.Errorf("error while checkpointing %#v: %s", workItem.selector, err.Error())
-					atomic.AddInt32(&errs, 1)
-				} else {
-					atomic.AddInt32(&n, 1)
-				}
-			}
-		}()
-	}
-
-	for i := range len(levels) {
-		dir := path.Join(dir, path.Join(selectors[i]...))
-		work <- workItem{
-			level:    levels[i],
-			dir:      dir,
-			selector: selectors[i],
-		}
-	}
-
-	close(work)
-	wg.Wait()
-
-	if errs > 0 {
-		return int(n), fmt.Errorf("%d errors happend while creating avro checkpoints (%d successes)", errs, n)
-	}
-
-	startUp = false
-
-	return int(n), nil
-}
-
-// getTimestamp returns the timestamp from the directory name
-func getTimestamp(dir string) int64 {
-	// Extract the resolution and timestamp from the directory name
-	// The existing avro file will be in epoch timestamp format
-	// iterate over all the files in the directory and find the maximum timestamp
-	// and return it
-
-	resolution := path.Base(dir)
-	dir = path.Dir(dir)
-
-	files, err := os.ReadDir(dir)
-	if err != nil {
-		return 0
-	}
-	var maxTS int64 = 0
-
-	if len(files) == 0 {
-		return 0
-	}
-
-	for _, file := range files {
-		if file.IsDir() {
-			continue
-		}
-		name := file.Name()
-
-		if len(name) < 5 || !strings.HasSuffix(name, ".avro") || !strings.HasPrefix(name, resolution+"_") {
-			continue
-		}
-
-		ts, err := strconv.ParseInt(name[strings.Index(name, "_")+1:len(name)-5], 10, 64)
-		if err != nil {
-			fmt.Printf("error while parsing timestamp: %s\n", err.Error())
-			continue
-		}
-
-		if ts > maxTS {
-			maxTS = ts
-		}
-	}
-
-	interval, _ := time.ParseDuration(Keys.Checkpoints.Interval)
-	updateTime := time.Unix(maxTS, 0).Add(interval).Add(time.Duration(CheckpointBufferMinutes-1) * time.Minute).Unix()
-
-	if startUp {
-		return 0
-	}
-
-	if updateTime < time.Now().Unix() {
-		return 0
-	}
-
-	return maxTS
-}
-
-func (l *AvroLevel) toCheckpoint(dir string, from int64, dumpAll bool) error {
-	l.lock.Lock()
-	defer l.lock.Unlock()
-
-	// fmt.Printf("Checkpointing directory: %s\n", dir)
-	// filepath contains the resolution
-	intRes, _ := strconv.Atoi(path.Base(dir))
-
-	// find smallest overall timestamp in l.data map and delete it from l.data
-	minTS := int64(1<<63 - 1)
-	for ts, dat := range l.data {
-		if ts < minTS && len(dat) != 0 {
-			minTS = ts
-		}
-	}
-
-	if from == 0 && minTS != int64(1<<63-1) {
-		from = minTS
-	}
-
-	if from == 0 {
-		return ErrNoNewArchiveData
-	}
-
-	var schema string
-	var codec *goavro.Codec
-	recordList := make([]map[string]any, 0)
-
-	var f *os.File
-
-	filePath := dir + fmt.Sprintf("_%d.avro", from)
-
-	var err error
-
-	fp_, err_ := os.Stat(filePath)
-	if errors.Is(err_, os.ErrNotExist) {
-		err = os.MkdirAll(path.Dir(dir), 0o755)
-		if err != nil {
-			return fmt.Errorf("failed to create directory: %v", err)
-		}
-	} else if fp_.Size() != 0 {
-		f, err = os.Open(filePath)
-		if err != nil {
-			return fmt.Errorf("failed to open existing avro file: %v", err)
-		}
-
-		br := bufio.NewReader(f)
-
-		reader, err := goavro.NewOCFReader(br)
-		if err != nil {
-			return fmt.Errorf("failed to create OCF reader: %v", err)
-		}
-		codec = reader.Codec()
-		schema = codec.Schema()
-
-		f.Close()
-	}
-
-	timeRef := time.Now().Add(time.Duration(-CheckpointBufferMinutes+1) * time.Minute).Unix()
-
-	if dumpAll {
-		timeRef = time.Now().Unix()
-	}
-
-	// Empty values
-	if len(l.data) == 0 {
-		// we checkpoint avro files every 60 seconds
-		repeat := 60 / intRes
-
-		for range repeat {
-			recordList = append(recordList, make(map[string]any))
-		}
-	}
-
-	readFlag := true
-
-	for ts := range l.data {
-		flag := false
-		if ts < timeRef {
-			data := l.data[ts]
-
-			schemaGen, err := generateSchema(data)
-			if err != nil {
-				return err
-			}
-
-			flag, schema, err = compareSchema(schema, schemaGen)
-			if err != nil {
-				return fmt.Errorf("failed to compare read and generated schema: %v", err)
-			}
-			if flag && readFlag && !errors.Is(err_, os.ErrNotExist) {
-
-				f.Close()
-
-				f, err = os.Open(filePath)
-				if err != nil {
-					return fmt.Errorf("failed to open Avro file: %v", err)
-				}
-
-				br := bufio.NewReader(f)
-
-				ocfReader, err := goavro.NewOCFReader(br)
-				if err != nil {
-					return fmt.Errorf("failed to create OCF reader while changing schema: %v", err)
-				}
-
-				for ocfReader.Scan() {
-					record, err := ocfReader.Read()
-					if err != nil {
-						return fmt.Errorf("failed to read record: %v", err)
-					}
-
-					recordList = append(recordList, record.(map[string]any))
-				}
-
-				f.Close()
-
-				err = os.Remove(filePath)
-				if err != nil {
-					return fmt.Errorf("failed to delete file: %v", err)
-				}
-
-				readFlag = false
-			}
-			codec, err = goavro.NewCodec(schema)
-			if err != nil {
-				return fmt.Errorf("failed to create codec after merged schema: %v", err)
-			}
-
-			recordList = append(recordList, generateRecord(data))
-			delete(l.data, ts)
-		}
-	}
-
-	if len(recordList) == 0 {
-		return ErrNoNewArchiveData
-	}
-
-	f, err = os.OpenFile(filePath, os.O_CREATE|os.O_APPEND|os.O_RDWR, 0o644)
-	if err != nil {
-		return fmt.Errorf("failed to append new avro file: %v", err)
-	}
-
-	// fmt.Printf("Codec : %#v\n", codec)
-
-	writer, err := goavro.NewOCFWriter(goavro.OCFConfig{
-		W:               f,
-		Codec:           codec,
-		CompressionName: goavro.CompressionDeflateLabel,
-	})
-	if err != nil {
-		return fmt.Errorf("failed to create OCF writer: %v", err)
-	}
-
-	// Append the new record
-	if err := writer.Append(recordList); err != nil {
-		return fmt.Errorf("failed to append record: %v", err)
-	}
-
-	f.Close()
-
-	return nil
-}
-
-func compareSchema(schemaRead, schemaGen string) (bool, string, error) {
-	var genSchema, readSchema AvroSchema
-
-	if schemaRead == "" {
-		return false, schemaGen, nil
-	}
-
-	// Unmarshal the schema strings into AvroSchema structs
-	if err := json.Unmarshal([]byte(schemaGen), &genSchema); err != nil {
-		return false, "", fmt.Errorf("failed to parse generated schema: %v", err)
-	}
-	if err := json.Unmarshal([]byte(schemaRead), &readSchema); err != nil {
-		return false, "", fmt.Errorf("failed to parse read schema: %v", err)
-	}
-
-	sort.Slice(genSchema.Fields, func(i, j int) bool {
-		return genSchema.Fields[i].Name < genSchema.Fields[j].Name
-	})
-
-	sort.Slice(readSchema.Fields, func(i, j int) bool {
-		return readSchema.Fields[i].Name < readSchema.Fields[j].Name
-	})
-
-	// Check if schemas are identical
-	schemasEqual := true
-	if len(genSchema.Fields) <= len(readSchema.Fields) {
-
-		for i := range genSchema.Fields {
-			if genSchema.Fields[i].Name != readSchema.Fields[i].Name {
-				schemasEqual = false
-				break
-			}
-		}
-
-		// If schemas are identical, return the read schema
-		if schemasEqual {
-			return false, schemaRead, nil
-		}
-	}
-
-	// Create a map to hold unique fields from both schemas
-	fieldMap := make(map[string]AvroField)
-
-	// Add fields from the read schema
-	for _, field := range readSchema.Fields {
-		fieldMap[field.Name] = field
-	}
-
-	// Add or update fields from the generated schema
-	for _, field := range genSchema.Fields {
-		fieldMap[field.Name] = field
-	}
-
-	// Create a union schema by collecting fields from the map
-	var mergedFields []AvroField
-	for _, field := range fieldMap {
-		mergedFields = append(mergedFields, field)
-	}
-
-	// Sort fields by name for consistency
-	sort.Slice(mergedFields, func(i, j int) bool {
-		return mergedFields[i].Name < mergedFields[j].Name
-	})
-
-	// Create the merged schema
-	mergedSchema := AvroSchema{
-		Type:   "record",
-		Name:   genSchema.Name,
-		Fields: mergedFields,
-	}
-
-	// Check if schemas are identical
-	schemasEqual = len(mergedSchema.Fields) == len(readSchema.Fields)
-	if schemasEqual {
-		for i := range mergedSchema.Fields {
-			if mergedSchema.Fields[i].Name != readSchema.Fields[i].Name {
-				schemasEqual = false
-				break
-			}
-		}
-
-		if schemasEqual {
-			return false, schemaRead, nil
-		}
-	}
-
-	// Marshal the merged schema back to JSON
-	mergedSchemaJSON, err := json.Marshal(mergedSchema)
-	if err != nil {
-		return false, "", fmt.Errorf("failed to marshal merged schema: %v", err)
-	}
-
-	return true, string(mergedSchemaJSON), nil
-}
-
-func generateSchema(data map[string]schema.Float) (string, error) {
-	// Define the Avro schema structure
-	schema := map[string]any{
-		"type":   "record",
-		"name":   "DataRecord",
-		"fields": []map[string]any{},
-	}
-
-	fieldTracker := make(map[string]struct{})
-
-	for key := range data {
-		if _, exists := fieldTracker[key]; !exists {
-			key = correctKey(key)
-
-			field := map[string]any{
-				"name":    key,
-				"type":    "double",
-				"default": -1.0,
-			}
-			schema["fields"] = append(schema["fields"].([]map[string]any), field)
-			fieldTracker[key] = struct{}{}
-		}
-	}
-
-	schemaString, err := json.Marshal(schema)
-	if err != nil {
-		return "", fmt.Errorf("failed to marshal schema: %v", err)
-	}
-
-	return string(schemaString), nil
-}
-
-func generateRecord(data map[string]schema.Float) map[string]any {
-	record := make(map[string]any)
-
-	// Iterate through each map in data
-	for key, value := range data {
-		key = correctKey(key)
-
-		// Set the value in the record
-		// avro only accepts basic types
-		record[key] = value.Double()
-	}
-
-	return record
-}
-
-func correctKey(key string) string {
-	// Replace any invalid characters in the key
-	// For example, replace spaces with underscores
-	key = strings.ReplaceAll(key, ":", "___")
-	key = strings.ReplaceAll(key, ".", "__")
-
-	return key
-}
-
-func ReplaceKey(key string) string {
-	// Replace any invalid characters in the key
-	// For example, replace spaces with underscores
-	key = strings.ReplaceAll(key, "___", ":")
-	key = strings.ReplaceAll(key, "__", ".")
-
-	return key
-}

internal/memorystore/avroHelper.go

@@ -1,84 +0,0 @@
-// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved. This file is part of cc-backend.
-// Use of this source code is governed by a MIT-style
-// license that can be found in the LICENSE file.
-
-package memorystore
-
-import (
-	"context"
-	"slices"
-	"strconv"
-	"sync"
-
-	cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
-)
-
-func DataStaging(wg *sync.WaitGroup, ctx context.Context) {
-	// AvroPool is a pool of Avro writers.
-	go func() {
-		if Keys.Checkpoints.FileFormat == "json" {
-			wg.Done() // Mark this goroutine as done
-			return    // Exit the goroutine
-		}
-
-		defer wg.Done()
-
-		var avroLevel *AvroLevel
-		oldSelector := make([]string, 0)
-
-		for {
-			select {
-			case <-ctx.Done():
-				return
-			case val := <-LineProtocolMessages:
-				// Fetch the frequency of the metric from the global configuration
-				freq, err := GetMetricFrequency(val.MetricName)
-				if err != nil {
-					cclog.Errorf("Error fetching metric frequency: %s\n", err)
-					continue
-				}
-
-				metricName := ""
-
-				for _, selectorName := range val.Selector {
-					metricName += selectorName + Delimiter
-				}
-
-				metricName += val.MetricName
-
-				// Create a new selector for the Avro level
-				// The selector is a slice of strings that represents the path to the
-				// Avro level. It is created by appending the cluster, node, and metric
-				// name to the selector.
-				var selector []string
-				selector = append(selector, val.Cluster, val.Node, strconv.FormatInt(freq, 10))
-
-				if !testEq(oldSelector, selector) {
-					// Get the Avro level for the metric
-					avroLevel = avroStore.root.findAvroLevelOrCreate(selector)
-
-					// If the Avro level is nil, create a new one
-					if avroLevel == nil {
-						cclog.Errorf("Error creating or finding the level with cluster : %s, node : %s, metric : %s\n", val.Cluster, val.Node, val.MetricName)
-					}
-					oldSelector = slices.Clone(selector)
-				}
-
-				avroLevel.addMetric(metricName, val.Value, val.Timestamp, int(freq))
-			}
-		}
-	}()
-}
-
-func testEq(a, b []string) bool {
-	if len(a) != len(b) {
-		return false
-	}
-	for i := range a {
-		if a[i] != b[i] {
-			return false
-		}
-	}
-	return true
-}

internal/memorystore/avroStruct.go

@@ -1,168 +0,0 @@
-// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved. This file is part of cc-backend.
-// Use of this source code is governed by a MIT-style
-// license that can be found in the LICENSE file.
-
-package memorystore
-
-import (
-	"sync"
-
-	"github.com/ClusterCockpit/cc-lib/schema"
-)
-
-var (
-	LineProtocolMessages = make(chan *AvroStruct)
-	Delimiter            = "ZZZZZ"
-)
-
-// CheckpointBufferMinutes should always be in minutes.
-// Its controls the amount of data to hold for given amount of time.
-var CheckpointBufferMinutes = 3
-
-type AvroStruct struct {
-	MetricName string
-	Cluster    string
-	Node       string
-	Selector   []string
-	Value      schema.Float
-	Timestamp  int64
-}
-
-type AvroStore struct {
-	root AvroLevel
-}
-
-var avroStore AvroStore
-
-type AvroLevel struct {
-	children map[string]*AvroLevel
-	data     map[int64]map[string]schema.Float
-	lock     sync.RWMutex
-}
-
-type AvroField struct {
-	Name    string `json:"name"`
-	Type    any    `json:"type"`
-	Default any    `json:"default,omitempty"`
-}
-
-type AvroSchema struct {
-	Type   string      `json:"type"`
-	Name   string      `json:"name"`
-	Fields []AvroField `json:"fields"`
-}
-
-func (l *AvroLevel) findAvroLevelOrCreate(selector []string) *AvroLevel {
-	if len(selector) == 0 {
-		return l
-	}
-
-	// Allow concurrent reads:
-	l.lock.RLock()
-	var child *AvroLevel
-	var ok bool
-	if l.children == nil {
-		// Children map needs to be created...
-		l.lock.RUnlock()
-	} else {
-		child, ok := l.children[selector[0]]
-		l.lock.RUnlock()
-		if ok {
-			return child.findAvroLevelOrCreate(selector[1:])
-		}
-	}
-
-	// The level does not exist, take write lock for unqiue access:
-	l.lock.Lock()
-	// While this thread waited for the write lock, another thread
-	// could have created the child node.
-	if l.children != nil {
-		child, ok = l.children[selector[0]]
-		if ok {
-			l.lock.Unlock()
-			return child.findAvroLevelOrCreate(selector[1:])
-		}
-	}
-
-	child = &AvroLevel{
-		data:     make(map[int64]map[string]schema.Float, 0),
-		children: nil,
-	}
-
-	if l.children != nil {
-		l.children[selector[0]] = child
-	} else {
-		l.children = map[string]*AvroLevel{selector[0]: child}
-	}
-	l.lock.Unlock()
-	return child.findAvroLevelOrCreate(selector[1:])
-}
-
-func (l *AvroLevel) addMetric(metricName string, value schema.Float, timestamp int64, Freq int) {
-	l.lock.Lock()
-	defer l.lock.Unlock()
-
-	KeyCounter := int(CheckpointBufferMinutes * 60 / Freq)
-
-	// Create keys in advance for the given amount of time
-	if len(l.data) != KeyCounter {
-		if len(l.data) == 0 {
-			for i := range KeyCounter {
-				l.data[timestamp+int64(i*Freq)] = make(map[string]schema.Float, 0)
-			}
-		} else {
-			// Get the last timestamp
-			var lastTS int64
-			for ts := range l.data {
-				if ts > lastTS {
-					lastTS = ts
-				}
-			}
-			// Create keys for the next KeyCounter timestamps
-			l.data[lastTS+int64(Freq)] = make(map[string]schema.Float, 0)
-		}
-	}
-
-	closestTS := int64(0)
-	minDiff := int64(Freq) + 1 // Start with diff just outside the valid range
-	found := false
-
-	// Iterate over timestamps and choose the one which is within range.
-	// Since its epoch time, we check if the difference is less than 60 seconds.
-	for ts, dat := range l.data {
-		// Check if timestamp is within range
-		diff := timestamp - ts
-		if diff < -int64(Freq) || diff > int64(Freq) {
-			continue
-		}
-
-		// Metric already present at this timestamp — skip
-		if _, ok := dat[metricName]; ok {
-			continue
-		}
-
-		// Check if this is the closest timestamp so far
-		if Abs(diff) < minDiff {
-			minDiff = Abs(diff)
-			closestTS = ts
-			found = true
-		}
-	}
-
-	if found {
-		l.data[closestTS][metricName] = value
-	}
-}
-
-func GetAvroStore() *AvroStore {
-	return &avroStore
-}
-
-// Abs returns the absolute value of x.
-func Abs(x int64) int64 {
-	if x < 0 {
-		return -x
-	}
-	return x
-}

internal/memorystore/buffer.go

@@ -1,238 +0,0 @@
-// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved. This file is part of cc-backend.
-// Use of this source code is governed by a MIT-style
-// license that can be found in the LICENSE file.
-
-package memorystore
-
-import (
-	"errors"
-	"sync"
-
-	"github.com/ClusterCockpit/cc-lib/schema"
-)
-
-// Default buffer capacity.
-// `buffer.data` will only ever grow up to it's capacity and a new link
-// in the buffer chain will be created if needed so that no copying
-// of data or reallocation needs to happen on writes.
-const (
-	BufferCap int = 512
-)
-
-// So that we can reuse allocations
-var bufferPool sync.Pool = sync.Pool{
-	New: func() any {
-		return &buffer{
-			data: make([]schema.Float, 0, BufferCap),
-		}
-	},
-}
-
-var (
-	ErrNoData           error = errors.New("[METRICSTORE]> no data for this metric/level")
-	ErrDataDoesNotAlign error = errors.New("[METRICSTORE]> data from lower granularities does not align")
-)
-
-// Each metric on each level has it's own buffer.
-// This is where the actual values go.
-// If `cap(data)` is reached, a new buffer is created and
-// becomes the new head of a buffer list.
-type buffer struct {
-	prev      *buffer
-	next      *buffer
-	data      []schema.Float
-	frequency int64
-	start     int64
-	archived  bool
-	closed    bool
-}
-
-func newBuffer(ts, freq int64) *buffer {
-	b := bufferPool.Get().(*buffer)
-	b.frequency = freq
-	b.start = ts - (freq / 2)
-	b.prev = nil
-	b.next = nil
-	b.archived = false
-	b.closed = false
-	b.data = b.data[:0]
-	return b
-}
-
-// If a new buffer was created, the new head is returnd.
-// Otherwise, the existing buffer is returnd.
-// Normaly, only "newer" data should be written, but if the value would
-// end up in the same buffer anyways it is allowed.
-func (b *buffer) write(ts int64, value schema.Float) (*buffer, error) {
-	if ts < b.start {
-		return nil, errors.New("[METRICSTORE]> cannot write value to buffer from past")
-	}
-
-	// idx := int((ts - b.start + (b.frequency / 3)) / b.frequency)
-	idx := int((ts - b.start) / b.frequency)
-	if idx >= cap(b.data) {
-		newbuf := newBuffer(ts, b.frequency)
-		newbuf.prev = b
-		b.next = newbuf
-		b.close()
-		b = newbuf
-		idx = 0
-	}
-
-	// Overwriting value or writing value from past
-	if idx < len(b.data) {
-		b.data[idx] = value
-		return b, nil
-	}
-
-	// Fill up unwritten slots with NaN
-	for i := len(b.data); i < idx; i++ {
-		b.data = append(b.data, schema.NaN)
-	}
-
-	b.data = append(b.data, value)
-	return b, nil
-}
-
-func (b *buffer) end() int64 {
-	return b.firstWrite() + int64(len(b.data))*b.frequency
-}
-
-func (b *buffer) firstWrite() int64 {
-	return b.start + (b.frequency / 2)
-}
-
-func (b *buffer) close() {}
-
-/*
-func (b *buffer) close() {
-	if b.closed {
-		return
-	}
-
-	b.closed = true
-	n, sum, min, max := 0, 0., math.MaxFloat64, -math.MaxFloat64
-	for _, x := range b.data {
-		if x.IsNaN() {
-			continue
-		}
-
-		n += 1
-		f := float64(x)
-		sum += f
-		min = math.Min(min, f)
-		max = math.Max(max, f)
-	}
-
-	b.statisticts.samples = n
-	if n > 0 {
-		b.statisticts.avg = Float(sum / float64(n))
-		b.statisticts.min = Float(min)
-		b.statisticts.max = Float(max)
-	} else {
-		b.statisticts.avg = NaN
-		b.statisticts.min = NaN
-		b.statisticts.max = NaN
-	}
-}
-*/
-
-// func interpolate(idx int, data []Float) Float {
-// 	if idx == 0 || idx+1 == len(data) {
-// 		return NaN
-// 	}
-// 	return (data[idx-1] + data[idx+1]) / 2.0
-// }
-
-// Return all known values from `from` to `to`. Gaps of information are represented as NaN.
-// Simple linear interpolation is done between the two neighboring cells if possible.
-// If values at the start or end are missing, instead of NaN values, the second and thrid
-// return values contain the actual `from`/`to`.
-// This function goes back the buffer chain if `from` is older than the currents buffer start.
-// The loaded values are added to `data` and `data` is returned, possibly with a shorter length.
-// If `data` is not long enough to hold all values, this function will panic!
-func (b *buffer) read(from, to int64, data []schema.Float) ([]schema.Float, int64, int64, error) {
-	if from < b.firstWrite() {
-		if b.prev != nil {
-			return b.prev.read(from, to, data)
-		}
-		from = b.firstWrite()
-	}
-
-	i := 0
-	t := from
-	for ; t < to; t += b.frequency {
-		idx := int((t - b.start) / b.frequency)
-		if idx >= cap(b.data) {
-			if b.next == nil {
-				break
-			}
-			b = b.next
-			idx = 0
-		}
-
-		if idx >= len(b.data) {
-			if b.next == nil || to <= b.next.start {
-				break
-			}
-			data[i] += schema.NaN
-		} else if t < b.start {
-			data[i] += schema.NaN
-			// } else if b.data[idx].IsNaN() {
-			// 	data[i] += interpolate(idx, b.data)
-		} else {
-			data[i] += b.data[idx]
-		}
-		i++
-	}
-
-	return data[:i], from, t, nil
-}
-
-// Returns true if this buffer needs to be freed.
-func (b *buffer) free(t int64) (delme bool, n int) {
-	if b.prev != nil {
-		delme, m := b.prev.free(t)
-		n += m
-		if delme {
-			b.prev.next = nil
-			if cap(b.prev.data) == BufferCap {
-				bufferPool.Put(b.prev)
-			}
-			b.prev = nil
-		}
-	}
-
-	end := b.end()
-	if end < t {
-		return true, n + 1
-	}
-
-	return false, n
-}
-
-// Call `callback` on every buffer that contains data in the range from `from` to `to`.
-func (b *buffer) iterFromTo(from, to int64, callback func(b *buffer) error) error {
-	if b == nil {
-		return nil
-	}
-
-	if err := b.prev.iterFromTo(from, to, callback); err != nil {
-		return err
-	}
-
-	if from <= b.end() && b.start <= to {
-		return callback(b)
-	}
-
-	return nil
-}
-
-func (b *buffer) count() int64 {
-	res := int64(len(b.data))
-	if b.prev != nil {
-		res += b.prev.count()
-	}
-	return res
-}

internal/memorystore/checkpoint.go

@@ -1,768 +0,0 @@
-// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved. This file is part of cc-backend.
-// Use of this source code is governed by a MIT-style
-// license that can be found in the LICENSE file.
-
-package memorystore
-
-import (
-	"bufio"
-	"context"
-	"encoding/json"
-	"errors"
-	"fmt"
-	"io/fs"
-	"os"
-	"path"
-	"path/filepath"
-	"runtime"
-	"sort"
-	"strconv"
-	"strings"
-	"sync"
-	"sync/atomic"
-	"time"
-
-	cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
-	"github.com/ClusterCockpit/cc-lib/schema"
-	"github.com/linkedin/goavro/v2"
-)
-
-// Whenever changed, update MarshalJSON as well!
-type CheckpointMetrics struct {
-	Data      []schema.Float `json:"data"`
-	Frequency int64          `json:"frequency"`
-	Start     int64          `json:"start"`
-}
-
-type CheckpointFile struct {
-	Metrics  map[string]*CheckpointMetrics `json:"metrics"`
-	Children map[string]*CheckpointFile    `json:"children"`
-	From     int64                         `json:"from"`
-	To       int64                         `json:"to"`
-}
-
-var lastCheckpoint time.Time
-
-func Checkpointing(wg *sync.WaitGroup, ctx context.Context) {
-	lastCheckpoint = time.Now()
-
-	if Keys.Checkpoints.FileFormat == "json" {
-		ms := GetMemoryStore()
-
-		go func() {
-			defer wg.Done()
-			d, err := time.ParseDuration(Keys.Checkpoints.Interval)
-			if err != nil {
-				cclog.Fatal(err)
-			}
-			if d <= 0 {
-				return
-			}
-
-			ticks := func() <-chan time.Time {
-				if d <= 0 {
-					return nil
-				}
-				return time.NewTicker(d).C
-			}()
-			for {
-				select {
-				case <-ctx.Done():
-					return
-				case <-ticks:
-					cclog.Printf("[METRICSTORE]> start checkpointing (starting at %s)...\n", lastCheckpoint.Format(time.RFC3339))
-					now := time.Now()
-					n, err := ms.ToCheckpoint(Keys.Checkpoints.RootDir,
-						lastCheckpoint.Unix(), now.Unix())
-					if err != nil {
-						cclog.Printf("[METRICSTORE]> checkpointing failed: %s\n", err.Error())
-					} else {
-						cclog.Printf("[METRICSTORE]> done: %d checkpoint files created\n", n)
-						lastCheckpoint = now
-					}
-				}
-			}
-		}()
-	} else {
-		go func() {
-			defer wg.Done()
-			d, _ := time.ParseDuration("1m")
-
-			select {
-			case <-ctx.Done():
-				return
-			case <-time.After(time.Duration(CheckpointBufferMinutes) * time.Minute):
-				// This is the first tick untill we collect the data for given minutes.
-				GetAvroStore().ToCheckpoint(Keys.Checkpoints.RootDir, false)
-				// log.Printf("Checkpointing %d avro files", count)
-
-			}
-
-			ticks := func() <-chan time.Time {
-				if d <= 0 {
-					return nil
-				}
-				return time.NewTicker(d).C
-			}()
-
-			for {
-				select {
-				case <-ctx.Done():
-					return
-				case <-ticks:
-					// Regular ticks of 1 minute to write data.
-					GetAvroStore().ToCheckpoint(Keys.Checkpoints.RootDir, false)
-					// log.Printf("Checkpointing %d avro files", count)
-				}
-			}
-		}()
-	}
-}
-
-// As `Float` implements a custom MarshalJSON() function,
-// serializing an array of such types has more overhead
-// than one would assume (because of extra allocations, interfaces and so on).
-func (cm *CheckpointMetrics) MarshalJSON() ([]byte, error) {
-	buf := make([]byte, 0, 128+len(cm.Data)*8)
-	buf = append(buf, `{"frequency":`...)
-	buf = strconv.AppendInt(buf, cm.Frequency, 10)
-	buf = append(buf, `,"start":`...)
-	buf = strconv.AppendInt(buf, cm.Start, 10)
-	buf = append(buf, `,"data":[`...)
-	for i, x := range cm.Data {
-		if i != 0 {
-			buf = append(buf, ',')
-		}
-		if x.IsNaN() {
-			buf = append(buf, `null`...)
-		} else {
-			buf = strconv.AppendFloat(buf, float64(x), 'f', 1, 32)
-		}
-	}
-	buf = append(buf, `]}`...)
-	return buf, nil
-}
-
-// Metrics stored at the lowest 2 levels are not stored away (root and cluster)!
-// On a per-host basis a new JSON file is created. I have no idea if this will scale.
-// The good thing: Only a host at a time is locked, so this function can run
-// in parallel to writes/reads.
-func (m *MemoryStore) ToCheckpoint(dir string, from, to int64) (int, error) {
-	levels := make([]*Level, 0)
-	selectors := make([][]string, 0)
-	m.root.lock.RLock()
-	for sel1, l1 := range m.root.children {
-		l1.lock.RLock()
-		for sel2, l2 := range l1.children {
-			levels = append(levels, l2)
-			selectors = append(selectors, []string{sel1, sel2})
-		}
-		l1.lock.RUnlock()
-	}
-	m.root.lock.RUnlock()
-
-	type workItem struct {
-		level    *Level
-		dir      string
-		selector []string
-	}
-
-	n, errs := int32(0), int32(0)
-
-	var wg sync.WaitGroup
-	wg.Add(NumWorkers)
-	work := make(chan workItem, NumWorkers*2)
-	for worker := 0; worker < NumWorkers; worker++ {
-		go func() {
-			defer wg.Done()
-
-			for workItem := range work {
-				if err := workItem.level.toCheckpoint(workItem.dir, from, to, m); err != nil {
-					if err == ErrNoNewArchiveData {
-						continue
-					}
-
-					cclog.Printf("[METRICSTORE]> error while checkpointing %#v: %s", workItem.selector, err.Error())
-					atomic.AddInt32(&errs, 1)
-				} else {
-					atomic.AddInt32(&n, 1)
-				}
-			}
-		}()
-	}
-
-	for i := 0; i < len(levels); i++ {
-		dir := path.Join(dir, path.Join(selectors[i]...))
-		work <- workItem{
-			level:    levels[i],
-			dir:      dir,
-			selector: selectors[i],
-		}
-	}
-
-	close(work)
-	wg.Wait()
-
-	if errs > 0 {
-		return int(n), fmt.Errorf("[METRICSTORE]> %d errors happend while creating checkpoints (%d successes)", errs, n)
-	}
-	return int(n), nil
-}
-
-func (l *Level) toCheckpointFile(from, to int64, m *MemoryStore) (*CheckpointFile, error) {
-	l.lock.RLock()
-	defer l.lock.RUnlock()
-
-	retval := &CheckpointFile{
-		From:     from,
-		To:       to,
-		Metrics:  make(map[string]*CheckpointMetrics),
-		Children: make(map[string]*CheckpointFile),
-	}
-
-	for metric, minfo := range m.Metrics {
-		b := l.metrics[minfo.offset]
-		if b == nil {
-			continue
-		}
-
-		allArchived := true
-		b.iterFromTo(from, to, func(b *buffer) error {
-			if !b.archived {
-				allArchived = false
-			}
-			return nil
-		})
-
-		if allArchived {
-			continue
-		}
-
-		data := make([]schema.Float, (to-from)/b.frequency+1)
-		data, start, end, err := b.read(from, to, data)
-		if err != nil {
-			return nil, err
-		}
-
-		for i := int((end - start) / b.frequency); i < len(data); i++ {
-			data[i] = schema.NaN
-		}
-
-		retval.Metrics[metric] = &CheckpointMetrics{
-			Frequency: b.frequency,
-			Start:     start,
-			Data:      data,
-		}
-	}
-
-	for name, child := range l.children {
-		val, err := child.toCheckpointFile(from, to, m)
-		if err != nil {
-			return nil, err
-		}
-
-		if val != nil {
-			retval.Children[name] = val
-		}
-	}
-
-	if len(retval.Children) == 0 && len(retval.Metrics) == 0 {
-		return nil, nil
-	}
-
-	return retval, nil
-}
-
-func (l *Level) toCheckpoint(dir string, from, to int64, m *MemoryStore) error {
-	cf, err := l.toCheckpointFile(from, to, m)
-	if err != nil {
-		return err
-	}
-
-	if cf == nil {
-		return ErrNoNewArchiveData
-	}
-
-	filepath := path.Join(dir, fmt.Sprintf("%d.json", from))
-	f, err := os.OpenFile(filepath, os.O_CREATE|os.O_WRONLY, 0o644)
-	if err != nil && os.IsNotExist(err) {
-		err = os.MkdirAll(dir, 0o755)
-		if err == nil {
-			f, err = os.OpenFile(filepath, os.O_CREATE|os.O_WRONLY, 0o644)
-		}
-	}
-	if err != nil {
-		return err
-	}
-	defer f.Close()
-
-	bw := bufio.NewWriter(f)
-	if err = json.NewEncoder(bw).Encode(cf); err != nil {
-		return err
-	}
-
-	return bw.Flush()
-}
-
-func (m *MemoryStore) FromCheckpoint(dir string, from int64, extension string) (int, error) {
-	var wg sync.WaitGroup
-	work := make(chan [2]string, NumWorkers)
-	n, errs := int32(0), int32(0)
-
-	wg.Add(NumWorkers)
-	for worker := 0; worker < NumWorkers; worker++ {
-		go func() {
-			defer wg.Done()
-			for host := range work {
-				lvl := m.root.findLevelOrCreate(host[:], len(m.Metrics))
-				nn, err := lvl.fromCheckpoint(m, filepath.Join(dir, host[0], host[1]), from, extension)
-				if err != nil {
-					cclog.Fatalf("[METRICSTORE]> error while loading checkpoints: %s", err.Error())
-					atomic.AddInt32(&errs, 1)
-				}
-				atomic.AddInt32(&n, int32(nn))
-			}
-		}()
-	}
-
-	i := 0
-	clustersDir, err := os.ReadDir(dir)
-	for _, clusterDir := range clustersDir {
-		if !clusterDir.IsDir() {
-			err = errors.New("[METRICSTORE]> expected only directories at first level of checkpoints/ directory")
-			goto done
-		}
-
-		hostsDir, e := os.ReadDir(filepath.Join(dir, clusterDir.Name()))
-		if e != nil {
-			err = e
-			goto done
-		}
-
-		for _, hostDir := range hostsDir {
-			if !hostDir.IsDir() {
-				err = errors.New("[METRICSTORE]> expected only directories at second level of checkpoints/ directory")
-				goto done
-			}
-
-			i++
-			if i%NumWorkers == 0 && i > 100 {
-				// Forcing garbage collection runs here regulary during the loading of checkpoints
-				// will decrease the total heap size after loading everything back to memory is done.
-				// While loading data, the heap will grow fast, so the GC target size will double
-				// almost always. By forcing GCs here, we can keep it growing more slowly so that
-				// at the end, less memory is wasted.
-				runtime.GC()
-			}
-
-			work <- [2]string{clusterDir.Name(), hostDir.Name()}
-		}
-	}
-done:
-	close(work)
-	wg.Wait()
-
-	if err != nil {
-		return int(n), err
-	}
-
-	if errs > 0 {
-		return int(n), fmt.Errorf("[METRICSTORE]> %d errors happend while creating checkpoints (%d successes)", errs, n)
-	}
-	return int(n), nil
-}
-
-// Metrics stored at the lowest 2 levels are not loaded (root and cluster)!
-// This function can only be called once and before the very first write or read.
-// Different host's data is loaded to memory in parallel.
-func (m *MemoryStore) FromCheckpointFiles(dir string, from int64) (int, error) {
-	if _, err := os.Stat(dir); os.IsNotExist(err) {
-		// The directory does not exist, so create it using os.MkdirAll()
-		err := os.MkdirAll(dir, 0o755) // 0755 sets the permissions for the directory
-		if err != nil {
-			cclog.Fatalf("[METRICSTORE]> Error creating directory: %#v\n", err)
-		}
-		cclog.Printf("[METRICSTORE]> %#v Directory created successfully.\n", dir)
-	}
-
-	// Config read (replace with your actual config read)
-	fileFormat := Keys.Checkpoints.FileFormat
-	if fileFormat == "" {
-		fileFormat = "avro"
-	}
-
-	// Map to easily get the fallback format
-	oppositeFormat := map[string]string{
-		"json": "avro",
-		"avro": "json",
-	}
-
-	// First, attempt to load the specified format
-	if found, err := checkFilesWithExtension(dir, fileFormat); err != nil {
-		return 0, fmt.Errorf("[METRICSTORE]> error checking files with extension: %v", err)
-	} else if found {
-		cclog.Printf("[METRICSTORE]> Loading %s files because fileformat is %s\n", fileFormat, fileFormat)
-		return m.FromCheckpoint(dir, from, fileFormat)
-	}
-
-	// If not found, attempt the opposite format
-	altFormat := oppositeFormat[fileFormat]
-	if found, err := checkFilesWithExtension(dir, altFormat); err != nil {
-		return 0, fmt.Errorf("[METRICSTORE]> error checking files with extension: %v", err)
-	} else if found {
-		cclog.Printf("[METRICSTORE]> Loading %s files but fileformat is %s\n", altFormat, fileFormat)
-		return m.FromCheckpoint(dir, from, altFormat)
-	}
-
-	cclog.Print("[METRICSTORE]> No valid checkpoint files found in the directory")
-	return 0, nil
-}
-
-func checkFilesWithExtension(dir string, extension string) (bool, error) {
-	found := false
-
-	err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
-		if err != nil {
-			return fmt.Errorf("[METRICSTORE]> error accessing path %s: %v", path, err)
-		}
-		if !info.IsDir() && filepath.Ext(info.Name()) == "."+extension {
-			found = true
-			return nil
-		}
-		return nil
-	})
-	if err != nil {
-		return false, fmt.Errorf("[METRICSTORE]> error walking through directories: %s", err)
-	}
-
-	return found, nil
-}
-
-func (l *Level) loadAvroFile(m *MemoryStore, f *os.File, from int64) error {
-	br := bufio.NewReader(f)
-
-	fileName := f.Name()[strings.LastIndex(f.Name(), "/")+1:]
-	resolution, err := strconv.ParseInt(fileName[0:strings.Index(fileName, "_")], 10, 64)
-	if err != nil {
-		return fmt.Errorf("[METRICSTORE]> error while reading avro file (resolution parsing) : %s", err)
-	}
-
-	fromTimestamp, err := strconv.ParseInt(fileName[strings.Index(fileName, "_")+1:len(fileName)-5], 10, 64)
-
-	// Same logic according to lineprotocol
-	fromTimestamp -= (resolution / 2)
-
-	if err != nil {
-		return fmt.Errorf("[METRICSTORE]> error converting timestamp from the avro file : %s", err)
-	}
-
-	// fmt.Printf("File : %s with resolution : %d\n", fileName, resolution)
-
-	var recordCounter int64 = 0
-
-	// Create a new OCF reader from the buffered reader
-	ocfReader, err := goavro.NewOCFReader(br)
-	if err != nil {
-		panic(err)
-	}
-
-	metricsData := make(map[string]schema.FloatArray)
-
-	for ocfReader.Scan() {
-		datum, err := ocfReader.Read()
-		if err != nil {
-			return fmt.Errorf("[METRICSTORE]> error while reading avro file : %s", err)
-		}
-
-		record, ok := datum.(map[string]any)
-		if !ok {
-			panic("[METRICSTORE]> failed to assert datum as map[string]interface{}")
-		}
-
-		for key, value := range record {
-			metricsData[key] = append(metricsData[key], schema.ConvertToFloat(value.(float64)))
-		}
-
-		recordCounter += 1
-	}
-
-	to := (fromTimestamp + (recordCounter / (60 / resolution) * 60))
-	if to < from {
-		return nil
-	}
-
-	for key, floatArray := range metricsData {
-		metricName := ReplaceKey(key)
-
-		if strings.Contains(metricName, Delimiter) {
-			subString := strings.Split(metricName, Delimiter)
-
-			lvl := l
-
-			for i := 0; i < len(subString)-1; i++ {
-
-				sel := subString[i]
-
-				if lvl.children == nil {
-					lvl.children = make(map[string]*Level)
-				}
-
-				child, ok := lvl.children[sel]
-				if !ok {
-					child = &Level{
-						metrics:  make([]*buffer, len(m.Metrics)),
-						children: nil,
-					}
-					lvl.children[sel] = child
-				}
-				lvl = child
-			}
-
-			leafMetricName := subString[len(subString)-1]
-			err = lvl.createBuffer(m, leafMetricName, floatArray, fromTimestamp, resolution)
-			if err != nil {
-				return fmt.Errorf("[METRICSTORE]> error while creating buffers from avroReader : %s", err)
-			}
-		} else {
-			err = l.createBuffer(m, metricName, floatArray, fromTimestamp, resolution)
-			if err != nil {
-				return fmt.Errorf("[METRICSTORE]> error while creating buffers from avroReader : %s", err)
-			}
-		}
-
-	}
-
-	return nil
-}
-
-func (l *Level) createBuffer(m *MemoryStore, metricName string, floatArray schema.FloatArray, from int64, resolution int64) error {
-	n := len(floatArray)
-	b := &buffer{
-		frequency: resolution,
-		start:     from,
-		data:      floatArray[0:n:n],
-		prev:      nil,
-		next:      nil,
-		archived:  true,
-	}
-	b.close()
-
-	minfo, ok := m.Metrics[metricName]
-	if !ok {
-		return nil
-		// return errors.New("Unkown metric: " + name)
-	}
-
-	prev := l.metrics[minfo.offset]
-	if prev == nil {
-		l.metrics[minfo.offset] = b
-	} else {
-		if prev.start > b.start {
-			return errors.New("wooops")
-		}
-
-		b.prev = prev
-		prev.next = b
-
-		missingCount := ((int(b.start) - int(prev.start)) - len(prev.data)*int(b.frequency))
-		if missingCount > 0 {
-			missingCount /= int(b.frequency)
-
-			for range missingCount {
-				prev.data = append(prev.data, schema.NaN)
-			}
-
-			prev.data = prev.data[0:len(prev.data):len(prev.data)]
-		}
-	}
-	l.metrics[minfo.offset] = b
-
-	return nil
-}
-
-func (l *Level) loadJSONFile(m *MemoryStore, f *os.File, from int64) error {
-	br := bufio.NewReader(f)
-	cf := &CheckpointFile{}
-	if err := json.NewDecoder(br).Decode(cf); err != nil {
-		return err
-	}
-
-	if cf.To != 0 && cf.To < from {
-		return nil
-	}
-
-	if err := l.loadFile(cf, m); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func (l *Level) loadFile(cf *CheckpointFile, m *MemoryStore) error {
-	for name, metric := range cf.Metrics {
-		n := len(metric.Data)
-		b := &buffer{
-			frequency: metric.Frequency,
-			start:     metric.Start,
-			data:      metric.Data[0:n:n], // Space is wasted here :(
-			prev:      nil,
-			next:      nil,
-			archived:  true,
-		}
-		b.close()
-
-		minfo, ok := m.Metrics[name]
-		if !ok {
-			continue
-			// return errors.New("Unkown metric: " + name)
-		}
-
-		prev := l.metrics[minfo.offset]
-		if prev == nil {
-			l.metrics[minfo.offset] = b
-		} else {
-			if prev.start > b.start {
-				return errors.New("wooops")
-			}
-
-			b.prev = prev
-			prev.next = b
-		}
-		l.metrics[minfo.offset] = b
-	}
-
-	if len(cf.Children) > 0 && l.children == nil {
-		l.children = make(map[string]*Level)
-	}
-
-	for sel, childCf := range cf.Children {
-		child, ok := l.children[sel]
-		if !ok {
-			child = &Level{
-				metrics:  make([]*buffer, len(m.Metrics)),
-				children: nil,
-			}
-			l.children[sel] = child
-		}
-
-		if err := child.loadFile(childCf, m); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func (l *Level) fromCheckpoint(m *MemoryStore, dir string, from int64, extension string) (int, error) {
-	direntries, err := os.ReadDir(dir)
-	if err != nil {
-		if os.IsNotExist(err) {
-			return 0, nil
-		}
-
-		return 0, err
-	}
-
-	allFiles := make([]fs.DirEntry, 0)
-	filesLoaded := 0
-	for _, e := range direntries {
-		if e.IsDir() {
-			child := &Level{
-				metrics:  make([]*buffer, len(m.Metrics)),
-				children: make(map[string]*Level),
-			}
-
-			files, err := child.fromCheckpoint(m, path.Join(dir, e.Name()), from, extension)
-			filesLoaded += files
-			if err != nil {
-				return filesLoaded, err
-			}
-
-			l.children[e.Name()] = child
-		} else if strings.HasSuffix(e.Name(), "."+extension) {
-			allFiles = append(allFiles, e)
-		} else {
-			continue
-		}
-	}
-
-	files, err := findFiles(allFiles, from, extension, true)
-	if err != nil {
-		return filesLoaded, err
-	}
-
-	loaders := map[string]func(*MemoryStore, *os.File, int64) error{
-		"json": l.loadJSONFile,
-		"avro": l.loadAvroFile,
-	}
-
-	loader := loaders[extension]
-
-	for _, filename := range files {
-		f, err := os.Open(path.Join(dir, filename))
-		if err != nil {
-			return filesLoaded, err
-		}
-		defer f.Close()
-
-		if err = loader(m, f, from); err != nil {
-			return filesLoaded, err
-		}
-
-		filesLoaded += 1
-	}
-
-	return filesLoaded, nil
-}
-
-// This will probably get very slow over time!
-// A solution could be some sort of an index file in which all other files
-// and the timespan they contain is listed.
-func findFiles(direntries []fs.DirEntry, t int64, extension string, findMoreRecentFiles bool) ([]string, error) {
-	nums := map[string]int64{}
-	for _, e := range direntries {
-		if !strings.HasSuffix(e.Name(), "."+extension) {
-			continue
-		}
-
-		ts, err := strconv.ParseInt(e.Name()[strings.Index(e.Name(), "_")+1:len(e.Name())-5], 10, 64)
-		if err != nil {
-			return nil, err
-		}
-		nums[e.Name()] = ts
-	}
-
-	sort.Slice(direntries, func(i, j int) bool {
-		a, b := direntries[i], direntries[j]
-		return nums[a.Name()] < nums[b.Name()]
-	})
-
-	filenames := make([]string, 0)
-	for i := range direntries {
-		e := direntries[i]
-		ts1 := nums[e.Name()]
-
-		if findMoreRecentFiles && t <= ts1 {
-			filenames = append(filenames, e.Name())
-		}
-		if i == len(direntries)-1 {
-			continue
-		}
-
-		enext := direntries[i+1]
-		ts2 := nums[enext.Name()]
-
-		if findMoreRecentFiles {
-			if ts1 < t && t < ts2 {
-				filenames = append(filenames, e.Name())
-			}
-		} else {
-			if ts2 < t {
-				filenames = append(filenames, e.Name())
-			}
-		}
-	}
-
-	return filenames, nil
-}

internal/memorystore/config.go

@@ -1,118 +0,0 @@
-// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved. This file is part of cc-backend.
-// Use of this source code is governed by a MIT-style
-// license that can be found in the LICENSE file.
-
-package memorystore
-
-import (
-	"fmt"
-)
-
-var InternalCCMSFlag bool = false
-
-type MetricStoreConfig struct {
-	Checkpoints struct {
-		FileFormat string `json:"file-format"`
-		Interval   string `json:"interval"`
-		RootDir    string `json:"directory"`
-		Restore    string `json:"restore"`
-	} `json:"checkpoints"`
-	Debug struct {
-		DumpToFile string `json:"dump-to-file"`
-		EnableGops bool   `json:"gops"`
-	} `json:"debug"`
-	RetentionInMemory string `json:"retention-in-memory"`
-	Archive           struct {
-		Interval      string `json:"interval"`
-		RootDir       string `json:"directory"`
-		DeleteInstead bool   `json:"delete-instead"`
-	} `json:"archive"`
-	Nats []*NatsConfig `json:"nats"`
-}
-
-type NatsConfig struct {
-	// Address of the nats server
-	Address string `json:"address"`
-
-	// Username/Password, optional
-	Username string `json:"username"`
-	Password string `json:"password"`
-
-	// Creds file path
-	Credsfilepath string `json:"creds-file-path"`
-
-	Subscriptions []struct {
-		// Channel name
-		SubscribeTo string `json:"subscribe-to"`
-
-		// Allow lines without a cluster tag, use this as default, optional
-		ClusterTag string `json:"cluster-tag"`
-	} `json:"subscriptions"`
-}
-
-var Keys MetricStoreConfig
-
-// AggregationStrategy for aggregation over multiple values at different cpus/sockets/..., not time!
-type AggregationStrategy int
-
-const (
-	NoAggregation AggregationStrategy = iota
-	SumAggregation
-	AvgAggregation
-)
-
-func AssignAggregationStratergy(str string) (AggregationStrategy, error) {
-	switch str {
-	case "":
-		return NoAggregation, nil
-	case "sum":
-		return SumAggregation, nil
-	case "avg":
-		return AvgAggregation, nil
-	default:
-		return NoAggregation, fmt.Errorf("[METRICSTORE]> unknown aggregation strategy: %s", str)
-	}
-}
-
-type MetricConfig struct {
-	// Interval in seconds at which measurements are stored
-	Frequency int64
-
-	// Can be 'sum', 'avg' or null. Describes how to aggregate metrics from the same timestep over the hierarchy.
-	Aggregation AggregationStrategy
-
-	// Private, used internally...
-	offset int
-}
-
-var Metrics map[string]MetricConfig
-
-func GetMetricFrequency(metricName string) (int64, error) {
-	if metric, ok := Metrics[metricName]; ok {
-		return metric.Frequency, nil
-	}
-	return 0, fmt.Errorf("[METRICSTORE]> metric %s not found", metricName)
-}
-
-// AddMetric adds logic to add metrics. Redundant metrics should be updated with max frequency.
-// use metric.Name to check if the metric already exists.
-// if not, add it to the Metrics map.
-func AddMetric(name string, metric MetricConfig) error {
-	if Metrics == nil {
-		Metrics = make(map[string]MetricConfig, 0)
-	}
-
-	if existingMetric, ok := Metrics[name]; ok {
-		if existingMetric.Frequency != metric.Frequency {
-			if existingMetric.Frequency < metric.Frequency {
-				existingMetric.Frequency = metric.Frequency
-				Metrics[name] = existingMetric
-			}
-		}
-	} else {
-		Metrics[name] = metric
-	}
-
-	return nil
-}

internal/memorystore/configSchema.go

@@ -1,89 +0,0 @@
-// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved. This file is part of cc-backend.
-// Use of this source code is governed by a MIT-style
-// license that can be found in the LICENSE file.
-
-package memorystore
-
-const configSchema = `{
-    "type": "object",
-    "description": "Configuration specific to built-in metric-store.",
-    "properties": {
-        "checkpoints": {
-            "description": "Configuration for checkpointing the metrics within metric-store",
-            "type": "object",
-            "properties": {
-                "file-format": {
-                    "description": "Specify the type of checkpoint file. There are 2 variants: 'avro' and 'json'. If nothing is specified, 'avro' is default.",
-                    "type": "string"
-                },
-                "interval": {
-                    "description": "Interval at which the metrics should be checkpointed.",
-                    "type": "string"
-                },
-                "directory": {
-                    "description": "Specify the parent directy in which the checkpointed files should be placed.",
-                    "type": "string"
-                },
-                "restore": {
-                    "description": "When cc-backend starts up, look for checkpointed files that are less than X hours old and load metrics from these selected checkpoint files.",
-                    "type": "string"
-                }
-            }
-        },
-        "archive": {
-            "description": "Configuration for archiving the already checkpointed files.",
-            "type": "object",
-            "properties": {
-                "interval": {
-                    "description": "Interval at which the checkpointed files should be archived.",
-                    "type": "string"
-                },
-                "directory": {
-                    "description": "Specify the parent directy in which the archived files should be placed.",
-                    "type": "string"
-                }
-            }
-        },
-        "retention-in-memory": {
-            "description": "Keep the metrics within memory for given time interval. Retention for X hours, then the metrics would be freed.",
-            "type": "string"
-        },
-        "nats": {
-            "description": "Configuration for accepting published data through NATS.",
-            "type": "object",
-            "properties": {
-                "address": {
-                    "description": "Address of the NATS server.",
-                    "type": "string"
-                },
-                "username": {
-                    "description": "Optional: If configured with username/password method.",
-                    "type": "string"
-                },
-                "password": {
-                    "description": "Optional: If configured with username/password method.",
-                    "type": "string"
-                },
-                "creds-file-path": {
-                    "description": "Optional: If configured with Credential File method. Path to your NATS cred file.",
-                    "type": "string"
-                },
-                "subscriptions": {
-                    "description": "Array of various subscriptions. Allows to subscibe to different subjects and publishers.",
-                    "type": "object",
-                    "properties": {
-                        "subscribe-to": {
-                            "description": "Channel name",
-                            "type": "string"
-                        },
-                        "cluster-tag": {
-                            "description": "Optional: Allow lines without a cluster tag, use this as default",
-                            "type": "string"
-                        }
-                    }
-                }
-            }
-        }
-    }
-}`

internal/memorystore/debug.go

@@ -1,112 +0,0 @@
-// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved. This file is part of cc-backend.
-// Use of this source code is governed by a MIT-style
-// license that can be found in the LICENSE file.
-
-package memorystore
-
-import (
-	"bufio"
-	"fmt"
-	"strconv"
-)
-
-func (b *buffer) debugDump(buf []byte) []byte {
-	if b.prev != nil {
-		buf = b.prev.debugDump(buf)
-	}
-
-	start, len, end := b.start, len(b.data), b.start+b.frequency*int64(len(b.data))
-	buf = append(buf, `{"start":`...)
-	buf = strconv.AppendInt(buf, start, 10)
-	buf = append(buf, `,"len":`...)
-	buf = strconv.AppendInt(buf, int64(len), 10)
-	buf = append(buf, `,"end":`...)
-	buf = strconv.AppendInt(buf, end, 10)
-	if b.archived {
-		buf = append(buf, `,"saved":true`...)
-	}
-	if b.next != nil {
-		buf = append(buf, `},`...)
-	} else {
-		buf = append(buf, `}`...)
-	}
-	return buf
-}
-
-func (l *Level) debugDump(m *MemoryStore, w *bufio.Writer, lvlname string, buf []byte, depth int) ([]byte, error) {
-	l.lock.RLock()
-	defer l.lock.RUnlock()
-	for i := 0; i < depth; i++ {
-		buf = append(buf, '\t')
-	}
-	buf = append(buf, '"')
-	buf = append(buf, lvlname...)
-	buf = append(buf, "\":{\n"...)
-	depth += 1
-	objitems := 0
-	for name, mc := range m.Metrics {
-		if b := l.metrics[mc.offset]; b != nil {
-			for i := 0; i < depth; i++ {
-				buf = append(buf, '\t')
-			}
-
-			buf = append(buf, '"')
-			buf = append(buf, name...)
-			buf = append(buf, `":[`...)
-			buf = b.debugDump(buf)
-			buf = append(buf, "],\n"...)
-			objitems++
-		}
-	}
-
-	for name, lvl := range l.children {
-		_, err := w.Write(buf)
-		if err != nil {
-			return nil, err
-		}
-
-		buf = buf[0:0]
-		buf, err = lvl.debugDump(m, w, name, buf, depth)
-		if err != nil {
-			return nil, err
-		}
-
-		buf = append(buf, ',', '\n')
-		objitems++
-	}
-
-	// remove final `,`:
-	if objitems > 0 {
-		buf = append(buf[0:len(buf)-1], '\n')
-	}
-
-	depth -= 1
-	for i := 0; i < depth; i++ {
-		buf = append(buf, '\t')
-	}
-	buf = append(buf, '}')
-	return buf, nil
-}
-
-func (m *MemoryStore) DebugDump(w *bufio.Writer, selector []string) error {
-	lvl := m.root.findLevel(selector)
-	if lvl == nil {
-		return fmt.Errorf("[METRICSTORE]> not found: %#v", selector)
-	}
-
-	buf := make([]byte, 0, 2048)
-	buf = append(buf, "{"...)
-
-	buf, err := lvl.debugDump(m, w, "data", buf, 0)
-	if err != nil {
-		return err
-	}
-
-	buf = append(buf, "}\n"...)
-	if _, err = w.Write(buf); err != nil {
-		return err
-	}
-
-	return w.Flush()
-}

internal/memorystore/healthcheck.go

@@ -1,92 +0,0 @@
-// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved. This file is part of cc-backend.
-// Use of this source code is governed by a MIT-style
-// license that can be found in the LICENSE file.
-
-package memorystore
-
-import (
-	"bufio"
-	"fmt"
-	"time"
-)
-
-// MaxMissingDataPoints is a threshold that allows a node to be healthy with certain number of data points missing.
-// Suppose a node does not receive last 5 data points, then healthCheck endpoint will still say a
-// node is healthy. Anything more than 5 missing points in metrics of the node will deem the node unhealthy.
-const MaxMissingDataPoints int64 = 5
-
-// MaxUnhealthyMetrics is a threshold which allows upto certain number of metrics in a node to be unhealthly.
-// Works with MaxMissingDataPoints. Say 5 metrics (including submetrics) do not receive the last
-// MaxMissingDataPoints data points, then the node will be deemed healthy. Any more metrics that does
-// not receive data for MaxMissingDataPoints data points will deem the node unhealthy.
-const MaxUnhealthyMetrics int64 = 5
-
-func (b *buffer) healthCheck() int64 {
-	// Check if the buffer is empty
-	if b.data == nil {
-		return 1
-	}
-
-	bufferEnd := b.start + b.frequency*int64(len(b.data))
-	t := time.Now().Unix()
-
-	// Check if the buffer is too old
-	if t-bufferEnd > MaxMissingDataPoints*b.frequency {
-		return 1
-	}
-
-	return 0
-}
-
-func (l *Level) healthCheck(m *MemoryStore, count int64) (int64, error) {
-	l.lock.RLock()
-	defer l.lock.RUnlock()
-
-	for _, mc := range m.Metrics {
-		if b := l.metrics[mc.offset]; b != nil {
-			count += b.healthCheck()
-		}
-	}
-
-	for _, lvl := range l.children {
-		c, err := lvl.healthCheck(m, 0)
-		if err != nil {
-			return 0, err
-		}
-		count += c
-	}
-
-	return count, nil
-}
-
-func (m *MemoryStore) HealthCheck(w *bufio.Writer, selector []string) error {
-	lvl := m.root.findLevel(selector)
-	if lvl == nil {
-		return fmt.Errorf("[METRICSTORE]> not found: %#v", selector)
-	}
-
-	buf := make([]byte, 0, 25)
-	// buf = append(buf, "{"...)
-
-	var count int64 = 0
-
-	unhealthyMetricsCount, err := lvl.healthCheck(m, count)
-	if err != nil {
-		return err
-	}
-
-	if unhealthyMetricsCount < MaxUnhealthyMetrics {
-		buf = append(buf, "Healthy"...)
-	} else {
-		buf = append(buf, "Unhealthy"...)
-	}
-
-	// buf = append(buf, "}\n"...)
-
-	if _, err = w.Write(buf); err != nil {
-		return err
-	}
-
-	return w.Flush()
-}

internal/memorystore/level.go

@@ -1,192 +0,0 @@
-// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved. This file is part of cc-backend.
-// Use of this source code is governed by a MIT-style
-// license that can be found in the LICENSE file.
-
-package memorystore
-
-import (
-	"sync"
-	"unsafe"
-
-	"github.com/ClusterCockpit/cc-lib/util"
-)
-
-// Could also be called "node" as this forms a node in a tree structure.
-// Called Level because "node" might be confusing here.
-// Can be both a leaf or a inner node. In this tree structue, inner nodes can
-// also hold data (in `metrics`).
-type Level struct {
-	children map[string]*Level
-	metrics  []*buffer
-	lock     sync.RWMutex
-}
-
-// Find the correct level for the given selector, creating it if
-// it does not exist. Example selector in the context of the
-// ClusterCockpit could be: []string{ "emmy", "host123", "cpu0" }.
-// This function would probably benefit a lot from `level.children` beeing a `sync.Map`?
-func (l *Level) findLevelOrCreate(selector []string, nMetrics int) *Level {
-	if len(selector) == 0 {
-		return l
-	}
-
-	// Allow concurrent reads:
-	l.lock.RLock()
-	var child *Level
-	var ok bool
-	if l.children == nil {
-		// Children map needs to be created...
-		l.lock.RUnlock()
-	} else {
-		child, ok := l.children[selector[0]]
-		l.lock.RUnlock()
-		if ok {
-			return child.findLevelOrCreate(selector[1:], nMetrics)
-		}
-	}
-
-	// The level does not exist, take write lock for unqiue access:
-	l.lock.Lock()
-	// While this thread waited for the write lock, another thread
-	// could have created the child node.
-	if l.children != nil {
-		child, ok = l.children[selector[0]]
-		if ok {
-			l.lock.Unlock()
-			return child.findLevelOrCreate(selector[1:], nMetrics)
-		}
-	}
-
-	child = &Level{
-		metrics:  make([]*buffer, nMetrics),
-		children: nil,
-	}
-
-	if l.children != nil {
-		l.children[selector[0]] = child
-	} else {
-		l.children = map[string]*Level{selector[0]: child}
-	}
-	l.lock.Unlock()
-	return child.findLevelOrCreate(selector[1:], nMetrics)
-}
-
-func (l *Level) free(t int64) (int, error) {
-	l.lock.Lock()
-	defer l.lock.Unlock()
-
-	n := 0
-	for i, b := range l.metrics {
-		if b != nil {
-			delme, m := b.free(t)
-			n += m
-			if delme {
-				if cap(b.data) == BufferCap {
-					bufferPool.Put(b)
-				}
-				l.metrics[i] = nil
-			}
-		}
-	}
-
-	for _, l := range l.children {
-		m, err := l.free(t)
-		n += m
-		if err != nil {
-			return n, err
-		}
-	}
-
-	return n, nil
-}
-
-func (l *Level) sizeInBytes() int64 {
-	l.lock.RLock()
-	defer l.lock.RUnlock()
-	size := int64(0)
-
-	for _, b := range l.metrics {
-		if b != nil {
-			size += b.count() * int64(unsafe.Sizeof(util.Float(0)))
-		}
-	}
-
-	for _, child := range l.children {
-		size += child.sizeInBytes()
-	}
-
-	return size
-}
-
-func (l *Level) findLevel(selector []string) *Level {
-	if len(selector) == 0 {
-		return l
-	}
-
-	l.lock.RLock()
-	defer l.lock.RUnlock()
-
-	lvl := l.children[selector[0]]
-	if lvl == nil {
-		return nil
-	}
-
-	return lvl.findLevel(selector[1:])
-}
-
-func (l *Level) findBuffers(selector util.Selector, offset int, f func(b *buffer) error) error {
-	l.lock.RLock()
-	defer l.lock.RUnlock()
-
-	if len(selector) == 0 {
-		b := l.metrics[offset]
-		if b != nil {
-			return f(b)
-		}
-
-		for _, lvl := range l.children {
-			err := lvl.findBuffers(nil, offset, f)
-			if err != nil {
-				return err
-			}
-		}
-		return nil
-	}
-
-	sel := selector[0]
-	if len(sel.String) != 0 && l.children != nil {
-		lvl, ok := l.children[sel.String]
-		if ok {
-			err := lvl.findBuffers(selector[1:], offset, f)
-			if err != nil {
-				return err
-			}
-		}
-		return nil
-	}
-
-	if sel.Group != nil && l.children != nil {
-		for _, key := range sel.Group {
-			lvl, ok := l.children[key]
-			if ok {
-				err := lvl.findBuffers(selector[1:], offset, f)
-				if err != nil {
-					return err
-				}
-			}
-		}
-		return nil
-	}
-
-	if sel.Any && l.children != nil {
-		for _, lvl := range l.children {
-			if err := lvl.findBuffers(selector[1:], offset, f); err != nil {
-				return err
-			}
-		}
-		return nil
-	}
-
-	return nil
-}

internal/memorystore/lineprotocol.go

@@ -1,351 +0,0 @@
-// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved. This file is part of cc-backend.
-// Use of this source code is governed by a MIT-style
-// license that can be found in the LICENSE file.
-
-package memorystore
-
-import (
-	"context"
-	"fmt"
-	"sync"
-	"time"
-
-	cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
-	"github.com/ClusterCockpit/cc-lib/schema"
-	"github.com/influxdata/line-protocol/v2/lineprotocol"
-	"github.com/nats-io/nats.go"
-)
-
-// Each connection is handled in it's own goroutine. This is a blocking function.
-// func ReceiveRaw(ctx context.Context,
-// 	listener net.Listener,
-// 	handleLine func(*lineprotocol.Decoder, string) error,
-// ) error {
-// 	var wg sync.WaitGroup
-
-// 	wg.Add(1)
-// 	go func() {
-// 		defer wg.Done()
-// 		<-ctx.Done()
-// 		if err := listener.Close(); err != nil {
-// 			log.Printf("listener.Close(): %s", err.Error())
-// 		}
-// 	}()
-
-// 	for {
-// 		conn, err := listener.Accept()
-// 		if err != nil {
-// 			if errors.Is(err, net.ErrClosed) {
-// 				break
-// 			}
-
-// 			log.Printf("listener.Accept(): %s", err.Error())
-// 		}
-
-// 		wg.Add(2)
-// 		go func() {
-// 			defer wg.Done()
-// 			defer conn.Close()
-
-// 			dec := lineprotocol.NewDecoder(conn)
-// 			connctx, cancel := context.WithCancel(context.Background())
-// 			defer cancel()
-// 			go func() {
-// 				defer wg.Done()
-// 				select {
-// 				case <-connctx.Done():
-// 					conn.Close()
-// 				case <-ctx.Done():
-// 					conn.Close()
-// 				}
-// 			}()
-
-// 			if err := handleLine(dec, "default"); err != nil {
-// 				if errors.Is(err, net.ErrClosed) {
-// 					return
-// 				}
-
-// 				log.Printf("%s: %s", conn.RemoteAddr().String(), err.Error())
-// 				errmsg := make([]byte, 128)
-// 				errmsg = append(errmsg, `error: `...)
-// 				errmsg = append(errmsg, err.Error()...)
-// 				errmsg = append(errmsg, '\n')
-// 				conn.Write(errmsg)
-// 			}
-// 		}()
-// 	}
-
-// 	wg.Wait()
-// 	return nil
-// }
-
-// ReceiveNats connects to a nats server and subscribes to "updates". This is a
-// blocking function. handleLine will be called for each line recieved via
-// nats. Send `true` through the done channel for gracefull termination.
-func ReceiveNats(conf *(NatsConfig),
-	ms *MemoryStore,
-	workers int,
-	ctx context.Context,
-) error {
-	var opts []nats.Option
-	if conf.Username != "" && conf.Password != "" {
-		opts = append(opts, nats.UserInfo(conf.Username, conf.Password))
-	}
-
-	if conf.Credsfilepath != "" {
-		opts = append(opts, nats.UserCredentials(conf.Credsfilepath))
-	}
-
-	nc, err := nats.Connect(conf.Address, opts...)
-	if err != nil {
-		return err
-	}
-	defer nc.Close()
-
-	var wg sync.WaitGroup
-	var subs []*nats.Subscription
-
-	msgs := make(chan *nats.Msg, workers*2)
-
-	for _, sc := range conf.Subscriptions {
-		clusterTag := sc.ClusterTag
-		var sub *nats.Subscription
-		if workers > 1 {
-			wg.Add(workers)
-
-			for range workers {
-				go func() {
-					for m := range msgs {
-						dec := lineprotocol.NewDecoderWithBytes(m.Data)
-						if err := DecodeLine(dec, ms, clusterTag); err != nil {
-							cclog.Printf("error: %s\n", err.Error())
-						}
-					}
-
-					wg.Done()
-				}()
-			}
-
-			sub, err = nc.Subscribe(sc.SubscribeTo, func(m *nats.Msg) {
-				msgs <- m
-			})
-		} else {
-			sub, err = nc.Subscribe(sc.SubscribeTo, func(m *nats.Msg) {
-				dec := lineprotocol.NewDecoderWithBytes(m.Data)
-				if err := DecodeLine(dec, ms, clusterTag); err != nil {
-					cclog.Printf("error: %s\n", err.Error())
-				}
-			})
-		}
-
-		if err != nil {
-			return err
-		}
-		cclog.Printf("NATS subscription to '%s' on '%s' established\n", sc.SubscribeTo, conf.Address)
-		subs = append(subs, sub)
-	}
-
-	<-ctx.Done()
-	for _, sub := range subs {
-		err = sub.Unsubscribe()
-		if err != nil {
-			cclog.Printf("NATS unsubscribe failed: %s", err.Error())
-		}
-	}
-	close(msgs)
-	wg.Wait()
-
-	nc.Close()
-	cclog.Print("NATS connection closed")
-	return nil
-}
-
-// Place `prefix` in front of `buf` but if possible,
-// do that inplace in `buf`.
-func reorder(buf, prefix []byte) []byte {
-	n := len(prefix)
-	m := len(buf)
-	if cap(buf) < m+n {
-		return append(prefix[:n:n], buf...)
-	} else {
-		buf = buf[:n+m]
-		for i := m - 1; i >= 0; i-- {
-			buf[i+n] = buf[i]
-		}
-		for i := range n {
-			buf[i] = prefix[i]
-		}
-		return buf
-	}
-}
-
-// Decode lines using dec and make write calls to the MemoryStore.
-// If a line is missing its cluster tag, use clusterDefault as default.
-func DecodeLine(dec *lineprotocol.Decoder,
-	ms *MemoryStore,
-	clusterDefault string,
-) error {
-	// Reduce allocations in loop:
-	t := time.Now()
-	metric, metricBuf := Metric{}, make([]byte, 0, 16)
-	selector := make([]string, 0, 4)
-	typeBuf, subTypeBuf := make([]byte, 0, 16), make([]byte, 0)
-
-	// Optimize for the case where all lines in a "batch" are about the same
-	// cluster and host. By using `WriteToLevel` (level = host), we do not need
-	// to take the root- and cluster-level lock as often.
-	var lvl *Level = nil
-	prevCluster, prevHost := "", ""
-
-	var ok bool
-	for dec.Next() {
-		rawmeasurement, err := dec.Measurement()
-		if err != nil {
-			return err
-		}
-
-		// Needs to be copied because another call to dec.* would
-		// invalidate the returned slice.
-		metricBuf = append(metricBuf[:0], rawmeasurement...)
-
-		// The go compiler optimizes map[string(byteslice)] lookups:
-		metric.MetricConfig, ok = ms.Metrics[string(rawmeasurement)]
-		if !ok {
-			continue
-		}
-
-		typeBuf, subTypeBuf := typeBuf[:0], subTypeBuf[:0]
-		cluster, host := clusterDefault, ""
-		for {
-			key, val, err := dec.NextTag()
-			if err != nil {
-				return err
-			}
-			if key == nil {
-				break
-			}
-
-			// The go compiler optimizes string([]byte{...}) == "...":
-			switch string(key) {
-			case "cluster":
-				if string(val) == prevCluster {
-					cluster = prevCluster
-				} else {
-					cluster = string(val)
-					lvl = nil
-				}
-			case "hostname", "host":
-				if string(val) == prevHost {
-					host = prevHost
-				} else {
-					host = string(val)
-					lvl = nil
-				}
-			case "type":
-				if string(val) == "node" {
-					break
-				}
-
-				// We cannot be sure that the "type" tag comes before the "type-id" tag:
-				if len(typeBuf) == 0 {
-					typeBuf = append(typeBuf, val...)
-				} else {
-					typeBuf = reorder(typeBuf, val)
-				}
-			case "type-id":
-				typeBuf = append(typeBuf, val...)
-			case "subtype":
-				// We cannot be sure that the "subtype" tag comes before the "stype-id" tag:
-				if len(subTypeBuf) == 0 {
-					subTypeBuf = append(subTypeBuf, val...)
-				} else {
-					subTypeBuf = reorder(subTypeBuf, val)
-					// subTypeBuf = reorder(typeBuf, val)
-				}
-			case "stype-id":
-				subTypeBuf = append(subTypeBuf, val...)
-			default:
-				// Ignore unkown tags (cc-metric-collector might send us a unit for example that we do not need)
-				// return fmt.Errorf("unkown tag: '%s' (value: '%s')", string(key), string(val))
-			}
-		}
-
-		// If the cluster or host changed, the lvl was set to nil
-		if lvl == nil {
-			selector = selector[:2]
-			selector[0], selector[1] = cluster, host
-			lvl = ms.GetLevel(selector)
-			prevCluster, prevHost = cluster, host
-		}
-
-		// subtypes:
-		selector = selector[:0]
-		if len(typeBuf) > 0 {
-			selector = append(selector, string(typeBuf)) // <- Allocation :(
-			if len(subTypeBuf) > 0 {
-				selector = append(selector, string(subTypeBuf))
-			}
-		}
-
-		for {
-			key, val, err := dec.NextField()
-			if err != nil {
-				return err
-			}
-
-			if key == nil {
-				break
-			}
-
-			if string(key) != "value" {
-				return fmt.Errorf("host %s: unknown field: '%s' (value: %#v)", host, string(key), val)
-			}
-
-			if val.Kind() == lineprotocol.Float {
-				metric.Value = schema.Float(val.FloatV())
-			} else if val.Kind() == lineprotocol.Int {
-				metric.Value = schema.Float(val.IntV())
-			} else if val.Kind() == lineprotocol.Uint {
-				metric.Value = schema.Float(val.UintV())
-			} else {
-				return fmt.Errorf("host %s: unsupported value type in message: %s", host, val.Kind().String())
-			}
-		}
-
-		if t, err = dec.Time(lineprotocol.Second, t); err != nil {
-			t = time.Now()
-			if t, err = dec.Time(lineprotocol.Millisecond, t); err != nil {
-				t = time.Now()
-				if t, err = dec.Time(lineprotocol.Microsecond, t); err != nil {
-					t = time.Now()
-					if t, err = dec.Time(lineprotocol.Nanosecond, t); err != nil {
-						return fmt.Errorf("host %s: timestamp : %#v with error : %#v", host, t, err.Error())
-					}
-				}
-			}
-		}
-
-		if err != nil {
-			return fmt.Errorf("host %s: timestamp : %#v with error : %#v", host, t, err.Error())
-		}
-
-		time := t.Unix()
-
-		if Keys.Checkpoints.FileFormat != "json" {
-			LineProtocolMessages <- &AvroStruct{
-				MetricName: string(metricBuf),
-				Cluster:    cluster,
-				Node:       host,
-				Selector:   append([]string{}, selector...),
-				Value:      metric.Value,
-				Timestamp:  time,
-			}
-		}
-
-		if err := ms.WriteToLevel(lvl, selector, time, []Metric{metric}); err != nil {
-			return err
-		}
-	}
-	return nil
-}

internal/memorystore/memorystore.go

@@ -1,486 +0,0 @@
-// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved. This file is part of cc-backend.
-// Use of this source code is governed by a MIT-style
-// license that can be found in the LICENSE file.
-
-package memorystore
-
-import (
-	"bytes"
-	"context"
-	"encoding/json"
-	"errors"
-	"os"
-	"os/signal"
-	"runtime"
-	"sync"
-	"syscall"
-	"time"
-
-	"github.com/ClusterCockpit/cc-backend/internal/config"
-	"github.com/ClusterCockpit/cc-backend/pkg/archive"
-	cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
-	"github.com/ClusterCockpit/cc-lib/resampler"
-	"github.com/ClusterCockpit/cc-lib/runtimeEnv"
-	"github.com/ClusterCockpit/cc-lib/schema"
-	"github.com/ClusterCockpit/cc-lib/util"
-)
-
-var (
-	singleton  sync.Once
-	msInstance *MemoryStore
-)
-
-var NumWorkers int = 4
-
-func init() {
-	maxWorkers := 10
-	NumWorkers = min(runtime.NumCPU()/2+1, maxWorkers)
-}
-
-type Metric struct {
-	Name         string
-	Value        schema.Float
-	MetricConfig MetricConfig
-}
-
-type MemoryStore struct {
-	Metrics map[string]MetricConfig
-	root    Level
-}
-
-func Init(rawConfig json.RawMessage, wg *sync.WaitGroup) {
-	startupTime := time.Now()
-
-	if rawConfig != nil {
-		config.Validate(configSchema, rawConfig)
-		dec := json.NewDecoder(bytes.NewReader(rawConfig))
-		// dec.DisallowUnknownFields()
-		if err := dec.Decode(&Keys); err != nil {
-			cclog.Abortf("[METRICSTORE]> Metric Store Config Init: Could not decode config file '%s'.\nError: %s\n", rawConfig, err.Error())
-		}
-	}
-
-	for _, c := range archive.Clusters {
-		for _, mc := range c.MetricConfig {
-			agg, err := AssignAggregationStratergy(mc.Aggregation)
-			if err != nil {
-				cclog.Warnf("Could not find aggregation stratergy for metric config '%s': %s", mc.Name, err.Error())
-			}
-
-			AddMetric(mc.Name, MetricConfig{
-				Frequency:   int64(mc.Timestep),
-				Aggregation: agg,
-			})
-		}
-
-		for _, sc := range c.SubClusters {
-			for _, mc := range sc.MetricConfig {
-				agg, err := AssignAggregationStratergy(mc.Aggregation)
-				if err != nil {
-					cclog.Warnf("Could not find aggregation stratergy for metric config '%s': %s", mc.Name, err.Error())
-				}
-
-				AddMetric(mc.Name, MetricConfig{
-					Frequency:   int64(mc.Timestep),
-					Aggregation: agg,
-				})
-			}
-		}
-	}
-
-	// Pass the config.MetricStoreKeys
-	InitMetrics(Metrics)
-
-	ms := GetMemoryStore()
-
-	d, err := time.ParseDuration(Keys.Checkpoints.Restore)
-	if err != nil {
-		cclog.Fatal(err)
-	}
-
-	restoreFrom := startupTime.Add(-d)
-	cclog.Infof("[METRICSTORE]> Loading checkpoints newer than %s\n", restoreFrom.Format(time.RFC3339))
-	files, err := ms.FromCheckpointFiles(Keys.Checkpoints.RootDir, restoreFrom.Unix())
-	loadedData := ms.SizeInBytes() / 1024 / 1024 // In MB
-	if err != nil {
-		cclog.Fatalf("[METRICSTORE]> Loading checkpoints failed: %s\n", err.Error())
-	} else {
-		cclog.Infof("[METRICSTORE]> Checkpoints loaded (%d files, %d MB, that took %fs)\n", files, loadedData, time.Since(startupTime).Seconds())
-	}
-
-	// Try to use less memory by forcing a GC run here and then
-	// lowering the target percentage. The default of 100 means
-	// that only once the ratio of new allocations execeds the
-	// previously active heap, a GC is triggered.
-	// Forcing a GC here will set the "previously active heap"
-	// to a minumum.
-	runtime.GC()
-
-	ctx, shutdown := context.WithCancel(context.Background())
-
-	wg.Add(4)
-
-	Retention(wg, ctx)
-	Checkpointing(wg, ctx)
-	Archiving(wg, ctx)
-	DataStaging(wg, ctx)
-
-	wg.Add(1)
-	sigs := make(chan os.Signal, 1)
-	signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
-	go func() {
-		defer wg.Done()
-		<-sigs
-		runtimeEnv.SystemdNotifiy(false, "[METRICSTORE]> Shutting down ...")
-		shutdown()
-	}()
-
-	if Keys.Nats != nil {
-		for _, natsConf := range Keys.Nats {
-			// TODO: When multiple nats configs share a URL, do a single connect.
-			wg.Add(1)
-			nc := natsConf
-			go func() {
-				// err := ReceiveNats(conf.Nats, decodeLine, runtime.NumCPU()-1, ctx)
-				err := ReceiveNats(nc, ms, 1, ctx)
-				if err != nil {
-					cclog.Fatal(err)
-				}
-				wg.Done()
-			}()
-		}
-	}
-}
-
-// InitMetrics creates a new, initialized instance of a MemoryStore.
-// Will panic if values in the metric configurations are invalid.
-func InitMetrics(metrics map[string]MetricConfig) {
-	singleton.Do(func() {
-		offset := 0
-		for key, cfg := range metrics {
-			if cfg.Frequency == 0 {
-				panic("[METRICSTORE]> invalid frequency")
-			}
-
-			metrics[key] = MetricConfig{
-				Frequency:   cfg.Frequency,
-				Aggregation: cfg.Aggregation,
-				offset:      offset,
-			}
-			offset += 1
-		}
-
-		msInstance = &MemoryStore{
-			root: Level{
-				metrics:  make([]*buffer, len(metrics)),
-				children: make(map[string]*Level),
-			},
-			Metrics: metrics,
-		}
-	})
-}
-
-func GetMemoryStore() *MemoryStore {
-	if msInstance == nil {
-		cclog.Fatalf("[METRICSTORE]> MemoryStore not initialized!")
-	}
-
-	return msInstance
-}
-
-func Shutdown() {
-	cclog.Infof("[METRICSTORE]> Writing to '%s'...\n", Keys.Checkpoints.RootDir)
-	var files int
-	var err error
-
-	ms := GetMemoryStore()
-
-	if Keys.Checkpoints.FileFormat == "json" {
-		files, err = ms.ToCheckpoint(Keys.Checkpoints.RootDir, lastCheckpoint.Unix(), time.Now().Unix())
-	} else {
-		files, err = GetAvroStore().ToCheckpoint(Keys.Checkpoints.RootDir, true)
-		close(LineProtocolMessages)
-	}
-
-	if err != nil {
-		cclog.Errorf("[METRICSTORE]> Writing checkpoint failed: %s\n", err.Error())
-	}
-	cclog.Infof("[METRICSTORE]> Done! (%d files written)\n", files)
-
-	// ms.PrintHeirarchy()
-}
-
-// func (m *MemoryStore) PrintHeirarchy() {
-// 	m.root.lock.Lock()
-// 	defer m.root.lock.Unlock()
-
-// 	fmt.Printf("Root : \n")
-
-// 	for lvl1, sel1 := range m.root.children {
-// 		fmt.Printf("\t%s\n", lvl1)
-// 		for lvl2, sel2 := range sel1.children {
-// 			fmt.Printf("\t\t%s\n", lvl2)
-// 			if lvl1 == "fritz" && lvl2 == "f0201" {
-
-// 				for name, met := range m.Metrics {
-// 					mt := sel2.metrics[met.Offset]
-
-// 					fmt.Printf("\t\t\t\t%s\n", name)
-// 					fmt.Printf("\t\t\t\t")
-
-// 					for mt != nil {
-// 						// if name == "cpu_load" {
-// 						fmt.Printf("%d(%d) -> %#v", mt.start, len(mt.data), mt.data)
-// 						// }
-// 						mt = mt.prev
-// 					}
-// 					fmt.Printf("\n")
-
-// 				}
-// 			}
-// 			for lvl3, sel3 := range sel2.children {
-// 				if lvl1 == "fritz" && lvl2 == "f0201" && lvl3 == "hwthread70" {
-
-// 					fmt.Printf("\t\t\t\t\t%s\n", lvl3)
-
-// 					for name, met := range m.Metrics {
-// 						mt := sel3.metrics[met.Offset]
-
-// 						fmt.Printf("\t\t\t\t\t\t%s\n", name)
-
-// 						fmt.Printf("\t\t\t\t\t\t")
-
-// 						for mt != nil {
-// 							// if name == "clock" {
-// 							fmt.Printf("%d(%d) -> %#v", mt.start, len(mt.data), mt.data)
-
-// 							mt = mt.prev
-// 						}
-// 						fmt.Printf("\n")
-
-// 					}
-
-// 					// for i, _ := range sel3.metrics {
-// 					// 	fmt.Printf("\t\t\t\t\t%s\n", getName(configmetrics, i))
-// 					// }
-// 				}
-// 			}
-// 		}
-// 	}
-
-// }
-
-func getName(m *MemoryStore, i int) string {
-	for key, val := range m.Metrics {
-		if val.offset == i {
-			return key
-		}
-	}
-	return ""
-}
-
-func Retention(wg *sync.WaitGroup, ctx context.Context) {
-	ms := GetMemoryStore()
-
-	go func() {
-		defer wg.Done()
-		d, err := time.ParseDuration(Keys.RetentionInMemory)
-		if err != nil {
-			cclog.Fatal(err)
-		}
-		if d <= 0 {
-			return
-		}
-
-		ticks := func() <-chan time.Time {
-			d := d / 2
-			if d <= 0 {
-				return nil
-			}
-			return time.NewTicker(d).C
-		}()
-		for {
-			select {
-			case <-ctx.Done():
-				return
-			case <-ticks:
-				t := time.Now().Add(-d)
-				cclog.Infof("[METRICSTORE]> start freeing buffers (older than %s)...\n", t.Format(time.RFC3339))
-				freed, err := ms.Free(nil, t.Unix())
-				if err != nil {
-					cclog.Errorf("[METRICSTORE]> freeing up buffers failed: %s\n", err.Error())
-				} else {
-					cclog.Infof("[METRICSTORE]> done: %d buffers freed\n", freed)
-				}
-			}
-		}
-	}()
-}
-
-// Write all values in `metrics` to the level specified by `selector` for time `ts`.
-// Look at `findLevelOrCreate` for how selectors work.
-func (m *MemoryStore) Write(selector []string, ts int64, metrics []Metric) error {
-	var ok bool
-	for i, metric := range metrics {
-		if metric.MetricConfig.Frequency == 0 {
-			metric.MetricConfig, ok = m.Metrics[metric.Name]
-			if !ok {
-				metric.MetricConfig.Frequency = 0
-			}
-			metrics[i] = metric
-		}
-	}
-
-	return m.WriteToLevel(&m.root, selector, ts, metrics)
-}
-
-func (m *MemoryStore) GetLevel(selector []string) *Level {
-	return m.root.findLevelOrCreate(selector, len(m.Metrics))
-}
-
-// WriteToLevel assumes that `minfo` in `metrics` is filled in
-func (m *MemoryStore) WriteToLevel(l *Level, selector []string, ts int64, metrics []Metric) error {
-	l = l.findLevelOrCreate(selector, len(m.Metrics))
-	l.lock.Lock()
-	defer l.lock.Unlock()
-
-	for _, metric := range metrics {
-		if metric.MetricConfig.Frequency == 0 {
-			continue
-		}
-
-		b := l.metrics[metric.MetricConfig.offset]
-		if b == nil {
-			// First write to this metric and level
-			b = newBuffer(ts, metric.MetricConfig.Frequency)
-			l.metrics[metric.MetricConfig.offset] = b
-		}
-
-		nb, err := b.write(ts, metric.Value)
-		if err != nil {
-			return err
-		}
-
-		// Last write created a new buffer...
-		if b != nb {
-			l.metrics[metric.MetricConfig.offset] = nb
-		}
-	}
-	return nil
-}
-
-// Read returns all values for metric `metric` from `from` to `to` for the selected level(s).
-// If the level does not hold the metric itself, the data will be aggregated recursively from the children.
-// The second and third return value are the actual from/to for the data. Those can be different from
-// the range asked for if no data was available.
-func (m *MemoryStore) Read(selector util.Selector, metric string, from, to, resolution int64) ([]schema.Float, int64, int64, int64, error) {
-	if from > to {
-		return nil, 0, 0, 0, errors.New("[METRICSTORE]> invalid time range")
-	}
-
-	minfo, ok := m.Metrics[metric]
-	if !ok {
-		return nil, 0, 0, 0, errors.New("[METRICSTORE]> unkown metric: " + metric)
-	}
-
-	n, data := 0, make([]schema.Float, (to-from)/minfo.Frequency+1)
-
-	err := m.root.findBuffers(selector, minfo.offset, func(b *buffer) error {
-		cdata, cfrom, cto, err := b.read(from, to, data)
-		if err != nil {
-			return err
-		}
-
-		if n == 0 {
-			from, to = cfrom, cto
-		} else if from != cfrom || to != cto || len(data) != len(cdata) {
-			missingfront, missingback := int((from-cfrom)/minfo.Frequency), int((to-cto)/minfo.Frequency)
-			if missingfront != 0 {
-				return ErrDataDoesNotAlign
-			}
-
-			newlen := len(cdata) - missingback
-			if newlen < 1 {
-				return ErrDataDoesNotAlign
-			}
-			cdata = cdata[0:newlen]
-			if len(cdata) != len(data) {
-				return ErrDataDoesNotAlign
-			}
-
-			from, to = cfrom, cto
-		}
-
-		data = cdata
-		n += 1
-		return nil
-	})
-
-	if err != nil {
-		return nil, 0, 0, 0, err
-	} else if n == 0 {
-		return nil, 0, 0, 0, errors.New("[METRICSTORE]> metric or host not found")
-	} else if n > 1 {
-		if minfo.Aggregation == AvgAggregation {
-			normalize := 1. / schema.Float(n)
-			for i := 0; i < len(data); i++ {
-				data[i] *= normalize
-			}
-		} else if minfo.Aggregation != SumAggregation {
-			return nil, 0, 0, 0, errors.New("[METRICSTORE]> invalid aggregation")
-		}
-	}
-
-	data, resolution, err = resampler.LargestTriangleThreeBucket(data, minfo.Frequency, resolution)
-	if err != nil {
-		return nil, 0, 0, 0, err
-	}
-
-	return data, from, to, resolution, nil
-}
-
-// Free releases all buffers for the selected level and all its children that
-// contain only values older than `t`.
-func (m *MemoryStore) Free(selector []string, t int64) (int, error) {
-	return m.GetLevel(selector).free(t)
-}
-
-func (m *MemoryStore) FreeAll() error {
-	for k := range m.root.children {
-		delete(m.root.children, k)
-	}
-
-	return nil
-}
-
-func (m *MemoryStore) SizeInBytes() int64 {
-	return m.root.sizeInBytes()
-}
-
-// ListChildren , given a selector, returns a list of all children of the level
-// selected.
-func (m *MemoryStore) ListChildren(selector []string) []string {
-	lvl := &m.root
-	for lvl != nil && len(selector) != 0 {
-		lvl.lock.RLock()
-		next := lvl.children[selector[0]]
-		lvl.lock.RUnlock()
-		lvl = next
-		selector = selector[1:]
-	}
-
-	if lvl == nil {
-		return nil
-	}
-
-	lvl.lock.RLock()
-	defer lvl.lock.RUnlock()
-
-	children := make([]string, 0, len(lvl.children))
-	for child := range lvl.children {
-		children = append(children, child)
-	}
-
-	return children
-}

internal/memorystore/stats.go

@@ -1,124 +0,0 @@
-// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved. This file is part of cc-backend.
-// Use of this source code is governed by a MIT-style
-// license that can be found in the LICENSE file.
-
-package memorystore
-
-import (
-	"errors"
-	"math"
-
-	"github.com/ClusterCockpit/cc-lib/util"
-)
-
-type Stats struct {
-	Samples int
-	Avg     util.Float
-	Min     util.Float
-	Max     util.Float
-}
-
-func (b *buffer) stats(from, to int64) (Stats, int64, int64, error) {
-	if from < b.start {
-		if b.prev != nil {
-			return b.prev.stats(from, to)
-		}
-		from = b.start
-	}
-
-	// TODO: Check if b.closed and if so and the full buffer is queried,
-	// use b.statistics instead of iterating over the buffer.
-
-	samples := 0
-	sum, min, max := 0.0, math.MaxFloat32, -math.MaxFloat32
-
-	var t int64
-	for t = from; t < to; t += b.frequency {
-		idx := int((t - b.start) / b.frequency)
-		if idx >= cap(b.data) {
-			b = b.next
-			if b == nil {
-				break
-			}
-			idx = 0
-		}
-
-		if t < b.start || idx >= len(b.data) {
-			continue
-		}
-
-		xf := float64(b.data[idx])
-		if math.IsNaN(xf) {
-			continue
-		}
-
-		samples += 1
-		sum += xf
-		min = math.Min(min, xf)
-		max = math.Max(max, xf)
-	}
-
-	return Stats{
-		Samples: samples,
-		Avg:     util.Float(sum) / util.Float(samples),
-		Min:     util.Float(min),
-		Max:     util.Float(max),
-	}, from, t, nil
-}
-
-// Returns statistics for the requested metric on the selected node/level.
-// Data is aggregated to the selected level the same way as in `MemoryStore.Read`.
-// If `Stats.Samples` is zero, the statistics should not be considered as valid.
-func (m *MemoryStore) Stats(selector util.Selector, metric string, from, to int64) (*Stats, int64, int64, error) {
-	if from > to {
-		return nil, 0, 0, errors.New("invalid time range")
-	}
-
-	minfo, ok := m.Metrics[metric]
-	if !ok {
-		return nil, 0, 0, errors.New("unkown metric: " + metric)
-	}
-
-	n, samples := 0, 0
-	avg, min, max := util.Float(0), math.MaxFloat32, -math.MaxFloat32
-	err := m.root.findBuffers(selector, minfo.offset, func(b *buffer) error {
-		stats, cfrom, cto, err := b.stats(from, to)
-		if err != nil {
-			return err
-		}
-
-		if n == 0 {
-			from, to = cfrom, cto
-		} else if from != cfrom || to != cto {
-			return ErrDataDoesNotAlign
-		}
-
-		samples += stats.Samples
-		avg += stats.Avg
-		min = math.Min(min, float64(stats.Min))
-		max = math.Max(max, float64(stats.Max))
-		n += 1
-		return nil
-	})
-	if err != nil {
-		return nil, 0, 0, err
-	}
-
-	if n == 0 {
-		return nil, 0, 0, ErrNoData
-	}
-
-	if minfo.Aggregation == AvgAggregation {
-		avg /= util.Float(n)
-	} else if n > 1 && minfo.Aggregation != SumAggregation {
-		return nil, 0, 0, errors.New("invalid aggregation")
-	}
-
-	return &Stats{
-		Samples: samples,
-		Avg:     avg,
-		Min:     util.Float(min),
-		Max:     util.Float(max),
-	}, from, to, nil
-}

internal/metricDataDispatcher/dataLoader.go

@@ -1,5 +1,5 @@
 // Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved. This file is part of cc-backend.
+// All rights reserved.
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 package metricDataDispatcher
@@ -7,17 +7,16 @@ package metricDataDispatcher
 import (
 	"context"
 	"fmt"
-	"math"
 	"time"
 
 	"github.com/ClusterCockpit/cc-backend/internal/config"
 	"github.com/ClusterCockpit/cc-backend/internal/graph/model"
 	"github.com/ClusterCockpit/cc-backend/internal/metricdata"
 	"github.com/ClusterCockpit/cc-backend/pkg/archive"
-	cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
-	"github.com/ClusterCockpit/cc-lib/lrucache"
-	"github.com/ClusterCockpit/cc-lib/resampler"
-	"github.com/ClusterCockpit/cc-lib/schema"
+	"github.com/ClusterCockpit/cc-backend/pkg/log"
+	"github.com/ClusterCockpit/cc-backend/pkg/lrucache"
+	"github.com/ClusterCockpit/cc-backend/pkg/resampler"
+	"github.com/ClusterCockpit/cc-backend/pkg/schema"
 )
 
 var cache *lrucache.Cache = lrucache.New(128 * 1024 * 1024)
@@ -41,7 +40,7 @@ func LoadData(job *schema.Job,
 	ctx context.Context,
 	resolution int,
 ) (schema.JobData, error) {
-	data := cache.Get(cacheKey(job, metrics, scopes, resolution), func() (_ any, ttl time.Duration, size int) {
+	data := cache.Get(cacheKey(job, metrics, scopes, resolution), func() (_ interface{}, ttl time.Duration, size int) {
 		var jd schema.JobData
 		var err error
 
@@ -68,10 +67,10 @@ func LoadData(job *schema.Job,
 			jd, err = repo.LoadData(job, metrics, scopes, ctx, resolution)
 			if err != nil {
 				if len(jd) != 0 {
-					cclog.Warnf("partial error: %s", err.Error())
+					log.Warnf("partial error: %s", err.Error())
 					// return err, 0, 0 // Reactivating will block archiving on one partial error
 				} else {
-					cclog.Error("Error while loading job data from metric repository")
+					log.Error("Error while loading job data from metric repository")
 					return err, 0, 0
 				}
 			}
@@ -80,25 +79,25 @@ func LoadData(job *schema.Job,
 			var jd_temp schema.JobData
 			jd_temp, err = archive.GetHandle().LoadJobData(job)
 			if err != nil {
-				cclog.Error("Error while loading job data from archive")
+				log.Error("Error while loading job data from archive")
 				return err, 0, 0
 			}
 
-			// Deep copy the cached archive hashmap
+			//Deep copy the cached archive hashmap
 			jd = metricdata.DeepCopy(jd_temp)
 
-			// Resampling for archived data.
-			// Pass the resolution from frontend here.
+			//Resampling for archived data.
+			//Pass the resolution from frontend here.
 			for _, v := range jd {
 				for _, v_ := range v {
-					timestep := int64(0)
+					timestep := 0
 					for i := 0; i < len(v_.Series); i += 1 {
-						v_.Series[i].Data, timestep, err = resampler.LargestTriangleThreeBucket(v_.Series[i].Data, int64(v_.Timestep), int64(resolution))
+						v_.Series[i].Data, timestep, err = resampler.LargestTriangleThreeBucket(v_.Series[i].Data, v_.Timestep, resolution)
 						if err != nil {
 							return err, 0, 0
 						}
 					}
-					v_.Timestep = int(timestep)
+					v_.Timestep = timestep
 				}
 			}
 
@@ -171,14 +170,11 @@ func LoadData(job *schema.Job,
 			jd.AddNodeScope("mem_bw")
 		}
 
-		// Round Resulting Stat Values
-		jd.RoundMetricStats()
-
 		return jd, ttl, size
 	})
 
 	if err, ok := data.(error); ok {
-		cclog.Error("Error in returned dataset")
+		log.Error("Error in returned dataset")
 		return nil, err
 	}
 
@@ -203,7 +199,7 @@ func LoadAverages(
 
 	stats, err := repo.LoadStats(job, metrics, ctx) // #166 how to handle stats for acc normalizazion?
 	if err != nil {
-		cclog.Errorf("Error while loading statistics for job %v (User %v, Project %v)", job.JobID, job.User, job.Project)
+		log.Errorf("Error while loading statistics for job %v (User %v, Project %v)", job.JobID, job.User, job.Project)
 		return err
 	}
 
@@ -224,77 +220,6 @@ func LoadAverages(
 	return nil
 }
 
-// Used for statsTable in frontend: Return scoped statistics by metric.
-func LoadScopedJobStats(
-	job *schema.Job,
-	metrics []string,
-	scopes []schema.MetricScope,
-	ctx context.Context,
-) (schema.ScopedJobStats, error) {
-	if job.State != schema.JobStateRunning && !config.Keys.DisableArchive {
-		return archive.LoadScopedStatsFromArchive(job, metrics, scopes)
-	}
-
-	repo, err := metricdata.GetMetricDataRepo(job.Cluster)
-	if err != nil {
-		return nil, fmt.Errorf("job %d: no metric data repository configured for '%s'", job.JobID, job.Cluster)
-	}
-
-	scopedStats, err := repo.LoadScopedStats(job, metrics, scopes, ctx)
-	if err != nil {
-		cclog.Errorf("error while loading scoped statistics for job %d (User %s, Project %s)", job.JobID, job.User, job.Project)
-		return nil, err
-	}
-
-	return scopedStats, nil
-}
-
-// Used for polar plots in frontend: Aggregates statistics for all nodes to single values for job per metric.
-func LoadJobStats(
-	job *schema.Job,
-	metrics []string,
-	ctx context.Context,
-) (map[string]schema.MetricStatistics, error) {
-	if job.State != schema.JobStateRunning && !config.Keys.DisableArchive {
-		return archive.LoadStatsFromArchive(job, metrics)
-	}
-
-	data := make(map[string]schema.MetricStatistics, len(metrics))
-	repo, err := metricdata.GetMetricDataRepo(job.Cluster)
-	if err != nil {
-		return data, fmt.Errorf("job %d: no metric data repository configured for '%s'", job.JobID, job.Cluster)
-	}
-
-	stats, err := repo.LoadStats(job, metrics, ctx)
-	if err != nil {
-		cclog.Errorf("error while loading statistics for job %d (User %s, Project %s)", job.JobID, job.User, job.Project)
-		return data, err
-	}
-
-	for _, m := range metrics {
-		sum, avg, min, max := 0.0, 0.0, 0.0, 0.0
-		nodes, ok := stats[m]
-		if !ok {
-			data[m] = schema.MetricStatistics{Min: min, Avg: avg, Max: max}
-			continue
-		}
-
-		for _, node := range nodes {
-			sum += node.Avg
-			min = math.Min(min, node.Min)
-			max = math.Max(max, node.Max)
-		}
-
-		data[m] = schema.MetricStatistics{
-			Avg: (math.Round((sum/float64(job.NumNodes))*100) / 100),
-			Min: (math.Round(min*100) / 100),
-			Max: (math.Round(max*100) / 100),
-		}
-	}
-
-	return data, nil
-}
-
 // Used for the classic node/system view. Returns a map of nodes to a map of metrics.
 func LoadNodeData(
 	cluster string,
@@ -317,9 +242,9 @@ func LoadNodeData(
 	data, err := repo.LoadNodeData(cluster, metrics, nodes, scopes, from, to, ctx)
 	if err != nil {
 		if len(data) != 0 {
-			cclog.Warnf("partial error: %s", err.Error())
+			log.Warnf("partial error: %s", err.Error())
 		} else {
-			cclog.Error("Error while loading node data from metric repository")
+			log.Error("Error while loading node data from metric repository")
 			return nil, err
 		}
 	}
@@ -354,9 +279,9 @@ func LoadNodeListData(
 	data, totalNodes, hasNextPage, err := repo.LoadNodeListData(cluster, subCluster, nodeFilter, metrics, scopes, resolution, from, to, page, ctx)
 	if err != nil {
 		if len(data) != 0 {
-			cclog.Warnf("partial error: %s", err.Error())
+			log.Warnf("partial error: %s", err.Error())
 		} else {
-			cclog.Error("Error while loading node data from metric repository")
+			log.Error("Error while loading node data from metric repository")
 			return nil, totalNodes, hasNextPage, err
 		}
 	}

internal/metricdata/cc-metric-store.go

@@ -12,13 +12,14 @@ import (
 	"fmt"
 	"net/http"
 	"sort"
+	"strconv"
 	"strings"
 	"time"
 
 	"github.com/ClusterCockpit/cc-backend/internal/graph/model"
 	"github.com/ClusterCockpit/cc-backend/pkg/archive"
-	cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
-	"github.com/ClusterCockpit/cc-lib/schema"
+	"github.com/ClusterCockpit/cc-backend/pkg/log"
+	"github.com/ClusterCockpit/cc-backend/pkg/schema"
 )
 
 type CCMetricStoreConfig struct {
@@ -81,7 +82,7 @@ type ApiMetricData struct {
 func (ccms *CCMetricStore) Init(rawConfig json.RawMessage) error {
 	var config CCMetricStoreConfig
 	if err := json.Unmarshal(rawConfig, &config); err != nil {
-		cclog.Warn("Error while unmarshaling raw json config")
+		log.Warn("Error while unmarshaling raw json config")
 		return err
 	}
 
@@ -128,13 +129,13 @@ func (ccms *CCMetricStore) doRequest(
 ) (*ApiQueryResponse, error) {
 	buf := &bytes.Buffer{}
 	if err := json.NewEncoder(buf).Encode(body); err != nil {
-		cclog.Errorf("Error while encoding request body: %s", err.Error())
+		log.Warn("Error while encoding request body")
 		return nil, err
 	}
 
 	req, err := http.NewRequestWithContext(ctx, http.MethodGet, ccms.queryEndpoint, buf)
 	if err != nil {
-		cclog.Errorf("Error while building request body: %s", err.Error())
+		log.Warn("Error while building request body")
 		return nil, err
 	}
 	if ccms.jwt != "" {
@@ -150,7 +151,7 @@ func (ccms *CCMetricStore) doRequest(
 
 	res, err := ccms.client.Do(req)
 	if err != nil {
-		cclog.Errorf("Error while performing request: %s", err.Error())
+		log.Error("Error while performing request")
 		return nil, err
 	}
 
@@ -160,7 +161,7 @@ func (ccms *CCMetricStore) doRequest(
 
 	var resBody ApiQueryResponse
 	if err := json.NewDecoder(bufio.NewReader(res.Body)).Decode(&resBody); err != nil {
-		cclog.Errorf("Error while decoding result body: %s", err.Error())
+		log.Warn("Error while decoding result body")
 		return nil, err
 	}
 
@@ -176,14 +177,14 @@ func (ccms *CCMetricStore) LoadData(
 ) (schema.JobData, error) {
 	queries, assignedScope, err := ccms.buildQueries(job, metrics, scopes, resolution)
 	if err != nil {
-		cclog.Errorf("Error while building queries for jobId %d, Metrics %v, Scopes %v: %s", job.JobID, metrics, scopes, err.Error())
+		log.Warn("Error while building queries")
 		return nil, err
 	}
 
 	req := ApiQueryRequest{
 		Cluster:   job.Cluster,
-		From:      job.StartTime,
-		To:        job.StartTime + int64(job.Duration),
+		From:      job.StartTime.Unix(),
+		To:        job.StartTime.Add(time.Duration(job.Duration) * time.Second).Unix(),
 		Queries:   queries,
 		WithStats: true,
 		WithData:  true,
@@ -191,7 +192,7 @@ func (ccms *CCMetricStore) LoadData(
 
 	resBody, err := ccms.doRequest(ctx, &req)
 	if err != nil {
-		cclog.Errorf("Error while performing request: %s", err.Error())
+		log.Error("Error while performing request")
 		return nil, err
 	}
 
@@ -269,6 +270,14 @@ func (ccms *CCMetricStore) LoadData(
 	return jobData, nil
 }
 
+var (
+	hwthreadString     = string(schema.MetricScopeHWThread)
+	coreString         = string(schema.MetricScopeCore)
+	memoryDomainString = string(schema.MetricScopeMemoryDomain)
+	socketString       = string(schema.MetricScopeSocket)
+	acceleratorString  = string(schema.MetricScopeAccelerator)
+)
+
 func (ccms *CCMetricStore) buildQueries(
 	job *schema.Job,
 	metrics []string,
@@ -289,24 +298,10 @@ func (ccms *CCMetricStore) buildQueries(
 		mc := archive.GetMetricConfig(job.Cluster, metric)
 		if mc == nil {
 			// return nil, fmt.Errorf("METRICDATA/CCMS > metric '%s' is not specified for cluster '%s'", metric, job.Cluster)
-			cclog.Infof("metric '%s' is not specified for cluster '%s'", metric, job.Cluster)
+			log.Infof("metric '%s' is not specified for cluster '%s'", metric, job.Cluster)
 			continue
 		}
 
-		// Skip if metric is removed for subcluster
-		if len(mc.SubClusters) != 0 {
-			isRemoved := false
-			for _, scConfig := range mc.SubClusters {
-				if scConfig.Name == job.SubCluster && scConfig.Remove {
-					isRemoved = true
-					break
-				}
-			}
-			if isRemoved {
-				continue
-			}
-		}
-
 		// Avoid duplicates...
 		handledScopes := make([]schema.MetricScope, 0, 3)
 
@@ -445,23 +440,6 @@ func (ccms *CCMetricStore) buildQueries(
 					continue
 				}
 
-				// Core -> Socket
-				if nativeScope == schema.MetricScopeCore && scope == schema.MetricScopeSocket {
-					sockets, _ := topology.GetSocketsFromCores(hwthreads)
-					for _, socket := range sockets {
-						queries = append(queries, ApiQuery{
-							Metric:     remoteName,
-							Hostname:   host.Hostname,
-							Aggregate:  true,
-							Type:       &coreString,
-							TypeIds:    intToStringSlice(topology.Socket[socket]),
-							Resolution: resolution,
-						})
-						assignedScope = append(assignedScope, scope)
-					}
-					continue
-				}
-
 				// Core -> Node
 				if nativeScope == schema.MetricScopeCore && scope == schema.MetricScopeNode {
 					cores, _ := topology.GetCoresFromHWThreads(hwthreads)
@@ -562,16 +540,23 @@ func (ccms *CCMetricStore) LoadStats(
 	ctx context.Context,
 ) (map[string]map[string]schema.MetricStatistics, error) {
 
+	// metricConfigs := archive.GetCluster(job.Cluster).MetricConfig
+	// resolution := 9000
+
+	// for _, mc := range metricConfigs {
+	// 	resolution = min(resolution, mc.Timestep)
+	// }
+
 	queries, _, err := ccms.buildQueries(job, metrics, []schema.MetricScope{schema.MetricScopeNode}, 0) // #166 Add scope shere for analysis view accelerator normalization?
 	if err != nil {
-		cclog.Errorf("Error while building queries for jobId %d, Metrics %v: %s", job.JobID, metrics, err.Error())
+		log.Warn("Error while building query")
 		return nil, err
 	}
 
 	req := ApiQueryRequest{
 		Cluster:   job.Cluster,
-		From:      job.StartTime,
-		To:        job.StartTime + int64(job.Duration),
+		From:      job.StartTime.Unix(),
+		To:        job.StartTime.Add(time.Duration(job.Duration) * time.Second).Unix(),
 		Queries:   queries,
 		WithStats: true,
 		WithData:  false,
@@ -579,7 +564,7 @@ func (ccms *CCMetricStore) LoadStats(
 
 	resBody, err := ccms.doRequest(ctx, &req)
 	if err != nil {
-		cclog.Errorf("Error while performing request: %s", err.Error())
+		log.Error("Error while performing request")
 		return nil, err
 	}
 
@@ -589,8 +574,9 @@ func (ccms *CCMetricStore) LoadStats(
 		metric := ccms.toLocalName(query.Metric)
 		data := res[0]
 		if data.Error != nil {
-			cclog.Errorf("fetching %s for node %s failed: %s", metric, query.Hostname, *data.Error)
+			log.Infof("fetching %s for node %s failed: %s", metric, query.Hostname, *data.Error)
 			continue
+			// return nil, fmt.Errorf("METRICDATA/CCMS > fetching %s for node %s failed: %s", metric, query.Hostname, *data.Error)
 		}
 
 		metricdata, ok := stats[metric]
@@ -600,8 +586,9 @@ func (ccms *CCMetricStore) LoadStats(
 		}
 
 		if data.Avg.IsNaN() || data.Min.IsNaN() || data.Max.IsNaN() {
-			cclog.Warnf("fetching %s for node %s failed: one of avg/min/max is NaN", metric, query.Hostname)
+			log.Infof("fetching %s for node %s failed: one of avg/min/max is NaN", metric, query.Hostname)
 			continue
+			// return nil, fmt.Errorf("METRICDATA/CCMS > fetching %s for node %s failed: %s", metric, query.Hostname, "avg/min/max is NaN")
 		}
 
 		metricdata[query.Hostname] = schema.MetricStatistics{
@@ -614,98 +601,7 @@ func (ccms *CCMetricStore) LoadStats(
 	return stats, nil
 }
 
-// Used for Job-View Statistics Table
-func (ccms *CCMetricStore) LoadScopedStats(
-	job *schema.Job,
-	metrics []string,
-	scopes []schema.MetricScope,
-	ctx context.Context,
-) (schema.ScopedJobStats, error) {
-	queries, assignedScope, err := ccms.buildQueries(job, metrics, scopes, 0)
-	if err != nil {
-		cclog.Errorf("Error while building queries for jobId %d, Metrics %v, Scopes %v: %s", job.JobID, metrics, scopes, err.Error())
-		return nil, err
-	}
-
-	req := ApiQueryRequest{
-		Cluster:   job.Cluster,
-		From:      job.StartTime,
-		To:        job.StartTime + int64(job.Duration),
-		Queries:   queries,
-		WithStats: true,
-		WithData:  false,
-	}
-
-	resBody, err := ccms.doRequest(ctx, &req)
-	if err != nil {
-		cclog.Errorf("Error while performing request: %s", err.Error())
-		return nil, err
-	}
-
-	var errors []string
-	scopedJobStats := make(schema.ScopedJobStats)
-
-	for i, row := range resBody.Results {
-		query := req.Queries[i]
-		metric := ccms.toLocalName(query.Metric)
-		scope := assignedScope[i]
-
-		if _, ok := scopedJobStats[metric]; !ok {
-			scopedJobStats[metric] = make(map[schema.MetricScope][]*schema.ScopedStats)
-		}
-
-		if _, ok := scopedJobStats[metric][scope]; !ok {
-			scopedJobStats[metric][scope] = make([]*schema.ScopedStats, 0)
-		}
-
-		for ndx, res := range row {
-			if res.Error != nil {
-				/* Build list for "partial errors", if any */
-				errors = append(errors, fmt.Sprintf("failed to fetch '%s' from host '%s': %s", query.Metric, query.Hostname, *res.Error))
-				continue
-			}
-
-			id := (*string)(nil)
-			if query.Type != nil {
-				id = new(string)
-				*id = query.TypeIds[ndx]
-			}
-
-			if res.Avg.IsNaN() || res.Min.IsNaN() || res.Max.IsNaN() {
-				// "schema.Float()" because regular float64 can not be JSONed when NaN.
-				res.Avg = schema.Float(0)
-				res.Min = schema.Float(0)
-				res.Max = schema.Float(0)
-			}
-
-			scopedJobStats[metric][scope] = append(scopedJobStats[metric][scope], &schema.ScopedStats{
-				Hostname: query.Hostname,
-				Id:       id,
-				Data: &schema.MetricStatistics{
-					Avg: float64(res.Avg),
-					Min: float64(res.Min),
-					Max: float64(res.Max),
-				},
-			})
-		}
-
-		// So that one can later check len(scopedJobStats[metric][scope]): Remove from map if empty
-		if len(scopedJobStats[metric][scope]) == 0 {
-			delete(scopedJobStats[metric], scope)
-			if len(scopedJobStats[metric]) == 0 {
-				delete(scopedJobStats, metric)
-			}
-		}
-	}
-
-	if len(errors) != 0 {
-		/* Returns list for "partial errors" */
-		return scopedJobStats, fmt.Errorf("METRICDATA/CCMS > Errors: %s", strings.Join(errors, ", "))
-	}
-	return scopedJobStats, nil
-}
-
-// Used for Systems-View Node-Overview
+// TODO: Support sub-node-scope metrics! For this, the partition of a node needs to be known!
 func (ccms *CCMetricStore) LoadNodeData(
 	cluster string,
 	metrics, nodes []string,
@@ -731,7 +627,7 @@ func (ccms *CCMetricStore) LoadNodeData(
 				req.Queries = append(req.Queries, ApiQuery{
 					Hostname:   node,
 					Metric:     ccms.toRemoteName(metric),
-					Resolution: 0, // Default for Node Queries: Will return metric $Timestep Resolution
+					Resolution: 60, // Default for Node Queries
 				})
 			}
 		}
@@ -739,7 +635,7 @@ func (ccms *CCMetricStore) LoadNodeData(
 
 	resBody, err := ccms.doRequest(ctx, &req)
 	if err != nil {
-		cclog.Errorf("Error while performing request: %s", err.Error())
+		log.Error(fmt.Sprintf("Error while performing request %#v\n", err))
 		return nil, err
 	}
 
@@ -797,7 +693,6 @@ func (ccms *CCMetricStore) LoadNodeData(
 	return data, nil
 }
 
-// Used for Systems-View Node-List
 func (ccms *CCMetricStore) LoadNodeListData(
 	cluster, subCluster, nodeFilter string,
 	metrics []string,
@@ -856,7 +751,7 @@ func (ccms *CCMetricStore) LoadNodeListData(
 
 	queries, assignedScope, err := ccms.buildNodeQueries(cluster, subCluster, nodes, metrics, scopes, resolution)
 	if err != nil {
-		cclog.Errorf("Error while building node queries for Cluster %s, SubCLuster %s, Metrics %v, Scopes %v: %s", cluster, subCluster, metrics, scopes, err.Error())
+		log.Warn("Error while building queries")
 		return nil, totalNodes, hasNextPage, err
 	}
 
@@ -871,7 +766,7 @@ func (ccms *CCMetricStore) LoadNodeListData(
 
 	resBody, err := ccms.doRequest(ctx, &req)
 	if err != nil {
-		cclog.Errorf("Error while performing request: %s", err.Error())
+		log.Error(fmt.Sprintf("Error while performing request %#v\n", err))
 		return nil, totalNodes, hasNextPage, err
 	}
 
@@ -976,7 +871,7 @@ func (ccms *CCMetricStore) buildNodeQueries(
 	if subCluster != "" {
 		subClusterTopol, scterr = archive.GetSubCluster(cluster, subCluster)
 		if scterr != nil {
-			cclog.Errorf("could not load cluster %s subCluster %s topology: %s", cluster, subCluster, scterr.Error())
+			// TODO: Log
 			return nil, nil, scterr
 		}
 	}
@@ -986,24 +881,10 @@ func (ccms *CCMetricStore) buildNodeQueries(
 		mc := archive.GetMetricConfig(cluster, metric)
 		if mc == nil {
 			// return nil, fmt.Errorf("METRICDATA/CCMS > metric '%s' is not specified for cluster '%s'", metric, cluster)
-			cclog.Warnf("metric '%s' is not specified for cluster '%s'", metric, cluster)
+			log.Infof("metric '%s' is not specified for cluster '%s'", metric, cluster)
 			continue
 		}
 
-		// Skip if metric is removed for subcluster
-		if mc.SubClusters != nil {
-			isRemoved := false
-			for _, scConfig := range mc.SubClusters {
-				if scConfig.Name == subCluster && scConfig.Remove {
-					isRemoved = true
-					break
-				}
-			}
-			if isRemoved {
-				continue
-			}
-		}
-
 		// Avoid duplicates...
 		handledScopes := make([]schema.MetricScope, 0, 3)
 
@@ -1157,23 +1038,6 @@ func (ccms *CCMetricStore) buildNodeQueries(
 					continue
 				}
 
-				// Core -> Socket
-				if nativeScope == schema.MetricScopeCore && scope == schema.MetricScopeSocket {
-					sockets, _ := topology.GetSocketsFromCores(topology.Node)
-					for _, socket := range sockets {
-						queries = append(queries, ApiQuery{
-							Metric:     remoteName,
-							Hostname:   hostname,
-							Aggregate:  true,
-							Type:       &coreString,
-							TypeIds:    intToStringSlice(topology.Socket[socket]),
-							Resolution: resolution,
-						})
-						assignedScope = append(assignedScope, scope)
-					}
-					continue
-				}
-
 				// Core -> Node
 				if nativeScope == schema.MetricScopeCore && scope == schema.MetricScopeNode {
 					cores, _ := topology.GetCoresFromHWThreads(topology.Node)
@@ -1267,3 +1131,11 @@ func (ccms *CCMetricStore) buildNodeQueries(
 
 	return queries, assignedScope, nil
 }
+
+func intToStringSlice(is []int) []string {
+	ss := make([]string, len(is))
+	for i, x := range is {
+		ss[i] = strconv.Itoa(x)
+	}
+	return ss
+}

internal/metricdata/influxdb-v2.go

@@ -0,0 +1,333 @@
+// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
+// All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+package metricdata
+
+import (
+	"context"
+	"crypto/tls"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/ClusterCockpit/cc-backend/internal/graph/model"
+	"github.com/ClusterCockpit/cc-backend/pkg/archive"
+	"github.com/ClusterCockpit/cc-backend/pkg/log"
+	"github.com/ClusterCockpit/cc-backend/pkg/schema"
+	influxdb2 "github.com/influxdata/influxdb-client-go/v2"
+	influxdb2Api "github.com/influxdata/influxdb-client-go/v2/api"
+)
+
+type InfluxDBv2DataRepositoryConfig struct {
+	Url     string `json:"url"`
+	Token   string `json:"token"`
+	Bucket  string `json:"bucket"`
+	Org     string `json:"org"`
+	SkipTls bool   `json:"skiptls"`
+}
+
+type InfluxDBv2DataRepository struct {
+	client              influxdb2.Client
+	queryClient         influxdb2Api.QueryAPI
+	bucket, measurement string
+}
+
+func (idb *InfluxDBv2DataRepository) Init(rawConfig json.RawMessage) error {
+	var config InfluxDBv2DataRepositoryConfig
+	if err := json.Unmarshal(rawConfig, &config); err != nil {
+		log.Warn("Error while unmarshaling raw json config")
+		return err
+	}
+
+	idb.client = influxdb2.NewClientWithOptions(config.Url, config.Token, influxdb2.DefaultOptions().SetTLSConfig(&tls.Config{InsecureSkipVerify: config.SkipTls}))
+	idb.queryClient = idb.client.QueryAPI(config.Org)
+	idb.bucket = config.Bucket
+
+	return nil
+}
+
+func (idb *InfluxDBv2DataRepository) formatTime(t time.Time) string {
+	return t.Format(time.RFC3339) // Like “2006-01-02T15:04:05Z07:00”
+}
+
+func (idb *InfluxDBv2DataRepository) epochToTime(epoch int64) time.Time {
+	return time.Unix(epoch, 0)
+}
+
+func (idb *InfluxDBv2DataRepository) LoadData(
+	job *schema.Job,
+	metrics []string,
+	scopes []schema.MetricScope,
+	ctx context.Context,
+	resolution int) (schema.JobData, error) {
+
+	measurementsConds := make([]string, 0, len(metrics))
+	for _, m := range metrics {
+		measurementsConds = append(measurementsConds, fmt.Sprintf(`r["_measurement"] == "%s"`, m))
+	}
+	measurementsCond := strings.Join(measurementsConds, " or ")
+
+	hostsConds := make([]string, 0, len(job.Resources))
+	for _, h := range job.Resources {
+		if h.HWThreads != nil || h.Accelerators != nil {
+			// TODO
+			return nil, errors.New("METRICDATA/INFLUXV2 > the InfluxDB metric data repository does not yet support HWThreads or Accelerators")
+		}
+		hostsConds = append(hostsConds, fmt.Sprintf(`r["hostname"] == "%s"`, h.Hostname))
+	}
+	hostsCond := strings.Join(hostsConds, " or ")
+
+	jobData := make(schema.JobData) // Empty Schema: map[<string>FIELD]map[<MetricScope>SCOPE]<*JobMetric>METRIC
+	// Requested Scopes
+	for _, scope := range scopes {
+		query := ""
+		switch scope {
+		case "node":
+			// Get Finest Granularity, Groupy By Measurement and Hostname (== Metric / Node), Calculate Mean for 60s windows
+			// log.Info("Scope 'node' requested. ")
+			query = fmt.Sprintf(`
+								from(bucket: "%s")
+								|> range(start: %s, stop: %s)
+								|> filter(fn: (r) => (%s) and (%s) )
+								|> drop(columns: ["_start", "_stop"])
+								|> group(columns: ["hostname", "_measurement"])
+		            |> aggregateWindow(every: 60s, fn: mean)
+								|> drop(columns: ["_time"])`,
+				idb.bucket,
+				idb.formatTime(job.StartTime), idb.formatTime(idb.epochToTime(job.StartTimeUnix+int64(job.Duration)+int64(1))),
+				measurementsCond, hostsCond)
+		case "socket":
+			log.Info("Scope 'socket' requested, but not yet supported: Will return 'node' scope only. ")
+			continue
+		case "core":
+			log.Info(" Scope 'core' requested, but not yet supported: Will return 'node' scope only. ")
+			continue
+			// Get Finest Granularity only, Set NULL to 0.0
+			// query = fmt.Sprintf(`
+			//  	from(bucket: "%s")
+			//  	|> range(start: %s, stop: %s)
+			//  	|> filter(fn: (r) => %s )
+			//  	|> filter(fn: (r) => %s )
+			//  	|> drop(columns: ["_start", "_stop", "cluster"])
+			//  	|> map(fn: (r) => (if exists r._value then {r with _value: r._value} else {r with _value: 0.0}))`,
+			//  	idb.bucket,
+			//  	idb.formatTime(job.StartTime), idb.formatTime(idb.epochToTime(job.StartTimeUnix + int64(job.Duration) + int64(1) )),
+			//  	measurementsCond, hostsCond)
+		default:
+			log.Infof("Unknown scope '%s' requested: Will return 'node' scope.", scope)
+			continue
+			// return nil, errors.New("METRICDATA/INFLUXV2 > the InfluxDB metric data repository does not yet support other scopes than 'node'")
+		}
+
+		rows, err := idb.queryClient.Query(ctx, query)
+		if err != nil {
+			log.Error("Error while performing query")
+			return nil, err
+		}
+
+		// Init Metrics: Only Node level now -> TODO: Matching /check on scope level ...
+		for _, metric := range metrics {
+			jobMetric, ok := jobData[metric]
+			if !ok {
+				mc := archive.GetMetricConfig(job.Cluster, metric)
+				jobMetric = map[schema.MetricScope]*schema.JobMetric{
+					scope: { // uses scope var from above!
+						Unit:             mc.Unit,
+						Timestep:         mc.Timestep,
+						Series:           make([]schema.Series, 0, len(job.Resources)),
+						StatisticsSeries: nil, // Should be: &schema.StatsSeries{},
+					},
+				}
+			}
+			jobData[metric] = jobMetric
+		}
+
+		// Process Result: Time-Data
+		field, host, hostSeries := "", "", schema.Series{}
+		// typeId := 0
+		switch scope {
+		case "node":
+			for rows.Next() {
+				row := rows.Record()
+				if host == "" || host != row.ValueByKey("hostname").(string) || rows.TableChanged() {
+					if host != "" {
+						// Append Series before reset
+						jobData[field][scope].Series = append(jobData[field][scope].Series, hostSeries)
+					}
+					field, host = row.Measurement(), row.ValueByKey("hostname").(string)
+					hostSeries = schema.Series{
+						Hostname:   host,
+						Statistics: schema.MetricStatistics{}, //TODO Add Statistics
+						Data:       make([]schema.Float, 0),
+					}
+				}
+				val, ok := row.Value().(float64)
+				if ok {
+					hostSeries.Data = append(hostSeries.Data, schema.Float(val))
+				} else {
+					hostSeries.Data = append(hostSeries.Data, schema.Float(0))
+				}
+			}
+		case "socket":
+			continue
+		case "core":
+			continue
+			// Include Series.Id in hostSeries
+			// for rows.Next() {
+			// 		row := rows.Record()
+			// 		if ( host == "" || host != row.ValueByKey("hostname").(string) || typeId != row.ValueByKey("type-id").(int) || rows.TableChanged() ) {
+			// 		 		if ( host != "" ) {
+			// 						// Append Series before reset
+			// 		 		  	jobData[field][scope].Series = append(jobData[field][scope].Series, hostSeries)
+			// 		 		}
+			// 		 		field, host, typeId = row.Measurement(), row.ValueByKey("hostname").(string), row.ValueByKey("type-id").(int)
+			// 		 		hostSeries  = schema.Series{
+			// 		 				Hostname:   host,
+			// 						Id:					&typeId,
+			// 		 				Statistics: nil,
+			// 		 				Data:       make([]schema.Float, 0),
+			// 		 		}
+			// 		}
+			// 		val := row.Value().(float64)
+			// 		hostSeries.Data = append(hostSeries.Data, schema.Float(val))
+			// }
+		default:
+			log.Infof("Unknown scope '%s' requested: Will return 'node' scope.", scope)
+			continue
+			// return nil, errors.New("the InfluxDB metric data repository does not yet support other scopes than 'node, core'")
+		}
+		// Append last Series
+		jobData[field][scope].Series = append(jobData[field][scope].Series, hostSeries)
+	}
+
+	// Get Stats
+	stats, err := idb.LoadStats(job, metrics, ctx)
+	if err != nil {
+		log.Warn("Error while loading statistics")
+		return nil, err
+	}
+
+	for _, scope := range scopes {
+		if scope == "node" { // No 'socket/core' support yet
+			for metric, nodes := range stats {
+				for node, stats := range nodes {
+					for index, _ := range jobData[metric][scope].Series {
+						if jobData[metric][scope].Series[index].Hostname == node {
+							jobData[metric][scope].Series[index].Statistics = schema.MetricStatistics{Avg: stats.Avg, Min: stats.Min, Max: stats.Max}
+						}
+					}
+				}
+			}
+		}
+	}
+
+	return jobData, nil
+}
+
+func (idb *InfluxDBv2DataRepository) LoadStats(
+	job *schema.Job,
+	metrics []string,
+	ctx context.Context) (map[string]map[string]schema.MetricStatistics, error) {
+
+	stats := map[string]map[string]schema.MetricStatistics{}
+
+	hostsConds := make([]string, 0, len(job.Resources))
+	for _, h := range job.Resources {
+		if h.HWThreads != nil || h.Accelerators != nil {
+			// TODO
+			return nil, errors.New("METRICDATA/INFLUXV2 > the InfluxDB metric data repository does not yet support HWThreads or Accelerators")
+		}
+		hostsConds = append(hostsConds, fmt.Sprintf(`r["hostname"] == "%s"`, h.Hostname))
+	}
+	hostsCond := strings.Join(hostsConds, " or ")
+
+	// lenMet := len(metrics)
+
+	for _, metric := range metrics {
+		// log.Debugf("<< You are here: %s (Index %d of %d metrics)", metric, index, lenMet)
+
+		query := fmt.Sprintf(`
+				  data = from(bucket: "%s")
+				  |> range(start: %s, stop: %s)
+				  |> filter(fn: (r) => r._measurement == "%s" and r._field == "value" and (%s))
+				  union(tables: [data |> mean(column: "_value") |> set(key: "_field", value: "avg"),
+				                 data |>  min(column: "_value") |> set(key: "_field", value: "min"),
+				                 data |>  max(column: "_value") |> set(key: "_field", value: "max")])
+				  |> pivot(rowKey: ["hostname"], columnKey: ["_field"], valueColumn: "_value")
+				  |> group()`,
+			idb.bucket,
+			idb.formatTime(job.StartTime), idb.formatTime(idb.epochToTime(job.StartTimeUnix+int64(job.Duration)+int64(1))),
+			metric, hostsCond)
+
+		rows, err := idb.queryClient.Query(ctx, query)
+		if err != nil {
+			log.Error("Error while performing query")
+			return nil, err
+		}
+
+		nodes := map[string]schema.MetricStatistics{}
+		for rows.Next() {
+			row := rows.Record()
+			host := row.ValueByKey("hostname").(string)
+
+			avg, avgok := row.ValueByKey("avg").(float64)
+			if !avgok {
+				// log.Debugf(">> Assertion error for metric %s, statistic AVG. Expected 'float64', got %v", metric, avg)
+				avg = 0.0
+			}
+			min, minok := row.ValueByKey("min").(float64)
+			if !minok {
+				// log.Debugf(">> Assertion error for metric %s, statistic MIN. Expected 'float64', got %v", metric, min)
+				min = 0.0
+			}
+			max, maxok := row.ValueByKey("max").(float64)
+			if !maxok {
+				// log.Debugf(">> Assertion error for metric %s, statistic MAX. Expected 'float64', got %v", metric, max)
+				max = 0.0
+			}
+
+			nodes[host] = schema.MetricStatistics{
+				Avg: avg,
+				Min: min,
+				Max: max,
+			}
+		}
+		stats[metric] = nodes
+	}
+
+	return stats, nil
+}
+
+func (idb *InfluxDBv2DataRepository) LoadNodeData(
+	cluster string,
+	metrics, nodes []string,
+	scopes []schema.MetricScope,
+	from, to time.Time,
+	ctx context.Context) (map[string]map[string][]*schema.JobMetric, error) {
+
+	// TODO : Implement to be used in Analysis- und System/Node-View
+	log.Infof("LoadNodeData unimplemented for InfluxDBv2DataRepository, Args: cluster %s, metrics %v, nodes %v, scopes %v", cluster, metrics, nodes, scopes)
+
+	return nil, errors.New("METRICDATA/INFLUXV2 > unimplemented for InfluxDBv2DataRepository")
+}
+
+func (idb *InfluxDBv2DataRepository) LoadNodeListData(
+	cluster, subCluster, nodeFilter string,
+	metrics []string,
+	scopes []schema.MetricScope,
+	resolution int,
+	from, to time.Time,
+	page *model.PageRequest,
+	ctx context.Context,
+) (map[string]schema.JobData, int, bool, error) {
+
+	var totalNodes int = 0
+	var hasNextPage bool = false
+	// TODO : Implement to be used in NodeList-View
+	log.Infof("LoadNodeListData unimplemented for InfluxDBv2DataRepository, Args: cluster %s, metrics %v, nodeFilter %v, scopes %v", cluster, metrics, nodeFilter, scopes)
+
+	return nil, totalNodes, hasNextPage, errors.New("METRICDATA/INFLUXV2 > unimplemented for InfluxDBv2DataRepository")
+}

internal/metricdata/metricdata.go

@@ -1,8 +1,7 @@
 // Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved. This file is part of cc-backend.
+// All rights reserved.
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
-
 package metricdata
 
 import (
@@ -13,9 +12,8 @@ import (
 
 	"github.com/ClusterCockpit/cc-backend/internal/config"
 	"github.com/ClusterCockpit/cc-backend/internal/graph/model"
-	"github.com/ClusterCockpit/cc-backend/internal/memorystore"
-	cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
-	"github.com/ClusterCockpit/cc-lib/schema"
+	"github.com/ClusterCockpit/cc-backend/pkg/log"
+	"github.com/ClusterCockpit/cc-backend/pkg/schema"
 )
 
 type MetricDataRepository interface {
@@ -26,12 +24,9 @@ type MetricDataRepository interface {
 	// Return the JobData for the given job, only with the requested metrics.
 	LoadData(job *schema.Job, metrics []string, scopes []schema.MetricScope, ctx context.Context, resolution int) (schema.JobData, error)
 
-	// Return a map of metrics to a map of nodes to the metric statistics of the job. node scope only.
+	// Return a map of metrics to a map of nodes to the metric statistics of the job. node scope assumed for now.
 	LoadStats(job *schema.Job, metrics []string, ctx context.Context) (map[string]map[string]schema.MetricStatistics, error)
 
-	// Return a map of metrics to a map of scopes to the scoped metric statistics of the job.
-	LoadScopedStats(job *schema.Job, metrics []string, scopes []schema.MetricScope, ctx context.Context) (schema.ScopedJobStats, error)
-
 	// Return a map of hosts to a map of metrics at the requested scopes (currently only node) for that node.
 	LoadNodeData(cluster string, metrics, nodes []string, scopes []schema.MetricScope, from, to time.Time, ctx context.Context) (map[string]map[string][]*schema.JobMetric, error)
 
@@ -42,13 +37,13 @@ type MetricDataRepository interface {
 var metricDataRepos map[string]MetricDataRepository = map[string]MetricDataRepository{}
 
 func Init() error {
-	for _, cluster := range config.Clusters {
+	for _, cluster := range config.Keys.Clusters {
 		if cluster.MetricDataRepository != nil {
 			var kind struct {
 				Kind string `json:"kind"`
 			}
 			if err := json.Unmarshal(cluster.MetricDataRepository, &kind); err != nil {
-				cclog.Warn("Error while unmarshaling raw json MetricDataRepository")
+				log.Warn("Error while unmarshaling raw json MetricDataRepository")
 				return err
 			}
 
@@ -56,9 +51,8 @@ func Init() error {
 			switch kind.Kind {
 			case "cc-metric-store":
 				mdr = &CCMetricStore{}
-			case "cc-metric-store-internal":
-				mdr = &CCMetricStoreInternal{}
-				memorystore.InternalCCMSFlag = true
+			case "influxdb":
+				mdr = &InfluxDBv2DataRepository{}
 			case "prometheus":
 				mdr = &PrometheusDataRepository{}
 			case "test":
@@ -68,7 +62,7 @@ func Init() error {
 			}
 
 			if err := mdr.Init(cluster.MetricDataRepository); err != nil {
-				cclog.Errorf("Error initializing MetricDataRepository %v for cluster %v", kind.Kind, cluster.Name)
+				log.Errorf("Error initializing MetricDataRepository %v for cluster %v", kind.Kind, cluster.Name)
 				return err
 			}
 			metricDataRepos[cluster.Name] = mdr

internal/metricdata/prometheus.go

@@ -1,5 +1,5 @@
 // Copyright (C) 2022 DKRZ
-// All rights reserved. This file is part of cc-backend.
+// All rights reserved.
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 package metricdata
@@ -22,8 +22,8 @@ import (
 
 	"github.com/ClusterCockpit/cc-backend/internal/graph/model"
 	"github.com/ClusterCockpit/cc-backend/pkg/archive"
-	cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
-	"github.com/ClusterCockpit/cc-lib/schema"
+	"github.com/ClusterCockpit/cc-backend/pkg/log"
+	"github.com/ClusterCockpit/cc-backend/pkg/schema"
 	promapi "github.com/prometheus/client_golang/api"
 	promv1 "github.com/prometheus/client_golang/api/prometheus/v1"
 	promcfg "github.com/prometheus/common/config"
@@ -160,7 +160,7 @@ func (pdb *PrometheusDataRepository) Init(rawConfig json.RawMessage) error {
 	var config PrometheusDataRepositoryConfig
 	// parse config
 	if err := json.Unmarshal(rawConfig, &config); err != nil {
-		cclog.Warn("Error while unmarshaling raw json config")
+		log.Warn("Error while unmarshaling raw json config")
 		return err
 	}
 	// support basic authentication
@@ -179,7 +179,7 @@ func (pdb *PrometheusDataRepository) Init(rawConfig json.RawMessage) error {
 		RoundTripper: rt,
 	})
 	if err != nil {
-		cclog.Error("Error while initializing new prometheus client")
+		log.Error("Error while initializing new prometheus client")
 		return err
 	}
 	// init query client
@@ -192,9 +192,9 @@ func (pdb *PrometheusDataRepository) Init(rawConfig json.RawMessage) error {
 	for metric, templ := range config.Templates {
 		pdb.templates[metric], err = template.New(metric).Parse(templ)
 		if err == nil {
-			cclog.Debugf("Added PromQL template for %s: %s", metric, templ)
+			log.Debugf("Added PromQL template for %s: %s", metric, templ)
 		} else {
-			cclog.Warnf("Failed to parse PromQL template %s for metric %s", templ, metric)
+			log.Warnf("Failed to parse PromQL template %s for metric %s", templ, metric)
 		}
 	}
 	return nil
@@ -221,7 +221,7 @@ func (pdb *PrometheusDataRepository) FormatQuery(
 			return "", errors.New(fmt.Sprintf("METRICDATA/PROMETHEUS > Error compiling template %v", templ))
 		} else {
 			query := buf.String()
-			cclog.Debugf("PromQL: %s", query)
+			log.Debugf("PromQL: %s", query)
 			return query, nil
 		}
 	} else {
@@ -279,13 +279,13 @@ func (pdb *PrometheusDataRepository) LoadData(
 	for i, resource := range job.Resources {
 		nodes[i] = resource.Hostname
 	}
-	from := time.Unix(job.StartTime, 0)
-	to := time.Unix(job.StartTime+int64(job.Duration), 0)
+	from := job.StartTime
+	to := job.StartTime.Add(time.Duration(job.Duration) * time.Second)
 
 	for _, scope := range scopes {
 		if scope != schema.MetricScopeNode {
 			logOnce.Do(func() {
-				cclog.Infof("Scope '%s' requested, but not yet supported: Will return 'node' scope only.", scope)
+				log.Infof("Scope '%s' requested, but not yet supported: Will return 'node' scope only.", scope)
 			})
 			continue
 		}
@@ -293,12 +293,12 @@ func (pdb *PrometheusDataRepository) LoadData(
 		for _, metric := range metrics {
 			metricConfig := archive.GetMetricConfig(job.Cluster, metric)
 			if metricConfig == nil {
-				cclog.Warnf("Error in LoadData: Metric %s for cluster %s not configured", metric, job.Cluster)
+				log.Warnf("Error in LoadData: Metric %s for cluster %s not configured", metric, job.Cluster)
 				return nil, errors.New("Prometheus config error")
 			}
 			query, err := pdb.FormatQuery(metric, scope, nodes, job.Cluster)
 			if err != nil {
-				cclog.Warn("Error while formatting prometheus query")
+				log.Warn("Error while formatting prometheus query")
 				return nil, err
 			}
 
@@ -310,11 +310,11 @@ func (pdb *PrometheusDataRepository) LoadData(
 			}
 			result, warnings, err := pdb.queryClient.QueryRange(ctx, query, r)
 			if err != nil {
-				cclog.Errorf("Prometheus query error in LoadData: %v\nQuery: %s", err, query)
+				log.Errorf("Prometheus query error in LoadData: %v\nQuery: %s", err, query)
 				return nil, errors.New("Prometheus query error")
 			}
 			if len(warnings) > 0 {
-				cclog.Warnf("Warnings: %v\n", warnings)
+				log.Warnf("Warnings: %v\n", warnings)
 			}
 
 			// init data structures
@@ -360,7 +360,7 @@ func (pdb *PrometheusDataRepository) LoadStats(
 
 	data, err := pdb.LoadData(job, metrics, []schema.MetricScope{schema.MetricScopeNode}, ctx, 0 /*resolution here*/)
 	if err != nil {
-		cclog.Warn("Error while loading job for stats")
+		log.Warn("Error while loading job for stats")
 		return nil, err
 	}
 	for metric, metricData := range data {
@@ -391,19 +391,19 @@ func (pdb *PrometheusDataRepository) LoadNodeData(
 	for _, scope := range scopes {
 		if scope != schema.MetricScopeNode {
 			logOnce.Do(func() {
-				cclog.Infof("Note: Scope '%s' requested, but not yet supported: Will return 'node' scope only.", scope)
+				log.Infof("Note: Scope '%s' requested, but not yet supported: Will return 'node' scope only.", scope)
 			})
 			continue
 		}
 		for _, metric := range metrics {
 			metricConfig := archive.GetMetricConfig(cluster, metric)
 			if metricConfig == nil {
-				cclog.Warnf("Error in LoadNodeData: Metric %s for cluster %s not configured", metric, cluster)
+				log.Warnf("Error in LoadNodeData: Metric %s for cluster %s not configured", metric, cluster)
 				return nil, errors.New("Prometheus config error")
 			}
 			query, err := pdb.FormatQuery(metric, scope, nodes, cluster)
 			if err != nil {
-				cclog.Warn("Error while formatting prometheus query")
+				log.Warn("Error while formatting prometheus query")
 				return nil, err
 			}
 
@@ -415,11 +415,11 @@ func (pdb *PrometheusDataRepository) LoadNodeData(
 			}
 			result, warnings, err := pdb.queryClient.QueryRange(ctx, query, r)
 			if err != nil {
-				cclog.Errorf("Prometheus query error in LoadNodeData: %v\n", err)
+				log.Errorf("Prometheus query error in LoadNodeData: %v\n", err)
 				return nil, errors.New("Prometheus query error")
 			}
 			if len(warnings) > 0 {
-				cclog.Warnf("Warnings: %v\n", warnings)
+				log.Warnf("Warnings: %v\n", warnings)
 			}
 
 			step := int64(metricConfig.Timestep)
@@ -444,55 +444,10 @@ func (pdb *PrometheusDataRepository) LoadNodeData(
 		}
 	}
 	t1 := time.Since(t0)
-	cclog.Debugf("LoadNodeData of %v nodes took %s", len(data), t1)
+	log.Debugf("LoadNodeData of %v nodes took %s", len(data), t1)
 	return data, nil
 }
 
-// Implemented by NHR@FAU; Used in Job-View StatsTable
-func (pdb *PrometheusDataRepository) LoadScopedStats(
-	job *schema.Job,
-	metrics []string,
-	scopes []schema.MetricScope,
-	ctx context.Context,
-) (schema.ScopedJobStats, error) {
-	// Assumption: pdb.loadData() only returns series node-scope - use node scope for statsTable
-	scopedJobStats := make(schema.ScopedJobStats)
-	data, err := pdb.LoadData(job, metrics, []schema.MetricScope{schema.MetricScopeNode}, ctx, 0 /*resolution here*/)
-	if err != nil {
-		cclog.Warn("Error while loading job for scopedJobStats")
-		return nil, err
-	}
-
-	for metric, metricData := range data {
-		for _, scope := range scopes {
-			if scope != schema.MetricScopeNode {
-				logOnce.Do(func() {
-					cclog.Infof("Note: Scope '%s' requested, but not yet supported: Will return 'node' scope only.", scope)
-				})
-				continue
-			}
-
-			if _, ok := scopedJobStats[metric]; !ok {
-				scopedJobStats[metric] = make(map[schema.MetricScope][]*schema.ScopedStats)
-			}
-
-			if _, ok := scopedJobStats[metric][scope]; !ok {
-				scopedJobStats[metric][scope] = make([]*schema.ScopedStats, 0)
-			}
-
-			for _, series := range metricData[scope].Series {
-				scopedJobStats[metric][scope] = append(scopedJobStats[metric][scope], &schema.ScopedStats{
-					Hostname: series.Hostname,
-					Data:     &series.Statistics,
-				})
-			}
-		}
-	}
-
-	return scopedJobStats, nil
-}
-
-// Implemented by NHR@FAU; Used in NodeList-View
 func (pdb *PrometheusDataRepository) LoadNodeListData(
 	cluster, subCluster, nodeFilter string,
 	metrics []string,
@@ -502,132 +457,11 @@ func (pdb *PrometheusDataRepository) LoadNodeListData(
 	page *model.PageRequest,
 	ctx context.Context,
 ) (map[string]schema.JobData, int, bool, error) {
-	// Assumption: pdb.loadData() only returns series node-scope - use node scope for NodeList
 
-	// 0) Init additional vars
 	var totalNodes int = 0
 	var hasNextPage bool = false
+	// TODO : Implement to be used in NodeList-View
+	log.Infof("LoadNodeListData unimplemented for PrometheusDataRepository, Args: cluster %s, metrics %v, nodeFilter %v, scopes %v", cluster, metrics, nodeFilter, scopes)
 
-	// 1) Get list of all nodes
-	var nodes []string
-	if subCluster != "" {
-		scNodes := archive.NodeLists[cluster][subCluster]
-		nodes = scNodes.PrintList()
-	} else {
-		subClusterNodeLists := archive.NodeLists[cluster]
-		for _, nodeList := range subClusterNodeLists {
-			nodes = append(nodes, nodeList.PrintList()...)
-		}
-	}
-
-	// 2) Filter nodes
-	if nodeFilter != "" {
-		filteredNodes := []string{}
-		for _, node := range nodes {
-			if strings.Contains(node, nodeFilter) {
-				filteredNodes = append(filteredNodes, node)
-			}
-		}
-		nodes = filteredNodes
-	}
-
-	// 2.1) Count total nodes && Sort nodes -> Sorting invalidated after return ...
-	totalNodes = len(nodes)
-	sort.Strings(nodes)
-
-	// 3) Apply paging
-	if len(nodes) > page.ItemsPerPage {
-		start := (page.Page - 1) * page.ItemsPerPage
-		end := start + page.ItemsPerPage
-		if end >= len(nodes) {
-			end = len(nodes)
-			hasNextPage = false
-		} else {
-			hasNextPage = true
-		}
-		nodes = nodes[start:end]
-	}
-
-	// 4) Fetch Data, based on pdb.LoadNodeData()
-
-	t0 := time.Now()
-	// Map of hosts of jobData
-	data := make(map[string]schema.JobData)
-
-	// query db for each metric
-	// TODO: scopes seems to be always empty
-	if len(scopes) == 0 || !contains(scopes, schema.MetricScopeNode) {
-		scopes = append(scopes, schema.MetricScopeNode)
-	}
-
-	for _, scope := range scopes {
-		if scope != schema.MetricScopeNode {
-			logOnce.Do(func() {
-				cclog.Infof("Note: Scope '%s' requested, but not yet supported: Will return 'node' scope only.", scope)
-			})
-			continue
-		}
-
-		for _, metric := range metrics {
-			metricConfig := archive.GetMetricConfig(cluster, metric)
-			if metricConfig == nil {
-				cclog.Warnf("Error in LoadNodeListData: Metric %s for cluster %s not configured", metric, cluster)
-				return nil, totalNodes, hasNextPage, errors.New("Prometheus config error")
-			}
-			query, err := pdb.FormatQuery(metric, scope, nodes, cluster)
-			if err != nil {
-				cclog.Warn("Error while formatting prometheus query")
-				return nil, totalNodes, hasNextPage, err
-			}
-
-			// ranged query over all nodes
-			r := promv1.Range{
-				Start: from,
-				End:   to,
-				Step:  time.Duration(metricConfig.Timestep * 1e9),
-			}
-			result, warnings, err := pdb.queryClient.QueryRange(ctx, query, r)
-			if err != nil {
-				cclog.Errorf("Prometheus query error in LoadNodeData: %v\n", err)
-				return nil, totalNodes, hasNextPage, errors.New("Prometheus query error")
-			}
-			if len(warnings) > 0 {
-				cclog.Warnf("Warnings: %v\n", warnings)
-			}
-
-			step := int64(metricConfig.Timestep)
-			steps := int64(to.Sub(from).Seconds()) / step
-
-			// iter rows of host, metric, values
-			for _, row := range result.(promm.Matrix) {
-				hostname := strings.TrimSuffix(string(row.Metric["exported_instance"]), pdb.suffix)
-
-				hostdata, ok := data[hostname]
-				if !ok {
-					hostdata = make(schema.JobData)
-					data[hostname] = hostdata
-				}
-
-				metricdata, ok := hostdata[metric]
-				if !ok {
-					metricdata = make(map[schema.MetricScope]*schema.JobMetric)
-					data[hostname][metric] = metricdata
-				}
-
-				// output per host, metric and scope
-				scopeData, ok := metricdata[scope]
-				if !ok {
-					scopeData = &schema.JobMetric{
-						Unit:     metricConfig.Unit,
-						Timestep: metricConfig.Timestep,
-						Series:   []schema.Series{pdb.RowToSeries(from, step, steps, row)},
-					}
-					data[hostname][metric][scope] = scopeData
-				}
-			}
-		}
-	}
-	t1 := time.Since(t0)
-	cclog.Debugf("LoadNodeListData of %v nodes took %s", len(data), t1)
-	return data, totalNodes, hasNextPage, nil
+	return nil, totalNodes, hasNextPage, errors.New("METRICDATA/INFLUXV2 > unimplemented for PrometheusDataRepository")
 }

internal/metricdata/utils.go

@@ -1,8 +1,7 @@
 // Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved. This file is part of cc-backend.
+// All rights reserved.
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
-
 package metricdata
 
 import (
@@ -11,14 +10,14 @@ import (
 	"time"
 
 	"github.com/ClusterCockpit/cc-backend/internal/graph/model"
-	"github.com/ClusterCockpit/cc-lib/schema"
+	"github.com/ClusterCockpit/cc-backend/pkg/schema"
 )
 
 var TestLoadDataCallback func(job *schema.Job, metrics []string, scopes []schema.MetricScope, ctx context.Context, resolution int) (schema.JobData, error) = func(job *schema.Job, metrics []string, scopes []schema.MetricScope, ctx context.Context, resolution int) (schema.JobData, error) {
 	panic("TODO")
 }
 
-// TestMetricDataRepository is only a mock for unit-testing.
+// Only a mock for unit-testing.
 type TestMetricDataRepository struct{}
 
 func (tmdr *TestMetricDataRepository) Init(_ json.RawMessage) error {
@@ -30,25 +29,15 @@ func (tmdr *TestMetricDataRepository) LoadData(
 	metrics []string,
 	scopes []schema.MetricScope,
 	ctx context.Context,
-	resolution int,
-) (schema.JobData, error) {
+	resolution int) (schema.JobData, error) {
+
 	return TestLoadDataCallback(job, metrics, scopes, ctx, resolution)
 }
 
 func (tmdr *TestMetricDataRepository) LoadStats(
 	job *schema.Job,
-	metrics []string,
-	ctx context.Context,
-) (map[string]map[string]schema.MetricStatistics, error) {
-	panic("TODO")
-}
+	metrics []string, ctx context.Context) (map[string]map[string]schema.MetricStatistics, error) {
 
-func (tmdr *TestMetricDataRepository) LoadScopedStats(
-	job *schema.Job,
-	metrics []string,
-	scopes []schema.MetricScope,
-	ctx context.Context,
-) (schema.ScopedJobStats, error) {
 	panic("TODO")
 }
 
@@ -57,8 +46,8 @@ func (tmdr *TestMetricDataRepository) LoadNodeData(
 	metrics, nodes []string,
 	scopes []schema.MetricScope,
 	from, to time.Time,
-	ctx context.Context,
-) (map[string]map[string][]*schema.JobMetric, error) {
+	ctx context.Context) (map[string]map[string][]*schema.JobMetric, error) {
+
 	panic("TODO")
 }
 
@@ -71,13 +60,16 @@ func (tmdr *TestMetricDataRepository) LoadNodeListData(
 	page *model.PageRequest,
 	ctx context.Context,
 ) (map[string]schema.JobData, int, bool, error) {
+
 	panic("TODO")
 }
 
-func DeepCopy(jdTemp schema.JobData) schema.JobData {
-	jd := make(schema.JobData, len(jdTemp))
-	for k, v := range jdTemp {
-		jd[k] = make(map[schema.MetricScope]*schema.JobMetric, len(jdTemp[k]))
+func DeepCopy(jd_temp schema.JobData) schema.JobData {
+	var jd schema.JobData
+
+	jd = make(schema.JobData, len(jd_temp))
+	for k, v := range jd_temp {
+		jd[k] = make(map[schema.MetricScope]*schema.JobMetric, len(jd_temp[k]))
 		for k_, v_ := range v {
 			jd[k][k_] = new(schema.JobMetric)
 			jd[k][k_].Series = make([]schema.Series, len(v_.Series))

internal/repository/dbConnection.go

@@ -1,17 +1,15 @@
 // Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved. This file is part of cc-backend.
+// All rights reserved.
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 package repository
 
 import (
 	"database/sql"
-	"fmt"
-	"net/url"
 	"sync"
 	"time"
 
-	cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
+	"github.com/ClusterCockpit/cc-backend/pkg/log"
 	"github.com/jmoiron/sqlx"
 	"github.com/mattn/go-sqlite3"
 	"github.com/qustavo/sqlhooks/v2"
@@ -35,27 +33,6 @@ type DatabaseOptions struct {
 	ConnectionMaxIdleTime time.Duration
 }
 
-func setupSqlite(db *sql.DB) (err error) {
-	pragmas := []string{
-		// "journal_mode = WAL",
-		// "busy_timeout = 5000",
-		// "synchronous = NORMAL",
-		// "cache_size = 1000000000", // 1GB
-		// "foreign_keys = true",
-		"temp_store = memory",
-		// "mmap_size = 3000000000",
-	}
-
-	for _, pragma := range pragmas {
-		_, err = db.Exec("PRAGMA " + pragma)
-		if err != nil {
-			return
-		}
-	}
-
-	return nil
-}
-
 func Connect(driver string, db string) {
 	var err error
 	var dbHandle *sqlx.DB
@@ -71,34 +48,28 @@ func Connect(driver string, db string) {
 
 		switch driver {
 		case "sqlite3":
-			// TODO: Have separate DB handles for Writes and Reads
-			// Optimize SQLite connection: https://kerkour.com/sqlite-for-servers
-			connectionUrlParams := make(url.Values)
-			connectionUrlParams.Add("_txlock", "immediate")
-			connectionUrlParams.Add("_journal_mode", "WAL")
-			connectionUrlParams.Add("_busy_timeout", "5000")
-			connectionUrlParams.Add("_synchronous", "NORMAL")
-			connectionUrlParams.Add("_cache_size", "1000000000")
-			connectionUrlParams.Add("_foreign_keys", "true")
-			opts.URL = fmt.Sprintf("file:%s?%s", opts.URL, connectionUrlParams.Encode())
+			// - Set WAL mode (not strictly necessary each time because it's persisted in the database, but good for first run)
+			// - Set busy timeout, so concurrent writers wait on each other instead of erroring immediately
+			// - Enable foreign key checks
+			opts.URL += "?_journal=WAL&_timeout=5000&_fk=true"
 
-			if cclog.Loglevel() == "debug" {
+			if log.Loglevel() == "debug" {
 				sql.Register("sqlite3WithHooks", sqlhooks.Wrap(&sqlite3.SQLiteDriver{}, &Hooks{}))
 				dbHandle, err = sqlx.Open("sqlite3WithHooks", opts.URL)
 			} else {
 				dbHandle, err = sqlx.Open("sqlite3", opts.URL)
 			}
-
-			setupSqlite(dbHandle.DB)
+			if err != nil {
+				log.Fatal(err)
+			}
 		case "mysql":
 			opts.URL += "?multiStatements=true"
 			dbHandle, err = sqlx.Open("mysql", opts.URL)
+			if err != nil {
+				log.Fatalf("sqlx.Open() error: %v", err)
+			}
 		default:
-			cclog.Abortf("DB Connection: Unsupported database driver '%s'.\n", driver)
-		}
-
-		if err != nil {
-			cclog.Abortf("DB Connection: Could not connect to '%s' database with sqlx.Open().\nError: %s\n", driver, err.Error())
+			log.Fatalf("unsupported database driver: %s", driver)
 		}
 
 		dbHandle.SetMaxOpenConns(opts.MaxOpenConnections)
@@ -109,14 +80,14 @@ func Connect(driver string, db string) {
 		dbConnInstance = &DBConnection{DB: dbHandle, Driver: driver}
 		err = checkDBVersion(driver, dbHandle.DB)
 		if err != nil {
-			cclog.Abortf("DB Connection: Failed DB version check.\nError: %s\n", err.Error())
+			log.Fatal(err)
 		}
 	})
 }
 
 func GetConnection() *DBConnection {
 	if dbConnInstance == nil {
-		cclog.Fatalf("Database connection not initialized!")
+		log.Fatalf("Database connection not initialized!")
 	}
 
 	return dbConnInstance

internal/repository/hooks.go

@@ -1,5 +1,5 @@
 // Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved. This file is part of cc-backend.
+// All rights reserved.
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 package repository
@@ -8,21 +8,21 @@ import (
 	"context"
 	"time"
 
-	cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
+	"github.com/ClusterCockpit/cc-backend/pkg/log"
 )
 
 // Hooks satisfies the sqlhook.Hooks interface
 type Hooks struct{}
 
 // Before hook will print the query with it's args and return the context with the timestamp
-func (h *Hooks) Before(ctx context.Context, query string, args ...any) (context.Context, error) {
-	cclog.Debugf("SQL query %s %q", query, args)
+func (h *Hooks) Before(ctx context.Context, query string, args ...interface{}) (context.Context, error) {
+	log.Debugf("SQL query %s %q", query, args)
 	return context.WithValue(ctx, "begin", time.Now()), nil
 }
 
 // After hook will get the timestamp registered on the Before hook and print the elapsed time
-func (h *Hooks) After(ctx context.Context, query string, args ...any) (context.Context, error) {
+func (h *Hooks) After(ctx context.Context, query string, args ...interface{}) (context.Context, error) {
 	begin := ctx.Value("begin").(time.Time)
-	cclog.Debugf("Took: %s\n", time.Since(begin))
+	log.Debugf("Took: %s\n", time.Since(begin))
 	return ctx, nil
 }

internal/repository/job.go

@@ -1,5 +1,5 @@
 // Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved. This file is part of cc-backend.
+// All rights reserved.
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 package repository
@@ -9,16 +9,16 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"maps"
 	"math"
 	"strconv"
 	"sync"
 	"time"
 
+	"github.com/ClusterCockpit/cc-backend/internal/graph/model"
 	"github.com/ClusterCockpit/cc-backend/pkg/archive"
-	cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
-	"github.com/ClusterCockpit/cc-lib/lrucache"
-	"github.com/ClusterCockpit/cc-lib/schema"
+	"github.com/ClusterCockpit/cc-backend/pkg/log"
+	"github.com/ClusterCockpit/cc-backend/pkg/lrucache"
+	"github.com/ClusterCockpit/cc-backend/pkg/schema"
 	sq "github.com/Masterminds/squirrel"
 	"github.com/jmoiron/sqlx"
 )
@@ -33,7 +33,6 @@ type JobRepository struct {
 	stmtCache *sq.StmtCache
 	cache     *lrucache.Cache
 	driver    string
-	Mutex     sync.Mutex
 }
 
 func GetJobRepository() *JobRepository {
@@ -52,49 +51,38 @@ func GetJobRepository() *JobRepository {
 }
 
 var jobColumns []string = []string{
-	"job.id", "job.job_id", "job.hpc_user", "job.project", "job.cluster", "job.subcluster",
-	"job.start_time", "job.cluster_partition", "job.array_job_id", "job.num_nodes",
-	"job.num_hwthreads", "job.num_acc", "job.shared", "job.monitoring_status",
-	"job.smt", "job.job_state", "job.duration", "job.walltime", "job.resources",
-	"job.footprint", "job.energy",
+	"job.id", "job.job_id", "job.hpc_user", "job.project", "job.cluster", "job.subcluster", "job.start_time", "job.cluster_partition", "job.array_job_id",
+	"job.num_nodes", "job.num_hwthreads", "job.num_acc", "job.exclusive", "job.monitoring_status", "job.smt", "job.job_state",
+	"job.duration", "job.walltime", "job.resources", "job.footprint", "job.energy",
 }
 
-var jobCacheColumns []string = []string{
-	"job_cache.id", "job_cache.job_id", "job_cache.hpc_user", "job_cache.project", "job_cache.cluster",
-	"job_cache.subcluster", "job_cache.start_time", "job_cache.cluster_partition",
-	"job_cache.array_job_id", "job_cache.num_nodes", "job_cache.num_hwthreads",
-	"job_cache.num_acc", "job_cache.shared", "job_cache.monitoring_status", "job_cache.smt",
-	"job_cache.job_state", "job_cache.duration", "job_cache.walltime", "job_cache.resources",
-	"job_cache.footprint", "job_cache.energy",
-}
-
-func scanJob(row interface{ Scan(...any) error }) (*schema.Job, error) {
+func scanJob(row interface{ Scan(...interface{}) error }) (*schema.Job, error) {
 	job := &schema.Job{}
 
 	if err := row.Scan(
-		&job.ID, &job.JobID, &job.User, &job.Project, &job.Cluster, &job.SubCluster,
-		&job.StartTime, &job.Partition, &job.ArrayJobID, &job.NumNodes, &job.NumHWThreads,
-		&job.NumAcc, &job.Shared, &job.MonitoringStatus, &job.SMT, &job.State,
+		&job.ID, &job.JobID, &job.User, &job.Project, &job.Cluster, &job.SubCluster, &job.StartTimeUnix, &job.Partition, &job.ArrayJobId,
+		&job.NumNodes, &job.NumHWThreads, &job.NumAcc, &job.Exclusive, &job.MonitoringStatus, &job.SMT, &job.State,
 		&job.Duration, &job.Walltime, &job.RawResources, &job.RawFootprint, &job.Energy); err != nil {
-		cclog.Warnf("Error while scanning rows (Job): %v", err)
+		log.Warnf("Error while scanning rows (Job): %v", err)
 		return nil, err
 	}
 
 	if err := json.Unmarshal(job.RawResources, &job.Resources); err != nil {
-		cclog.Warn("Error while unmarshaling raw resources json")
+		log.Warn("Error while unmarshaling raw resources json")
 		return nil, err
 	}
 	job.RawResources = nil
 
 	if err := json.Unmarshal(job.RawFootprint, &job.Footprint); err != nil {
-		cclog.Warnf("Error while unmarshaling raw footprint json: %v", err)
+		log.Warnf("Error while unmarshaling raw footprint json: %v", err)
 		return nil, err
 	}
 	job.RawFootprint = nil
 
+	job.StartTime = time.Unix(job.StartTimeUnix, 0)
 	// Always ensure accurate duration for running jobs
 	if job.State == schema.JobStateRunning {
-		job.Duration = int32(time.Now().Unix() - job.StartTime)
+		job.Duration = int32(time.Since(job.StartTime).Seconds())
 	}
 
 	return job, nil
@@ -109,7 +97,7 @@ func (r *JobRepository) Optimize() error {
 			return err
 		}
 	case "mysql":
-		cclog.Info("Optimize currently not supported for mysql driver")
+		log.Info("Optimize currently not supported for mysql driver")
 	}
 
 	return nil
@@ -150,6 +138,17 @@ func (r *JobRepository) Flush() error {
 	return nil
 }
 
+func scanJobLink(row interface{ Scan(...interface{}) error }) (*model.JobLink, error) {
+	jobLink := &model.JobLink{}
+	if err := row.Scan(
+		&jobLink.ID, &jobLink.JobID); err != nil {
+		log.Warn("Error while scanning rows (jobLink)")
+		return nil, err
+	}
+
+	return jobLink, nil
+}
+
 func (r *JobRepository) FetchMetadata(job *schema.Job) (map[string]string, error) {
 	start := time.Now()
 	cachekey := fmt.Sprintf("metadata:%d", job.ID)
@@ -160,7 +159,7 @@ func (r *JobRepository) FetchMetadata(job *schema.Job) (map[string]string, error
 
 	if err := sq.Select("job.meta_data").From("job").Where("job.id = ?", job.ID).
 		RunWith(r.stmtCache).QueryRow().Scan(&job.RawMetaData); err != nil {
-		cclog.Warn("Error while scanning for job metadata")
+		log.Warn("Error while scanning for job metadata")
 		return nil, err
 	}
 
@@ -169,12 +168,12 @@ func (r *JobRepository) FetchMetadata(job *schema.Job) (map[string]string, error
 	}
 
 	if err := json.Unmarshal(job.RawMetaData, &job.MetaData); err != nil {
-		cclog.Warn("Error while unmarshaling raw metadata json")
+		log.Warn("Error while unmarshaling raw metadata json")
 		return nil, err
 	}
 
 	r.cache.Put(cachekey, job.MetaData, len(job.RawMetaData), 24*time.Hour)
-	cclog.Debugf("Timer FetchMetadata %s", time.Since(start))
+	log.Debugf("Timer FetchMetadata %s", time.Since(start))
 	return job.MetaData, nil
 }
 
@@ -183,14 +182,16 @@ func (r *JobRepository) UpdateMetadata(job *schema.Job, key, val string) (err er
 	r.cache.Del(cachekey)
 	if job.MetaData == nil {
 		if _, err = r.FetchMetadata(job); err != nil {
-			cclog.Warnf("Error while fetching metadata for job, DB ID '%v'", job.ID)
+			log.Warnf("Error while fetching metadata for job, DB ID '%v'", job.ID)
 			return err
 		}
 	}
 
 	if job.MetaData != nil {
 		cpy := make(map[string]string, len(job.MetaData)+1)
-		maps.Copy(cpy, job.MetaData)
+		for k, v := range job.MetaData {
+			cpy[k] = v
+		}
 		cpy[key] = val
 		job.MetaData = cpy
 	} else {
@@ -198,7 +199,7 @@ func (r *JobRepository) UpdateMetadata(job *schema.Job, key, val string) (err er
 	}
 
 	if job.RawMetaData, err = json.Marshal(job.MetaData); err != nil {
-		cclog.Warnf("Error while marshaling metadata for job, DB ID '%v'", job.ID)
+		log.Warnf("Error while marshaling metadata for job, DB ID '%v'", job.ID)
 		return err
 	}
 
@@ -206,7 +207,7 @@ func (r *JobRepository) UpdateMetadata(job *schema.Job, key, val string) (err er
 		Set("meta_data", job.RawMetaData).
 		Where("job.id = ?", job.ID).
 		RunWith(r.stmtCache).Exec(); err != nil {
-		cclog.Warnf("Error while updating metadata for job, DB ID '%v'", job.ID)
+		log.Warnf("Error while updating metadata for job, DB ID '%v'", job.ID)
 		return err
 	}
 
@@ -216,10 +217,15 @@ func (r *JobRepository) UpdateMetadata(job *schema.Job, key, val string) (err er
 
 func (r *JobRepository) FetchFootprint(job *schema.Job) (map[string]float64, error) {
 	start := time.Now()
+	cachekey := fmt.Sprintf("footprint:%d", job.ID)
+	if cached := r.cache.Get(cachekey, nil); cached != nil {
+		job.Footprint = cached.(map[string]float64)
+		return job.Footprint, nil
+	}
 
 	if err := sq.Select("job.footprint").From("job").Where("job.id = ?", job.ID).
 		RunWith(r.stmtCache).QueryRow().Scan(&job.RawFootprint); err != nil {
-		cclog.Warn("Error while scanning for job footprint")
+		log.Warn("Error while scanning for job footprint")
 		return nil, err
 	}
 
@@ -228,11 +234,12 @@ func (r *JobRepository) FetchFootprint(job *schema.Job) (map[string]float64, err
 	}
 
 	if err := json.Unmarshal(job.RawFootprint, &job.Footprint); err != nil {
-		cclog.Warn("Error while unmarshaling raw footprint json")
+		log.Warn("Error while unmarshaling raw footprint json")
 		return nil, err
 	}
 
-	cclog.Debugf("Timer FetchFootprint %s", time.Since(start))
+	r.cache.Put(cachekey, job.Footprint, len(job.Footprint), 24*time.Hour)
+	log.Debugf("Timer FetchFootprint %s", time.Since(start))
 	return job.Footprint, nil
 }
 
@@ -246,7 +253,7 @@ func (r *JobRepository) FetchEnergyFootprint(job *schema.Job) (map[string]float6
 
 	if err := sq.Select("job.energy_footprint").From("job").Where("job.id = ?", job.ID).
 		RunWith(r.stmtCache).QueryRow().Scan(&job.RawEnergyFootprint); err != nil {
-		cclog.Warn("Error while scanning for job energy_footprint")
+		log.Warn("Error while scanning for job energy_footprint")
 		return nil, err
 	}
 
@@ -255,12 +262,12 @@ func (r *JobRepository) FetchEnergyFootprint(job *schema.Job) (map[string]float6
 	}
 
 	if err := json.Unmarshal(job.RawEnergyFootprint, &job.EnergyFootprint); err != nil {
-		cclog.Warn("Error while unmarshaling raw energy footprint json")
+		log.Warn("Error while unmarshaling raw energy footprint json")
 		return nil, err
 	}
 
 	r.cache.Put(cachekey, job.EnergyFootprint, len(job.EnergyFootprint), 24*time.Hour)
-	cclog.Debugf("Timer FetchEnergyFootprint %s", time.Since(start))
+	log.Debugf("Timer FetchEnergyFootprint %s", time.Since(start))
 	return job.EnergyFootprint, nil
 }
 
@@ -273,9 +280,9 @@ func (r *JobRepository) DeleteJobsBefore(startTime int64) (int, error) {
 
 	if err != nil {
 		s, _, _ := qd.ToSql()
-		cclog.Errorf(" DeleteJobsBefore(%d) with %s: error %#v", startTime, s, err)
+		log.Errorf(" DeleteJobsBefore(%d) with %s: error %#v", startTime, s, err)
 	} else {
-		cclog.Debugf("DeleteJobsBefore(%d): Deleted %d jobs", startTime, cnt)
+		log.Debugf("DeleteJobsBefore(%d): Deleted %d jobs", startTime, cnt)
 	}
 	return cnt, err
 }
@@ -286,9 +293,9 @@ func (r *JobRepository) DeleteJobById(id int64) error {
 
 	if err != nil {
 		s, _, _ := qd.ToSql()
-		cclog.Errorf("DeleteJobById(%d) with %s : error %#v", id, s, err)
+		log.Errorf("DeleteJobById(%d) with %s : error %#v", id, s, err)
 	} else {
-		cclog.Debugf("DeleteJobById(%d): Success", id)
+		log.Debugf("DeleteJobById(%d): Success", id)
 	}
 	return err
 }
@@ -337,10 +344,10 @@ func (r *JobRepository) FindColumnValue(user *schema.User, searchterm string, ta
 
 		// theSql, args, theErr := theQuery.ToSql()
 		// if theErr != nil {
-		// 	cclog.Warn("Error while converting query to sql")
+		// 	log.Warn("Error while converting query to sql")
 		// 	return "", err
 		// }
-		// cclog.Debugf("SQL query (FindColumnValue): `%s`, args: %#v", theSql, args)
+		// log.Debugf("SQL query (FindColumnValue): `%s`, args: %#v", theSql, args)
 
 		err := theQuery.RunWith(r.stmtCache).QueryRow().Scan(&result)
 
@@ -351,7 +358,7 @@ func (r *JobRepository) FindColumnValue(user *schema.User, searchterm string, ta
 		}
 		return "", ErrNotFound
 	} else {
-		cclog.Infof("Non-Admin User %s : Requested Query '%s' on table '%s' : Forbidden", user.Name, query, table)
+		log.Infof("Non-Admin User %s : Requested Query '%s' on table '%s' : Forbidden", user.Name, query, table)
 		return "", ErrForbidden
 	}
 }
@@ -370,7 +377,7 @@ func (r *JobRepository) FindColumnValues(user *schema.User, query string, table
 				err := rows.Scan(&result)
 				if err != nil {
 					rows.Close()
-					cclog.Warnf("Error while scanning rows: %v", err)
+					log.Warnf("Error while scanning rows: %v", err)
 					return emptyResult, err
 				}
 				results = append(results, result)
@@ -380,7 +387,7 @@ func (r *JobRepository) FindColumnValues(user *schema.User, query string, table
 		return emptyResult, ErrNotFound
 
 	} else {
-		cclog.Infof("Non-Admin User %s : Requested Query '%s' on table '%s' : Forbidden", user.Name, query, table)
+		log.Infof("Non-Admin User %s : Requested Query '%s' on table '%s' : Forbidden", user.Name, query, table)
 		return emptyResult, ErrForbidden
 	}
 }
@@ -388,7 +395,7 @@ func (r *JobRepository) FindColumnValues(user *schema.User, query string, table
 func (r *JobRepository) Partitions(cluster string) ([]string, error) {
 	var err error
 	start := time.Now()
-	partitions := r.cache.Get("partitions:"+cluster, func() (any, time.Duration, int) {
+	partitions := r.cache.Get("partitions:"+cluster, func() (interface{}, time.Duration, int) {
 		parts := []string{}
 		if err = r.DB.Select(&parts, `SELECT DISTINCT job.cluster_partition FROM job WHERE job.cluster = ?;`, cluster); err != nil {
 			return nil, 0, 1000
@@ -399,7 +406,7 @@ func (r *JobRepository) Partitions(cluster string) ([]string, error) {
 	if err != nil {
 		return nil, err
 	}
-	cclog.Debugf("Timer Partitions %s", time.Since(start))
+	log.Debugf("Timer Partitions %s", time.Since(start))
 	return partitions.([]string), nil
 }
 
@@ -413,7 +420,7 @@ func (r *JobRepository) AllocatedNodes(cluster string) (map[string]map[string]in
 		Where("job.cluster = ?", cluster).
 		RunWith(r.stmtCache).Query()
 	if err != nil {
-		cclog.Error("Error while running query")
+		log.Error("Error while running query")
 		return nil, err
 	}
 
@@ -424,11 +431,11 @@ func (r *JobRepository) AllocatedNodes(cluster string) (map[string]map[string]in
 		var resources []*schema.Resource
 		var subcluster string
 		if err := rows.Scan(&raw, &subcluster); err != nil {
-			cclog.Warn("Error while scanning rows")
+			log.Warn("Error while scanning rows")
 			return nil, err
 		}
 		if err := json.Unmarshal(raw, &resources); err != nil {
-			cclog.Warn("Error while unmarshaling raw resources json")
+			log.Warn("Error while unmarshaling raw resources json")
 			return nil, err
 		}
 
@@ -443,7 +450,7 @@ func (r *JobRepository) AllocatedNodes(cluster string) (map[string]map[string]in
 		}
 	}
 
-	cclog.Debugf("Timer AllocatedNodes %s", time.Since(start))
+	log.Debugf("Timer AllocatedNodes %s", time.Since(start))
 	return subclusters, nil
 }
 
@@ -459,50 +466,23 @@ func (r *JobRepository) StopJobsExceedingWalltimeBy(seconds int) error {
 		Where(fmt.Sprintf("(%d - job.start_time) > (job.walltime + %d)", time.Now().Unix(), seconds)).
 		RunWith(r.DB).Exec()
 	if err != nil {
-		cclog.Warn("Error while stopping jobs exceeding walltime")
+		log.Warn("Error while stopping jobs exceeding walltime")
 		return err
 	}
 
 	rowsAffected, err := res.RowsAffected()
 	if err != nil {
-		cclog.Warn("Error while fetching affected rows after stopping due to exceeded walltime")
+		log.Warn("Error while fetching affected rows after stopping due to exceeded walltime")
 		return err
 	}
 
 	if rowsAffected > 0 {
-		cclog.Infof("%d jobs have been marked as failed due to running too long", rowsAffected)
+		log.Infof("%d jobs have been marked as failed due to running too long", rowsAffected)
 	}
-	cclog.Debugf("Timer StopJobsExceedingWalltimeBy %s", time.Since(start))
+	log.Debugf("Timer StopJobsExceedingWalltimeBy %s", time.Since(start))
 	return nil
 }
 
-func (r *JobRepository) FindJobIdsByTag(tagId int64) ([]int64, error) {
-	query := sq.Select("job.id").From("job").
-		Join("jobtag ON jobtag.job_id = job.id").
-		Where(sq.Eq{"jobtag.tag_id": tagId}).Distinct()
-	rows, err := query.RunWith(r.stmtCache).Query()
-	if err != nil {
-		cclog.Error("Error while running query")
-		return nil, err
-	}
-	jobIds := make([]int64, 0, 100)
-
-	for rows.Next() {
-		var jobId int64
-
-		if err := rows.Scan(&jobId); err != nil {
-			rows.Close()
-			cclog.Warn("Error while scanning rows")
-			return nil, err
-		}
-
-		jobIds = append(jobIds, jobId)
-	}
-
-	return jobIds, nil
-}
-
-// FIXME: Reconsider filtering short jobs with harcoded threshold
 func (r *JobRepository) FindRunningJobs(cluster string) ([]*schema.Job, error) {
 	query := sq.Select(jobColumns...).From("job").
 		Where(fmt.Sprintf("job.cluster = '%s'", cluster)).
@@ -511,7 +491,7 @@ func (r *JobRepository) FindRunningJobs(cluster string) ([]*schema.Job, error) {
 
 	rows, err := query.RunWith(r.stmtCache).Query()
 	if err != nil {
-		cclog.Error("Error while running query")
+		log.Error("Error while running query")
 		return nil, err
 	}
 
@@ -520,13 +500,13 @@ func (r *JobRepository) FindRunningJobs(cluster string) ([]*schema.Job, error) {
 		job, err := scanJob(rows)
 		if err != nil {
 			rows.Close()
-			cclog.Warn("Error while scanning rows")
+			log.Warn("Error while scanning rows")
 			return nil, err
 		}
 		jobs = append(jobs, job)
 	}
 
-	cclog.Infof("Return job count %d", len(jobs))
+	log.Infof("Return job count %d", len(jobs))
 	return jobs, nil
 }
 
@@ -551,18 +531,18 @@ func (r *JobRepository) FindJobsBetween(startTimeBegin int64, startTimeEnd int64
 	}
 
 	if startTimeBegin == 0 {
-		cclog.Infof("Find jobs before %d", startTimeEnd)
+		log.Infof("Find jobs before %d", startTimeEnd)
 		query = sq.Select(jobColumns...).From("job").Where(fmt.Sprintf(
 			"job.start_time < %d", startTimeEnd))
 	} else {
-		cclog.Infof("Find jobs between %d and %d", startTimeBegin, startTimeEnd)
+		log.Infof("Find jobs between %d and %d", startTimeBegin, startTimeEnd)
 		query = sq.Select(jobColumns...).From("job").Where(fmt.Sprintf(
 			"job.start_time BETWEEN %d AND %d", startTimeBegin, startTimeEnd))
 	}
 
 	rows, err := query.RunWith(r.stmtCache).Query()
 	if err != nil {
-		cclog.Error("Error while running query")
+		log.Error("Error while running query")
 		return nil, err
 	}
 
@@ -571,13 +551,13 @@ func (r *JobRepository) FindJobsBetween(startTimeBegin int64, startTimeEnd int64
 		job, err := scanJob(rows)
 		if err != nil {
 			rows.Close()
-			cclog.Warn("Error while scanning rows")
+			log.Warn("Error while scanning rows")
 			return nil, err
 		}
 		jobs = append(jobs, job)
 	}
 
-	cclog.Infof("Return job count %d", len(jobs))
+	log.Infof("Return job count %d", len(jobs))
 	return jobs, nil
 }
 
@@ -587,7 +567,7 @@ func (r *JobRepository) UpdateMonitoringStatus(job int64, monitoringStatus int32
 		Where("job.id = ?", job)
 
 	_, err = stmt.RunWith(r.stmtCache).Exec()
-	return err
+	return
 }
 
 func (r *JobRepository) Execute(stmt sq.UpdateBuilder) error {
@@ -607,62 +587,53 @@ func (r *JobRepository) MarkArchived(
 
 func (r *JobRepository) UpdateEnergy(
 	stmt sq.UpdateBuilder,
-	jobMeta *schema.Job,
+	jobMeta *schema.JobMeta,
 ) (sq.UpdateBuilder, error) {
 	/* Note: Only Called for Running Jobs during Intermediate Update or on Archiving */
 	sc, err := archive.GetSubCluster(jobMeta.Cluster, jobMeta.SubCluster)
 	if err != nil {
-		cclog.Errorf("cannot get subcluster: %s", err.Error())
+		log.Errorf("cannot get subcluster: %s", err.Error())
 		return stmt, err
 	}
 	energyFootprint := make(map[string]float64)
+	var totalEnergy float64
+	var energy float64
 
-	// Total Job Energy Outside Loop
-	totalEnergy := 0.0
 	for _, fp := range sc.EnergyFootprint {
-		// Always Init Metric Energy Inside Loop
-		metricEnergy := 0.0
 		if i, err := archive.MetricIndex(sc.MetricConfig, fp); err == nil {
 			// Note: For DB data, calculate and save as kWh
 			if sc.MetricConfig[i].Energy == "energy" { // this metric has energy as unit (Joules or Wh)
-				cclog.Warnf("Update EnergyFootprint for Job %d and Metric %s on cluster %s: Set to 'energy' in cluster.json: Not implemented, will return 0.0", jobMeta.JobID, jobMeta.Cluster, fp)
 				// FIXME: Needs sum as stats type
 			} else if sc.MetricConfig[i].Energy == "power" { // this metric has power as unit (Watt)
 				// Energy: Power (in Watts) * Time (in Seconds)
-				// Unit: (W * (s / 3600)) / 1000 = kWh
-				// Round 2 Digits: round(Energy * 100) / 100
-				// Here: (All-Node Metric Average * Number of Nodes) * (Job Duration in Seconds / 3600) / 1000
-				// Note: Shared Jobs handled correctly since "Node Average" is based on partial resources, while "numNodes" factor is 1
-				rawEnergy := ((LoadJobStat(jobMeta, fp, "avg") * float64(jobMeta.NumNodes)) * (float64(jobMeta.Duration) / 3600.0)) / 1000.0
-				metricEnergy = math.Round(rawEnergy*100.0) / 100.0
+				// Unit: ( W * s ) / 3600 / 1000 = kWh ; Rounded to 2 nearest digits
+				energy = math.Round(((LoadJobStat(jobMeta, fp, "avg")*float64(jobMeta.Duration))/3600/1000)*100) / 100
 			}
 		} else {
-			cclog.Warnf("Error while collecting energy metric %s for job, DB ID '%v', return '0.0'", fp, jobMeta.ID)
+			log.Warnf("Error while collecting energy metric %s for job, DB ID '%v', return '0.0'", fp, jobMeta.ID)
 		}
 
-		energyFootprint[fp] = metricEnergy
-		totalEnergy += metricEnergy
-
-		// cclog.Infof("Metric %s Average %f -> %f kWh | Job %d Total -> %f kWh", fp, LoadJobStat(jobMeta, fp, "avg"), energy, jobMeta.JobID, totalEnergy)
+		energyFootprint[fp] = energy
+		totalEnergy += energy
 	}
 
 	var rawFootprint []byte
 	if rawFootprint, err = json.Marshal(energyFootprint); err != nil {
-		cclog.Warnf("Error while marshaling energy footprint for job INTO BYTES, DB ID '%v'", jobMeta.ID)
+		log.Warnf("Error while marshaling energy footprint for job INTO BYTES, DB ID '%v'", jobMeta.ID)
 		return stmt, err
 	}
 
-	return stmt.Set("energy_footprint", string(rawFootprint)).Set("energy", (math.Round(totalEnergy*100.0) / 100.0)), nil
+	return stmt.Set("energy_footprint", string(rawFootprint)).Set("energy", (math.Round(totalEnergy*100) / 100)), nil
 }
 
 func (r *JobRepository) UpdateFootprint(
 	stmt sq.UpdateBuilder,
-	jobMeta *schema.Job,
+	jobMeta *schema.JobMeta,
 ) (sq.UpdateBuilder, error) {
 	/* Note: Only Called for Running Jobs during Intermediate Update or on Archiving */
 	sc, err := archive.GetSubCluster(jobMeta.Cluster, jobMeta.SubCluster)
 	if err != nil {
-		cclog.Errorf("cannot get subcluster: %s", err.Error())
+		log.Errorf("cannot get subcluster: %s", err.Error())
 		return stmt, err
 	}
 	footprint := make(map[string]float64)
@@ -676,7 +647,7 @@ func (r *JobRepository) UpdateFootprint(
 		}
 
 		if statType != "avg" && statType != "min" && statType != "max" {
-			cclog.Warnf("unknown statType for footprint update: %s", statType)
+			log.Warnf("unknown statType for footprint update: %s", statType)
 			return stmt, fmt.Errorf("unknown statType for footprint update: %s", statType)
 		}
 
@@ -690,7 +661,7 @@ func (r *JobRepository) UpdateFootprint(
 
 	var rawFootprint []byte
 	if rawFootprint, err = json.Marshal(footprint); err != nil {
-		cclog.Warnf("Error while marshaling footprint for job INTO BYTES, DB ID '%v'", jobMeta.ID)
+		log.Warnf("Error while marshaling footprint for job INTO BYTES, DB ID '%v'", jobMeta.ID)
 		return stmt, err
 	}
 

internal/repository/jobCreate.go

@@ -1,5 +1,5 @@
 // Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved. This file is part of cc-backend.
+// All rights reserved.
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 package repository
@@ -8,86 +8,37 @@ import (
 	"encoding/json"
 	"fmt"
 
-	cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
-	"github.com/ClusterCockpit/cc-lib/schema"
+	"github.com/ClusterCockpit/cc-backend/pkg/log"
+	"github.com/ClusterCockpit/cc-backend/pkg/schema"
 	sq "github.com/Masterminds/squirrel"
 )
 
-const NamedJobCacheInsert string = `INSERT INTO job_cache (
-	job_id, hpc_user, project, cluster, subcluster, cluster_partition, array_job_id, num_nodes, num_hwthreads, num_acc,
-	shared, monitoring_status, smt, job_state, start_time, duration, walltime, footprint, energy, energy_footprint, resources, meta_data
-) VALUES (
-	:job_id, :hpc_user, :project, :cluster, :subcluster, :cluster_partition, :array_job_id, :num_nodes, :num_hwthreads, :num_acc,
-  :shared, :monitoring_status, :smt, :job_state, :start_time, :duration, :walltime, :footprint,  :energy, :energy_footprint, :resources, :meta_data
-);`
-
 const NamedJobInsert string = `INSERT INTO job (
 	job_id, hpc_user, project, cluster, subcluster, cluster_partition, array_job_id, num_nodes, num_hwthreads, num_acc,
-	shared, monitoring_status, smt, job_state, start_time, duration, walltime, footprint, energy, energy_footprint, resources, meta_data
+	exclusive, monitoring_status, smt, job_state, start_time, duration, walltime, footprint, energy, energy_footprint, resources, meta_data
 ) VALUES (
 	:job_id, :hpc_user, :project, :cluster, :subcluster, :cluster_partition, :array_job_id, :num_nodes, :num_hwthreads, :num_acc,
-  :shared, :monitoring_status, :smt, :job_state, :start_time, :duration, :walltime, :footprint,  :energy, :energy_footprint, :resources, :meta_data
+  :exclusive, :monitoring_status, :smt, :job_state, :start_time, :duration, :walltime, :footprint,  :energy, :energy_footprint, :resources, :meta_data
 );`
 
-func (r *JobRepository) InsertJob(job *schema.Job) (int64, error) {
-	r.Mutex.Lock()
-	res, err := r.DB.NamedExec(NamedJobCacheInsert, job)
-	r.Mutex.Unlock()
+func (r *JobRepository) InsertJob(job *schema.JobMeta) (int64, error) {
+	res, err := r.DB.NamedExec(NamedJobInsert, job)
 	if err != nil {
-		cclog.Warn("Error while NamedJobInsert")
+		log.Warn("Error while NamedJobInsert")
 		return 0, err
 	}
 	id, err := res.LastInsertId()
 	if err != nil {
-		cclog.Warn("Error while getting last insert ID")
+		log.Warn("Error while getting last insert ID")
 		return 0, err
 	}
 
 	return id, nil
 }
 
-func (r *JobRepository) SyncJobs() ([]*schema.Job, error) {
-	r.Mutex.Lock()
-	defer r.Mutex.Unlock()
-
-	query := sq.Select(jobCacheColumns...).From("job_cache")
-
-	rows, err := query.RunWith(r.stmtCache).Query()
-	if err != nil {
-		cclog.Errorf("Error while running query %v", err)
-		return nil, err
-	}
-
-	jobs := make([]*schema.Job, 0, 50)
-	for rows.Next() {
-		job, err := scanJob(rows)
-		if err != nil {
-			rows.Close()
-			cclog.Warn("Error while scanning rows")
-			return nil, err
-		}
-		jobs = append(jobs, job)
-	}
-
-	_, err = r.DB.Exec(
-		"INSERT INTO job (job_id, cluster, subcluster, start_time, hpc_user, project, cluster_partition, array_job_id, num_nodes, num_hwthreads, num_acc, shared, monitoring_status, smt, job_state, duration, walltime, footprint, energy, energy_footprint, resources, meta_data) SELECT job_id, cluster, subcluster, start_time, hpc_user, project, cluster_partition, array_job_id, num_nodes, num_hwthreads, num_acc, shared, monitoring_status, smt, job_state, duration, walltime, footprint, energy, energy_footprint, resources, meta_data FROM job_cache")
-	if err != nil {
-		cclog.Warnf("Error while Job sync: %v", err)
-		return nil, err
-	}
-
-	_, err = r.DB.Exec("DELETE FROM job_cache")
-	if err != nil {
-		cclog.Warnf("Error while Job cache clean: %v", err)
-		return nil, err
-	}
-
-	return jobs, nil
-}
-
 // Start inserts a new job in the table, returning the unique job ID.
 // Statistics are not transfered!
-func (r *JobRepository) Start(job *schema.Job) (id int64, err error) {
+func (r *JobRepository) Start(job *schema.JobMeta) (id int64, err error) {
 	job.RawFootprint, err = json.Marshal(job.Footprint)
 	if err != nil {
 		return -1, fmt.Errorf("REPOSITORY/JOB > encoding footprint field failed: %w", err)
@@ -120,21 +71,5 @@ func (r *JobRepository) Stop(
 		Where("job.id = ?", jobId)
 
 	_, err = stmt.RunWith(r.stmtCache).Exec()
-	return err
-}
-
-func (r *JobRepository) StopCached(
-	jobId int64,
-	duration int32,
-	state schema.JobState,
-	monitoringStatus int32,
-) (err error) {
-	stmt := sq.Update("job_cache").
-		Set("job_state", state).
-		Set("duration", duration).
-		Set("monitoring_status", monitoringStatus).
-		Where("job.id = ?", jobId)
-
-	_, err = stmt.RunWith(r.stmtCache).Exec()
-	return err
+	return
 }

internal/repository/jobFind.go

@@ -1,5 +1,5 @@
 // Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved. This file is part of cc-backend.
+// All rights reserved.
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 package repository
@@ -11,8 +11,8 @@ import (
 	"time"
 
 	"github.com/ClusterCockpit/cc-backend/internal/graph/model"
-	cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
-	"github.com/ClusterCockpit/cc-lib/schema"
+	"github.com/ClusterCockpit/cc-backend/pkg/log"
+	"github.com/ClusterCockpit/cc-backend/pkg/schema"
 	sq "github.com/Masterminds/squirrel"
 )
 
@@ -39,27 +39,7 @@ func (r *JobRepository) Find(
 
 	q = q.OrderBy("job.id DESC") // always use newest matching job by db id if more than one match
 
-	cclog.Debugf("Timer Find %s", time.Since(start))
-	return scanJob(q.RunWith(r.stmtCache).QueryRow())
-}
-
-func (r *JobRepository) FindCached(
-	jobId *int64,
-	cluster *string,
-	startTime *int64,
-) (*schema.Job, error) {
-	q := sq.Select(jobCacheColumns...).From("job_cache").
-		Where("job_cache.job_id = ?", *jobId)
-
-	if cluster != nil {
-		q = q.Where("job_cache.cluster = ?", *cluster)
-	}
-	if startTime != nil {
-		q = q.Where("job_cache.start_time = ?", *startTime)
-	}
-
-	q = q.OrderBy("job_cache.id DESC") // always use newest matching job by db id if more than one match
-
+	log.Debugf("Timer Find %s", time.Since(start))
 	return scanJob(q.RunWith(r.stmtCache).QueryRow())
 }
 
@@ -86,7 +66,7 @@ func (r *JobRepository) FindAll(
 
 	rows, err := q.RunWith(r.stmtCache).Query()
 	if err != nil {
-		cclog.Error("Error while running query")
+		log.Error("Error while running query")
 		return nil, err
 	}
 
@@ -94,44 +74,15 @@ func (r *JobRepository) FindAll(
 	for rows.Next() {
 		job, err := scanJob(rows)
 		if err != nil {
-			cclog.Warn("Error while scanning rows")
+			log.Warn("Error while scanning rows")
 			return nil, err
 		}
 		jobs = append(jobs, job)
 	}
-	cclog.Debugf("Timer FindAll %s", time.Since(start))
+	log.Debugf("Timer FindAll %s", time.Since(start))
 	return jobs, nil
 }
 
-// Get complete joblist only consisting of db ids.
-// This is useful to process large job counts and intended to be used
-// together with FindById to process jobs one by one
-func (r *JobRepository) GetJobList() ([]int64, error) {
-	query := sq.Select("id").From("job").
-		Where("job.job_state != 'running'")
-
-	rows, err := query.RunWith(r.stmtCache).Query()
-	if err != nil {
-		cclog.Error("Error while running query")
-		return nil, err
-	}
-
-	jl := make([]int64, 0, 1000)
-	for rows.Next() {
-		var id int64
-		err := rows.Scan(&id)
-		if err != nil {
-			rows.Close()
-			cclog.Warn("Error while scanning rows")
-			return nil, err
-		}
-		jl = append(jl, id)
-	}
-
-	cclog.Infof("Return job count %d", len(jl))
-	return jl, nil
-}
-
 // FindById executes a SQL query to find a specific batch job.
 // The job is queried using the database id.
 // It returns a pointer to a schema.Job data structure and an error variable.
@@ -227,7 +178,7 @@ func (r *JobRepository) FindConcurrentJobs(
 	var startTime int64
 	var stopTime int64
 
-	startTime = job.StartTime
+	startTime = job.StartTimeUnix
 	hostname := job.Resources[0].Hostname
 
 	if job.State == schema.JobStateRunning {
@@ -243,17 +194,15 @@ func (r *JobRepository) FindConcurrentJobs(
 
 	queryRunning := query.Where("job.job_state = ?").Where("(job.start_time BETWEEN ? AND ? OR job.start_time < ?)",
 		"running", startTimeTail, stopTimeTail, startTime)
-	// Get At Least One Exact Hostname Match from JSON Resources Array in Database
-	queryRunning = queryRunning.Where("EXISTS (SELECT 1 FROM json_each(job.resources) WHERE json_extract(value, '$.hostname') = ?)", hostname)
+	queryRunning = queryRunning.Where("job.resources LIKE ?", fmt.Sprint("%", hostname, "%"))
 
 	query = query.Where("job.job_state != ?").Where("((job.start_time BETWEEN ? AND ?) OR (job.start_time + job.duration) BETWEEN ? AND ? OR (job.start_time < ?) AND (job.start_time + job.duration) > ?)",
 		"running", startTimeTail, stopTimeTail, startTimeFront, stopTimeTail, startTime, stopTime)
-	// Get At Least One Exact Hostname Match from JSON Resources Array in Database
-	query = query.Where("EXISTS (SELECT 1 FROM json_each(job.resources) WHERE json_extract(value, '$.hostname') = ?)", hostname)
+	query = query.Where("job.resources LIKE ?", fmt.Sprint("%", hostname, "%"))
 
 	rows, err := query.RunWith(r.stmtCache).Query()
 	if err != nil {
-		cclog.Errorf("Error while running query: %v", err)
+		log.Errorf("Error while running query: %v", err)
 		return nil, err
 	}
 
@@ -264,7 +213,7 @@ func (r *JobRepository) FindConcurrentJobs(
 		var id, jobId, startTime sql.NullInt64
 
 		if err = rows.Scan(&id, &jobId, &startTime); err != nil {
-			cclog.Warn("Error while scanning rows")
+			log.Warn("Error while scanning rows")
 			return nil, err
 		}
 
@@ -280,7 +229,7 @@ func (r *JobRepository) FindConcurrentJobs(
 
 	rows, err = queryRunning.RunWith(r.stmtCache).Query()
 	if err != nil {
-		cclog.Errorf("Error while running query: %v", err)
+		log.Errorf("Error while running query: %v", err)
 		return nil, err
 	}
 
@@ -288,7 +237,7 @@ func (r *JobRepository) FindConcurrentJobs(
 		var id, jobId, startTime sql.NullInt64
 
 		if err := rows.Scan(&id, &jobId, &startTime); err != nil {
-			cclog.Warn("Error while scanning rows")
+			log.Warn("Error while scanning rows")
 			return nil, err
 		}
 

internal/repository/jobHooks.go

@@ -1,57 +0,0 @@
-// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved. This file is part of cc-backend.
-// Use of this source code is governed by a MIT-style
-// license that can be found in the LICENSE file.
-package repository
-
-import (
-	"sync"
-
-	"github.com/ClusterCockpit/cc-lib/schema"
-)
-
-type JobHook interface {
-	JobStartCallback(job *schema.Job)
-	JobStopCallback(job *schema.Job)
-}
-
-var (
-	initOnce sync.Once
-	hooks    []JobHook
-)
-
-func RegisterJobJook(hook JobHook) {
-	initOnce.Do(func() {
-		hooks = make([]JobHook, 0)
-	})
-
-	if hook != nil {
-		hooks = append(hooks, hook)
-	}
-}
-
-func CallJobStartHooks(jobs []*schema.Job) {
-	if hooks == nil {
-		return
-	}
-
-	for _, hook := range hooks {
-		if hook != nil {
-			for _, job := range jobs {
-				hook.JobStartCallback(job)
-			}
-		}
-	}
-}
-
-func CallJobStopHooks(job *schema.Job) {
-	if hooks == nil {
-		return
-	}
-
-	for _, hook := range hooks {
-		if hook != nil {
-			hook.JobStopCallback(job)
-		}
-	}
-}

internal/repository/jobQuery.go

@@ -1,5 +1,5 @@
 // Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved. This file is part of cc-backend.
+// All rights reserved.
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 package repository
@@ -12,10 +12,9 @@ import (
 	"strings"
 	"time"
 
-	"github.com/ClusterCockpit/cc-backend/internal/config"
 	"github.com/ClusterCockpit/cc-backend/internal/graph/model"
-	cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
-	"github.com/ClusterCockpit/cc-lib/schema"
+	"github.com/ClusterCockpit/cc-backend/pkg/log"
+	"github.com/ClusterCockpit/cc-backend/pkg/schema"
 	sq "github.com/Masterminds/squirrel"
 )
 
@@ -68,8 +67,7 @@ func (r *JobRepository) QueryJobs(
 
 	rows, err := query.RunWith(r.stmtCache).Query()
 	if err != nil {
-		queryString, queryVars, _ := query.ToSql()
-		cclog.Errorf("Error while running query '%s' %v: %v", queryString, queryVars, err)
+		log.Errorf("Error while running query: %v", err)
 		return nil, err
 	}
 
@@ -78,7 +76,7 @@ func (r *JobRepository) QueryJobs(
 		job, err := scanJob(rows)
 		if err != nil {
 			rows.Close()
-			cclog.Warn("Error while scanning rows (Jobs)")
+			log.Warn("Error while scanning rows (Jobs)")
 			return nil, err
 		}
 		jobs = append(jobs, job)
@@ -124,7 +122,7 @@ func SecurityCheckWithUser(user *schema.User, query sq.SelectBuilder) (sq.Select
 		if len(user.Projects) != 0 {
 			return query.Where(sq.Or{sq.Eq{"job.project": user.Projects}, sq.Eq{"job.hpc_user": user.Username}}), nil
 		} else {
-			cclog.Debugf("Manager-User '%s' has no defined projects to lookup! Query only personal jobs ...", user.Username)
+			log.Debugf("Manager-User '%s' has no defined projects to lookup! Query only personal jobs ...", user.Username)
 			return query.Where("job.hpc_user = ?", user.Username), nil
 		}
 	case user.HasRole(schema.RoleUser): // User : Only personal jobs
@@ -147,11 +145,6 @@ func BuildWhereClause(filter *model.JobFilter, query sq.SelectBuilder) sq.Select
 		// This is an OR-Logic query: Returns all distinct jobs with at least one of the requested tags; TODO: AND-Logic query?
 		query = query.Join("jobtag ON jobtag.job_id = job.id").Where(sq.Eq{"jobtag.tag_id": filter.Tags}).Distinct()
 	}
-	if filter.DbID != nil {
-		dbIDs := make([]string, len(filter.DbID))
-		copy(dbIDs, filter.DbID)
-		query = query.Where(sq.Eq{"job.id": dbIDs})
-	}
 	if filter.JobID != nil {
 		query = buildStringCondition("job.job_id", filter.JobID, query)
 	}
@@ -183,9 +176,6 @@ func BuildWhereClause(filter *model.JobFilter, query sq.SelectBuilder) sq.Select
 		now := time.Now().Unix() // There does not seam to be a portable way to get the current unix timestamp accross different DBs.
 		query = query.Where("(job.job_state != 'running' OR (? - job.start_time) > ?)", now, *filter.MinRunningFor)
 	}
-	if filter.Shared != nil {
-		query = query.Where("job.shared = ?", *filter.Shared)
-	}
 	if filter.State != nil {
 		states := make([]string, len(filter.State))
 		for i, val := range filter.State {
@@ -204,7 +194,7 @@ func BuildWhereClause(filter *model.JobFilter, query sq.SelectBuilder) sq.Select
 		query = buildIntCondition("job.num_hwthreads", filter.NumHWThreads, query)
 	}
 	if filter.Node != nil {
-		query = buildResourceJsonCondition("hostname", filter.Node, query)
+		query = buildStringCondition("job.resources", filter.Node, query)
 	}
 	if filter.Energy != nil {
 		query = buildFloatCondition("job.energy", filter.Energy, query)
@@ -217,7 +207,7 @@ func BuildWhereClause(filter *model.JobFilter, query sq.SelectBuilder) sq.Select
 	return query
 }
 
-func buildIntCondition(field string, cond *config.IntRange, query sq.SelectBuilder) sq.SelectBuilder {
+func buildIntCondition(field string, cond *schema.IntRange, query sq.SelectBuilder) sq.SelectBuilder {
 	return query.Where(field+" BETWEEN ? AND ?", cond.From, cond.To)
 }
 
@@ -225,7 +215,7 @@ func buildFloatCondition(field string, cond *model.FloatRange, query sq.SelectBu
 	return query.Where(field+" BETWEEN ? AND ?", cond.From, cond.To)
 }
 
-func buildTimeCondition(field string, cond *config.TimeRange, query sq.SelectBuilder) sq.SelectBuilder {
+func buildTimeCondition(field string, cond *schema.TimeRange, query sq.SelectBuilder) sq.SelectBuilder {
 	if cond.From != nil && cond.To != nil {
 		return query.Where(field+" BETWEEN ? AND ?", cond.From.Unix(), cond.To.Unix())
 	} else if cond.From != nil {
@@ -245,7 +235,7 @@ func buildTimeCondition(field string, cond *config.TimeRange, query sq.SelectBui
 		case "last30d":
 			then = now - (60 * 60 * 24 * 30)
 		default:
-			cclog.Debugf("No known named timeRange: startTime.range = %s", cond.Range)
+			log.Debugf("No known named timeRange: startTime.range = %s", cond.Range)
 			return query
 		}
 		return query.Where(field+" BETWEEN ? AND ?", then, now)
@@ -306,28 +296,6 @@ func buildMetaJsonCondition(jsonField string, cond *model.StringInput, query sq.
 	return query
 }
 
-func buildResourceJsonCondition(jsonField string, cond *model.StringInput, query sq.SelectBuilder) sq.SelectBuilder {
-	// Verify and Search Only in Valid Jsons
-	query = query.Where("JSON_VALID(resources)")
-	// add "AND" Sql query Block for field match
-	if cond.Eq != nil {
-		return query.Where("EXISTS (SELECT 1 FROM json_each(job.resources) WHERE json_extract(value, \"$."+jsonField+"\") = ?)", *cond.Eq)
-	}
-	if cond.Neq != nil { // Currently Unused
-		return query.Where("EXISTS (SELECT 1 FROM json_each(job.resources) WHERE json_extract(value, \"$."+jsonField+"\") != ?)", *cond.Neq)
-	}
-	if cond.StartsWith != nil { // Currently Unused
-		return query.Where("EXISTS (SELECT 1 FROM json_each(job.resources) WHERE json_extract(value, \"$."+jsonField+"\")) LIKE ?)", fmt.Sprint(*cond.StartsWith, "%"))
-	}
-	if cond.EndsWith != nil { // Currently Unused
-		return query.Where("EXISTS (SELECT 1 FROM json_each(job.resources) WHERE json_extract(value, \"$."+jsonField+"\") LIKE ?)", fmt.Sprint("%", *cond.EndsWith))
-	}
-	if cond.Contains != nil {
-		return query.Where("EXISTS (SELECT 1 FROM json_each(job.resources) WHERE json_extract(value, \"$."+jsonField+"\") LIKE ?)", fmt.Sprint("%", *cond.Contains, "%"))
-	}
-	return query
-}
-
 var (
 	matchFirstCap = regexp.MustCompile("(.)([A-Z][a-z]+)")
 	matchAllCap   = regexp.MustCompile("([a-z0-9])([A-Z])")
@@ -336,7 +304,7 @@ var (
 func toSnakeCase(str string) string {
 	for _, c := range str {
 		if c == '\'' || c == '\\' {
-			cclog.Panic("toSnakeCase() attack vector!")
+			log.Panic("toSnakeCase() attack vector!")
 		}
 	}
 

internal/repository/job_test.go

@@ -1,5 +1,5 @@
 // Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved. This file is part of cc-backend.
+// All rights reserved.
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 package repository
@@ -9,38 +9,38 @@ import (
 	"fmt"
 	"testing"
 
-	"github.com/ClusterCockpit/cc-lib/schema"
+	"github.com/ClusterCockpit/cc-backend/pkg/schema"
 	_ "github.com/mattn/go-sqlite3"
 )
 
 func TestFind(t *testing.T) {
 	r := setup(t)
 
-	jobID, cluster, startTime := int64(398800), "fritz", int64(1675954712)
-	job, err := r.Find(&jobID, &cluster, &startTime)
+	jobId, cluster, startTime := int64(398998), "fritz", int64(1675957496)
+	job, err := r.Find(&jobId, &cluster, &startTime)
 	if err != nil {
 		t.Fatal(err)
 	}
 
 	// fmt.Printf("%+v", job)
 
-	if *job.ID != 345 {
-		t.Errorf("wrong summary for diagnostic \ngot: %d \nwant: 345", job.JobID)
+	if job.ID != 5 {
+		t.Errorf("wrong summary for diagnostic 3\ngot: %d \nwant: 1366", job.JobID)
 	}
 }
 
 func TestFindById(t *testing.T) {
 	r := setup(t)
 
-	job, err := r.FindById(getContext(t), 338)
+	job, err := r.FindById(getContext(t), 5)
 	if err != nil {
 		t.Fatal(err)
 	}
 
 	// fmt.Printf("%+v", job)
 
-	if job.JobID != 398793 {
-		t.Errorf("wrong summary for diagnostic \ngot: %d \nwant: 1404396", job.JobID)
+	if job.JobID != 398998 {
+		t.Errorf("wrong summary for diagnostic 3\ngot: %d \nwant: 1404396", job.JobID)
 	}
 }
 

internal/repository/migration.go

@@ -1,8 +1,7 @@
 // Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved. This file is part of cc-backend.
+// All rights reserved.
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
-
 package repository
 
 import (
@@ -10,14 +9,14 @@ import (
 	"embed"
 	"fmt"
 
-	cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
+	"github.com/ClusterCockpit/cc-backend/pkg/log"
 	"github.com/golang-migrate/migrate/v4"
 	"github.com/golang-migrate/migrate/v4/database/mysql"
 	"github.com/golang-migrate/migrate/v4/database/sqlite3"
 	"github.com/golang-migrate/migrate/v4/source/iofs"
 )
 
-const Version uint = 10
+const Version uint = 8
 
 //go:embed migrations/*
 var migrationFiles embed.FS
@@ -55,13 +54,13 @@ func checkDBVersion(backend string, db *sql.DB) error {
 			return err
 		}
 	default:
-		cclog.Abortf("Migration: Unsupported database backend '%s'.\n", backend)
+		log.Fatalf("unsupported database backend: %s", backend)
 	}
 
 	v, dirty, err := m.Version()
 	if err != nil {
 		if err == migrate.ErrNilVersion {
-			cclog.Warn("Legacy database without version or missing database file!")
+			log.Warn("Legacy database without version or missing database file!")
 		} else {
 			return err
 		}
@@ -85,7 +84,7 @@ func getMigrateInstance(backend string, db string) (m *migrate.Migrate, err erro
 	case "sqlite3":
 		d, err := iofs.New(migrationFiles, "migrations/sqlite3")
 		if err != nil {
-			cclog.Fatal(err)
+			log.Fatal(err)
 		}
 
 		m, err = migrate.NewWithSourceInstance("iofs", d, fmt.Sprintf("sqlite3://%s?_foreign_keys=on", db))
@@ -103,7 +102,7 @@ func getMigrateInstance(backend string, db string) (m *migrate.Migrate, err erro
 			return m, err
 		}
 	default:
-		cclog.Abortf("Migration: Unsupported database backend '%s'.\n", backend)
+		log.Fatalf("unsupported database backend: %s", backend)
 	}
 
 	return m, nil
@@ -116,17 +115,8 @@ func MigrateDB(backend string, db string) error {
 	}
 
 	v, dirty, err := m.Version()
-	if err != nil {
-		if err == migrate.ErrNilVersion {
-			cclog.Warn("Legacy database without version or missing database file!")
-		} else {
-			return err
-		}
-	}
 
-	if v < Version {
-		cclog.Infof("unsupported database version %d, need %d.\nPlease backup your database file and run cc-backend -migrate-db", v, Version)
-	}
+	log.Infof("unsupported database version %d, need %d.\nPlease backup your database file and run cc-backend -migrate-db", v, Version)
 
 	if dirty {
 		return fmt.Errorf("last migration to version %d has failed, please fix the db manually and force version with -force-db flag", Version)
@@ -134,7 +124,7 @@ func MigrateDB(backend string, db string) error {
 
 	if err := m.Up(); err != nil {
 		if err == migrate.ErrNoChange {
-			cclog.Info("DB already up to date!")
+			log.Info("DB already up to date!")
 		} else {
 			return err
 		}
@@ -152,7 +142,7 @@ func RevertDB(backend string, db string) error {
 
 	if err := m.Migrate(Version - 1); err != nil {
 		if err == migrate.ErrNoChange {
-			cclog.Info("DB already up to date!")
+			log.Info("DB already up to date!")
 		} else {
 			return err
 		}

internal/repository/migrations/sqlite3/09_add-job-cache.down.sql

@@ -1 +0,0 @@
-DROP TABLE IF EXISTS job_cache;

internal/repository/migrations/sqlite3/09_add-job-cache.up.sql

@@ -1,119 +0,0 @@
-CREATE TABLE "job_cache" (
-    id INTEGER PRIMARY KEY,
-    job_id BIGINT NOT NULL,
-    cluster VARCHAR(255) NOT NULL,
-    subcluster VARCHAR(255) NOT NULL,
-    submit_time BIGINT NOT NULL DEFAULT 0, -- Unix timestamp
-    start_time BIGINT NOT NULL DEFAULT 0, -- Unix timestamp
-    hpc_user VARCHAR(255) NOT NULL,
-    project VARCHAR(255) NOT NULL,
-    cluster_partition VARCHAR(255),
-    array_job_id BIGINT,
-    duration INT NOT NULL,
-    walltime INT NOT NULL,
-    job_state VARCHAR(255) NOT NULL
-    CHECK (job_state IN (
-        'boot_fail', 'cancelled', 'completed', 'deadline',
-        'failed', 'node_fail', 'out-of-memory', 'pending',
-        'preempted', 'running', 'suspended', 'timeout'
-    )),
-    meta_data TEXT,          -- JSON
-    resources TEXT NOT NULL, -- JSON
-    num_nodes INT NOT NULL,
-    num_hwthreads INT,
-    num_acc INT,
-    smt TINYINT NOT NULL DEFAULT 1 CHECK (smt IN (0, 1)),
-    shared TEXT NOT NULL
-    CHECK (shared IN ("none", "single_user", "multi_user")),
-    monitoring_status TINYINT NOT NULL DEFAULT 1
-    CHECK (monitoring_status IN (0, 1, 2, 3)),
-    energy REAL NOT NULL DEFAULT 0.0,
-    energy_footprint TEXT DEFAULT NULL,
-    footprint TEXT DEFAULT NULL,
-    UNIQUE (job_id, cluster, start_time)
-);
-
-CREATE TABLE "job_new" (
-    id INTEGER PRIMARY KEY,
-    job_id BIGINT NOT NULL,
-    cluster TEXT NOT NULL,
-    subcluster TEXT NOT NULL,
-    submit_time BIGINT NOT NULL DEFAULT 0, -- Unix timestamp
-    start_time BIGINT NOT NULL DEFAULT 0, -- Unix timestamp
-    hpc_user TEXT NOT NULL,
-    project TEXT NOT NULL,
-    cluster_partition TEXT,
-    array_job_id BIGINT,
-    duration INT NOT NULL,
-    walltime INT NOT NULL,
-    job_state TEXT NOT NULL
-    CHECK (job_state IN (
-        'boot_fail', 'cancelled', 'completed', 'deadline',
-        'failed', 'node_fail', 'out-of-memory', 'pending',
-        'preempted', 'running', 'suspended', 'timeout'
-    )),
-    meta_data TEXT,          -- JSON
-    resources TEXT NOT NULL, -- JSON
-    num_nodes INT NOT NULL,
-    num_hwthreads INT,
-    num_acc INT,
-    smt INT NOT NULL DEFAULT 1,
-    shared TEXT NOT NULL
-    CHECK (shared IN ("none", "single_user", "multi_user")),
-    monitoring_status TINYINT NOT NULL DEFAULT 1
-    CHECK (monitoring_status IN (0, 1, 2, 3)),
-    energy REAL NOT NULL DEFAULT 0.0,
-    energy_footprint TEXT DEFAULT NULL,
-    footprint TEXT DEFAULT NULL,
-    UNIQUE (job_id, cluster, start_time)
-);
-
-
-CREATE TABLE IF NOT EXISTS lookup_exclusive (
-    id INTEGER PRIMARY KEY,
-    name TEXT NOT NULL UNIQUE
-);
-
-INSERT INTO lookup_exclusive (id, name) VALUES
-(0, 'multi_user'),
-(1, 'none'),
-(2, 'single_user');
-
-INSERT INTO job_new (
-    id, job_id, cluster, subcluster, submit_time, start_time, hpc_user, project,
-    cluster_partition, array_job_id, duration, walltime, job_state, meta_data, resources,
-    num_nodes, num_hwthreads, num_acc, smt, shared, monitoring_status, energy,
-    energy_footprint, footprint
-) SELECT
-    id,
-    job_id,
-    cluster,
-    subcluster,
-    0,
-    start_time,
-    hpc_user,
-    project,
-    cluster_partition,
-    array_job_id,
-    duration,
-    walltime,
-    job_state,
-    meta_data,
-    resources,
-    num_nodes,
-    num_hwthreads,
-    num_acc,
-    smt,
-    (
-        SELECT name FROM lookup_exclusive
-        WHERE id = job.exclusive
-    ),
-    monitoring_status,
-    energy,
-    energy_footprint,
-    footprint
-FROM job;
-
-DROP TABLE lookup_exclusive;
-DROP TABLE job;
-ALTER TABLE job_new RENAME TO job;

internal/repository/migrations/sqlite3/10_node-table.down.sql

@@ -1 +0,0 @@
-DROP TABLE IF EXISTS node;

internal/repository/migrations/sqlite3/10_node-table.up.sql

@@ -1,50 +0,0 @@
-CREATE TABLE "node" (
-    id INTEGER PRIMARY KEY,
-    hostname VARCHAR(255) NOT NULL,
-    cluster VARCHAR(255) NOT NULL,
-    subcluster VARCHAR(255) NOT NULL,
-    meta_data TEXT,          -- JSON
-    UNIQUE (hostname, cluster)
-);
-
-CREATE TABLE "node_state" (
-    id INTEGER PRIMARY KEY,
-    time_stamp INTEGER NOT NULL,
-    jobs_running INTEGER DEFAULT 0 NOT NULL,
-    cpus_allocated INTEGER DEFAULT 0 NOT NULL,
-    memory_allocated INTEGER DEFAULT 0 NOT NULL,
-    gpus_allocated INTEGER DEFAULT 0 NOT NULL,
-    node_state VARCHAR(255) NOT NULL
-    CHECK (node_state IN (
-        'allocated', 'reserved', 'idle', 'mixed',
-        'down', 'unknown'
-    )),
-    health_state VARCHAR(255) NOT NULL
-    CHECK (health_state IN (
-        'full', 'partial', 'failed'
-    )),
-    node_id INTEGER,
-    FOREIGN KEY (node_id) REFERENCES node (id)
-);
-
--- Add NEW Indices For New Job Table Columns
-CREATE INDEX IF NOT EXISTS jobs_cluster_submittime ON job (cluster, submit_time);
-CREATE INDEX IF NOT EXISTS jobs_cluster_partition_submittime ON job (cluster, cluster_partition, submit_time);
-CREATE INDEX IF NOT EXISTS jobs_cluster_partition_jobstate_submittime ON job (
-    cluster, cluster_partition, job_state, submit_time
-);
-CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_submittime ON job (cluster, job_state, submit_time);
-
--- Add NEW Indices For New Node Table VARCHAR Fields
-CREATE INDEX IF NOT EXISTS nodes_cluster ON node (cluster);
-CREATE INDEX IF NOT EXISTS nodes_cluster_subcluster ON node (cluster, subcluster);
-
--- Add NEW Indices For New Node_State Table Fields
-CREATE INDEX IF NOT EXISTS nodestates_state ON node_state (node_state);
-CREATE INDEX IF NOT EXISTS nodestates_health ON node_state (health_state);
-CREATE INDEX IF NOT EXISTS nodestates_nodeid_state ON node_state (node_id, node_state);
-CREATE INDEX IF NOT EXISTS nodestates_nodeid_health ON node_state (node_id, health_state);
-
--- Add NEW Indices For Increased Amounts of Tags
-CREATE INDEX IF NOT EXISTS tags_jobid ON jobtag (job_id);
-CREATE INDEX IF NOT EXISTS tags_tagid ON jobtag (tag_id);

internal/repository/node.go

@@ -1,498 +0,0 @@
-// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved. This file is part of cc-backend.
-// Use of this source code is governed by a MIT-style
-// license that can be found in the LICENSE file.
-
-package repository
-
-import (
-	"context"
-	"database/sql"
-	"encoding/json"
-	"fmt"
-	"sync"
-	"time"
-
-	"github.com/ClusterCockpit/cc-backend/internal/graph/model"
-	"github.com/ClusterCockpit/cc-backend/pkg/archive"
-	cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
-	"github.com/ClusterCockpit/cc-lib/lrucache"
-	"github.com/ClusterCockpit/cc-lib/schema"
-	sq "github.com/Masterminds/squirrel"
-	"github.com/jmoiron/sqlx"
-)
-
-var (
-	nodeRepoOnce     sync.Once
-	nodeRepoInstance *NodeRepository
-)
-
-type NodeRepository struct {
-	DB        *sqlx.DB
-	stmtCache *sq.StmtCache
-	cache     *lrucache.Cache
-	driver    string
-}
-
-func GetNodeRepository() *NodeRepository {
-	nodeRepoOnce.Do(func() {
-		db := GetConnection()
-
-		nodeRepoInstance = &NodeRepository{
-			DB:     db.DB,
-			driver: db.Driver,
-
-			stmtCache: sq.NewStmtCache(db.DB),
-			cache:     lrucache.New(1024 * 1024),
-		}
-	})
-	return nodeRepoInstance
-}
-
-func (r *NodeRepository) FetchMetadata(hostname string, cluster string) (map[string]string, error) {
-	start := time.Now()
-
-	RawMetaData := make([]byte, 0)
-
-	if err := sq.Select("node.meta_data").From("node").
-		Where("node.hostname = ?", hostname).
-		Where("node.cluster = ?", cluster).
-		RunWith(r.stmtCache).QueryRow().Scan(&RawMetaData); err != nil {
-		cclog.Warn("Error while scanning for node metadata")
-		return nil, err
-	}
-
-	if len(RawMetaData) == 0 {
-		return nil, nil
-	}
-
-	MetaData := make(map[string]string)
-
-	if err := json.Unmarshal(RawMetaData, &MetaData); err != nil {
-		cclog.Warn("Error while unmarshaling raw metadata json")
-		return nil, err
-	}
-
-	cclog.Debugf("Timer FetchMetadata %s", time.Since(start))
-	return MetaData, nil
-}
-
-//
-// func (r *NodeRepository) UpdateMetadata(node *schema.Node, key, val string) (err error) {
-// 	cachekey := fmt.Sprintf("metadata:%d", node.ID)
-// 	r.cache.Del(cachekey)
-// 	if node.MetaData == nil {
-// 		if _, err = r.FetchMetadata(node); err != nil {
-// 			cclog.Warnf("Error while fetching metadata for node, DB ID '%v'", node.ID)
-// 			return err
-// 		}
-// 	}
-//
-// 	if node.MetaData != nil {
-// 		cpy := make(map[string]string, len(node.MetaData)+1)
-// 		maps.Copy(cpy, node.MetaData)
-// 		cpy[key] = val
-// 		node.MetaData = cpy
-// 	} else {
-// 		node.MetaData = map[string]string{key: val}
-// 	}
-//
-// 	if node.RawMetaData, err = json.Marshal(node.MetaData); err != nil {
-// 		cclog.Warnf("Error while marshaling metadata for node, DB ID '%v'", node.ID)
-// 		return err
-// 	}
-//
-// 	if _, err = sq.Update("node").
-// 		Set("meta_data", node.RawMetaData).
-// 		Where("node.id = ?", node.ID).
-// 		RunWith(r.stmtCache).Exec(); err != nil {
-// 		cclog.Warnf("Error while updating metadata for node, DB ID '%v'", node.ID)
-// 		return err
-// 	}
-//
-// 	r.cache.Put(cachekey, node.MetaData, len(node.RawMetaData), 24*time.Hour)
-// 	return nil
-// }
-
-func (r *NodeRepository) GetNode(hostname string, cluster string, withMeta bool) (*schema.Node, error) {
-	node := &schema.Node{}
-	if err := sq.Select("node.hostname", "node.cluster", "node.subcluster", "node_state.node_state",
-		"node_state.health_state", "MAX(node_state.time_stamp) as time").
-		From("node_state").
-		Join("node ON nodes_state.node_id = node.id").
-		Where("node.hostname = ?", hostname).
-		Where("node.cluster = ?", cluster).
-		GroupBy("node_state.node_id").
-		RunWith(r.DB).
-		QueryRow().Scan(&node.Hostname, &node.Cluster, &node.SubCluster, &node.NodeState, &node.HealthState); err != nil {
-		cclog.Warnf("Error while querying node '%s' from database: %v", hostname, err)
-		return nil, err
-	}
-
-	if withMeta {
-		var err error
-		var meta map[string]string
-		if meta, err = r.FetchMetadata(hostname, cluster); err != nil {
-			cclog.Warnf("Error while fetching metadata for node '%s'", hostname)
-			return nil, err
-		}
-		node.MetaData = meta
-	}
-
-	return node, nil
-}
-
-func (r *NodeRepository) GetNodeById(id int64, withMeta bool) (*schema.Node, error) {
-	node := &schema.Node{}
-	if err := sq.Select("node.hostname", "node.cluster", "node.subcluster", "node_state.node_state",
-		"node_state.health_state", "MAX(node_state.time_stamp) as time").
-		From("node_state").
-		Join("node ON nodes_state.node_id = node.id").
-		Where("node.id = ?", id).
-		GroupBy("node_state.node_id").
-		RunWith(r.DB).
-		QueryRow().Scan(&node.Hostname, &node.Cluster, &node.SubCluster, &node.NodeState, &node.HealthState); err != nil {
-		cclog.Warnf("Error while querying node ID '%d' from database: %v", id, err)
-		return nil, err
-	}
-
-	// NEEDS METADATA BY ID
-	// if withMeta {
-	// 	var err error
-	// 	var meta map[string]string
-	// 	if meta, err = r.FetchMetadata(hostname, cluster); err != nil {
-	// 		cclog.Warnf("Error while fetching metadata for node '%s'", hostname)
-	// 		return nil, err
-	// 	}
-	// 	node.MetaData = meta
-	// }
-
-	return node, nil
-}
-
-// const NamedNodeInsert string = `
-// INSERT INTO node (time_stamp, hostname, cluster, subcluster, node_state, health_state,
-//
-//	cpus_allocated, cpus_total, memory_allocated, memory_total, gpus_allocated, gpus_total)
-//	VALUES (:time_stamp, :hostname, :cluster, :subcluster, :node_state, :health_state,
-//	:cpus_allocated, :cpus_total, :memory_allocated, :memory_total, :gpus_allocated, :gpus_total);`
-
-const NamedNodeInsert string = `
-INSERT INTO node (hostname, cluster, subcluster)
-	VALUES (:hostname, :cluster, :subcluster);`
-
-// AddNode adds a Node to the node table. This can be triggered by a node collector registration or
-// from a nodestate update from the job scheduler.
-func (r *NodeRepository) AddNode(node *schema.NodeDB) (int64, error) {
-	var err error
-
-	res, err := r.DB.NamedExec(NamedNodeInsert, node)
-	if err != nil {
-		cclog.Errorf("Error while adding node '%v' to database", node.Hostname)
-		return 0, err
-	}
-	node.ID, err = res.LastInsertId()
-	if err != nil {
-		cclog.Errorf("Error while getting last insert id for node '%v' from database", node.Hostname)
-		return 0, err
-	}
-
-	return node.ID, nil
-}
-
-const NamedNodeStateInsert string = `
-INSERT INTO node_state (time_stamp, node_state, health_state, cpus_allocated,
-	memory_allocated, gpus_allocated, jobs_running, node_id)
-	VALUES (:time_stamp, :node_state, :health_state, :cpus_allocated, :memory_allocated, :gpus_allocated, :jobs_running, :node_id);`
-
-// TODO: Add real Monitoring Health State
-
-// UpdateNodeState is called from the Node REST API to add a row in the node state table
-func (r *NodeRepository) UpdateNodeState(hostname string, cluster string, nodeState *schema.NodeStateDB) error {
-	var id int64
-
-	if err := sq.Select("id").From("node").
-		Where("node.hostname = ?", hostname).Where("node.cluster = ?", cluster).RunWith(r.DB).
-		QueryRow().Scan(&id); err != nil {
-		if err == sql.ErrNoRows {
-			subcluster, err := archive.GetSubClusterByNode(cluster, hostname)
-			if err != nil {
-				cclog.Errorf("Error while getting subcluster for node '%s' in cluster '%s': %v", hostname, cluster, err)
-				return err
-			}
-			node := schema.NodeDB{
-				Hostname: hostname, Cluster: cluster, SubCluster: subcluster,
-			}
-			id, err = r.AddNode(&node)
-			if err != nil {
-				cclog.Errorf("Error while adding node '%s' to database: %v", hostname, err)
-				return err
-			}
-
-			cclog.Infof("Added node '%s' to database", hostname)
-			return nil
-		} else {
-			cclog.Warnf("Error while querying node '%v' from database", id)
-			return err
-		}
-	}
-
-	nodeState.NodeID = id
-
-	_, err := r.DB.NamedExec(NamedNodeStateInsert, nodeState)
-	if err != nil {
-		cclog.Errorf("Error while adding node state for '%v' to database", hostname)
-		return err
-	}
-	cclog.Infof("Updated node state for '%s' in database", hostname)
-	return nil
-}
-
-// func (r *NodeRepository) UpdateHealthState(hostname string, healthState *schema.MonitoringState) error {
-// 	if _, err := sq.Update("node").Set("health_state", healthState).Where("node.id = ?", id).RunWith(r.DB).Exec(); err != nil {
-// 		cclog.Errorf("error while updating node '%d'", id)
-// 		return err
-// 	}
-//
-// 	return nil
-// }
-
-func (r *NodeRepository) DeleteNode(id int64) error {
-	_, err := r.DB.Exec(`DELETE FROM node WHERE node.id = ?`, id)
-	if err != nil {
-		cclog.Errorf("Error while deleting node '%d' from DB", id)
-		return err
-	}
-	cclog.Infof("deleted node '%d' from DB", id)
-	return nil
-}
-
-// QueryNodes returns a list of nodes based on a node filter. It always operates
-// on the last state (largest timestamp).
-func (r *NodeRepository) QueryNodes(
-	ctx context.Context,
-	filters []*model.NodeFilter,
-	order *model.OrderByInput, // Currently unused!
-) ([]*schema.Node, error) {
-	query, qerr := AccessCheck(ctx,
-		sq.Select("node.hostname", "node.cluster", "node.subcluster", "node_state.node_state",
-			"node_state.health_state", "MAX(node_state.time_stamp) as time").
-			From("node").
-			Join("node_state ON nodes_state.node_id = node.id"))
-	if qerr != nil {
-		return nil, qerr
-	}
-
-	for _, f := range filters {
-		if f.Hostname != nil {
-			query = buildStringCondition("node.hostname", f.Hostname, query)
-		}
-		if f.Cluster != nil {
-			query = buildStringCondition("node.cluster", f.Cluster, query)
-		}
-		if f.Subcluster != nil {
-			query = buildStringCondition("node.subcluster", f.Subcluster, query)
-		}
-		if f.SchedulerState != nil {
-			query = query.Where("node.node_state = ?", f.SchedulerState)
-		}
-		if f.HealthState != nil {
-			query = query.Where("node.health_state = ?", f.HealthState)
-		}
-	}
-
-	// Add Grouping after filters
-	query = query.GroupBy("node_state.node_id")
-
-	rows, err := query.RunWith(r.stmtCache).Query()
-	if err != nil {
-		queryString, queryVars, _ := query.ToSql()
-		cclog.Errorf("Error while running query '%s' %v: %v", queryString, queryVars, err)
-		return nil, err
-	}
-
-	nodes := make([]*schema.Node, 0, 50)
-	for rows.Next() {
-		node := schema.Node{}
-
-		if err := rows.Scan(&node.Hostname, &node.Cluster, &node.SubCluster,
-			&node.NodeState, &node.HealthState); err != nil {
-			rows.Close()
-			cclog.Warn("Error while scanning rows (Nodes)")
-			return nil, err
-		}
-		nodes = append(nodes, &node)
-	}
-
-	return nodes, nil
-}
-
-func (r *NodeRepository) ListNodes(cluster string) ([]*schema.Node, error) {
-	q := sq.Select("node.hostname", "node.cluster", "node.subcluster", "node_state.node_state",
-		"node_state.health_state", "MAX(node_state.time_stamp) as time").
-		From("node").
-		Join("node_state ON node_state.node_id = node.id").
-		Where("node.cluster = ?", cluster).
-		GroupBy("node_state.node_id").
-		OrderBy("node.hostname ASC")
-
-	rows, err := q.RunWith(r.DB).Query()
-	if err != nil {
-		cclog.Warn("Error while querying node list")
-		return nil, err
-	}
-	nodeList := make([]*schema.Node, 0, 100)
-	defer rows.Close()
-	for rows.Next() {
-		node := &schema.Node{}
-		if err := rows.Scan(&node.Hostname, &node.Cluster,
-			&node.SubCluster, &node.NodeState, &node.HealthState); err != nil {
-			cclog.Warn("Error while scanning node list")
-			return nil, err
-		}
-
-		nodeList = append(nodeList, node)
-	}
-
-	return nodeList, nil
-}
-
-func (r *NodeRepository) CountNodeStates(ctx context.Context, filters []*model.NodeFilter) ([]*model.NodeStates, error) {
-	query, qerr := AccessCheck(ctx, sq.Select("hostname", "node_state", "MAX(time_stamp) as time").From("node"))
-	if qerr != nil {
-		return nil, qerr
-	}
-
-	query = query.Join("node_state ON node_state.node_id = node.id")
-
-	for _, f := range filters {
-		if f.Hostname != nil {
-			query = buildStringCondition("hostname", f.Hostname, query)
-		}
-		if f.Cluster != nil {
-			query = buildStringCondition("cluster", f.Cluster, query)
-		}
-		if f.Subcluster != nil {
-			query = buildStringCondition("subcluster", f.Subcluster, query)
-		}
-		if f.SchedulerState != nil {
-			query = query.Where("node_state = ?", f.SchedulerState)
-		}
-		if f.HealthState != nil {
-			query = query.Where("health_state = ?", f.HealthState)
-		}
-	}
-
-	// Add Group and Order
-	query = query.GroupBy("hostname").OrderBy("hostname DESC")
-
-	rows, err := query.RunWith(r.stmtCache).Query()
-	if err != nil {
-		queryString, queryVars, _ := query.ToSql()
-		cclog.Errorf("Error while running query '%s' %v: %v", queryString, queryVars, err)
-		return nil, err
-	}
-
-	stateMap := map[string]int{}
-	for rows.Next() {
-		var hostname, node_state string
-		var timestamp int64
-
-		if err := rows.Scan(&hostname, &node_state, &timestamp); err != nil {
-			rows.Close()
-			cclog.Warn("Error while scanning rows (NodeStates)")
-			return nil, err
-		}
-
-		stateMap[node_state] += 1
-	}
-
-	nodes := make([]*model.NodeStates, 0)
-	for state, counts := range stateMap {
-		node := model.NodeStates{State: state, Count: counts}
-		nodes = append(nodes, &node)
-	}
-
-	return nodes, nil
-}
-
-func (r *NodeRepository) CountHealthStates(ctx context.Context, filters []*model.NodeFilter) ([]*model.NodeStates, error) {
-	query, qerr := AccessCheck(ctx, sq.Select("hostname", "health_state", "MAX(time_stamp) as time").From("node"))
-	if qerr != nil {
-		return nil, qerr
-	}
-
-	query = query.Join("node_state ON node_state.node_id = node.id")
-
-	for _, f := range filters {
-		if f.Hostname != nil {
-			query = buildStringCondition("hostname", f.Hostname, query)
-		}
-		if f.Cluster != nil {
-			query = buildStringCondition("cluster", f.Cluster, query)
-		}
-		if f.Subcluster != nil {
-			query = buildStringCondition("subcluster", f.Subcluster, query)
-		}
-		if f.SchedulerState != nil {
-			query = query.Where("node_state = ?", f.SchedulerState)
-		}
-		if f.HealthState != nil {
-			query = query.Where("health_state = ?", f.HealthState)
-		}
-	}
-
-	// Add Group and Order
-	query = query.GroupBy("hostname").OrderBy("hostname DESC")
-
-	rows, err := query.RunWith(r.stmtCache).Query()
-	if err != nil {
-		queryString, queryVars, _ := query.ToSql()
-		cclog.Errorf("Error while running query '%s' %v: %v", queryString, queryVars, err)
-		return nil, err
-	}
-
-	stateMap := map[string]int{}
-	for rows.Next() {
-		var hostname, health_state string
-		var timestamp int64
-
-		if err := rows.Scan(&hostname, &health_state, &timestamp); err != nil {
-			rows.Close()
-			cclog.Warn("Error while scanning rows (NodeStates)")
-			return nil, err
-		}
-
-		stateMap[health_state] += 1
-	}
-
-	nodes := make([]*model.NodeStates, 0)
-	for state, counts := range stateMap {
-		node := model.NodeStates{State: state, Count: counts}
-		nodes = append(nodes, &node)
-	}
-
-	return nodes, nil
-}
-
-func AccessCheck(ctx context.Context, query sq.SelectBuilder) (sq.SelectBuilder, error) {
-	user := GetUserFromContext(ctx)
-	return AccessCheckWithUser(user, query)
-}
-
-func AccessCheckWithUser(user *schema.User, query sq.SelectBuilder) (sq.SelectBuilder, error) {
-	if user == nil {
-		var qnil sq.SelectBuilder
-		return qnil, fmt.Errorf("user context is nil")
-	}
-
-	switch {
-	// case len(user.Roles) == 1 && user.HasRole(schema.RoleApi): // API-User : Access NodeInfos
-	// 	return query, nil
-	case user.HasAnyRole([]schema.Role{schema.RoleAdmin, schema.RoleSupport}): // Admin & Support : Access NodeInfos
-		return query, nil
-	default: // No known Role: No Access, return error
-		var qnil sq.SelectBuilder
-		return qnil, fmt.Errorf("user has no or unknown roles")
-	}
-}

internal/repository/node_test.go

@@ -1,190 +0,0 @@
-// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved. This file is part of cc-backend.
-// Use of this source code is governed by a MIT-style
-// license that can be found in the LICENSE file.
-
-package repository
-
-import (
-	"encoding/json"
-	"fmt"
-	"os"
-	"path/filepath"
-	"testing"
-	"time"
-
-	"github.com/ClusterCockpit/cc-backend/internal/config"
-	"github.com/ClusterCockpit/cc-backend/pkg/archive"
-	ccconf "github.com/ClusterCockpit/cc-lib/ccConfig"
-	cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
-	"github.com/ClusterCockpit/cc-lib/schema"
-	_ "github.com/mattn/go-sqlite3"
-)
-
-func nodeTestSetup(t *testing.T) {
-	const testconfig = `{
-		"main": {
-	"addr":            "0.0.0.0:8080",
-	"validate": false,
-  "apiAllowedIPs": [
-    "*"
-  ]
-	},
-	"archive": {
-		"kind": "file",
-		"path": "./var/job-archive"
-	},
-	"auth": {
-  "jwts": {
-      "max-age": "2m"
-  }
-	},
-	"clusters": [
-	{
-	   "name": "testcluster",
-	   "metricDataRepository": {"kind": "test", "url": "bla:8081"},
-	   "filterRanges": {
-		"numNodes": { "from": 1, "to": 64 },
-		"duration": { "from": 0, "to": 86400 },
-		"startTime": { "from": "2022-01-01T00:00:00Z", "to": null }
-	   }
-	}
-	]
-}`
-	const testclusterJSON = `{
-        "name": "testcluster",
-		"subClusters": [
-			{
-				"name": "sc1",
-				"nodes": "host123,host124,host125",
-				"processorType": "Intel Core i7-4770",
-				"socketsPerNode": 1,
-				"coresPerSocket": 4,
-				"threadsPerCore": 2,
-                "flopRateScalar": {
-                  "unit": {
-                    "prefix": "G",
-                    "base": "F/s"
-                  },
-                  "value": 14
-                },
-                "flopRateSimd": {
-                  "unit": {
-                    "prefix": "G",
-                    "base": "F/s"
-                  },
-                  "value": 112
-                },
-                "memoryBandwidth": {
-                  "unit": {
-                    "prefix": "G",
-                    "base": "B/s"
-                  },
-                  "value": 24
-                },
-                "numberOfNodes": 70,
-				"topology": {
-					"node": [0, 1, 2, 3, 4, 5, 6, 7],
-					"socket": [[0, 1, 2, 3, 4, 5, 6, 7]],
-					"memoryDomain": [[0, 1, 2, 3, 4, 5, 6, 7]],
-					"die": [[0, 1, 2, 3, 4, 5, 6, 7]],
-					"core": [[0], [1], [2], [3], [4], [5], [6], [7]]
-				}
-			}
-		],
-		"metricConfig": [
-			{
-				"name": "load_one",
-			    "unit": { "base": ""},
-				"scope": "node",
-				"timestep": 60,
-        "aggregation": "avg",
-				"peak": 8,
-				"normal": 0,
-				"caution": 0,
-				"alert": 0
-			}
-		]
-	}`
-
-	cclog.Init("debug", true)
-	tmpdir := t.TempDir()
-	jobarchive := filepath.Join(tmpdir, "job-archive")
-	if err := os.Mkdir(jobarchive, 0o777); err != nil {
-		t.Fatal(err)
-	}
-
-	if err := os.WriteFile(filepath.Join(jobarchive, "version.txt"),
-		fmt.Appendf(nil, "%d", 2), 0o666); err != nil {
-		t.Fatal(err)
-	}
-
-	if err := os.Mkdir(filepath.Join(jobarchive, "testcluster"),
-		0o777); err != nil {
-		t.Fatal(err)
-	}
-
-	if err := os.WriteFile(filepath.Join(jobarchive, "testcluster", "cluster.json"),
-		[]byte(testclusterJSON), 0o666); err != nil {
-		t.Fatal(err)
-	}
-
-	dbfilepath := filepath.Join(tmpdir, "test.db")
-	err := MigrateDB("sqlite3", dbfilepath)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	cfgFilePath := filepath.Join(tmpdir, "config.json")
-	if err := os.WriteFile(cfgFilePath, []byte(testconfig), 0o666); err != nil {
-		t.Fatal(err)
-	}
-
-	ccconf.Init(cfgFilePath)
-
-	// Load and check main configuration
-	if cfg := ccconf.GetPackageConfig("main"); cfg != nil {
-		if clustercfg := ccconf.GetPackageConfig("clusters"); clustercfg != nil {
-			config.Init(cfg, clustercfg)
-		} else {
-			cclog.Abort("Cluster configuration must be present")
-		}
-	} else {
-		cclog.Abort("Main configuration must be present")
-	}
-	archiveCfg := fmt.Sprintf("{\"kind\": \"file\",\"path\": \"%s\"}", jobarchive)
-
-	Connect("sqlite3", dbfilepath)
-
-	if err := archive.Init(json.RawMessage(archiveCfg), config.Keys.DisableArchive); err != nil {
-		t.Fatal(err)
-	}
-}
-
-func TestUpdateNodeState(t *testing.T) {
-	nodeTestSetup(t)
-
-	nodeState := schema.NodeStateDB{
-		TimeStamp: time.Now().Unix(), NodeState: "allocated",
-		CpusAllocated:   72,
-		MemoryAllocated: 480,
-		GpusAllocated:   0,
-		HealthState:     schema.MonitoringStateFull,
-		JobsRunning:     1,
-	}
-
-	repo := GetNodeRepository()
-	err := repo.UpdateNodeState("host124", "testcluster", &nodeState)
-	if err != nil {
-		return
-	}
-
-	node, err := repo.GetNode("host124", "testcluster", false)
-	if err != nil {
-		return
-	}
-
-	if node.NodeState != "allocated" {
-		t.Errorf("wrong node state\ngot: %s \nwant: allocated ", node.NodeState)
-	}
-}

internal/repository/repository_test.go

@@ -1,5 +1,5 @@
 // Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved. This file is part of cc-backend.
+// All rights reserved.
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 package repository
@@ -9,8 +9,8 @@ import (
 	"testing"
 
 	"github.com/ClusterCockpit/cc-backend/internal/graph/model"
-	cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
-	"github.com/ClusterCockpit/cc-lib/schema"
+	"github.com/ClusterCockpit/cc-backend/pkg/log"
+	"github.com/ClusterCockpit/cc-backend/pkg/schema"
 	_ "github.com/mattn/go-sqlite3"
 )
 
@@ -65,7 +65,7 @@ func BenchmarkDB_FindJobById(b *testing.B) {
 func BenchmarkDB_FindJob(b *testing.B) {
 	var jobId int64 = 107266
 	var startTime int64 = 1657557241
-	cluster := "fritz"
+	var cluster = "fritz"
 
 	b.Run("FindJob", func(b *testing.B) {
 		db := setup(b)
@@ -147,7 +147,7 @@ func getContext(tb testing.TB) context.Context {
 
 func setup(tb testing.TB) *JobRepository {
 	tb.Helper()
-	cclog.Init("warn", true)
+	log.Init("warn", true)
 	dbfile := "testdata/job.db"
 	err := MigrateDB("sqlite3", dbfile)
 	noErr(tb, err)

internal/repository/stats.go

@@ -1,5 +1,5 @@
 // Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved. This file is part of cc-backend.
+// All rights reserved.
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 package repository
@@ -14,22 +14,20 @@ import (
 	"github.com/ClusterCockpit/cc-backend/internal/graph/model"
 	"github.com/ClusterCockpit/cc-backend/internal/metricDataDispatcher"
 	"github.com/ClusterCockpit/cc-backend/pkg/archive"
-	cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
-	"github.com/ClusterCockpit/cc-lib/schema"
+	"github.com/ClusterCockpit/cc-backend/pkg/log"
+	"github.com/ClusterCockpit/cc-backend/pkg/schema"
 	sq "github.com/Masterminds/squirrel"
 )
 
 // GraphQL validation should make sure that no unkown values can be specified.
 var groupBy2column = map[model.Aggregate]string{
-	model.AggregateUser:       "job.hpc_user",
-	model.AggregateProject:    "job.project",
-	model.AggregateCluster:    "job.cluster",
-	model.AggregateSubcluster: "job.subcluster",
+	model.AggregateUser:    "job.hpc_user",
+	model.AggregateProject: "job.project",
+	model.AggregateCluster: "job.cluster",
 }
 
 var sortBy2column = map[model.SortByAggregate]string{
 	model.SortByAggregateTotaljobs:      "totalJobs",
-	model.SortByAggregateTotalusers:     "totalUsers",
 	model.SortByAggregateTotalwalltime:  "totalWalltime",
 	model.SortByAggregateTotalnodes:     "totalNodes",
 	model.SortByAggregateTotalnodehours: "totalNodeHours",
@@ -78,12 +76,8 @@ func (r *JobRepository) buildStatsQuery(
 	// fmt.Sprintf(`CAST(ROUND((CASE WHEN job.job_state = "running" THEN %d - job.start_time ELSE job.duration END) / 3600) as %s) as value`, time.Now().Unix(), castType)
 
 	if col != "" {
-		// Scan columns: id, name, totalJobs, totalUsers, totalWalltime, totalNodes, totalNodeHours, totalCores, totalCoreHours, totalAccs, totalAccHours
-		query = sq.Select(
-			col,
-			"name",
-			"COUNT(job.id) as totalJobs",
-			"COUNT(DISTINCT job.hpc_user) AS totalUsers",
+		// Scan columns: id, totalJobs, name, totalWalltime, totalNodes, totalNodeHours, totalCores, totalCoreHours, totalAccs, totalAccHours
+		query = sq.Select(col, "COUNT(job.id) as totalJobs", "name",
 			fmt.Sprintf(`CAST(ROUND(SUM((CASE WHEN job.job_state = "running" THEN %d - job.start_time ELSE job.duration END)) / 3600) as %s) as totalWalltime`, time.Now().Unix(), castType),
 			fmt.Sprintf(`CAST(SUM(job.num_nodes) as %s) as totalNodes`, castType),
 			fmt.Sprintf(`CAST(ROUND(SUM((CASE WHEN job.job_state = "running" THEN %d - job.start_time ELSE job.duration END) * job.num_nodes) / 3600) as %s) as totalNodeHours`, time.Now().Unix(), castType),
@@ -93,10 +87,8 @@ func (r *JobRepository) buildStatsQuery(
 			fmt.Sprintf(`CAST(ROUND(SUM((CASE WHEN job.job_state = "running" THEN %d - job.start_time ELSE job.duration END) * job.num_acc) / 3600) as %s) as totalAccHours`, time.Now().Unix(), castType),
 		).From("job").LeftJoin("hpc_user ON hpc_user.username = job.hpc_user").GroupBy(col)
 	} else {
-		// Scan columns: totalJobs, totalUsers, totalWalltime, totalNodes, totalNodeHours, totalCores, totalCoreHours, totalAccs, totalAccHours
-		query = sq.Select(
-			"COUNT(job.id) as totalJobs",
-			"COUNT(DISTINCT job.hpc_user) AS totalUsers",
+		// Scan columns: totalJobs, name, totalWalltime, totalNodes, totalNodeHours, totalCores, totalCoreHours, totalAccs, totalAccHours
+		query = sq.Select("COUNT(job.id)",
 			fmt.Sprintf(`CAST(ROUND(SUM((CASE WHEN job.job_state = "running" THEN %d - job.start_time ELSE job.duration END)) / 3600) as %s)`, time.Now().Unix(), castType),
 			fmt.Sprintf(`CAST(SUM(job.num_nodes) as %s)`, castType),
 			fmt.Sprintf(`CAST(ROUND(SUM((CASE WHEN job.job_state = "running" THEN %d - job.start_time ELSE job.duration END) * job.num_nodes) / 3600) as %s)`, time.Now().Unix(), castType),
@@ -166,7 +158,7 @@ func (r *JobRepository) JobsStatsGrouped(
 
 	rows, err := query.RunWith(r.DB).Query()
 	if err != nil {
-		cclog.Warn("Error while querying DB for job statistics")
+		log.Warn("Error while querying DB for job statistics")
 		return nil, err
 	}
 
@@ -175,14 +167,14 @@ func (r *JobRepository) JobsStatsGrouped(
 	for rows.Next() {
 		var id sql.NullString
 		var name sql.NullString
-		var jobs, users, walltime, nodes, nodeHours, cores, coreHours, accs, accHours sql.NullInt64
-		if err := rows.Scan(&id, &name, &jobs, &users, &walltime, &nodes, &nodeHours, &cores, &coreHours, &accs, &accHours); err != nil {
-			cclog.Warnf("Error while scanning rows: %s", err.Error())
+		var jobs, walltime, nodes, nodeHours, cores, coreHours, accs, accHours sql.NullInt64
+		if err := rows.Scan(&id, &jobs, &name, &walltime, &nodes, &nodeHours, &cores, &coreHours, &accs, &accHours); err != nil {
+			log.Warn("Error while scanning rows")
 			return nil, err
 		}
 
 		if id.Valid {
-			var totalJobs, totalUsers, totalWalltime, totalNodes, totalNodeHours, totalCores, totalCoreHours, totalAccs, totalAccHours int
+			var totalJobs, totalWalltime, totalNodes, totalNodeHours, totalCores, totalCoreHours, totalAccs, totalAccHours int
 			var personName string
 
 			if name.Valid {
@@ -193,10 +185,6 @@ func (r *JobRepository) JobsStatsGrouped(
 				totalJobs = int(jobs.Int64)
 			}
 
-			if users.Valid {
-				totalUsers = int(users.Int64)
-			}
-
 			if walltime.Valid {
 				totalWalltime = int(walltime.Int64)
 			}
@@ -240,9 +228,8 @@ func (r *JobRepository) JobsStatsGrouped(
 				stats = append(stats,
 					&model.JobsStatistics{
 						ID:             id.String,
-						TotalJobs:      totalJobs,
-						TotalUsers:     totalUsers,
-						TotalWalltime:  totalWalltime,
+						TotalJobs:      int(jobs.Int64),
+						TotalWalltime:  int(walltime.Int64),
 						TotalNodes:     totalNodes,
 						TotalNodeHours: totalNodeHours,
 						TotalCores:     totalCores,
@@ -254,7 +241,7 @@ func (r *JobRepository) JobsStatsGrouped(
 		}
 	}
 
-	cclog.Debugf("Timer JobsStatsGrouped %s", time.Since(start))
+	log.Debugf("Timer JobsStatsGrouped %s", time.Since(start))
 	return stats, nil
 }
 
@@ -272,9 +259,9 @@ func (r *JobRepository) JobsStats(
 	row := query.RunWith(r.DB).QueryRow()
 	stats := make([]*model.JobsStatistics, 0, 1)
 
-	var jobs, users, walltime, nodes, nodeHours, cores, coreHours, accs, accHours sql.NullInt64
-	if err := row.Scan(&jobs, &users, &walltime, &nodes, &nodeHours, &cores, &coreHours, &accs, &accHours); err != nil {
-		cclog.Warn("Error while scanning rows")
+	var jobs, walltime, nodes, nodeHours, cores, coreHours, accs, accHours sql.NullInt64
+	if err := row.Scan(&jobs, &walltime, &nodes, &nodeHours, &cores, &coreHours, &accs, &accHours); err != nil {
+		log.Warn("Error while scanning rows")
 		return nil, err
 	}
 
@@ -293,7 +280,6 @@ func (r *JobRepository) JobsStats(
 		stats = append(stats,
 			&model.JobsStatistics{
 				TotalJobs:      int(jobs.Int64),
-				TotalUsers:     int(users.Int64),
 				TotalWalltime:  int(walltime.Int64),
 				TotalNodeHours: totalNodeHours,
 				TotalCoreHours: totalCoreHours,
@@ -301,11 +287,11 @@ func (r *JobRepository) JobsStats(
 			})
 	}
 
-	cclog.Debugf("Timer JobStats %s", time.Since(start))
+	log.Debugf("Timer JobStats %s", time.Since(start))
 	return stats, nil
 }
 
-func LoadJobStat(job *schema.Job, metric string, statType string) float64 {
+func LoadJobStat(job *schema.JobMeta, metric string, statType string) float64 {
 	if stats, ok := job.Statistics[metric]; ok {
 		switch statType {
 		case "avg":
@@ -315,7 +301,7 @@ func LoadJobStat(job *schema.Job, metric string, statType string) float64 {
 		case "min":
 			return stats.Min
 		default:
-			cclog.Errorf("Unknown stat type %s", statType)
+			log.Errorf("Unknown stat type %s", statType)
 		}
 	}
 
@@ -336,7 +322,7 @@ func (r *JobRepository) JobCountGrouped(
 	}
 	rows, err := query.RunWith(r.DB).Query()
 	if err != nil {
-		cclog.Warn("Error while querying DB for job statistics")
+		log.Warn("Error while querying DB for job statistics")
 		return nil, err
 	}
 
@@ -346,7 +332,7 @@ func (r *JobRepository) JobCountGrouped(
 		var id sql.NullString
 		var cnt sql.NullInt64
 		if err := rows.Scan(&id, &cnt); err != nil {
-			cclog.Warn("Error while scanning rows")
+			log.Warn("Error while scanning rows")
 			return nil, err
 		}
 		if id.Valid {
@@ -358,7 +344,7 @@ func (r *JobRepository) JobCountGrouped(
 		}
 	}
 
-	cclog.Debugf("Timer JobCountGrouped %s", time.Since(start))
+	log.Debugf("Timer JobCountGrouped %s", time.Since(start))
 	return stats, nil
 }
 
@@ -378,7 +364,7 @@ func (r *JobRepository) AddJobCountGrouped(
 	}
 	rows, err := query.RunWith(r.DB).Query()
 	if err != nil {
-		cclog.Warn("Error while querying DB for job statistics")
+		log.Warn("Error while querying DB for job statistics")
 		return nil, err
 	}
 
@@ -388,7 +374,7 @@ func (r *JobRepository) AddJobCountGrouped(
 		var id sql.NullString
 		var cnt sql.NullInt64
 		if err := rows.Scan(&id, &cnt); err != nil {
-			cclog.Warn("Error while scanning rows")
+			log.Warn("Error while scanning rows")
 			return nil, err
 		}
 		if id.Valid {
@@ -407,7 +393,7 @@ func (r *JobRepository) AddJobCountGrouped(
 		}
 	}
 
-	cclog.Debugf("Timer AddJobCountGrouped %s", time.Since(start))
+	log.Debugf("Timer AddJobCountGrouped %s", time.Since(start))
 	return stats, nil
 }
 
@@ -425,7 +411,7 @@ func (r *JobRepository) AddJobCount(
 	}
 	rows, err := query.RunWith(r.DB).Query()
 	if err != nil {
-		cclog.Warn("Error while querying DB for job statistics")
+		log.Warn("Error while querying DB for job statistics")
 		return nil, err
 	}
 
@@ -434,7 +420,7 @@ func (r *JobRepository) AddJobCount(
 	for rows.Next() {
 		var cnt sql.NullInt64
 		if err := rows.Scan(&cnt); err != nil {
-			cclog.Warn("Error while scanning rows")
+			log.Warn("Error while scanning rows")
 			return nil, err
 		}
 
@@ -452,7 +438,7 @@ func (r *JobRepository) AddJobCount(
 		}
 	}
 
-	cclog.Debugf("Timer AddJobCount %s", time.Since(start))
+	log.Debugf("Timer AddJobCount %s", time.Since(start))
 	return stats, nil
 }
 
@@ -493,29 +479,29 @@ func (r *JobRepository) AddHistograms(
 	value := fmt.Sprintf(`CAST(ROUND(((CASE WHEN job.job_state = "running" THEN %d - job.start_time ELSE job.duration END) / %d) + 1) as %s) as value`, time.Now().Unix(), targetBinSize, castType)
 	stat.HistDuration, err = r.jobsDurationStatisticsHistogram(ctx, value, filter, targetBinSize, &targetBinCount)
 	if err != nil {
-		cclog.Warn("Error while loading job statistics histogram: job duration")
+		log.Warn("Error while loading job statistics histogram: job duration")
 		return nil, err
 	}
 
 	stat.HistNumNodes, err = r.jobsStatisticsHistogram(ctx, "job.num_nodes as value", filter)
 	if err != nil {
-		cclog.Warn("Error while loading job statistics histogram: num nodes")
+		log.Warn("Error while loading job statistics histogram: num nodes")
 		return nil, err
 	}
 
 	stat.HistNumCores, err = r.jobsStatisticsHistogram(ctx, "job.num_hwthreads as value", filter)
 	if err != nil {
-		cclog.Warn("Error while loading job statistics histogram: num hwthreads")
+		log.Warn("Error while loading job statistics histogram: num hwthreads")
 		return nil, err
 	}
 
 	stat.HistNumAccs, err = r.jobsStatisticsHistogram(ctx, "job.num_acc as value", filter)
 	if err != nil {
-		cclog.Warn("Error while loading job statistics histogram: num acc")
+		log.Warn("Error while loading job statistics histogram: num acc")
 		return nil, err
 	}
 
-	cclog.Debugf("Timer AddHistograms %s", time.Since(start))
+	log.Debugf("Timer AddHistograms %s", time.Since(start))
 	return stat, nil
 }
 
@@ -534,7 +520,7 @@ func (r *JobRepository) AddMetricHistograms(
 		if f.State != nil {
 			if len(f.State) == 1 && f.State[0] == "running" {
 				stat.HistMetrics = r.runningJobsMetricStatisticsHistogram(ctx, metrics, filter, targetBinCount)
-				cclog.Debugf("Timer AddMetricHistograms %s", time.Since(start))
+				log.Debugf("Timer AddMetricHistograms %s", time.Since(start))
 				return stat, nil
 			}
 		}
@@ -544,13 +530,13 @@ func (r *JobRepository) AddMetricHistograms(
 	for _, m := range metrics {
 		metricHisto, err := r.jobsMetricStatisticsHistogram(ctx, m, filter, targetBinCount)
 		if err != nil {
-			cclog.Warnf("Error while loading job metric statistics histogram: %s", m)
+			log.Warnf("Error while loading job metric statistics histogram: %s", m)
 			continue
 		}
 		stat.HistMetrics = append(stat.HistMetrics, metricHisto)
 	}
 
-	cclog.Debugf("Timer AddMetricHistograms %s", time.Since(start))
+	log.Debugf("Timer AddMetricHistograms %s", time.Since(start))
 	return stat, nil
 }
 
@@ -574,7 +560,7 @@ func (r *JobRepository) jobsStatisticsHistogram(
 
 	rows, err := query.GroupBy("value").RunWith(r.DB).Query()
 	if err != nil {
-		cclog.Error("Error while running query")
+		log.Error("Error while running query")
 		return nil, err
 	}
 
@@ -583,13 +569,13 @@ func (r *JobRepository) jobsStatisticsHistogram(
 	for rows.Next() {
 		point := model.HistoPoint{}
 		if err := rows.Scan(&point.Value, &point.Count); err != nil {
-			cclog.Warn("Error while scanning rows")
+			log.Warn("Error while scanning rows")
 			return nil, err
 		}
 
 		points = append(points, &point)
 	}
-	cclog.Debugf("Timer jobsStatisticsHistogram %s", time.Since(start))
+	log.Debugf("Timer jobsStatisticsHistogram %s", time.Since(start))
 	return points, nil
 }
 
@@ -621,7 +607,7 @@ func (r *JobRepository) jobsDurationStatisticsHistogram(
 
 	rows, err := query.GroupBy("value").RunWith(r.DB).Query()
 	if err != nil {
-		cclog.Error("Error while running query")
+		log.Error("Error while running query")
 		return nil, err
 	}
 
@@ -629,7 +615,7 @@ func (r *JobRepository) jobsDurationStatisticsHistogram(
 	for rows.Next() {
 		point := model.HistoPoint{}
 		if err := rows.Scan(&point.Value, &point.Count); err != nil {
-			cclog.Warn("Error while scanning rows")
+			log.Warn("Error while scanning rows")
 			return nil, err
 		}
 
@@ -644,7 +630,7 @@ func (r *JobRepository) jobsDurationStatisticsHistogram(
 		}
 	}
 
-	cclog.Debugf("Timer jobsStatisticsHistogram %s", time.Since(start))
+	log.Debugf("Timer jobsStatisticsHistogram %s", time.Since(start))
 	return points, nil
 }
 
@@ -666,7 +652,7 @@ func (r *JobRepository) jobsMetricStatisticsHistogram(
 			peak = metricConfig.Peak
 			unit = metricConfig.Unit.Prefix + metricConfig.Unit.Base
 			footprintStat = metricConfig.Footprint
-			cclog.Debugf("Cluster %s filter found with peak %f for %s", *f.Cluster.Eq, peak, metric)
+			log.Debugf("Cluster %s filter found with peak %f for %s", *f.Cluster.Eq, peak, metric)
 		}
 	}
 
@@ -688,32 +674,57 @@ func (r *JobRepository) jobsMetricStatisticsHistogram(
 		}
 	}
 
-	// cclog.Debugf("Metric %s, Peak %f, Unit %s", metric, peak, unit)
-	// Make bins, see https://jereze.com/code/sql-histogram/ (Modified here)
-	start := time.Now()
+	// log.Debugf("Metric %s, Peak %f, Unit %s, Aggregation %s", metric, peak, unit, aggreg)
+	// Make bins, see https://jereze.com/code/sql-histogram/
 
-	// Find Jobs' Value Bin Number: Divide Value by Peak, Multiply by RequestedBins, then CAST to INT: Gets Bin-Number of Job
-	binQuery := fmt.Sprintf(`CAST(
-		((case when json_extract(footprint, "$.%s") = %f then %f*0.999999999 else json_extract(footprint, "$.%s") end) / %f)
-		* %v as INTEGER )`,
-		(metric + "_" + footprintStat), peak, peak, (metric + "_" + footprintStat), peak, *bins)
+	start := time.Now()
+	jm := fmt.Sprintf(`json_extract(footprint, "$.%s")`, (metric + "_" + footprintStat))
+
+	crossJoinQuery := sq.Select(
+		fmt.Sprintf(`max(%s) as max`, jm),
+		fmt.Sprintf(`min(%s) as min`, jm),
+	).From("job").Where(
+		"JSON_VALID(footprint)",
+	).Where(
+		fmt.Sprintf(`%s is not null`, jm),
+	).Where(
+		fmt.Sprintf(`%s <= %f`, jm, peak),
+	)
+
+	crossJoinQuery, cjqerr := SecurityCheck(ctx, crossJoinQuery)
+
+	if cjqerr != nil {
+		return nil, cjqerr
+	}
+
+	for _, f := range filters {
+		crossJoinQuery = BuildWhereClause(f, crossJoinQuery)
+	}
+
+	crossJoinQuerySql, crossJoinQueryArgs, sqlerr := crossJoinQuery.ToSql()
+	if sqlerr != nil {
+		return nil, sqlerr
+	}
+
+	binQuery := fmt.Sprintf(`CAST( (case when %s = value.max
+	   then value.max*0.999999999 else %s end - value.min) / (value.max -
+	   value.min) * %v as INTEGER )`, jm, jm, *bins)
 
 	mainQuery := sq.Select(
 		fmt.Sprintf(`%s + 1 as bin`, binQuery),
-		`count(*) as count`,
-		// For Debug: // fmt.Sprintf(`CAST((%f / %d) as INTEGER ) * %s as min`, peak, *bins, binQuery),
-		// For Debug: // fmt.Sprintf(`CAST((%f / %d) as INTEGER ) * (%s + 1) as max`, peak, *bins, binQuery),
-	).From("job").Where(
-		"JSON_VALID(footprint)",
-	).Where(fmt.Sprintf(`json_extract(footprint, "$.%s") is not null and json_extract(footprint, "$.%s") <= %f`, (metric + "_" + footprintStat), (metric + "_" + footprintStat), peak))
+		fmt.Sprintf(`count(%s) as count`, jm),
+		fmt.Sprintf(`CAST(((value.max / %d) * (%v     )) as INTEGER ) as min`, *bins, binQuery),
+		fmt.Sprintf(`CAST(((value.max / %d) * (%v + 1 )) as INTEGER ) as max`, *bins, binQuery),
+	).From("job").CrossJoin(
+		fmt.Sprintf(`(%s) as value`, crossJoinQuerySql), crossJoinQueryArgs...,
+	).Where(fmt.Sprintf(`%s is not null and %s <= %f`, jm, jm, peak))
 
-	// Only accessible Jobs...
 	mainQuery, qerr := SecurityCheck(ctx, mainQuery)
+
 	if qerr != nil {
 		return nil, qerr
 	}
 
-	// Filters...
 	for _, f := range filters {
 		mainQuery = BuildWhereClause(f, mainQuery)
 	}
@@ -723,38 +734,36 @@ func (r *JobRepository) jobsMetricStatisticsHistogram(
 
 	rows, err := mainQuery.RunWith(r.DB).Query()
 	if err != nil {
-		cclog.Errorf("Error while running mainQuery: %s", err)
+		log.Errorf("Error while running mainQuery: %s", err)
 		return nil, err
 	}
 
-	// Setup Return Array With Bin-Numbers for Match and Min/Max based on Peak
+	// Setup Array
 	points := make([]*model.MetricHistoPoint, 0)
-	binStep := int(peak) / *bins
 	for i := 1; i <= *bins; i++ {
-		binMin := (binStep * (i - 1))
-		binMax := (binStep * i)
-		epoint := model.MetricHistoPoint{Bin: &i, Count: 0, Min: &binMin, Max: &binMax}
-		points = append(points, &epoint)
+		binMax := ((int(peak) / *bins) * i)
+		binMin := ((int(peak) / *bins) * (i - 1))
+		point := model.MetricHistoPoint{Bin: &i, Count: 0, Min: &binMin, Max: &binMax}
+		points = append(points, &point)
 	}
 
-	for rows.Next() { // Fill Count if Bin-No. Matches (Not every Bin exists in DB!)
-		rpoint := model.MetricHistoPoint{}
-		if err := rows.Scan(&rpoint.Bin, &rpoint.Count); err != nil { // Required for Debug: &rpoint.Min, &rpoint.Max
-			cclog.Warnf("Error while scanning rows for %s", metric)
-			return nil, err // FIXME: Totally bricks cc-backend if returned and if all metrics requested?
+	for rows.Next() {
+		point := model.MetricHistoPoint{}
+		if err := rows.Scan(&point.Bin, &point.Count, &point.Min, &point.Max); err != nil {
+			log.Warnf("Error while scanning rows for %s", jm)
+			return nil, err // Totally bricks cc-backend if returned and if all metrics requested?
 		}
 
 		for _, e := range points {
-			if e.Bin != nil && rpoint.Bin != nil {
-				if *e.Bin == *rpoint.Bin {
-					e.Count = rpoint.Count
-					// Only Required For Debug: Check DB returned Min/Max against Backend Init above
-					// if rpoint.Min != nil {
-					// 	cclog.Warnf(">>>> Bin %d Min Set For %s to %d (Init'd with: %d)", *e.Bin, metric, *rpoint.Min, *e.Min)
-					// }
-					// if rpoint.Max != nil {
-					// 	cclog.Warnf(">>>> Bin %d Max Set For %s to %d (Init'd with: %d)", *e.Bin, metric, *rpoint.Max, *e.Max)
-					// }
+			if e.Bin != nil && point.Bin != nil {
+				if *e.Bin == *point.Bin {
+					e.Count = point.Count
+					if point.Min != nil {
+						e.Min = point.Min
+					}
+					if point.Max != nil {
+						e.Max = point.Max
+					}
 					break
 				}
 			}
@@ -763,7 +772,7 @@ func (r *JobRepository) jobsMetricStatisticsHistogram(
 
 	result := model.MetricHistoPoints{Metric: metric, Unit: unit, Stat: &footprintStat, Data: points}
 
-	cclog.Debugf("Timer jobsStatisticsHistogram %s", time.Since(start))
+	log.Debugf("Timer jobsStatisticsHistogram %s", time.Since(start))
 	return &result, nil
 }
 
@@ -773,14 +782,15 @@ func (r *JobRepository) runningJobsMetricStatisticsHistogram(
 	filters []*model.JobFilter,
 	bins *int,
 ) []*model.MetricHistoPoints {
+
 	// Get Jobs
 	jobs, err := r.QueryJobs(ctx, filters, &model.PageRequest{Page: 1, ItemsPerPage: 500 + 1}, nil)
 	if err != nil {
-		cclog.Errorf("Error while querying jobs for footprint: %s", err)
+		log.Errorf("Error while querying jobs for footprint: %s", err)
 		return nil
 	}
 	if len(jobs) > 500 {
-		cclog.Errorf("too many jobs matched (max: %d)", 500)
+		log.Errorf("too many jobs matched (max: %d)", 500)
 		return nil
 	}
 
@@ -796,7 +806,7 @@ func (r *JobRepository) runningJobsMetricStatisticsHistogram(
 		}
 
 		if err := metricDataDispatcher.LoadAverages(job, metrics, avgs, ctx); err != nil {
-			cclog.Errorf("Error while loading averages for histogram: %s", err)
+			log.Errorf("Error while loading averages for histogram: %s", err)
 			return nil
 		}
 	}

internal/repository/stats_test.go

@@ -1,8 +1,7 @@
 // Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved. This file is part of cc-backend.
+// All rights reserved.
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
-
 package repository
 
 import (
@@ -20,6 +19,7 @@ func TestBuildJobStatsQuery(t *testing.T) {
 	noErr(t, err)
 
 	fmt.Printf("SQL: %s\n", sql)
+
 }
 
 func TestJobStats(t *testing.T) {
@@ -29,7 +29,7 @@ func TestJobStats(t *testing.T) {
 	stats, err := r.JobsStats(getContext(t), []*model.JobFilter{filter})
 	noErr(t, err)
 
-	if stats[0].TotalJobs != 544 {
-		t.Fatalf("Want 544, Got %d", stats[0].TotalJobs)
+	if stats[0].TotalJobs != 6 {
+		t.Fatalf("Want 98, Got %d", stats[0].TotalJobs)
 	}
 }

internal/repository/tags.go

@@ -1,5 +1,5 @@
 // Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved. This file is part of cc-backend.
+// All rights reserved.
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 package repository
@@ -9,8 +9,8 @@ import (
 	"strings"
 
 	"github.com/ClusterCockpit/cc-backend/pkg/archive"
-	cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
-	"github.com/ClusterCockpit/cc-lib/schema"
+	"github.com/ClusterCockpit/cc-backend/pkg/log"
+	"github.com/ClusterCockpit/cc-backend/pkg/schema"
 	sq "github.com/Masterminds/squirrel"
 )
 
@@ -18,7 +18,7 @@ import (
 func (r *JobRepository) AddTag(user *schema.User, job int64, tag int64) ([]*schema.Tag, error) {
 	j, err := r.FindByIdWithUser(user, job)
 	if err != nil {
-		cclog.Warn("Error while finding job by id")
+		log.Warn("Error while finding job by id")
 		return nil, err
 	}
 
@@ -26,61 +26,30 @@ func (r *JobRepository) AddTag(user *schema.User, job int64, tag int64) ([]*sche
 
 	if _, err := q.RunWith(r.stmtCache).Exec(); err != nil {
 		s, _, _ := q.ToSql()
-		cclog.Errorf("Error adding tag with %s: %v", s, err)
+		log.Errorf("Error adding tag with %s: %v", s, err)
 		return nil, err
 	}
 
 	tags, err := r.GetTags(user, &job)
 	if err != nil {
-		cclog.Warn("Error while getting tags for job")
+		log.Warn("Error while getting tags for job")
 		return nil, err
 	}
 
 	archiveTags, err := r.getArchiveTags(&job)
 	if err != nil {
-		cclog.Warn("Error while getting tags for job")
+		log.Warn("Error while getting tags for job")
 		return nil, err
 	}
 
 	return tags, archive.UpdateTags(j, archiveTags)
 }
 
-func (r *JobRepository) AddTagDirect(job int64, tag int64) ([]*schema.Tag, error) {
-	j, err := r.FindByIdDirect(job)
-	if err != nil {
-		cclog.Warn("Error while finding job by id")
-		return nil, err
-	}
-
-	q := sq.Insert("jobtag").Columns("job_id", "tag_id").Values(job, tag)
-
-	if _, err := q.RunWith(r.stmtCache).Exec(); err != nil {
-		s, _, _ := q.ToSql()
-		cclog.Errorf("Error adding tag with %s: %v", s, err)
-		return nil, err
-	}
-
-	tags, err := r.GetTagsDirect(&job)
-	if err != nil {
-		cclog.Warn("Error while getting tags for job")
-		return nil, err
-	}
-
-	archiveTags, err := r.getArchiveTags(&job)
-	if err != nil {
-		cclog.Warn("Error while getting tags for job")
-		return nil, err
-	}
-
-	return tags, archive.UpdateTags(j, archiveTags)
-}
-
-// Removes a tag from a job by tag id.
-// Used by GraphQL API
+// Removes a tag from a job
 func (r *JobRepository) RemoveTag(user *schema.User, job, tag int64) ([]*schema.Tag, error) {
 	j, err := r.FindByIdWithUser(user, job)
 	if err != nil {
-		cclog.Warn("Error while finding job by id")
+		log.Warn("Error while finding job by id")
 		return nil, err
 	}
 
@@ -88,129 +57,25 @@ func (r *JobRepository) RemoveTag(user *schema.User, job, tag int64) ([]*schema.
 
 	if _, err := q.RunWith(r.stmtCache).Exec(); err != nil {
 		s, _, _ := q.ToSql()
-		cclog.Errorf("Error removing tag with %s: %v", s, err)
+		log.Errorf("Error removing tag with %s: %v", s, err)
 		return nil, err
 	}
 
 	tags, err := r.GetTags(user, &job)
 	if err != nil {
-		cclog.Warn("Error while getting tags for job")
+		log.Warn("Error while getting tags for job")
 		return nil, err
 	}
 
 	archiveTags, err := r.getArchiveTags(&job)
 	if err != nil {
-		cclog.Warn("Error while getting tags for job")
+		log.Warn("Error while getting tags for job")
 		return nil, err
 	}
 
 	return tags, archive.UpdateTags(j, archiveTags)
 }
 
-// Removes a tag from a job by tag info
-// Used by REST API
-func (r *JobRepository) RemoveJobTagByRequest(user *schema.User, job int64, tagType string, tagName string, tagScope string) ([]*schema.Tag, error) {
-	// Get Tag ID to delete
-	tagID, exists := r.TagId(tagType, tagName, tagScope)
-	if !exists {
-		cclog.Warnf("Tag does not exist (name, type, scope): %s, %s, %s", tagName, tagType, tagScope)
-		return nil, fmt.Errorf("tag does not exist (name, type, scope): %s, %s, %s", tagName, tagType, tagScope)
-	}
-
-	// Get Job
-	j, err := r.FindByIdWithUser(user, job)
-	if err != nil {
-		cclog.Warn("Error while finding job by id")
-		return nil, err
-	}
-
-	// Handle Delete
-	q := sq.Delete("jobtag").Where("jobtag.job_id = ?", job).Where("jobtag.tag_id = ?", tagID)
-
-	if _, err := q.RunWith(r.stmtCache).Exec(); err != nil {
-		s, _, _ := q.ToSql()
-		cclog.Errorf("Error removing tag from table 'jobTag' with %s: %v", s, err)
-		return nil, err
-	}
-
-	tags, err := r.GetTags(user, &job)
-	if err != nil {
-		cclog.Warn("Error while getting tags for job")
-		return nil, err
-	}
-
-	archiveTags, err := r.getArchiveTags(&job)
-	if err != nil {
-		cclog.Warn("Error while getting tags for job")
-		return nil, err
-	}
-
-	return tags, archive.UpdateTags(j, archiveTags)
-}
-
-func (r *JobRepository) removeTagFromArchiveJobs(jobIds []int64) {
-	for _, j := range jobIds {
-		tags, err := r.getArchiveTags(&j)
-		if err != nil {
-			cclog.Warnf("Error while getting tags for job %d", j)
-			continue
-		}
-
-		job, err := r.FindByIdDirect(j)
-		if err != nil {
-			cclog.Warnf("Error while getting job %d", j)
-			continue
-		}
-
-		archive.UpdateTags(job, tags)
-	}
-}
-
-// Removes a tag from db by tag info
-// Used by REST API. Does not update tagged jobs in Job archive.
-func (r *JobRepository) RemoveTagByRequest(tagType string, tagName string, tagScope string) error {
-	// Get Tag ID to delete
-	tagID, exists := r.TagId(tagType, tagName, tagScope)
-	if !exists {
-		cclog.Warnf("Tag does not exist (name, type, scope): %s, %s, %s", tagName, tagType, tagScope)
-		return fmt.Errorf("tag does not exist (name, type, scope): %s, %s, %s", tagName, tagType, tagScope)
-	}
-
-	return r.RemoveTagById(tagID)
-}
-
-// Removes a tag from db by tag id
-// Used by GraphQL API.
-func (r *JobRepository) RemoveTagById(tagID int64) error {
-	jobIds, err := r.FindJobIdsByTag(tagID)
-	if err != nil {
-		return err
-	}
-
-	// Handle Delete JobTagTable
-	qJobTag := sq.Delete("jobtag").Where("jobtag.tag_id = ?", tagID)
-
-	if _, err := qJobTag.RunWith(r.stmtCache).Exec(); err != nil {
-		s, _, _ := qJobTag.ToSql()
-		cclog.Errorf("Error removing tag from table 'jobTag' with %s: %v", s, err)
-		return err
-	}
-
-	// Handle Delete TagTable
-	qTag := sq.Delete("tag").Where("tag.id = ?", tagID)
-
-	if _, err := qTag.RunWith(r.stmtCache).Exec(); err != nil {
-		s, _, _ := qTag.ToSql()
-		cclog.Errorf("Error removing tag from table 'tag' with %s: %v", s, err)
-		return err
-	}
-
-	// asynchronously update archive jobs
-	go r.removeTagFromArchiveJobs(jobIds)
-
-	return nil
-}
-
 // CreateTag creates a new tag with the specified type and name and returns its database id.
 func (r *JobRepository) CreateTag(tagType string, tagName string, tagScope string) (tagId int64, err error) {
 	// Default to "Global" scope if none defined
@@ -223,7 +88,7 @@ func (r *JobRepository) CreateTag(tagType string, tagName string, tagScope strin
 	res, err := q.RunWith(r.stmtCache).Exec()
 	if err != nil {
 		s, _, _ := q.ToSql()
-		cclog.Errorf("Error inserting tag with %s: %v", s, err)
+		log.Errorf("Error inserting tag with %s: %v", s, err)
 		return 0, err
 	}
 
@@ -272,7 +137,7 @@ func (r *JobRepository) CountTags(user *schema.User) (tags []schema.Tag, counts
 
 	// Handle Job Ownership
 	if user != nil && user.HasAnyRole([]schema.Role{schema.RoleAdmin, schema.RoleSupport}) { // ADMIN || SUPPORT: Count all jobs
-		// cclog.Debug("CountTags: User Admin or Support -> Count all Jobs for Tags")
+		// log.Debug("CountTags: User Admin or Support -> Count all Jobs for Tags")
 		// Unchanged: Needs to be own case still, due to UserRole/NoRole compatibility handling in else case
 	} else if user != nil && user.HasRole(schema.RoleManager) { // MANAGER: Count own jobs plus project's jobs
 		// Build ("project1", "project2", ...) list of variable length directly in SQL string
@@ -333,38 +198,6 @@ func (r *JobRepository) AddTagOrCreate(user *schema.User, jobId int64, tagType s
 	return tagId, nil
 }
 
-// used in auto tagger plugins
-func (r *JobRepository) AddTagOrCreateDirect(jobId int64, tagType string, tagName string) (tagId int64, err error) {
-	tagScope := "global"
-
-	tagId, exists := r.TagId(tagType, tagName, tagScope)
-	if !exists {
-		tagId, err = r.CreateTag(tagType, tagName, tagScope)
-		if err != nil {
-			return 0, err
-		}
-	}
-
-	if _, err := r.AddTagDirect(jobId, tagId); err != nil {
-		return 0, err
-	}
-
-	return tagId, nil
-}
-
-func (r *JobRepository) HasTag(jobId int64, tagType string, tagName string) bool {
-	var id int64
-	q := sq.Select("id").From("tag").Join("jobtag ON jobtag.tag_id = tag.id").
-		Where("jobtag.job_id = ?", jobId).Where("tag.tag_type = ?", tagType).
-		Where("tag.tag_name = ?", tagName)
-	err := q.RunWith(r.stmtCache).QueryRow().Scan(&id)
-	if err != nil {
-		return false
-	} else {
-		return true
-	}
-}
-
 // TagId returns the database id of the tag with the specified type and name.
 func (r *JobRepository) TagId(tagType string, tagName string, tagScope string) (tagId int64, exists bool) {
 	exists = true
@@ -376,16 +209,6 @@ func (r *JobRepository) TagId(tagType string, tagName string, tagScope string) (
 	return
 }
 
-// TagInfo returns the database infos of the tag with the specified id.
-func (r *JobRepository) TagInfo(tagId int64) (tagType string, tagName string, tagScope string, exists bool) {
-	exists = true
-	if err := sq.Select("tag.tag_type", "tag.tag_name", "tag.tag_scope").From("tag").Where("tag.id = ?", tagId).
-		RunWith(r.stmtCache).QueryRow().Scan(&tagType, &tagName, &tagScope); err != nil {
-		exists = false
-	}
-	return
-}
-
 // GetTags returns a list of all scoped tags if job is nil or of the tags that the job with that database ID has.
 func (r *JobRepository) GetTags(user *schema.User, job *int64) ([]*schema.Tag, error) {
 	q := sq.Select("id", "tag_type", "tag_name", "tag_scope").From("tag")
@@ -396,7 +219,7 @@ func (r *JobRepository) GetTags(user *schema.User, job *int64) ([]*schema.Tag, e
 	rows, err := q.RunWith(r.stmtCache).Query()
 	if err != nil {
 		s, _, _ := q.ToSql()
-		cclog.Errorf("Error get tags with %s: %v", s, err)
+		log.Errorf("Error get tags with %s: %v", s, err)
 		return nil, err
 	}
 
@@ -404,7 +227,7 @@ func (r *JobRepository) GetTags(user *schema.User, job *int64) ([]*schema.Tag, e
 	for rows.Next() {
 		tag := &schema.Tag{}
 		if err := rows.Scan(&tag.ID, &tag.Type, &tag.Name, &tag.Scope); err != nil {
-			cclog.Warn("Error while scanning rows")
+			log.Warn("Error while scanning rows")
 			return nil, err
 		}
 		// Handle Scope Filtering: Tag Scope is Global, Private (== Username) or User is auth'd to view Admin Tags
@@ -420,32 +243,6 @@ func (r *JobRepository) GetTags(user *schema.User, job *int64) ([]*schema.Tag, e
 	return tags, nil
 }
 
-func (r *JobRepository) GetTagsDirect(job *int64) ([]*schema.Tag, error) {
-	q := sq.Select("id", "tag_type", "tag_name", "tag_scope").From("tag")
-	if job != nil {
-		q = q.Join("jobtag ON jobtag.tag_id = tag.id").Where("jobtag.job_id = ?", *job)
-	}
-
-	rows, err := q.RunWith(r.stmtCache).Query()
-	if err != nil {
-		s, _, _ := q.ToSql()
-		cclog.Errorf("Error get tags with %s: %v", s, err)
-		return nil, err
-	}
-
-	tags := make([]*schema.Tag, 0)
-	for rows.Next() {
-		tag := &schema.Tag{}
-		if err := rows.Scan(&tag.ID, &tag.Type, &tag.Name, &tag.Scope); err != nil {
-			cclog.Warn("Error while scanning rows")
-			return nil, err
-		}
-		tags = append(tags, tag)
-	}
-
-	return tags, nil
-}
-
 // GetArchiveTags returns a list of all tags *regardless of scope* for archiving if job is nil or of the tags that the job with that database ID has.
 func (r *JobRepository) getArchiveTags(job *int64) ([]*schema.Tag, error) {
 	q := sq.Select("id", "tag_type", "tag_name", "tag_scope").From("tag")
@@ -456,7 +253,7 @@ func (r *JobRepository) getArchiveTags(job *int64) ([]*schema.Tag, error) {
 	rows, err := q.RunWith(r.stmtCache).Query()
 	if err != nil {
 		s, _, _ := q.ToSql()
-		cclog.Errorf("Error get tags with %s: %v", s, err)
+		log.Errorf("Error get tags with %s: %v", s, err)
 		return nil, err
 	}
 
@@ -464,7 +261,7 @@ func (r *JobRepository) getArchiveTags(job *int64) ([]*schema.Tag, error) {
 	for rows.Next() {
 		tag := &schema.Tag{}
 		if err := rows.Scan(&tag.ID, &tag.Type, &tag.Name, &tag.Scope); err != nil {
-			cclog.Warn("Error while scanning rows")
+			log.Warn("Error while scanning rows")
 			return nil, err
 		}
 		tags = append(tags, tag)
@@ -488,7 +285,7 @@ func (r *JobRepository) ImportTag(jobId int64, tagType string, tagName string, t
 
 	if _, err := q.RunWith(r.stmtCache).Exec(); err != nil {
 		s, _, _ := q.ToSql()
-		cclog.Errorf("Error adding tag on import with %s: %v", s, err)
+		log.Errorf("Error adding tag on import with %s: %v", s, err)
 		return err
 	}