Merge pull request #519 from ClusterCockpit/hotfix

Hotfix

.claude/settings.json

@@ -0,0 +1,84 @@
+{
+  "hooks": {
+    "PostToolUse": [
+      {
+        "matcher": "Task",
+        "hooks": [
+          {
+            "type": "command",
+            "command": "entire hooks claude-code post-task"
+          }
+        ]
+      },
+      {
+        "matcher": "TodoWrite",
+        "hooks": [
+          {
+            "type": "command",
+            "command": "entire hooks claude-code post-todo"
+          }
+        ]
+      }
+    ],
+    "PreToolUse": [
+      {
+        "matcher": "Task",
+        "hooks": [
+          {
+            "type": "command",
+            "command": "entire hooks claude-code pre-task"
+          }
+        ]
+      }
+    ],
+    "SessionEnd": [
+      {
+        "matcher": "",
+        "hooks": [
+          {
+            "type": "command",
+            "command": "entire hooks claude-code session-end"
+          }
+        ]
+      }
+    ],
+    "SessionStart": [
+      {
+        "matcher": "",
+        "hooks": [
+          {
+            "type": "command",
+            "command": "entire hooks claude-code session-start"
+          }
+        ]
+      }
+    ],
+    "Stop": [
+      {
+        "matcher": "",
+        "hooks": [
+          {
+            "type": "command",
+            "command": "entire hooks claude-code stop"
+          }
+        ]
+      }
+    ],
+    "UserPromptSubmit": [
+      {
+        "matcher": "",
+        "hooks": [
+          {
+            "type": "command",
+            "command": "entire hooks claude-code user-prompt-submit"
+          }
+        ]
+      }
+    ]
+  },
+  "permissions": {
+    "deny": [
+      "Read(./.entire/metadata/**)"
+    ]
+  }
+}

.entire/.gitignore

@@ -0,0 +1,4 @@
+tmp/
+settings.local.json
+metadata/
+logs/

.entire/metadata/28147033-ddc8-4056-b064-e0558fbc614e/context.md

@@ -0,0 +1,29 @@
+# Session Context
+
+Session ID: 28147033-ddc8-4056-b064-e0558fbc614e
+Commit Message: There are memory leaks in production on queries that trigger a full scan
+
+## Prompts
+
+### Prompt 1
+
+There are memory leaks in production on queries that trigger a full scan. Review all queries for missing close and other bugs that could cause memory leaks in the sqlite-3 driver. Example for a query that triggers a massive allocation is a specific jobstate list with sorting on start_time on all jobs.
+
+## Summary
+
+All 31 `.Query()` calls now have proper `rows.Close()`. Build passes, tests pass, vet clean.
+
+### Summary of changes
+
+**`internal/repository/stats.go`** (7 fixes + 1 rewrite):
+- Added `defer rows.Close()` + `rows.Err()` to `JobsStatsGrouped`, `JobCountGrouped`, `AddJobCountGrouped`, `jobsStatisticsHistogram`, `jobsDurationStatisticsHistogram`, `jobsMetricStatisticsHistogram`
+- Rewrote `AddJobCount` to use `QueryRow()` instead of `Query()` (single-row result)
+
+**`internal/repository/tags.go`** (5 fixes):
+- Added `defer xrows.Close()` + `xrows.Err()` for `Queryx` in `CountTags`
+- Added `defer rows.Close()` for second query in `CountTags`
+- Added `defer rows.Close()` + `rows.Err()` to `GetTags`, `GetTagsDirect`, `getArchiveTags`
+
+**`internal/repository/node.go`** (4 fixes + 1 rewrite):
+- Added `defer rows.Close()` + `rows.Err()`, removed manual `rows.Close()` in `QueryNodes`, `QueryNodesWithMeta`, `CountStatesTimed`
+- Rewrote `CountStates` to use `COUNT(*) ... GROUP BY` in SQL instead of fetching every row and counting in Go

.entire/metadata/28147033-ddc8-4056-b064-e0558fbc614e/prompt.txt

@@ -0,0 +1 @@
+There are memory leaks in production on queries that trigger a full scan. Review all queries for missing close and other bugs that could cause memory leaks in the sqlite-3 driver. Example for a query that triggers a massive allocation is a specific jobstate list with sorting on start_time on all jobs.

.entire/metadata/28147033-ddc8-4056-b064-e0558fbc614e/summary.txt

@@ -0,0 +1,16 @@
+All 31 `.Query()` calls now have proper `rows.Close()`. Build passes, tests pass, vet clean.
+
+### Summary of changes
+
+**`internal/repository/stats.go`** (7 fixes + 1 rewrite):
+- Added `defer rows.Close()` + `rows.Err()` to `JobsStatsGrouped`, `JobCountGrouped`, `AddJobCountGrouped`, `jobsStatisticsHistogram`, `jobsDurationStatisticsHistogram`, `jobsMetricStatisticsHistogram`
+- Rewrote `AddJobCount` to use `QueryRow()` instead of `Query()` (single-row result)
+
+**`internal/repository/tags.go`** (5 fixes):
+- Added `defer xrows.Close()` + `xrows.Err()` for `Queryx` in `CountTags`
+- Added `defer rows.Close()` for second query in `CountTags`
+- Added `defer rows.Close()` + `rows.Err()` to `GetTags`, `GetTagsDirect`, `getArchiveTags`
+
+**`internal/repository/node.go`** (4 fixes + 1 rewrite):
+- Added `defer rows.Close()` + `rows.Err()`, removed manual `rows.Close()` in `QueryNodes`, `QueryNodesWithMeta`, `CountStatesTimed`
+- Rewrote `CountStates` to use `COUNT(*) ... GROUP BY` in SQL instead of fetching every row and counting in Go

.entire/metadata/42401d2e-7d1c-4c0e-abe6-356cb2d48747/context.md

@@ -0,0 +1,40 @@
+# Session Context
+
+Session ID: 42401d2e-7d1c-4c0e-abe6-356cb2d48747
+Commit Message: There also have to be bugs in jobQuery.go . Especially the following que
+
+## Prompts
+
+### Prompt 1
+
+There also have to be bugs in jobQuery.go . Especially the following query triggers the memory leak: SELECT * FROM job WHERE job.job_state IN ("completed", "running", "failed") ORDER BY job.start_time DESC LIMIT 1 OFFSET 10; Dig deeper to find the cause. Also investigate why no existing index is used for this query.
+
+## Summary
+
+All passes. Here's a summary of findings and fixes:
+
+---
+
+## Findings
+
+### Bug 1: Missing `defer rows.Close()` in `user.go:GetLdapUsernames()` (line 100)
+**Fixed.** Every call leaked a database connection and its buffers.
+
+### Bug 2: SQLite never runs `ANALYZE` — wrong index selection
+**Fixed.** Added `ANALYZE` to `setupSqlite()` in `dbConnection.go`.
+
+Without statistics in `sqlite_stat1`, SQLite's query planner **guessed** and chose `jobs_jobstate_energy` (a terrible index for this query) instead of scanning `jobs_starttime` in reverse. The result:
+
+| | Before ANALYZE | After ANALYZE |
+|---|---|---|
+| Index used | `jobs_jobstate_energy` | `jobs_starttime` |
+| Sort | **TEMP B-TREE** (materializes ALL rows) | None (index order) |
+| Memory | Proportional to total matching rows | Constant |
+| I/O | Full scan of all matching rows | Stops at OFFSET+LIMIT |
+
+### Bug 3: `IN` clause + `ORDER BY` is fundamentally incompatible with composite indexes
+
+Even with the "correct" index `(job_state, start_time)`, SQLite **cannot** merge-sort across 3 separate index range scans for `IN ('completed','running','failed')`. It always falls back to a temp B-tree sort. The only efficient plan is to use the standalone `jobs_starttime` index — which SQLite does automatically **after ANALYZE** because it realizes the 3 states cover virtually all rows, making the WHERE clause nearly a no-op.
+
+### Observation: 79 indexes on the `job` table
+This is excessive and actively harmful — it confuses the query planner (especially without ANALYZE) and slows writes. The `jobs_jobstate_starttime` index from migration 08 is also missing from the actual DB (only the 3-column `jobs_jobstate_starttime_duration` exists). This is worth investigating separately but is a schema/migration concern, not a code bug.

.entire/metadata/42401d2e-7d1c-4c0e-abe6-356cb2d48747/prompt.txt

@@ -0,0 +1 @@
+There also have to be bugs in jobQuery.go . Especially the following query triggers the memory leak: SELECT * FROM job WHERE job.job_state IN ("completed", "running", "failed") ORDER BY job.start_time DESC LIMIT 1 OFFSET 10; Dig deeper to find the cause. Also investigate why no existing index is used for this query.

.entire/metadata/42401d2e-7d1c-4c0e-abe6-356cb2d48747/summary.txt

@@ -0,0 +1,27 @@
+All passes. Here's a summary of findings and fixes:
+
+---
+
+## Findings
+
+### Bug 1: Missing `defer rows.Close()` in `user.go:GetLdapUsernames()` (line 100)
+**Fixed.** Every call leaked a database connection and its buffers.
+
+### Bug 2: SQLite never runs `ANALYZE` — wrong index selection
+**Fixed.** Added `ANALYZE` to `setupSqlite()` in `dbConnection.go`.
+
+Without statistics in `sqlite_stat1`, SQLite's query planner **guessed** and chose `jobs_jobstate_energy` (a terrible index for this query) instead of scanning `jobs_starttime` in reverse. The result:
+
+| | Before ANALYZE | After ANALYZE |
+|---|---|---|
+| Index used | `jobs_jobstate_energy` | `jobs_starttime` |
+| Sort | **TEMP B-TREE** (materializes ALL rows) | None (index order) |
+| Memory | Proportional to total matching rows | Constant |
+| I/O | Full scan of all matching rows | Stops at OFFSET+LIMIT |
+
+### Bug 3: `IN` clause + `ORDER BY` is fundamentally incompatible with composite indexes
+
+Even with the "correct" index `(job_state, start_time)`, SQLite **cannot** merge-sort across 3 separate index range scans for `IN ('completed','running','failed')`. It always falls back to a temp B-tree sort. The only efficient plan is to use the standalone `jobs_starttime` index — which SQLite does automatically **after ANALYZE** because it realizes the 3 states cover virtually all rows, making the WHERE clause nearly a no-op.
+
+### Observation: 79 indexes on the `job` table
+This is excessive and actively harmful — it confuses the query planner (especially without ANALYZE) and slows writes. The `jobs_jobstate_starttime` index from migration 08 is also missing from the actual DB (only the 3-column `jobs_jobstate_starttime_duration` exists). This is worth investigating separately but is a schema/migration concern, not a code bug.

.entire/settings.json

@@ -0,0 +1,4 @@
+{
+  "enabled": true,
+  "telemetry": true
+}

.github/workflows/Release.yml

@@ -1,331 +0,0 @@
-# See: https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions
-
-# Workflow name
-name: Release
-
-# Run on tag push
-on:
- push:
-  tags:
-  - '**'
-
-jobs:
-
-  #
-  # Build on AlmaLinux 8.5 using golang-1.18.2
-  #
-  AlmaLinux-RPM-build:
-    runs-on: ubuntu-latest
-    # See: https://hub.docker.com/_/almalinux
-    container: almalinux:8.5
-    # The job outputs link to the outputs of the 'rpmrename' step
-    # Only job outputs can be used in child jobs
-    outputs:
-      rpm : ${{steps.rpmrename.outputs.RPM}}
-      srpm : ${{steps.rpmrename.outputs.SRPM}}
-    steps:
-
-    # Use dnf to install development packages
-    - name: Install development packages
-      run: |
-          dnf --assumeyes group install "Development Tools" "RPM Development Tools"
-          dnf --assumeyes install wget openssl-devel diffutils delve which npm
-          dnf --assumeyes install 'dnf-command(builddep)'
-
-    # Checkout git repository and submodules
-    # fetch-depth must be 0 to use git describe
-    # See: https://github.com/marketplace/actions/checkout
-    - name: Checkout
-      uses: actions/checkout@v2
-      with:
-        submodules: recursive
-        fetch-depth: 0
-
-    # Use dnf to install build dependencies
-    - name: Install build dependencies
-      run: |
-          wget -q http://mirror.centos.org/centos/8-stream/AppStream/x86_64/os/Packages/golang-1.18.2-1.module_el8.7.0+1173+5d37c0fd.x86_64.rpm \
-                  http://mirror.centos.org/centos/8-stream/AppStream/x86_64/os/Packages/golang-bin-1.18.2-1.module_el8.7.0+1173+5d37c0fd.x86_64.rpm \
-                  http://mirror.centos.org/centos/8-stream/AppStream/x86_64/os/Packages/golang-src-1.18.2-1.module_el8.7.0+1173+5d37c0fd.noarch.rpm \
-                  http://mirror.centos.org/centos/8-stream/AppStream/x86_64/os/Packages/go-toolset-1.18.2-1.module_el8.7.0+1173+5d37c0fd.x86_64.rpm
-          rpm -i go*.rpm
-          npm install --global yarn rollup svelte rollup-plugin-svelte
-          #dnf --assumeyes builddep build/package/cc-backend.spec
-
-    - name: RPM build ClusterCockpit
-      id: rpmbuild
-      run: make RPM
-
-    # AlmaLinux 8.5 is a derivate of RedHat Enterprise Linux 8 (UBI8),
-    # so the created RPM both contain the substring 'el8' in the RPM file names
-    # This step replaces the substring 'el8' to 'alma85'. It uses the move operation
-    # because it is unclear whether the default AlmaLinux 8.5 container contains the 
-    # 'rename' command. This way we also get the new names for output.
-    - name: Rename RPMs (s/el8/alma85/)
-      id: rpmrename
-      run: |
-        OLD_RPM="${{steps.rpmbuild.outputs.RPM}}"
-        OLD_SRPM="${{steps.rpmbuild.outputs.SRPM}}"
-        NEW_RPM="${OLD_RPM/el8/alma85}"
-        NEW_SRPM=${OLD_SRPM/el8/alma85}
-        mv "${OLD_RPM}" "${NEW_RPM}"
-        mv "${OLD_SRPM}" "${NEW_SRPM}"
-        echo "::set-output name=SRPM::${NEW_SRPM}"
-        echo "::set-output name=RPM::${NEW_RPM}"
-
-    # See: https://github.com/actions/upload-artifact
-    - name: Save RPM as artifact
-      uses: actions/upload-artifact@v2
-      with:
-        name: cc-backend RPM for AlmaLinux 8.5
-        path: ${{ steps.rpmrename.outputs.RPM }}
-    - name: Save SRPM as artifact
-      uses: actions/upload-artifact@v2
-      with:
-        name: cc-backend SRPM for AlmaLinux 8.5
-        path: ${{ steps.rpmrename.outputs.SRPM }}
-
-  #
-  # Build on UBI 8 using golang-1.18.2
-  #
-  UBI-8-RPM-build:
-    runs-on: ubuntu-latest
-    # See: https://catalog.redhat.com/software/containers/ubi8/ubi/5c359854d70cc534b3a3784e?container-tabs=gti
-    container: registry.access.redhat.com/ubi8/ubi:8.5-226.1645809065
-    # The job outputs link to the outputs of the 'rpmbuild' step
-    outputs:
-      rpm : ${{steps.rpmbuild.outputs.RPM}}
-      srpm : ${{steps.rpmbuild.outputs.SRPM}}
-    steps:
-
-    # Use dnf to install development packages
-    - name: Install development packages
-      run: dnf --assumeyes --disableplugin=subscription-manager install rpm-build go-srpm-macros rpm-build-libs rpm-libs gcc make python38 git wget openssl-devel diffutils delve which
-
-    # Checkout git repository and submodules
-    # fetch-depth must be 0 to use git describe
-    # See: https://github.com/marketplace/actions/checkout
-    - name: Checkout
-      uses: actions/checkout@v2
-      with:
-        submodules: recursive
-        fetch-depth: 0
-
-    # Use dnf to install build dependencies
-    - name: Install build dependencies
-      run: |
-          wget -q http://mirror.centos.org/centos/8-stream/AppStream/x86_64/os/Packages/golang-1.18.2-1.module_el8.7.0+1173+5d37c0fd.x86_64.rpm \
-                  http://mirror.centos.org/centos/8-stream/AppStream/x86_64/os/Packages/golang-bin-1.18.2-1.module_el8.7.0+1173+5d37c0fd.x86_64.rpm \
-                  http://mirror.centos.org/centos/8-stream/AppStream/x86_64/os/Packages/golang-src-1.18.2-1.module_el8.7.0+1173+5d37c0fd.noarch.rpm \
-                  http://mirror.centos.org/centos/8-stream/AppStream/x86_64/os/Packages/go-toolset-1.18.2-1.module_el8.7.0+1173+5d37c0fd.x86_64.rpm
-          rpm -i go*.rpm
-          dnf --assumeyes --disableplugin=subscription-manager install npm
-          npm install --global yarn rollup svelte rollup-plugin-svelte
-          #dnf --assumeyes builddep build/package/cc-backend.spec
-
-    - name: RPM build ClusterCockpit
-      id: rpmbuild
-      run: make RPM
-
-    # See: https://github.com/actions/upload-artifact
-    - name: Save RPM as artifact
-      uses: actions/upload-artifact@v2
-      with:
-        name: cc-backend RPM for UBI 8
-        path: ${{ steps.rpmbuild.outputs.RPM }}
-    - name: Save SRPM as artifact
-      uses: actions/upload-artifact@v2
-      with:
-        name: cc-backend SRPM for UBI 8
-        path: ${{ steps.rpmbuild.outputs.SRPM }}
-
-  #
-  # Build on Ubuntu 20.04 using official go 1.19.1 package
-  #
-  Ubuntu-focal-build:
-    runs-on: ubuntu-latest
-    container: ubuntu:20.04
-    # The job outputs link to the outputs of the 'debrename' step
-    # Only job outputs can be used in child jobs
-    outputs:
-      deb : ${{steps.debrename.outputs.DEB}}
-    steps:
-    # Use apt to install development packages
-    - name: Install development packages
-      run: |
-          apt update && apt --assume-yes upgrade
-          apt --assume-yes install build-essential sed git wget bash
-          apt --assume-yes install npm
-          npm install --global yarn rollup svelte rollup-plugin-svelte
-    # Checkout git repository and submodules
-    # fetch-depth must be 0 to use git describe
-    # See: https://github.com/marketplace/actions/checkout
-    - name: Checkout
-      uses: actions/checkout@v2
-      with:
-        submodules: recursive
-        fetch-depth: 0
-    # Use official golang package
-    - name: Install Golang
-      run: |
-          wget -q https://go.dev/dl/go1.19.1.linux-amd64.tar.gz
-          tar -C /usr/local -xzf go1.19.1.linux-amd64.tar.gz
-          export PATH=/usr/local/go/bin:/usr/local/go/pkg/tool/linux_amd64:$PATH
-          go version
-    - name: DEB build ClusterCockpit
-      id: dpkg-build
-      run: |
-          ls -la
-          pwd
-          env
-          export PATH=/usr/local/go/bin:/usr/local/go/pkg/tool/linux_amd64:$PATH
-          git config --global --add safe.directory $(pwd)
-          make DEB
-    - name: Rename DEB (add '_ubuntu20.04')
-      id: debrename
-      run: |
-        OLD_DEB_NAME=$(echo "${{steps.dpkg-build.outputs.DEB}}" | rev | cut -d '.' -f 2- | rev)
-        NEW_DEB_FILE="${OLD_DEB_NAME}_ubuntu20.04.deb"
-        mv "${{steps.dpkg-build.outputs.DEB}}" "${NEW_DEB_FILE}"
-        echo "::set-output name=DEB::${NEW_DEB_FILE}"
-    # See: https://github.com/actions/upload-artifact
-    - name: Save DEB as artifact
-      uses: actions/upload-artifact@v2
-      with:
-        name: cc-backend DEB for Ubuntu 20.04
-        path: ${{ steps.debrename.outputs.DEB }}
-
-  #
-  # Build on Ubuntu 20.04 using official go 1.19.1 package
-  #
-  Ubuntu-jammy-build:
-    runs-on: ubuntu-latest
-    container: ubuntu:22.04
-    # The job outputs link to the outputs of the 'debrename' step
-    # Only job outputs can be used in child jobs
-    outputs:
-      deb : ${{steps.debrename.outputs.DEB}}
-    steps:
-    # Use apt to install development packages
-    - name: Install development packages
-      run: |
-          apt update && apt --assume-yes upgrade
-          apt --assume-yes install build-essential sed git wget bash npm
-          npm install --global yarn rollup svelte rollup-plugin-svelte
-    # Checkout git repository and submodules
-    # fetch-depth must be 0 to use git describe
-    # See: https://github.com/marketplace/actions/checkout
-    - name: Checkout
-      uses: actions/checkout@v2
-      with:
-        submodules: recursive
-        fetch-depth: 0
-    # Use official golang package
-    - name: Install Golang
-      run: |
-          wget -q https://go.dev/dl/go1.19.1.linux-amd64.tar.gz
-          tar -C /usr/local -xzf go1.19.1.linux-amd64.tar.gz
-          export PATH=/usr/local/go/bin:/usr/local/go/pkg/tool/linux_amd64:$PATH
-          go version
-    - name: DEB build ClusterCockpit
-      id: dpkg-build
-      run: |
-          ls -la
-          pwd
-          env
-          export PATH=/usr/local/go/bin:/usr/local/go/pkg/tool/linux_amd64:$PATH
-          git config --global --add safe.directory $(pwd)
-          make DEB
-    - name: Rename DEB (add '_ubuntu22.04')
-      id: debrename
-      run: |
-        OLD_DEB_NAME=$(echo "${{steps.dpkg-build.outputs.DEB}}" | rev | cut -d '.' -f 2- | rev)
-        NEW_DEB_FILE="${OLD_DEB_NAME}_ubuntu22.04.deb"
-        mv "${{steps.dpkg-build.outputs.DEB}}" "${NEW_DEB_FILE}"
-        echo "::set-output name=DEB::${NEW_DEB_FILE}"
-    # See: https://github.com/actions/upload-artifact
-    - name: Save DEB as artifact
-      uses: actions/upload-artifact@v2
-      with:
-        name: cc-backend DEB for Ubuntu 22.04
-        path: ${{ steps.debrename.outputs.DEB }}
-
-  #
-  # Create release with fresh RPMs
-  #
-  Release:
-    runs-on: ubuntu-latest
-    # We need the RPMs, so add dependency
-    needs: [AlmaLinux-RPM-build, UBI-8-RPM-build, Ubuntu-focal-build, Ubuntu-jammy-build]
-
-    steps:
-    # See: https://github.com/actions/download-artifact
-    - name: Download AlmaLinux 8.5 RPM
-      uses: actions/download-artifact@v2
-      with:
-        name: cc-backend RPM for AlmaLinux 8.5
-    - name: Download AlmaLinux 8.5 SRPM
-      uses: actions/download-artifact@v2
-      with:
-        name: cc-backend SRPM for AlmaLinux 8.5
-    
-    - name: Download UBI 8 RPM
-      uses: actions/download-artifact@v2
-      with:
-        name: cc-backend RPM for UBI 8
-    - name: Download UBI 8 SRPM
-      uses: actions/download-artifact@v2
-      with:
-        name: cc-backend SRPM for UBI 8
-
-    - name: Download Ubuntu 20.04 DEB
-      uses: actions/download-artifact@v2
-      with:
-        name: cc-backend DEB for Ubuntu 20.04
-
-    - name: Download Ubuntu 22.04 DEB
-      uses: actions/download-artifact@v2
-      with:
-        name: cc-backend DEB for Ubuntu 22.04
-
-    # The download actions do not publish the name of the downloaded file,
-    # so we re-use the job outputs of the parent jobs. The files are all
-    # downloaded to the current folder.
-    # The gh-release action afterwards does not accept file lists but all
-    # files have to be listed at 'files'. The step creates one output per
-    # RPM package (2 per distro)
-    - name: Set RPM variables
-      id: files
-      run: |
-        ALMA_85_RPM=$(basename "${{ needs.AlmaLinux-RPM-build.outputs.rpm}}")
-        ALMA_85_SRPM=$(basename "${{ needs.AlmaLinux-RPM-build.outputs.srpm}}")
-        UBI_8_RPM=$(basename "${{ needs.UBI-8-RPM-build.outputs.rpm}}")
-        UBI_8_SRPM=$(basename "${{ needs.UBI-8-RPM-build.outputs.srpm}}")
-        U_2004_DEB=$(basename "${{ needs.Ubuntu-focal-build.outputs.deb}}")
-        U_2204_DEB=$(basename "${{ needs.Ubuntu-jammy-build.outputs.deb}}")
-        echo "ALMA_85_RPM::${ALMA_85_RPM}"
-        echo "ALMA_85_SRPM::${ALMA_85_SRPM}"
-        echo "UBI_8_RPM::${UBI_8_RPM}"
-        echo "UBI_8_SRPM::${UBI_8_SRPM}"
-        echo "U_2004_DEB::${U_2004_DEB}"
-        echo "U_2204_DEB::${U_2204_DEB}"
-        echo "::set-output name=ALMA_85_RPM::${ALMA_85_RPM}"
-        echo "::set-output name=ALMA_85_SRPM::${ALMA_85_SRPM}"
-        echo "::set-output name=UBI_8_RPM::${UBI_8_RPM}"
-        echo "::set-output name=UBI_8_SRPM::${UBI_8_SRPM}"
-        echo "::set-output name=U_2004_DEB::${U_2004_DEB}"
-        echo "::set-output name=U_2204_DEB::${U_2204_DEB}"
-
-    # See: https://github.com/softprops/action-gh-release
-    - name: Release
-      uses: softprops/action-gh-release@v1
-      if: startsWith(github.ref, 'refs/tags/')
-      with:
-        name: cc-backend-${{github.ref_name}}
-        files: |
-         ${{ steps.files.outputs.ALMA_85_RPM }}
-         ${{ steps.files.outputs.ALMA_85_SRPM }}
-         ${{ steps.files.outputs.UBI_8_RPM }}
-         ${{ steps.files.outputs.UBI_8_SRPM }}
-         ${{ steps.files.outputs.U_2004_DEB }}
-         ${{ steps.files.outputs.U_2204_DEB }}

.github/workflows/test.yml

@@ -7,7 +7,7 @@ jobs:
       - name: Install Go
         uses: actions/setup-go@v4
         with:
-          go-version: 1.22.x
+          go-version: 1.25.x
       - name: Checkout code
         uses: actions/checkout@v3
       - name: Build, Vet & Test

.gitignore

@@ -1,14 +1,20 @@
 /cc-backend
 /.env
 /config.json
+/uiConfig.json
 
 /var/job-archive
 /var/machine-state
-/var/job.db-shm
-/var/job.db-wal
+/var/*.db-shm
+/var/*.db-wal
 /var/*.db
 /var/*.txt
 
+/var/checkpoints*
+
+migrateTimestamps.pl
+test_ccms_*
+
 /web/frontend/public/build
 /web/frontend/node_modules
 
@@ -21,3 +27,6 @@
 /.vscode/*
 dist/
 *.db
+.idea
+tools/archive-migration/archive-migration
+tools/archive-manager/archive-manager

.goreleaser.yaml

@@ -1,3 +1,4 @@
+version: 2
 before:
   hooks:
     - go mod tidy
@@ -34,6 +35,19 @@ builds:
     main: ./tools/archive-manager
     tags:
       - static_build
+  - env:
+      - CGO_ENABLED=0
+    goos:
+      - linux
+    goarch:
+      - amd64
+    goamd64:
+      - v3
+    id: "archive-migration"
+    binary: archive-migration
+    main: ./tools/archive-migration
+    tags:
+      - static_build
   - env:
       - CGO_ENABLED=0
     goos:
@@ -48,7 +62,7 @@ builds:
     tags:
       - static_build
 archives:
-  - format: tar.gz
+  - formats: tar.gz
     # this name template makes the OS and Arch compatible with the results of uname.
     name_template: >-
       {{ .ProjectName }}_
@@ -59,7 +73,7 @@ archives:
 checksum:
   name_template: "checksums.txt"
 snapshot:
-  name_template: "{{ incpatch .Version }}-next"
+  version_template: "{{ incpatch .Version }}-next"
 changelog:
   sort: asc
   filters:
@@ -87,7 +101,7 @@ changelog:
 release:
   draft: false
   footer: |
-    Supports job archive version 2 and database version 8.
+    Supports job archive version 3 and database version 11.
     Please check out the [Release Notes](https://github.com/ClusterCockpit/cc-backend/blob/master/ReleaseNotes.md) for further details on breaking changes.
 
 # vim: set ts=2 sw=2 tw=0 fo=cnqoj

AGENTS.md

@@ -0,0 +1,26 @@
+# ClusterCockpit Backend - Agent Guidelines
+
+## Build/Test Commands
+
+- Build: `make` or `go build ./cmd/cc-backend`
+- Run all tests: `make test` (runs: `go clean -testcache && go build ./... && go vet ./... && go test ./...`)
+- Run single test: `go test -run TestName ./path/to/package`
+- Run single test file: `go test ./path/to/package -run TestName`
+- Frontend build: `cd web/frontend && npm install && npm run build`
+- Generate GraphQL: `make graphql` (uses gqlgen)
+- Generate Swagger: `make swagger` (uses swaggo/swag)
+
+## Code Style
+
+- **Formatting**: Use `gofumpt` for all Go files (strict requirement)
+- **Copyright header**: All files must include copyright header (see existing files)
+- **Package docs**: Document packages with comprehensive package-level comments explaining purpose, usage, configuration
+- **Imports**: Standard library first, then external packages, then internal packages (grouped with blank lines)
+- **Naming**: Use camelCase for private, PascalCase for exported; descriptive names (e.g., `JobRepository`, `handleError`)
+- **Error handling**: Return errors, don't panic; use custom error types where appropriate; log with cclog package
+- **Logging**: Use `cclog` package (e.g., `cclog.Errorf()`, `cclog.Warnf()`, `cclog.Debugf()`)
+- **Testing**: Use standard `testing` package; use `testify/assert` for assertions; name tests `TestFunctionName`
+- **Comments**: Document all exported functions/types with godoc-style comments
+- **Structs**: Document fields with inline comments, especially for complex configurations
+- **HTTP handlers**: Return proper status codes; use `handleError()` helper for consistent error responses
+- **JSON**: Use struct tags for JSON marshaling; `DisallowUnknownFields()` for strict decoding

CLAUDE.md

@@ -0,0 +1,306 @@
+# CLAUDE.md
+
+This file provides guidance to Claude Code (claude.ai/code) when working with
+code in this repository.
+
+## Project Overview
+
+ClusterCockpit is a job-specific performance monitoring framework for HPC
+clusters. This is a Golang backend that provides REST and GraphQL APIs, serves a
+Svelte-based frontend, and manages job archives and metric data from various
+time-series databases.
+
+## Build and Development Commands
+
+### Building
+
+```bash
+# Build everything (frontend + backend)
+make
+
+# Build only the frontend
+make frontend
+
+# Build only the backend (requires frontend to be built first)
+go build -ldflags='-s -X main.date=$(date +"%Y-%m-%d:T%H:%M:%S") -X main.version=1.5.0 -X main.commit=$(git rev-parse --short HEAD)' ./cmd/cc-backend
+```
+
+### Testing
+
+```bash
+# Run all tests
+make test
+
+# Run tests with verbose output
+go test -v ./...
+
+# Run tests for a specific package
+go test ./internal/repository
+```
+
+### Code Generation
+
+```bash
+# Regenerate GraphQL schema and resolvers (after modifying api/schema.graphqls)
+make graphql
+
+# Regenerate Swagger/OpenAPI docs (after modifying API comments)
+make swagger
+```
+
+### Frontend Development
+
+```bash
+cd web/frontend
+
+# Install dependencies
+npm install
+
+# Build for production
+npm run build
+
+# Development mode with watch
+npm run dev
+```
+
+### Running
+
+```bash
+# Initialize database and create admin user
+./cc-backend -init-db -add-user demo:admin:demo
+
+# Start server in development mode (enables GraphQL Playground and Swagger UI)
+./cc-backend -server -dev -loglevel info
+
+# Start demo with sample data
+./startDemo.sh
+```
+
+## Architecture
+
+### Backend Structure
+
+The backend follows a layered architecture with clear separation of concerns:
+
+- **cmd/cc-backend**: Entry point, orchestrates initialization of all subsystems
+- **internal/repository**: Data access layer using repository pattern
+  - Abstracts database operations (SQLite3 only)
+  - Implements LRU caching for performance
+  - Provides repositories for Job, User, Node, and Tag entities
+  - Transaction support for batch operations
+- **internal/api**: REST API endpoints (Swagger/OpenAPI documented)
+- **internal/graph**: GraphQL API (uses gqlgen)
+  - Schema in `api/schema.graphqls`
+  - Generated code in `internal/graph/generated/`
+  - Resolvers in `internal/graph/schema.resolvers.go`
+- **internal/auth**: Authentication layer
+  - Supports local accounts, LDAP, OIDC, and JWT tokens
+  - Implements rate limiting for login attempts
+- **pkg/metricstore**: Metric store with data loading API
+  - In-memory metric storage with checkpointing
+  - Query API for loading job metric data
+- **internal/archiver**: Job archiving to file-based archive
+- **internal/api/nats.go**: NATS-based API for job and node operations
+  - Subscribes to NATS subjects for job events (start/stop)
+  - Handles node state updates via NATS
+  - Uses InfluxDB line protocol message format
+- **pkg/archive**: Job archive backend implementations
+  - File system backend (default)
+  - S3 backend
+  - SQLite backend (experimental)
+  - **parquet** sub-package: Parquet format support (schema, reader, writer, conversion)
+- **internal/metricstoreclient**: Client for cc-metric-store queries
+
+### Frontend Structure
+
+- **web/frontend**: Svelte 5 application
+  - Uses Rollup for building
+  - Components organized by feature (analysis, job, user, etc.)
+  - GraphQL client using @urql/svelte
+  - Bootstrap 5 + SvelteStrap for UI
+  - uPlot for time-series visualization
+- **web/templates**: Server-side Go templates
+
+### Key Concepts
+
+**Job Archive**: Completed jobs are stored in a file-based archive following the
+[ClusterCockpit job-archive
+specification](https://github.com/ClusterCockpit/cc-specifications/tree/master/job-archive).
+Each job has a `meta.json` file with metadata and metric data files.
+
+**Metric Data Repositories**: Time-series metric data is stored separately from
+job metadata. The system supports multiple backends (cc-metric-store is
+recommended). Configuration is per-cluster in `config.json`.
+
+**Authentication Flow**:
+
+1. Multiple authenticators can be configured (local, LDAP, OIDC, JWT)
+2. Each authenticator's `CanLogin` method is called to determine if it should handle the request
+3. The first authenticator that returns true performs the actual `Login`
+4. JWT tokens are used for API authentication
+
+**Database Migrations**: SQL migrations in `internal/repository/migrations/sqlite3/` are
+applied automatically on startup. Version tracking in `version` table.
+
+**Scopes**: Metrics can be collected at different scopes:
+
+- Node scope (always available)
+- Core scope (for jobs with ≤8 nodes)
+- Accelerator scope (for GPU/accelerator metrics)
+
+## Configuration
+
+- **config.json**: Main configuration (clusters, metric repositories, archive settings)
+  - `main.apiSubjects`: NATS subject configuration (optional)
+    - `subjectJobEvent`: Subject for job start/stop events (e.g., "cc.job.event")
+    - `subjectNodeState`: Subject for node state updates (e.g., "cc.node.state")
+  - `nats`: NATS client connection configuration (optional)
+    - `address`: NATS server address (e.g., "nats://localhost:4222")
+    - `username`: Authentication username (optional)
+    - `password`: Authentication password (optional)
+    - `creds-file-path`: Path to NATS credentials file (optional)
+- **.env**: Environment variables (secrets like JWT keys)
+  - Copy from `configs/env-template.txt`
+  - NEVER commit this file
+- **cluster.json**: Cluster topology and metric definitions (loaded from archive or config)
+
+## Database
+
+- Default: SQLite 3 (`./var/job.db`)
+- Connection managed by `internal/repository`
+- Schema version in `internal/repository/migration.go`
+
+## Code Generation
+
+**GraphQL** (gqlgen):
+
+- Schema: `api/schema.graphqls`
+- Config: `gqlgen.yml`
+- Generated code: `internal/graph/generated/`
+- Custom resolvers: `internal/graph/schema.resolvers.go`
+- Run `make graphql` after schema changes
+
+**Swagger/OpenAPI**:
+
+- Annotations in `internal/api/*.go`
+- Generated docs: `internal/api/docs.go`, `api/swagger.yaml`
+- Run `make swagger` after API changes
+
+## Testing Conventions
+
+- Test files use `_test.go` suffix
+- Test data in `testdata/` subdirectories
+- Repository tests use in-memory SQLite
+- API tests use httptest
+
+## Common Workflows
+
+### Adding a new GraphQL field
+
+1. Edit schema in `api/schema.graphqls`
+2. Run `make graphql`
+3. Implement resolver in `internal/graph/schema.resolvers.go`
+
+### Adding a new REST endpoint
+
+1. Add handler in `internal/api/*.go`
+2. Add route in `internal/api/rest.go`
+3. Add Swagger annotations
+4. Run `make swagger`
+
+### Adding a new metric data backend
+
+1. Implement metric loading functions in `pkg/metricstore/query.go`
+2. Add cluster configuration to metric store initialization
+3. Update config.json schema documentation
+
+### Modifying database schema
+
+1. Create new migration in `internal/repository/migrations/sqlite3/`
+2. Increment `repository.Version`
+3. Test with fresh database and existing database
+
+## NATS API
+
+The backend supports a NATS-based API as an alternative to the REST API for job and node operations.
+
+### Setup
+
+1. Configure NATS client connection in `config.json`:
+   ```json
+   {
+     "nats": {
+       "address": "nats://localhost:4222",
+       "username": "user",
+       "password": "pass"
+     }
+   }
+   ```
+
+2. Configure API subjects in `config.json` under `main`:
+   ```json
+   {
+     "main": {
+       "apiSubjects": {
+         "subjectJobEvent": "cc.job.event",
+         "subjectNodeState": "cc.node.state"
+       }
+     }
+   }
+   ```
+
+### Message Format
+
+Messages use **InfluxDB line protocol** format with the following structure:
+
+#### Job Events
+
+**Start Job:**
+```
+job,function=start_job event="{\"jobId\":123,\"user\":\"alice\",\"cluster\":\"test\", ...}" 1234567890000000000
+```
+
+**Stop Job:**
+```
+job,function=stop_job event="{\"jobId\":123,\"cluster\":\"test\",\"startTime\":1234567890,\"stopTime\":1234571490,\"jobState\":\"completed\"}" 1234571490000000000
+```
+
+**Tags:**
+- `function`: Either `start_job` or `stop_job`
+
+**Fields:**
+- `event`: JSON payload containing job data (see REST API documentation for schema)
+
+#### Node State Updates
+
+```json
+{
+  "cluster": "testcluster",
+  "nodes": [
+    {
+      "hostname": "node001",
+      "states": ["allocated"],
+      "cpusAllocated": 8,
+      "memoryAllocated": 16384,
+      "gpusAllocated": 0,
+      "jobsRunning": 1
+    }
+  ]
+}
+```
+
+### Implementation Notes
+
+- NATS API mirrors REST API functionality but uses messaging
+- Job start/stop events are processed asynchronously
+- Duplicate job detection is handled (same as REST API)
+- All validation rules from REST API apply
+- Messages are logged; no responses are sent back to publishers
+- If NATS client is unavailable, API subscriptions are skipped (logged as warning)
+
+## Dependencies
+
+- Go 1.24.0+ (check go.mod for exact version)
+- Node.js (for frontend builds)
+- SQLite 3 (only supported database)
+- Optional: NATS server for NATS API integration

Makefile

@@ -1,8 +1,6 @@
 TARGET = ./cc-backend
-VAR = ./var
-CFG = config.json .env
 FRONTEND = ./web/frontend
-VERSION = 1.4.3
+VERSION = 1.5.1
 GIT_HASH := $(shell git rev-parse --short HEAD || echo 'development')
 CURRENT_TIME = $(shell date +"%Y-%m-%d:T%H:%M:%S")
 LD_FLAGS = '-s -X main.date=${CURRENT_TIME} -X main.version=${VERSION} -X main.commit=${GIT_HASH}'
@@ -42,22 +40,22 @@ SVELTE_SRC = $(wildcard $(FRONTEND)/src/*.svelte)                 \
 
 .NOTPARALLEL:
 
-$(TARGET): $(VAR) $(CFG) $(SVELTE_TARGETS)
+$(TARGET): $(SVELTE_TARGETS)
 	$(info ===>  BUILD cc-backend)
 	@go build -ldflags=${LD_FLAGS} ./cmd/cc-backend
 
 frontend:
 	$(info ===>  BUILD frontend)
-	cd web/frontend && npm install && npm run build
+	cd web/frontend && npm ci && npm run build
 
 swagger:
 	$(info ===>  GENERATE swagger)
-	@go run github.com/swaggo/swag/cmd/swag init -d ./internal/api,./pkg/schema -g rest.go -o ./api
+	@go tool github.com/swaggo/swag/cmd/swag init  --parseDependency -d ./internal/api -g rest.go -o ./api
 	@mv ./api/docs.go ./internal/api/docs.go
 
 graphql:
 	$(info ===>  GENERATE graphql)
-	@go run github.com/99designs/gqlgen
+	@go tool github.com/99designs/gqlgen
 
 clean:
 	$(info ===>  CLEAN)
@@ -68,7 +66,7 @@ distclean:
 	@$(MAKE) clean
 	$(info ===>  DISTCLEAN)
 	@rm -rf $(FRONTEND)/node_modules
-	@rm -rf $(VAR)
+	@rm -rf ./var
 
 test:
 	$(info ===>  TESTING)
@@ -84,14 +82,6 @@ tags:
 $(VAR):
 	@mkdir -p $(VAR)
 
-config.json:
-	$(info ===>  Initialize config.json file)
-	@cp configs/config.json config.json
-
-.env:
-	$(info ===>  Initialize .env file)
-	@cp configs/env-template.txt .env
-
 $(SVELTE_TARGETS): $(SVELTE_SRC)
 	$(info ===>  BUILD frontend)
-	cd web/frontend && npm install && npm run build
+	cd web/frontend && npm ci && npm run build

README.md

@@ -1,5 +1,8 @@
 # NOTE
 
+While we do our best to keep the master branch in a usable state, there is no guarantee the master branch works.
+Please do not use it for production!
+
 Please have a look at the [Release
 Notes](https://github.com/ClusterCockpit/cc-backend/blob/master/ReleaseNotes.md)
 for breaking changes!
@@ -19,19 +22,23 @@ switching from PHP Symfony to a Golang based solution are explained
 ## Overview
 
 This is a Golang web backend for the ClusterCockpit job-specific performance
-monitoring framework. It provides a REST API for integrating ClusterCockpit with
-an HPC cluster batch system and external analysis scripts. Data exchange between
-the web front-end and the back-end is based on a GraphQL API. The web frontend
-is also served by the backend using [Svelte](https://svelte.dev/) components.
-Layout and styling are based on [Bootstrap 5](https://getbootstrap.com/) using
+monitoring framework. It provides a REST API and an optional NATS-based messaging
+API for integrating ClusterCockpit with an HPC cluster batch system and external
+analysis scripts. Data exchange between the web front-end and the back-end is
+based on a GraphQL API. The web frontend is also served by the backend using
+[Svelte](https://svelte.dev/) components. Layout and styling are based on
+[Bootstrap 5](https://getbootstrap.com/) using
 [Bootstrap Icons](https://icons.getbootstrap.com/).
 
-The backend uses [SQLite 3](https://sqlite.org/) as a relational SQL database by
-default. Optionally it can use a MySQL/MariaDB database server. While there are
-metric data  backends for the InfluxDB and Prometheus time series databases, the
-only tested and supported setup is to use cc-metric-store as the metric data
-backend. Documentation on how to integrate ClusterCockpit with other time series
-databases will be added in the future.
+The backend uses [SQLite 3](https://sqlite.org/) as the relational SQL database.
+While there are metric data backends for the InfluxDB and Prometheus time series
+databases, the only tested and supported setup is to use cc-metric-store as the
+metric data backend. Documentation on how to integrate ClusterCockpit with other
+time series databases will be added in the future.
+
+For real-time integration with HPC systems, the backend can subscribe to
+[NATS](https://nats.io/) subjects to receive job start/stop events and node
+state updates, providing an alternative to REST API polling.
 
 Completed batch jobs are stored in a file-based job archive according to
 [this specification](https://github.com/ClusterCockpit/cc-specifications/tree/master/job-archive).
@@ -69,7 +76,7 @@ You can also try the demo using the latest release binary.
 Create a folder and put the release binary `cc-backend` into this folder.
 Execute the following steps:
 
-``` shell
+```shell
 ./cc-backend -init
 vim config.json (Add a second cluster entry and name the clusters alex and fritz)
 wget https://hpc-mover.rrze.uni-erlangen.de/HPC-Data/0x7b58aefb/eig7ahyo6fo2bais0ephuf2aitohv1ai/job-archive-demo.tar
@@ -88,11 +95,11 @@ Analysis, Systems and Status views).
 There is a Makefile to automate the build of cc-backend. The Makefile supports
 the following targets:
 
-* `make`: Initialize `var` directory and build svelte frontend and backend
-binary. Note that there is no proper prerequisite handling. Any change of
-frontend source files will result in a complete rebuild.
-* `make clean`: Clean go build cache and remove binary.
-* `make test`: Run the tests that are also run in the GitHub workflow setup.
+- `make`: Initialize `var` directory and build svelte frontend and backend
+  binary. Note that there is no proper prerequisite handling. Any change of
+  frontend source files will result in a complete rebuild.
+- `make clean`: Clean go build cache and remove binary.
+- `make test`: Run the tests that are also run in the GitHub workflow setup.
 
 A common workflow for setting up cc-backend from scratch is:
 
@@ -126,43 +133,161 @@ ln -s <your-existing-job-archive> ./var/job-archive
 ./cc-backend -help
 ```
 
+## Database Configuration
+
+cc-backend uses SQLite as its database. For large installations, SQLite memory
+usage can be tuned via the optional `db-config` section in config.json under
+`main`:
+
+```json
+{
+  "main": {
+    "db": "./var/job.db",
+    "db-config": {
+      "cache-size-mb": 2048,
+      "soft-heap-limit-mb": 16384,
+      "max-open-connections": 4,
+      "max-idle-connections": 4,
+      "max-idle-time-minutes": 10
+    }
+  }
+}
+```
+
+All fields are optional. If `db-config` is omitted entirely, built-in defaults
+are used.
+
+### Options
+
+| Option                  | Default | Description                                                                                                                                                                             |
+| ----------------------- | ------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `cache-size-mb`         | 2048    | SQLite page cache size per connection in MB. Maps to `PRAGMA cache_size`. Total cache memory is up to `cache-size-mb × max-open-connections`.                                           |
+| `soft-heap-limit-mb`    | 16384   | Process-wide SQLite soft heap limit in MB. SQLite will try to release cache pages to stay under this limit. Queries won't fail if exceeded, but cache eviction becomes more aggressive. |
+| `max-open-connections`  | 4       | Maximum number of open database connections.                                                                                                                                            |
+| `max-idle-connections`  | 4       | Maximum number of idle database connections kept in the pool.                                                                                                                           |
+| `max-idle-time-minutes` | 10      | Maximum time in minutes a connection can sit idle before being closed.                                                                                                                  |
+
+### Sizing Guidelines
+
+SQLite's `cache_size` is a **per-connection** setting — each connection
+maintains its own independent page cache. With multiple connections, the total
+memory available for caching is the sum across all connections.
+
+In practice, different connections tend to cache **different pages** (e.g., one
+handles a job listing query while another runs a statistics aggregation), so
+their caches naturally spread across the database. The formula
+`DB_size / max-open-connections` gives enough per-connection cache that the
+combined caches can cover the entire database.
+
+However, this is a best-case estimate. Connections running similar queries will
+cache the same pages redundantly. In the worst case (all connections caching
+identical pages), only `cache-size-mb` worth of unique data is cached rather
+than `cache-size-mb × max-open-connections`. For workloads with diverse
+concurrent queries, cache overlap is typically low.
+
+**Rules of thumb:**
+
+- **cache-size-mb**: Set to `DB_size_in_MB / max-open-connections` to allow the
+  entire database to be cached in memory. For example, an 80GB database with 8
+  connections needs at least 10240 MB (10GB) per connection. If your workload
+  has many similar concurrent queries, consider setting it higher to account for
+  cache overlap between connections.
+
+- **soft-heap-limit-mb**: Should be >= `cache-size-mb × max-open-connections` to
+  avoid cache thrashing. This is the total SQLite memory budget for the process.
+- On small installations the defaults work well. On servers with large databases
+  (tens of GB) and plenty of RAM, increasing these values significantly improves
+  query performance by reducing disk I/O.
+
+### Example: Large Server (512GB RAM, 80GB database)
+
+```json
+{
+  "main": {
+    "db-config": {
+      "cache-size-mb": 16384,
+      "soft-heap-limit-mb": 131072,
+      "max-open-connections": 8,
+      "max-idle-time-minutes": 30
+    }
+  }
+}
+```
+
+This allows the entire 80GB database to be cached (8 × 16GB = 128GB page cache)
+with a 128GB soft heap limit, using about 25% of available RAM.
+
+The effective configuration is logged at startup for verification.
+
 ## Project file structure
 
-* [`api/`](https://github.com/ClusterCockpit/cc-backend/tree/master/api)
-contains the API schema files for the REST and GraphQL APIs. The REST API is
-documented in the OpenAPI 3.0 format in
-[./api/openapi.yaml](./api/openapi.yaml).
-* [`cmd/cc-backend`](https://github.com/ClusterCockpit/cc-backend/tree/master/cmd/cc-backend)
-contains `main.go` for the main application.
-* [`configs/`](https://github.com/ClusterCockpit/cc-backend/tree/master/configs)
-contains documentation about configuration and command line options and required
-environment variables. A sample configuration file is provided.
-* [`docs/`](https://github.com/ClusterCockpit/cc-backend/tree/master/docs)
-contains more in-depth documentation.
-* [`init/`](https://github.com/ClusterCockpit/cc-backend/tree/master/init)
-contains an example of setting up systemd for production use.
-* [`internal/`](https://github.com/ClusterCockpit/cc-backend/tree/master/internal)
-contains library source code that is not intended for use by others.
-* [`pkg/`](https://github.com/ClusterCockpit/cc-backend/tree/master/pkg)
-contains Go packages that can be used by other projects.
-* [`tools/`](https://github.com/ClusterCockpit/cc-backend/tree/master/tools)
-Additional command line helper tools.
-  * [`archive-manager`](https://github.com/ClusterCockpit/cc-backend/tree/master/tools/archive-manager)
-  Commands for getting infos about and existing job archive.
-  * [`convert-pem-pubkey`](https://github.com/ClusterCockpit/cc-backend/tree/master/tools/convert-pem-pubkey)
-  Tool to convert external pubkey for use in `cc-backend`.
-  * [`gen-keypair`](https://github.com/ClusterCockpit/cc-backend/tree/master/tools/gen-keypair)
-  contains a small application to generate a compatible JWT keypair. You find
-  documentation on how to use it
-  [here](https://github.com/ClusterCockpit/cc-backend/blob/master/docs/JWT-Handling.md).
-* [`web/`](https://github.com/ClusterCockpit/cc-backend/tree/master/web)
-Server-side templates and frontend-related files:
-  * [`frontend`](https://github.com/ClusterCockpit/cc-backend/tree/master/web/frontend)
-  Svelte components and static assets for the frontend UI
-  * [`templates`](https://github.com/ClusterCockpit/cc-backend/tree/master/web/templates)
-  Server-side Go templates
-* [`gqlgen.yml`](https://github.com/ClusterCockpit/cc-backend/blob/master/gqlgen.yml)
-Configures the behaviour and generation of
-[gqlgen](https://github.com/99designs/gqlgen).
-* [`startDemo.sh`](https://github.com/ClusterCockpit/cc-backend/blob/master/startDemo.sh)
-is a shell script that sets up demo data, and builds and starts `cc-backend`.
+- [`.github/`](https://github.com/ClusterCockpit/cc-backend/tree/master/.github)
+  GitHub Actions workflows and dependabot configuration for CI/CD.
+- [`api/`](https://github.com/ClusterCockpit/cc-backend/tree/master/api)
+  contains the API schema files for the REST and GraphQL APIs. The REST API is
+  documented in the OpenAPI 3.0 format in
+  [./api/swagger.yaml](./api/swagger.yaml). The GraphQL schema is in
+  [./api/schema.graphqls](./api/schema.graphqls).
+- [`cmd/cc-backend`](https://github.com/ClusterCockpit/cc-backend/tree/master/cmd/cc-backend)
+  contains the main application entry point and CLI implementation.
+- [`configs/`](https://github.com/ClusterCockpit/cc-backend/tree/master/configs)
+  contains documentation about configuration and command line options and required
+  environment variables. Sample configuration files are provided.
+- [`init/`](https://github.com/ClusterCockpit/cc-backend/tree/master/init)
+  contains an example of setting up systemd for production use.
+- [`internal/`](https://github.com/ClusterCockpit/cc-backend/tree/master/internal)
+  contains library source code that is not intended for use by others.
+  - [`api`](https://github.com/ClusterCockpit/cc-backend/tree/master/internal/api)
+    REST API handlers and NATS integration
+  - [`archiver`](https://github.com/ClusterCockpit/cc-backend/tree/master/internal/archiver)
+    Job archiving functionality
+  - [`auth`](https://github.com/ClusterCockpit/cc-backend/tree/master/internal/auth)
+    Authentication (local, LDAP, OIDC) and JWT token handling
+  - [`config`](https://github.com/ClusterCockpit/cc-backend/tree/master/internal/config)
+    Configuration management and validation
+  - [`graph`](https://github.com/ClusterCockpit/cc-backend/tree/master/internal/graph)
+    GraphQL schema and resolvers
+  - [`importer`](https://github.com/ClusterCockpit/cc-backend/tree/master/internal/importer)
+    Job data import and database initialization
+  - [`metricdispatch`](https://github.com/ClusterCockpit/cc-backend/tree/master/internal/metricdispatch)
+    Dispatches metric data loading to appropriate backends
+  - [`repository`](https://github.com/ClusterCockpit/cc-backend/tree/master/internal/repository)
+    Database repository layer for jobs and metadata
+  - [`routerConfig`](https://github.com/ClusterCockpit/cc-backend/tree/master/internal/routerConfig)
+    HTTP router configuration and middleware
+  - [`tagger`](https://github.com/ClusterCockpit/cc-backend/tree/master/internal/tagger)
+    Job classification and application detection
+  - [`taskmanager`](https://github.com/ClusterCockpit/cc-backend/tree/master/internal/taskmanager)
+    Background task management and scheduled jobs
+  - [`metricstoreclient`](https://github.com/ClusterCockpit/cc-backend/tree/master/internal/metricstoreclient)
+    Client for cc-metric-store queries
+- [`pkg/`](https://github.com/ClusterCockpit/cc-backend/tree/master/pkg)
+  contains Go packages that can be used by other projects.
+  - [`archive`](https://github.com/ClusterCockpit/cc-backend/tree/master/pkg/archive)
+    Job archive backend implementations (filesystem, S3, SQLite)
+  - [`metricstore`](https://github.com/ClusterCockpit/cc-backend/tree/master/pkg/metricstore)
+    In-memory metric data store with checkpointing and metric loading
+- [`tools/`](https://github.com/ClusterCockpit/cc-backend/tree/master/tools)
+  Additional command line helper tools.
+  - [`archive-manager`](https://github.com/ClusterCockpit/cc-backend/tree/master/tools/archive-manager)
+    Commands for getting infos about an existing job archive, importing jobs
+    between archive backends, and converting archives between JSON and Parquet formats.
+  - [`archive-migration`](https://github.com/ClusterCockpit/cc-backend/tree/master/tools/archive-migration)
+    Tool for migrating job archives between formats.
+  - [`convert-pem-pubkey`](https://github.com/ClusterCockpit/cc-backend/tree/master/tools/convert-pem-pubkey)
+    Tool to convert external pubkey for use in `cc-backend`.
+  - [`gen-keypair`](https://github.com/ClusterCockpit/cc-backend/tree/master/tools/gen-keypair)
+    contains a small application to generate a compatible JWT keypair. You find
+    documentation on how to use it
+    [here](https://github.com/ClusterCockpit/cc-backend/blob/master/docs/JWT-Handling.md).
+- [`web/`](https://github.com/ClusterCockpit/cc-backend/tree/master/web)
+  Server-side templates and frontend-related files:
+  - [`frontend`](https://github.com/ClusterCockpit/cc-backend/tree/master/web/frontend)
+    Svelte components and static assets for the frontend UI
+  - [`templates`](https://github.com/ClusterCockpit/cc-backend/tree/master/web/templates)
+    Server-side Go templates, including monitoring views
+- [`gqlgen.yml`](https://github.com/ClusterCockpit/cc-backend/blob/master/gqlgen.yml)
+  Configures the behaviour and generation of
+  [gqlgen](https://github.com/99designs/gqlgen).
+- [`startDemo.sh`](https://github.com/ClusterCockpit/cc-backend/blob/master/startDemo.sh)
+  is a shell script that sets up demo data, and builds and starts `cc-backend`.

ReleaseNotes.md

@@ -1,47 +1,320 @@
-# `cc-backend` version 1.4.3
+# `cc-backend` version 1.5.1
 
-Supports job archive version 2 and database version 8.
+Supports job archive version 3 and database version 11.
 
-This is a bug fix release of `cc-backend`, the API backend and frontend
+This is a bugfix release of `cc-backend`, the API backend and frontend
 implementation of ClusterCockpit.
 For release specific notes visit the [ClusterCockpit Documentation](https://clusterockpit.org/docs/release/).
+If you are upgrading from v1.5.0 you need to do another DB migration. This
+should not take long. For optimal database performance after the migration it is
+recommended to apply the new `optimize-db` flag, which runs the sqlite `ANALYZE`
+and `VACUUM` commands. Depending on your database size (more then 40GB) the
+`VACUUM` may take up to 2h.
 
-## Breaking changes for minor release 1.4.x
+## Changes in 1.5.1
 
-- You need to perform a database migration. Depending on your database size the
-  migration might require several hours!
-- You need to adapt the `cluster.json` configuration files in the job-archive,
-  add new required attributes to the metric list and after that edit
-  `./job-archive/version.txt` to version 2. Only metrics that have the footprint
-  attribute set can be filtered and show up in the footprint UI and polar plot.
-- Continuous scrolling is default now in all job lists. You can change this back
-  to paging globally, also every user can configure to use paging or continuous
-  scrolling individually.
-- Tags have a scope now. Existing tags will get global scope in the database
-  migration.
+### Database
 
-## New features
+- **New migration (version 11)**: Optimized database index count and added covering indexes for stats queries for significantly improved query performance
+- **Migration 9 fix**: Removed redundant indices from migration 9 that are superseded by migration 11
+- **Optional DB optimization flag**: Added `-optimize-db` CLI flag to run `ANALYZE` on demand; removed automatic ANALYZE on startup
+- **Selective stats queries**: Stats queries are now selective, reducing unnecessary computation
+- **User list paging**: Added paging support to the user list for better scalability
+- **SQLite configuration hardening**: Sanitized SQLite configuration with new configurable options; fixes large heap allocations in the SQLite driver
+- **Query cancellation**: Long-running database queries can now be cancelled
+- **Resource leak fix**: Added missing `defer Close()` calls for all query result sets
 
-- Detailed Node List
-  - Adds new routes `/systems/list/$cluster` and `/systems/list/$cluster/$subcluster`
-  - Displays live, scoped metric data requested from the nodes indepenent of jobs
-- Color Blind Mode
-  - Set on a per-user basis in options
-  - Applies to plot data, plot background color, statsseries colors, roofline timescale
-- Job-View metric selection is now persisted based on the jobs subcluster.
-Helpful for heterogeneous subcluster configurations.
-- Histogram Bin Select in User-View
-  - Metric-Histograms: `10 Bins` now default, selectable options `20, 50, 100`
-  - Job-Duration-Histogram: `48h in 1h Bins` now default, selectable options:
-    - `60 minutes in 1 minute Bins`
-    - `12 hours in 10 minute Bins`
-    - `3 days in 6 hour Bins`
-    - `7 days in 12 hour Bins`
+### Bug fixes
+
+- **Segfault when taggers misconfigured**: Fixed crash when `enable-job-taggers` is set but tagger rule directories are missing
+- **GroupBy stats query complexity**: Reduced complexity for `groupBy` statistics queries
+- **Ranged filter conditions**: Fixed GT and LT conditions in ranged filters
+- **Energy filter preset**: Reduced energy filter preset to a more practical default
+- **JSON validity check**: Fixed wrong field being checked for JSON validity
+- **Tagger float rounding**: Fixed rounding of floats in tagger messages
+- **Node view null safety**: Added null-safe checks in node view to prevent runtime errors
+- **Public dashboard null safety**: Added null-safe checks in the public dashboard to prevent runtime errors
+
+### Frontend
+
+- **Bumped patch versions**: Updated frontend dependencies to latest patch versions
+
+### Documentation
+
+- **New DB config options**: Added new database configuration options to README
+
+---
+
+_The sections below document all features and changes introduced in the 1.5.0 major release, which 1.5.1 is based on._
+
+## Breaking changes
+
+### Configuration changes
+
+- **JSON attribute naming**: All JSON configuration attributes now use `kebab-case`
+  style consistently (e.g., `api-allowed-ips` instead of `apiAllowedIPs`).
+  Update your `config.json` accordingly.
+- **Removed `disable-archive` option**: This obsolete configuration option has been removed.
+- **Removed `clusters` config section**: The separate clusters configuration section
+  has been removed. Cluster information is now derived from the job archive.
+- **`apiAllowedIPs` is now optional**: If not specified, defaults to not
+  restricted.
+
+### Architecture changes
+
+- **Web framework replaced**: Migrated from `gorilla/mux` to `chi` as the HTTP
+  router. This should be transparent to users but affects how middleware and
+  routes are composed. A proper 404 handler is now in place.
+- **MetricStore moved**: The `metricstore` package has been moved from `internal/`
+  to `pkg/` as it is now part of the public API.
+- **MySQL/MariaDB support removed**: Only SQLite is now supported as the database backend.
+- **Archive to Cleanup renaming**: Archive-related functions have been refactored
+  and renamed to "Cleanup" for clarity.
+- **`minRunningFor` filter removed**: This undocumented filter has been removed
+  from the API and frontend.
+
+### Dependency changes
+
+- **cc-lib v2.8.0**: Switched to cc-lib version 2 with updated APIs
+- **cclib NATS client**: Now using the cclib NATS client implementation
+- Removed obsolete `util.Float` usage from cclib
+
+## Major new features
+
+### NATS API Integration
+
+- **Real-time job events**: Subscribe to job start/stop events via NATS
+- **Node state updates**: Receive real-time node state changes via NATS
+- **Configurable subjects**: NATS API subjects are now configurable via `api-subjects`
+- **Deadlock fixes**: Improved NATS client stability and graceful shutdown
+
+### Public Dashboard
+
+- **Public-facing interface**: New public dashboard route for external users
+- **DoubleMetricPlot component**: New visualization component for comparing metrics
+- **Improved layout**: Reviewed and optimized dashboard layouts for better readability
+
+### Enhanced Node Management
+
+- **Node state tracking**: New node table in database with timestamp tracking
+- **Node state filtering**: Filter jobs by node state in systems view
+- **Node list enhancements**: Improved paging, filtering, and continuous scroll support
+- **Nodestate retention and archiving**: Node state data is now subject to configurable
+  retention policies and can be archived to Parquet format for long-term storage
+- **Faulty node metric tracking**: Faulty node state metric lists are persisted to the database
+
+### Health Monitoring
+
+- **Health status dashboard**: New dedicated "Health" tab in the status details view
+  showing per-node metric health across the cluster
+- **CCMS health check**: Support for querying health status of external
+  cc-metric-store (CCMS) instances via the API
+- **GraphQL health endpoints**: New GraphQL queries and resolvers for health data
+- **Cluster/subcluster filter**: Filter health status view by cluster or subcluster
+
+### Log Viewer
+
+- **Web-based log viewer**: New log viewer page in the admin interface for inspecting
+  backend log output directly from the browser without shell access
+- **Accessible from header**: Quick access link from the navigation header
+
+### MetricStore Improvements
+
+- **Memory tracking worker**: New worker for CCMS memory usage tracking
+- **Dynamic retention**: Support for job specific dynamic retention times
+- **Improved compression**: Transparent compression for job archive imports
+- **Parallel processing**: Parallelized Iter function in all archive backends
+
+### Job Tagging System
+
+- **Job tagger option**: Enable automatic job tagging via configuration flag
+- **Application detection**: Automatic detection of applications (MATLAB, GROMACS, etc.)
+- **Job classification**: Automatic detection of pathological jobs
+- **omit-tagged**: Option to exclude tagged jobs from retention/cleanup operations (`none`, `all`, or `user`)
+- **Admin UI trigger**: Taggers can be run on-demand from the admin web interface
+  without restarting the backend
+
+### Archive Backends
+
+- **Parquet archive format**: New Parquet file format for job archiving, providing
+  columnar storage with efficient compression for analytical workloads
+- **S3 backend**: Full support for S3-compatible object storage
+- **SQLite backend**: Full support for SQLite backend using blobs
+- **Performance improvements**: Fixed performance bugs in archive backends
+- **Better error handling**: Improved error messages and fallback handling
+- **Zstd compression**: Parquet writers use zstd compression for better
+  compression ratios compared to the previous snappy default
+- **Optimized sort order**: Job and nodestate Parquet files are sorted by
+  cluster, subcluster, and start time for efficient range queries
+
+### Unified Archive Retention and Format Conversion
+
+- **Uniform retention policy**: Job archive retention now supports both JSON and
+  Parquet as target formats under a single, consistent policy configuration
+- **Archive manager tool**: The `tools/archive-manager` utility now supports
+  format conversion between JSON and Parquet job archives
+- **Parquet reader**: Full Parquet archive reader implementation for reading back
+  archived job data
+
+## New features and improvements
+
+### Frontend
+
+- **Loading indicators**: Added loading indicators to status detail and job lists
+- **Job info layout**: Reviewed and improved job info row layout
+- **Metric selection**: Enhanced metric selection with drag-and-drop fixes
+- **Filter presets**: Move list filter preset to URL for easy sharing
+- **Job comparison**: Improved job comparison views and plots
+- **Subcluster reactivity**: Job list now reacts to subcluster filter changes
+- **Short jobs quick selection**: New "Short jobs" quick-filter button in job lists
+  replaces the removed undocumented `minRunningFor` filter
+- **Row plot cursor sync**: Cursor position is now synchronized across all metric
+  plots in a job list row for easier cross-metric comparison
+- **Disabled metrics handling**: Improved handling and display of disabled metrics
+  across job view, node view, and list rows
+- **"Not configured" info cards**: Informational cards shown when optional features
+  are not yet configured
+- **Frontend dependencies**: Bumped frontend dependencies to latest versions
+- **Svelte 5 compatibility**: Fixed Svelte state warnings and compatibility issues
+
+### Backend
+
+- **Progress bars**: Import function now shows progress during long operations
+- **Better logging**: Improved logging with appropriate log levels throughout
+- **Graceful shutdown**: Fixed shutdown timeout bugs and hanging issues
+- **Configuration defaults**: Sensible defaults for most configuration options
+- **Documentation**: Extensive documentation improvements across packages
+- **Server flag in systemd unit**: Example systemd unit now includes the `-server` flag
+
+### Security
+
+- **LDAP security hardening**: Improved input validation, connection handling, and
+  error reporting in the LDAP authenticator
+- **OIDC security hardening**: Stricter token validation and improved error handling
+  in the OIDC authenticator
+- **Auth schema extensions**: Additional schema fields for improved auth configuration
+
+### API improvements
+
+- **Role-based metric visibility**: Metrics can now have role-based access control
+- **Job exclusivity filter**: New filter for exclusive vs. shared jobs
+- **Improved error messages**: Better error messages and documentation in REST API
+- **GraphQL enhancements**: Improved GraphQL queries and resolvers
+- **Stop job lookup order**: Reversed lookup order in stop job requests for
+  more reliable job matching (cluster+jobId first, then jobId alone)
+
+### Performance
+
+- **Database indices**: Optimized SQLite indices for better query performance
+- **Job cache**: Introduced caching table for faster job inserts
+- **Parallel imports**: Archive imports now run in parallel where possible
+- **External tool integration**: Optimized use of external tools (fd) for better performance
+- **Node repository queries**: Reviewed and optimized node repository SQL queries
+- **Buffer pool**: Resized and pooled internal buffers for better memory reuse
+
+### Developer experience
+
+- **AI agent guidelines**: Added documentation for AI coding agents (AGENTS.md, CLAUDE.md)
+- **Example API payloads**: Added example JSON API payloads for testing
+- **Unit tests**: Added more unit tests for NATS API, node repository, and other components
+- **Test improvements**: Better test coverage; test DB is now copied before unit tests
+  to avoid state pollution between test runs
+- **Parquet writer tests**: Comprehensive tests for Parquet archive writing and conversion
+
+## Bug fixes
+
+- Fixed nodelist paging issues
+- Fixed metric select drag and drop functionality
+- Fixed render race conditions in nodeList
+- Fixed tag count grouping including type
+- Fixed wrong metricstore schema (missing comma)
+- Fixed configuration issues causing shutdown hangs
+- Fixed deadlock when NATS is not configured
+- Fixed archive backend performance bugs
+- Fixed continuous scroll buildup on refresh
+- Improved footprint calculation logic
+- Fixed polar plot data query decoupling
+- Fixed missing resolution parameter handling
+- Fixed node table initialization fallback
+- Fixed reactivity key placement in nodeList
+- Fixed nodeList resolver data handling and increased nodestate filter cutoff
+- Fixed job always being transferred to main job table before archiving
+- Fixed AppTagger error handling and logging
+- Fixed log endpoint formatting and correctness
+- Fixed automatic refresh in metric status tab
+- Fixed NULL value handling in `health_state` and `health_metrics` columns
+- Fixed bugs related to `job_cache` IDs being used in the main job table
+- Fixed SyncJobs bug causing start job hooks to be called with wrong (cache) IDs
+- Fixed 404 handler route for sub-routers
+
+## Configuration changes
+
+### New configuration options
+
+```json
+{
+  "main": {
+    "enable-job-taggers": true,
+    "resampling": {
+      "minimum-points": 600,
+      "trigger": 180,
+      "resolutions": [240, 60]
+    },
+    "api-subjects": {
+      "subject-job-event": "cc.job.event",
+      "subject-node-state": "cc.node.state"
+    }
+  },
+  "nats": {
+    "address": "nats://0.0.0.0:4222",
+    "username": "root",
+    "password": "root"
+  },
+  "cron": {
+    "commit-job-worker": "1m",
+    "duration-worker": "5m",
+    "footprint-worker": "10m"
+  },
+  "metric-store": {
+    "cleanup": {
+      "mode": "archive",
+      "interval": "48h",
+      "directory": "./var/archive"
+    }
+  },
+  "archive": {
+    "retention": {
+      "policy": "delete",
+      "age": "6months",
+      "target-format": "parquet"
+    }
+  },
+  "nodestate": {
+    "retention": {
+      "policy": "archive",
+      "age": "30d",
+      "archive-path": "./var/nodestate-archive"
+    }
+  }
+}
+```
+
+## Migration notes
+
+- Review and update your `config.json` to use kebab-case attribute names
+- If using NATS, configure the new `nats` and `api-subjects` sections
+- If using S3 archive backend, configure the new `archive` section options
+- Test the new public dashboard at `/public` route
+- Review cron worker configuration if you need different frequencies
+- If using the archive retention feature, configure the `target-format` option
+  to choose between `json` (default) and `parquet` output formats
+- Consider enabling nodestate retention if you track node states over time
 
 ## Known issues
 
+- The new dynamic memory management is not bullet proof yet across restarts. We
+  will fix that in a subsequent patch release
 - Currently energy footprint metrics of type energy are ignored for calculating
   total energy.
-- Resampling for running jobs only works with cc-metric-store
 - With energy footprint metrics of type power the unit is ignored and it is
   assumed the metric has the unit Watt.

api/schema.graphqls

@@ -4,61 +4,89 @@ scalar Any
 scalar NullableFloat
 scalar MetricScope
 scalar JobState
+scalar SchedulerState
+scalar MonitoringState
+
+type Node {
+  id: ID!
+  hostname: String!
+  cluster: String!
+  subCluster: String!
+  jobsRunning: Int!
+  cpusAllocated: Int
+  memoryAllocated: Int
+  gpusAllocated: Int
+  schedulerState: SchedulerState!
+  healthState: MonitoringState!
+  metaData: Any
+  healthData: Any
+}
+
+type NodeStates {
+  state: String!
+  count: Int!
+}
+
+type NodeStatesTimed {
+  state: String!
+  counts: [Int!]!
+  times: [Int!]!
+}
 
 type Job {
-  id:               ID!
-  jobId:            Int!
-  user:             String!
-  project:          String!
-  cluster:          String!
-  subCluster:       String!
-  startTime:        Time!
-  duration:         Int!
-  walltime:         Int!
-  numNodes:         Int!
-  numHWThreads:     Int!
-  numAcc:           Int!
-  energy:           Float!
-  SMT:              Int!
-  exclusive:        Int!
-  partition:        String!
-  arrayJobId:       Int!
+  id: ID!
+  jobId: Int!
+  user: String!
+  project: String!
+  cluster: String!
+  subCluster: String!
+  startTime: Time!
+  duration: Int!
+  walltime: Int!
+  numNodes: Int!
+  numHWThreads: Int!
+  numAcc: Int!
+  energy: Float!
+  SMT: Int!
+  shared: String!
+  partition: String!
+  arrayJobId: Int!
   monitoringStatus: Int!
-  state:            JobState!
-  tags:             [Tag!]!
-  resources:        [Resource!]!
-  concurrentJobs:   JobLinkResultList
-  footprint:        [FootprintValue]
-  energyFootprint:  [EnergyFootprintValue]
-  metaData:         Any
-  userData:         User
+  state: JobState!
+  tags: [Tag!]!
+  resources: [Resource!]!
+  concurrentJobs: JobLinkResultList
+  footprint: [FootprintValue]
+  energyFootprint: [EnergyFootprintValue]
+  metaData: Any
+  userData: User
 }
 
 type JobLink {
-  id:               ID!
-  jobId:            Int!
+  id: ID!
+  jobId: Int!
 }
 
 type Cluster {
-  name:         String!
-  partitions:   [String!]!        # Slurm partitions
-  subClusters:  [SubCluster!]!    # Hardware partitions/subclusters
+  name: String!
+  partitions: [String!]! # Slurm partitions
+  subClusters: [SubCluster!]! # Hardware partitions/subclusters
 }
 
 type SubCluster {
-  name:            String!
-  nodes:           String!
-  numberOfNodes:   Int!
-  processorType:   String!
-  socketsPerNode:  Int!
-  coresPerSocket:  Int!
-  threadsPerCore:  Int!
-  flopRateScalar:  MetricValue!
-  flopRateSimd:    MetricValue!
+  name: String!
+  nodes: String!
+  numberOfNodes: Int!
+  processorType: String!
+  socketsPerNode: Int!
+  coresPerSocket: Int!
+  threadsPerCore: Int!
+  flopRateScalar: MetricValue!
+  flopRateSimd: MetricValue!
   memoryBandwidth: MetricValue!
-  topology:        Topology!
-  metricConfig:    [MetricConfig!]!
-  footprint:       [String!]!
+  topology: Topology!
+  metricConfig: [MetricConfig!]!
+  footprint: [String!]!
 }
 
 type FootprintValue {
@@ -80,80 +108,119 @@ type MetricValue {
 }
 
 type Topology {
-  node:         [Int!]
-  socket:       [[Int!]!]
+  node: [Int!]
+  socket: [[Int!]!]
   memoryDomain: [[Int!]!]
-  die:          [[Int!]!]
-  core:         [[Int!]!]
+  die: [[Int!]!]
+  core: [[Int!]!]
   accelerators: [Accelerator!]
 }
 
 type Accelerator {
-  id:    String!
-  type:  String!
+  id: String!
+  type: String!
   model: String!
 }
 
 type SubClusterConfig {
-  name:    String!
-  peak:    Float
-  normal:  Float
+  name: String!
+  peak: Float
+  normal: Float
   caution: Float
-  alert:   Float
-  remove:  Boolean
+  alert: Float
+  remove: Boolean
 }
 
 type MetricConfig {
-  name:        String!
-  unit:        Unit!
-  scope:       MetricScope!
+  name: String!
+  unit: Unit!
+  scope: MetricScope!
   aggregation: String!
-  timestep:    Int!
-  peak:    Float!
-  normal:  Float
+  timestep: Int!
+  peak: Float!
+  normal: Float
   caution: Float!
-  alert:   Float!
+  alert: Float!
   lowerIsBetter: Boolean
   subClusters: [SubClusterConfig!]!
 }
 
 type Tag {
-  id:   ID!
+  id: ID!
   type: String!
   name: String!
   scope: String!
 }
 
 type Resource {
-  hostname:      String!
-  hwthreads:     [Int!]
-  accelerators:  [String!]
+  hostname: String!
+  hwthreads: [Int!]
+  accelerators: [String!]
   configuration: String
 }
 
 type JobMetricWithName {
-  name:   String!
-  scope:  MetricScope!
+  name: String!
+  scope: MetricScope!
   metric: JobMetric!
 }
 
-type JobMetricStatWithName {
-  name:   String!
-  stats:  MetricStatistics!
+type ClusterMetricWithName {
+  name: String!
+  unit: Unit
+  timestep: Int!
+  data: [NullableFloat!]!
 }
 
 type JobMetric {
-  unit:             Unit
-  timestep:         Int!
-  series:           [Series!]
+  unit: Unit
+  timestep: Int!
+  series: [Series!]
   statisticsSeries: StatsSeries
 }
 
 type Series {
-  hostname:   String!
-  id:         String
+  hostname: String!
+  id: String
   statistics: MetricStatistics
-  data:       [NullableFloat!]!
+  data: [NullableFloat!]!
+}
+
+type StatsSeries {
+  mean: [NullableFloat!]!
+  median: [NullableFloat!]!
+  min: [NullableFloat!]!
+  max: [NullableFloat!]!
+}
+
+type NamedStatsWithScope {
+  name: String!
+  scope: MetricScope!
+  stats: [ScopedStats!]!
+}
+
+type ScopedStats {
+  hostname: String!
+  id: String
+  data: MetricStatistics!
+}
+
+type JobStats {
+  id: Int!
+  jobId: String!
+  startTime: Int!
+  duration: Int!
+  cluster: String!
+  subCluster: String!
+  numNodes: Int!
+  numHWThreads: Int
+  numAccelerators: Int
+  stats: [NamedStats!]!
+}
+
+type NamedStats {
+  name: String!
+  data: MetricStatistics!
 }
 
 type Unit {
@@ -167,21 +234,14 @@ type MetricStatistics {
   max: Float!
 }
 
-type StatsSeries {
-  mean:   [NullableFloat!]!
-  median: [NullableFloat!]!
-  min:    [NullableFloat!]!
-  max:    [NullableFloat!]!
-}
-
 type MetricFootprints {
   metric: String!
-  data:   [NullableFloat!]!
+  data: [NullableFloat!]!
 }
 
 type Footprints {
   timeWeights: TimeWeights!
-  metrics:   [MetricFootprints!]!
+  metrics: [MetricFootprints!]!
 }
 
 type TimeWeights {
@@ -190,20 +250,41 @@ type TimeWeights {
   coreHours: [NullableFloat!]!
 }
 
-enum Aggregate { USER, PROJECT, CLUSTER }
-enum SortByAggregate { TOTALWALLTIME, TOTALJOBS, TOTALNODES, TOTALNODEHOURS, TOTALCORES, TOTALCOREHOURS, TOTALACCS, TOTALACCHOURS }
+enum Aggregate {
+  USER
+  PROJECT
+  CLUSTER
+  SUBCLUSTER
+}
+enum SortByAggregate {
+  TOTALWALLTIME
+  TOTALJOBS
+  TOTALUSERS
+  TOTALNODES
+  TOTALNODEHOURS
+  TOTALCORES
+  TOTALCOREHOURS
+  TOTALACCS
+  TOTALACCHOURS
+}
 
 type NodeMetrics {
-  host:       String!
+  host: String!
+  state: String!
   subCluster: String!
-  metrics:    [JobMetricWithName!]!
+  metrics: [JobMetricWithName!]!
+}
+
+type ClusterMetrics {
+  nodeCount: Int!
+  metrics: [ClusterMetricWithName!]!
 }
 
 type NodesResultList {
-  items:  [NodeMetrics!]!
+  items: [NodeMetrics!]!
   offset: Int
-  limit:  Int
-  count:  Int
+  limit: Int
+  count: Int
   totalNodes: Int
   hasNextPage: Boolean
 }
@@ -222,14 +303,14 @@ type GlobalMetricListItem {
 }
 
 type Count {
-  name:  String!
+  name: String!
   count: Int!
 }
 
 type User {
   username: String!
-  name:     String!
-  email:    String!
+  name: String!
+  email: String!
 }
 
 input MetricStatItem {
@@ -238,25 +319,93 @@ input MetricStatItem {
 }
 
 type Query {
-  clusters:     [Cluster!]!   # List of all clusters
-  tags:         [Tag!]!       # List of all tags
-  globalMetrics:   [GlobalMetricListItem!]!
+  clusters: [Cluster!]! # List of all clusters
+  tags: [Tag!]! # List of all tags
+  globalMetrics: [GlobalMetricListItem!]!
 
   user(username: String!): User
   allocatedNodes(cluster: String!): [Count!]!
 
+  ## Node Queries New
+  node(id: ID!): Node
+  nodes(filter: [NodeFilter!], order: OrderByInput): NodeStateResultList!
+  nodesWithMeta(filter: [NodeFilter!], order: OrderByInput): NodeStateResultList!
+  nodeStates(filter: [NodeFilter!]): [NodeStates!]!
+  nodeStatesTimed(filter: [NodeFilter!], type: String!): [NodeStatesTimed!]!
+
   job(id: ID!): Job
-  jobMetrics(id: ID!, metrics: [String!], scopes: [MetricScope!], resolution: Int): [JobMetricWithName!]!
-  jobMetricStats(id: ID!, metrics: [String!]): [JobMetricStatWithName!]!
+  jobMetrics(
+    id: ID!
+    metrics: [String!]
+    scopes: [MetricScope!]
+    resolution: Int
+  ): [JobMetricWithName!]!
+
+  jobStats(id: ID!, metrics: [String!]): [NamedStats!]!
+
+  scopedJobStats(
+    id: ID!
+    metrics: [String!]
+    scopes: [MetricScope!]
+  ): [NamedStatsWithScope!]!
+
+  jobs(
+    filter: [JobFilter!]
+    page: PageRequest
+    order: OrderByInput
+  ): JobResultList!
+
+  jobsStatistics(
+    filter: [JobFilter!]
+    metrics: [String!]
+    page: PageRequest
+    sortBy: SortByAggregate
+    groupBy: Aggregate
+    numDurationBins: String
+    numMetricBins: Int
+  ): [JobsStatistics!]!
+
+  jobsMetricStats(filter: [JobFilter!], metrics: [String!]): [JobStats!]!
   jobsFootprints(filter: [JobFilter!], metrics: [String!]!): Footprints
 
-  jobs(filter: [JobFilter!], page: PageRequest, order: OrderByInput): JobResultList!
-  jobsStatistics(filter: [JobFilter!], metrics: [String!], page: PageRequest, sortBy: SortByAggregate, groupBy: Aggregate, numDurationBins: String, numMetricBins: Int): [JobsStatistics!]!
+  rooflineHeatmap(
+    filter: [JobFilter!]!
+    rows: Int!
+    cols: Int!
+    minX: Float!
+    minY: Float!
+    maxX: Float!
+    maxY: Float!
+  ): [[Float!]!]!
 
-  rooflineHeatmap(filter: [JobFilter!]!, rows: Int!, cols: Int!, minX: Float!, minY: Float!, maxX: Float!, maxY: Float!): [[Float!]!]!
+  nodeMetrics(
+    cluster: String!
+    nodes: [String!]
+    scopes: [MetricScope!]
+    metrics: [String!]
+    from: Time!
+    to: Time!
+  ): [NodeMetrics!]!
 
-  nodeMetrics(cluster: String!, nodes: [String!], scopes: [MetricScope!], metrics: [String!], from: Time!, to: Time!): [NodeMetrics!]!
-  nodeMetricsList(cluster: String!, subCluster: String!, nodeFilter: String!, scopes: [MetricScope!], metrics: [String!], from: Time!, to: Time!, page: PageRequest, resolution: Int): NodesResultList!
+  nodeMetricsList(
+    cluster: String!
+    subCluster: String!
+    stateFilter: String!
+    nodeFilter: String!
+    scopes: [MetricScope!]
+    metrics: [String!]
+    from: Time!
+    to: Time!
+    page: PageRequest
+    resolution: Int
+  ): NodesResultList!
+
+  clusterMetrics(
+    cluster: String!
+    metrics: [String!]
+    from: Time!
+    to: Time!
+  ): ClusterMetrics!
 }
 
 type Mutation {
@@ -264,41 +413,61 @@ type Mutation {
   deleteTag(id: ID!): ID!
   addTagsToJob(job: ID!, tagIds: [ID!]!): [Tag!]!
   removeTagsFromJob(job: ID!, tagIds: [ID!]!): [Tag!]!
+  removeTagFromList(tagIds: [ID!]!): [Int!]!
 
   updateConfiguration(name: String!, value: String!): String
 }
 
-type IntRangeOutput { from: Int!, to: Int! }
-type TimeRangeOutput { range: String, from: Time!, to: Time! }
+type IntRangeOutput {
+  from: Int!
+  to: Int!
+}
+type TimeRangeOutput {
+  range: String
+  from: Time!
+  to: Time!
+}
+
+input NodeFilter {
+  hostname: StringInput
+  cluster: StringInput
+  subCluster: StringInput
+  schedulerState: SchedulerState
+  healthState: MonitoringState
+  timeStart: Int
+}
 
 input JobFilter {
-  tags:        [ID!]
-  jobId:       StringInput
-  arrayJobId:  Int
-  user:        StringInput
-  project:     StringInput
-  jobName:     StringInput
-  cluster:     StringInput
-  partition:   StringInput
-  duration:    IntRange
-  energy:      FloatRange
+  tags: [ID!]
+  dbId: [ID!]
+  jobId: StringInput
+  arrayJobId: Int
+  user: StringInput
+  project: StringInput
+  jobName: StringInput
+  cluster: StringInput
+  subCluster: StringInput
+  partition: StringInput
+  duration: IntRange
+  energy: FloatRange
 
   minRunningFor: Int
 
-  numNodes:        IntRange
+  numNodes: IntRange
   numAccelerators: IntRange
-  numHWThreads:    IntRange
+  numHWThreads: IntRange
 
-  startTime:   TimeRange
-  state:       [JobState!]
+  startTime: TimeRange
+  state: [JobState!]
   metricStats: [MetricStatItem!]
-  exclusive:     Int
-  node:    StringInput
+  shared: String
+  schedule: String
+  node: StringInput
 }
 
 input OrderByInput {
   field: String!
-  type: String!,
+  type: String!
   order: SortDirectionEnum! = ASC
 }
 
@@ -308,34 +477,46 @@ enum SortDirectionEnum {
 }
 
 input StringInput {
-  eq:         String
-  neq:        String
-  contains:   String
+  eq: String
+  neq: String
+  contains: String
   startsWith: String
-  endsWith:   String
-  in:         [String!]
+  endsWith: String
+  in: [String!]
 }
 
-input IntRange   { from: Int!, to: Int! }
-input TimeRange  { range: String, from: Time, to: Time }
+input IntRange {
+  from: Int!
+  to: Int!
+}
+input TimeRange {
+  range: String
+  from: Time
+  to: Time
+}
 
 input FloatRange {
   from: Float!
   to: Float!
 }
 
+type NodeStateResultList {
+  items: [Node!]!
+  count: Int
+}
+
 type JobResultList {
-  items:  [Job!]!
+  items: [Job!]!
   offset: Int
-  limit:  Int
-  count:  Int
+  limit: Int
+  count: Int
   hasNextPage: Boolean
 }
 
 type JobLinkResultList {
   listQuery: String
-  items:  [JobLink!]!
-  count:  Int
+  items: [JobLink!]!
+  count: Int
 }
 
 type HistoPoint {
@@ -357,27 +538,28 @@ type MetricHistoPoint {
   max: Int
 }
 
-type JobsStatistics  {
-  id:             ID!            # If `groupBy` was used, ID of the user/project/cluster
-  name:           String!        # if User-Statistics: Given Name of Account (ID) Owner
-  totalJobs:      Int!           # Number of jobs
-  runningJobs:    Int!           # Number of running jobs
-  shortJobs:      Int!           # Number of jobs with a duration of less than duration
-  totalWalltime:  Int!           # Sum of the duration of all matched jobs in hours
-  totalNodes:     Int!           # Sum of the nodes of all matched jobs
-  totalNodeHours: Int!           # Sum of the node hours of all matched jobs
-  totalCores:     Int!           # Sum of the cores of all matched jobs
-  totalCoreHours: Int!           # Sum of the core hours of all matched jobs
-  totalAccs:      Int!         # Sum of the accs of all matched jobs
-  totalAccHours:  Int!           # Sum of the gpu hours of all matched jobs
-  histDuration:   [HistoPoint!]! # value: hour, count: number of jobs with a rounded duration of value
-  histNumNodes:   [HistoPoint!]! # value: number of nodes, count: number of jobs with that number of nodes
-  histNumCores:   [HistoPoint!]! # value: number of cores, count: number of jobs with that number of cores
-  histNumAccs:    [HistoPoint!]! # value: number of accs, count: number of jobs with that number of accs
-  histMetrics:    [MetricHistoPoints!]! # metric: metricname, data array of histopoints: value: metric average bin, count: number of jobs with that metric average
+type JobsStatistics {
+  id: ID! # If `groupBy` was used, ID of the user/project/cluster/subcluster
+  name: String! # if User-Statistics: Given Name of Account (ID) Owner
+  totalUsers: Int! # if *not* User-Statistics: Number of active users (based on running jobs)
+  totalJobs: Int! # Number of jobs
+  runningJobs: Int! # Number of running jobs
+  shortJobs: Int! # Number of jobs with a duration of less than config'd ShortRunningJobsDuration
+  totalWalltime: Int! # Sum of the duration of all matched jobs in hours
+  totalNodes: Int! # Sum of the nodes of all matched jobs
+  totalNodeHours: Int! # Sum of the node hours of all matched jobs
+  totalCores: Int! # Sum of the cores of all matched jobs
+  totalCoreHours: Int! # Sum of the core hours of all matched jobs
+  totalAccs: Int! # Sum of the accs of all matched jobs
+  totalAccHours: Int! # Sum of the gpu hours of all matched jobs
+  histDuration: [HistoPoint!]! # value: hour, count: number of jobs with a rounded duration of value
+  histNumNodes: [HistoPoint!]! # value: number of nodes, count: number of jobs with that number of nodes
+  histNumCores: [HistoPoint!]! # value: number of cores, count: number of jobs with that number of cores
+  histNumAccs: [HistoPoint!]! # value: number of accs, count: number of jobs with that number of accs
+  histMetrics: [MetricHistoPoints!]! # metric: metricname, data array of histopoints: value: metric average bin, count: number of jobs with that metric average
 }
 
 input PageRequest {
   itemsPerPage: Int!
-  page:         Int!
+  page: Int!
 }

cmd/cc-backend/cli.go

@@ -1,18 +1,22 @@
 // Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved.
+// All rights reserved. This file is part of cc-backend.
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
+
+// Package main provides the entry point for the ClusterCockpit backend server.
+// This file defines all command-line flags and their default values.
 package main
 
 import "flag"
 
 var (
-	flagReinitDB, flagInit, flagServer, flagSyncLDAP, flagGops, flagMigrateDB, flagRevertDB, flagForceDB, flagDev, flagVersion, flagLogDateTime bool
-	flagNewUser, flagDelUser, flagGenJWT, flagConfigFile, flagImportJob, flagLogLevel                                                           string
+	flagReinitDB, flagInit, flagServer, flagSyncLDAP, flagGops, flagMigrateDB, flagRevertDB,
+	flagForceDB, flagDev, flagVersion, flagLogDateTime, flagApplyTags, flagOptimizeDB bool
+	flagNewUser, flagDelUser, flagGenJWT, flagConfigFile, flagImportJob, flagLogLevel string
 )
 
 func cliInit() {
-	flag.BoolVar(&flagInit, "init", false, "Setup var directory, initialize swlite database file, config.json and .env")
+	flag.BoolVar(&flagInit, "init", false, "Setup var directory, initialize sqlite database file, config.json and .env")
 	flag.BoolVar(&flagReinitDB, "init-db", false, "Go through job-archive and re-initialize the 'job', 'tag', and 'jobtag' tables (all running jobs will be lost!)")
 	flag.BoolVar(&flagSyncLDAP, "sync-ldap", false, "Sync the 'hpc_user' table with ldap")
 	flag.BoolVar(&flagServer, "server", false, "Start a server, continues listening on port after initialization and argument handling")
@@ -21,13 +25,15 @@ func cliInit() {
 	flag.BoolVar(&flagVersion, "version", false, "Show version information and exit")
 	flag.BoolVar(&flagMigrateDB, "migrate-db", false, "Migrate database to supported version and exit")
 	flag.BoolVar(&flagRevertDB, "revert-db", false, "Migrate database to previous version and exit")
+	flag.BoolVar(&flagApplyTags, "apply-tags", false, "Run taggers on all completed jobs and exit")
 	flag.BoolVar(&flagForceDB, "force-db", false, "Force database version, clear dirty flag and exit")
+	flag.BoolVar(&flagOptimizeDB, "optimize-db", false, "Optimize database: run VACUUM to reclaim space, then ANALYZE to update query planner statistics")
 	flag.BoolVar(&flagLogDateTime, "logdate", false, "Set this flag to add date and time to log messages")
 	flag.StringVar(&flagConfigFile, "config", "./config.json", "Specify alternative path to `config.json`")
-	flag.StringVar(&flagNewUser, "add-user", "", "Add a new user. Argument format: `<username>:[admin,support,manager,api,user]:<password>`")
-	flag.StringVar(&flagDelUser, "del-user", "", "Remove user by `username`")
+	flag.StringVar(&flagNewUser, "add-user", "", "Add a new user. Argument format: <username>:[admin,support,manager,api,user]:<password>")
+	flag.StringVar(&flagDelUser, "del-user", "", "Remove a existing user. Argument format: <username>")
 	flag.StringVar(&flagGenJWT, "jwt", "", "Generate and print a JWT for the user specified by its `username`")
 	flag.StringVar(&flagImportJob, "import-job", "", "Import a job. Argument format: `<path-to-meta.json>:<path-to-data.json>,...`")
-	flag.StringVar(&flagLogLevel, "loglevel", "warn", "Sets the logging level: `[debug,info,warn (default),err,fatal,crit]`")
+	flag.StringVar(&flagLogLevel, "loglevel", "warn", "Sets the logging level: `[debug, info , warn (default), err, crit]`")
 	flag.Parse()
 }

cmd/cc-backend/init.go

@@ -1,16 +1,21 @@
 // Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved.
+// All rights reserved. This file is part of cc-backend.
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
+
+// Package main provides the entry point for the ClusterCockpit backend server.
+// This file contains bootstrap logic for initializing the environment,
+// creating default configuration files, and setting up the database.
 package main
 
 import (
-	"fmt"
+	"encoding/json"
 	"os"
 
 	"github.com/ClusterCockpit/cc-backend/internal/repository"
-	"github.com/ClusterCockpit/cc-backend/internal/util"
-	"github.com/ClusterCockpit/cc-backend/pkg/log"
+	"github.com/ClusterCockpit/cc-backend/pkg/archive"
+	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
+	"github.com/ClusterCockpit/cc-lib/v2/util"
 )
 
 const envString = `
@@ -25,61 +30,65 @@ SESSION_KEY="67d829bf61dc5f87a73fd814e2c9f629"
 
 const configString = `
 {
+  "main": {
     "addr": "127.0.0.1:8080",
-    "archive": {
-        "kind": "file",
-        "path": "./var/job-archive"
+    "short-running-jobs-duration": 300,
+    "resampling": {
+      "minimum-points": 600,
+      "trigger": 300,
+      "resolutions": [
+        240,
+        60
+      ]
     },
+    "api-allowed-ips": [
+      "*"
+    ],
+    "emission-constant": 317
+  },
+  "cron": {
+    "commit-job-worker": "1m",
+    "duration-worker": "5m",
+    "footprint-worker": "10m"
+  },
+  "archive": {
+    "kind": "file",
+    "path": "./var/job-archive"
+  },
+  "auth": {
     "jwts": {
-        "max-age": "2000h"
-    },
-    "clusters": [
-        {
-            "name": "name",
-            "metricDataRepository": {
-                "kind": "cc-metric-store",
-                "url": "http://localhost:8082",
-                "token": ""
-            },
-            "filterRanges": {
-                "numNodes": {
-                    "from": 1,
-                    "to": 64
-                },
-                "duration": {
-                    "from": 0,
-                    "to": 86400
-                },
-                "startTime": {
-                    "from": "2023-01-01T00:00:00Z",
-                    "to": null
-                }
-            }
-        }
-    ]
+      "max-age": "2000h"
+    }
+  }
 }
 `
 
 func initEnv() {
 	if util.CheckFileExists("var") {
-		fmt.Print("Directory ./var already exists. Exiting!\n")
-		os.Exit(0)
+		cclog.Exit("Directory ./var already exists. Cautiously exiting application initialization.")
 	}
 
 	if err := os.WriteFile("config.json", []byte(configString), 0o666); err != nil {
-		log.Fatalf("Writing config.json failed: %s", err.Error())
+		cclog.Abortf("Could not write default ./config.json with permissions '0o666'. Application initialization failed, exited.\nError: %s\n", err.Error())
 	}
 
 	if err := os.WriteFile(".env", []byte(envString), 0o666); err != nil {
-		log.Fatalf("Writing .env failed: %s", err.Error())
+		cclog.Abortf("Could not write default ./.env file with permissions '0o666'. Application initialization failed, exited.\nError: %s\n", err.Error())
 	}
 
 	if err := os.Mkdir("var", 0o777); err != nil {
-		log.Fatalf("Mkdir var failed: %s", err.Error())
+		cclog.Abortf("Could not create default ./var folder with permissions '0o777'. Application initialization failed, exited.\nError: %s\n", err.Error())
 	}
 
-	err := repository.MigrateDB("sqlite3", "./var/job.db")
+	err := repository.MigrateDB("./var/job.db")
 	if err != nil {
-		log.Fatalf("Initialize job.db failed: %s", err.Error())
+		cclog.Abortf("Could not initialize default SQLite database as './var/job.db'. Application initialization failed, exited.\nError: %s\n", err.Error())
+	}
+	if err := os.Mkdir("var/job-archive", 0o777); err != nil {
+		cclog.Abortf("Could not create default ./var/job-archive folder with permissions '0o777'. Application initialization failed, exited.\nError: %s\n", err.Error())
+	}
+	archiveCfg := "{\"kind\": \"file\",\"path\": \"./var/job-archive\"}"
+	if err := archive.Init(json.RawMessage(archiveCfg)); err != nil {
+		cclog.Abortf("Could not initialize job-archive, exited.\nError: %s\n", err.Error())
 	}
 }

cmd/cc-backend/main.go

@@ -1,10 +1,16 @@
 // Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved.
+// All rights reserved. This file is part of cc-backend.
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
+
+// Package main provides the entry point for the ClusterCockpit backend server.
+// It orchestrates initialization of all subsystems including configuration,
+// database, authentication, and the HTTP server.
 package main
 
 import (
+	"context"
+	"encoding/json"
 	"fmt"
 	"os"
 	"os/signal"
@@ -12,21 +18,28 @@ import (
 	"strings"
 	"sync"
 	"syscall"
+	"time"
 
 	"github.com/ClusterCockpit/cc-backend/internal/archiver"
 	"github.com/ClusterCockpit/cc-backend/internal/auth"
 	"github.com/ClusterCockpit/cc-backend/internal/config"
 	"github.com/ClusterCockpit/cc-backend/internal/importer"
-	"github.com/ClusterCockpit/cc-backend/internal/metricdata"
+	"github.com/ClusterCockpit/cc-backend/internal/metricdispatch"
 	"github.com/ClusterCockpit/cc-backend/internal/repository"
-	"github.com/ClusterCockpit/cc-backend/internal/taskManager"
+	"github.com/ClusterCockpit/cc-backend/internal/tagger"
+	"github.com/ClusterCockpit/cc-backend/internal/taskmanager"
 	"github.com/ClusterCockpit/cc-backend/pkg/archive"
-	"github.com/ClusterCockpit/cc-backend/pkg/log"
-	"github.com/ClusterCockpit/cc-backend/pkg/runtimeEnv"
-	"github.com/ClusterCockpit/cc-backend/pkg/schema"
+	"github.com/ClusterCockpit/cc-backend/pkg/metricstore"
+	"github.com/ClusterCockpit/cc-backend/web"
+	ccconf "github.com/ClusterCockpit/cc-lib/v2/ccConfig"
+	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
+	"github.com/ClusterCockpit/cc-lib/v2/nats"
+	"github.com/ClusterCockpit/cc-lib/v2/runtime"
+	"github.com/ClusterCockpit/cc-lib/v2/schema"
+	"github.com/ClusterCockpit/cc-lib/v2/util"
 	"github.com/google/gops/agent"
+	"github.com/joho/godotenv"
 
-	_ "github.com/go-sql-driver/mysql"
 	_ "github.com/mattn/go-sqlite3"
 )
 
@@ -39,198 +52,502 @@ const logoString = `
                                                     |_|
 `
 
+// Environment variable names
+const (
+	envGOGC = "GOGC"
+)
+
+// Default configurations
+const (
+	defaultArchiveConfig = `{"kind":"file","path":"./var/job-archive"}`
+)
+
 var (
 	date    string
 	commit  string
 	version string
 )
 
-func main() {
-	cliInit()
+func printVersion() {
+	fmt.Print(logoString)
+	fmt.Printf("Version:\t%s\n", version)
+	fmt.Printf("Git hash:\t%s\n", commit)
+	fmt.Printf("Build time:\t%s\n", date)
+	fmt.Printf("SQL db version:\t%d\n", repository.Version)
+	fmt.Printf("Job archive version:\t%d\n", archive.Version)
+}
 
-	if flagVersion {
-		fmt.Print(logoString)
-		fmt.Printf("Version:\t%s\n", version)
-		fmt.Printf("Git hash:\t%s\n", commit)
-		fmt.Printf("Build time:\t%s\n", date)
-		fmt.Printf("SQL db version:\t%d\n", repository.Version)
-		fmt.Printf("Job archive version:\t%d\n", archive.Version)
-		os.Exit(0)
+func initGops() error {
+	if !flagGops {
+		return nil
 	}
 
-	// Apply config flags for pkg/log
-	log.Init(flagLogLevel, flagLogDateTime)
+	if err := agent.Listen(agent.Options{}); err != nil {
+		return fmt.Errorf("starting gops agent: %w", err)
+	}
+	return nil
+}
 
-	// See https://github.com/google/gops (Runtime overhead is almost zero)
-	if flagGops {
-		if err := agent.Listen(agent.Options{}); err != nil {
-			log.Fatalf("gops/agent.Listen failed: %s", err.Error())
+func loadEnvironment() error {
+	if err := godotenv.Load(); err != nil {
+		return fmt.Errorf("loading .env file: %w", err)
+	}
+	return nil
+}
+
+func initConfiguration() error {
+	ccconf.Init(flagConfigFile)
+
+	cfg := ccconf.GetPackageConfig("main")
+	if cfg == nil {
+		return fmt.Errorf("main configuration must be present")
+	}
+
+	config.Init(cfg)
+	return nil
+}
+
+func initDatabase() error {
+	if config.Keys.DbConfig != nil {
+		cfg := repository.DefaultConfig()
+		dc := config.Keys.DbConfig
+		if dc.CacheSizeMB > 0 {
+			cfg.DbCacheSizeMB = dc.CacheSizeMB
 		}
+		if dc.SoftHeapLimitMB > 0 {
+			cfg.DbSoftHeapLimitMB = dc.SoftHeapLimitMB
+		}
+		if dc.MaxOpenConnections > 0 {
+			cfg.MaxOpenConnections = dc.MaxOpenConnections
+		}
+		if dc.MaxIdleConnections > 0 {
+			cfg.MaxIdleConnections = dc.MaxIdleConnections
+		}
+		if dc.ConnectionMaxIdleTimeMins > 0 {
+			cfg.ConnectionMaxIdleTime = time.Duration(dc.ConnectionMaxIdleTimeMins) * time.Minute
+		}
+		repository.SetConfig(cfg)
 	}
+	repository.Connect(config.Keys.DB)
+	return nil
+}
 
-	if err := runtimeEnv.LoadEnv("./.env"); err != nil && !os.IsNotExist(err) {
-		log.Fatalf("parsing './.env' file failed: %s", err.Error())
-	}
-
-	// Initialize sub-modules and handle command line flags.
-	// The order here is important!
-	config.Init(flagConfigFile)
-
-	// As a special case for `db`, allow using an environment variable instead of the value
-	// stored in the config. This can be done for people having security concerns about storing
-	// the password for their mysql database in config.json.
-	if strings.HasPrefix(config.Keys.DB, "env:") {
-		envvar := strings.TrimPrefix(config.Keys.DB, "env:")
-		config.Keys.DB = os.Getenv(envvar)
-	}
-
+func handleDatabaseCommands() error {
 	if flagMigrateDB {
-		err := repository.MigrateDB(config.Keys.DBDriver, config.Keys.DB)
+		err := repository.MigrateDB(config.Keys.DB)
 		if err != nil {
-			log.Fatal(err)
+			return fmt.Errorf("migrating database to version %d: %w", repository.Version, err)
 		}
-		os.Exit(0)
+		cclog.Exitf("MigrateDB Success: Migrated SQLite database at '%s' to version %d.\n",
+			config.Keys.DB, repository.Version)
 	}
 
 	if flagRevertDB {
-		err := repository.RevertDB(config.Keys.DBDriver, config.Keys.DB)
+		err := repository.RevertDB(config.Keys.DB)
 		if err != nil {
-			log.Fatal(err)
+			return fmt.Errorf("reverting database to version %d: %w", repository.Version-1, err)
 		}
-		os.Exit(0)
+		cclog.Exitf("RevertDB Success: Reverted SQLite database at '%s' to version %d.\n",
+			config.Keys.DB, repository.Version-1)
 	}
 
 	if flagForceDB {
-		err := repository.ForceDB(config.Keys.DBDriver, config.Keys.DB)
+		err := repository.ForceDB(config.Keys.DB)
 		if err != nil {
-			log.Fatal(err)
+			return fmt.Errorf("forcing database to version %d: %w", repository.Version, err)
 		}
-		os.Exit(0)
+		cclog.Exitf("ForceDB Success: Forced SQLite database at '%s' to version %d.\n",
+			config.Keys.DB, repository.Version)
 	}
 
-	repository.Connect(config.Keys.DBDriver, config.Keys.DB)
+	return nil
+}
 
-	if flagInit {
-		initEnv()
-		fmt.Print("Successfully setup environment!\n")
-		fmt.Print("Please review config.json and .env and adjust it to your needs.\n")
-		fmt.Print("Add your job-archive at ./var/job-archive.\n")
-		os.Exit(0)
+func handleUserCommands() error {
+	if config.Keys.DisableAuthentication && (flagNewUser != "" || flagDelUser != "") {
+		return fmt.Errorf("--add-user and --del-user can only be used if authentication is enabled")
 	}
 
 	if !config.Keys.DisableAuthentication {
+		if cfg := ccconf.GetPackageConfig("auth"); cfg != nil {
+			auth.Init(&cfg)
+		} else {
+			cclog.Warn("Authentication disabled due to missing configuration")
+			auth.Init(nil)
+		}
 
-		auth.Init()
+		// Check for default security keys
+		checkDefaultSecurityKeys()
 
 		if flagNewUser != "" {
-			parts := strings.SplitN(flagNewUser, ":", 3)
-			if len(parts) != 3 || len(parts[0]) == 0 {
-				log.Fatal("invalid argument format for user creation")
-			}
-
-			ur := repository.GetUserRepository()
-			if err := ur.AddUser(&schema.User{
-				Username: parts[0], Projects: make([]string, 0), Password: parts[2], Roles: strings.Split(parts[1], ","),
-			}); err != nil {
-				log.Fatalf("adding '%s' user authentication failed: %v", parts[0], err)
+			if err := addUser(flagNewUser); err != nil {
+				return err
 			}
 		}
+
 		if flagDelUser != "" {
-			ur := repository.GetUserRepository()
-			if err := ur.DelUser(flagDelUser); err != nil {
-				log.Fatalf("deleting user failed: %v", err)
+			if err := delUser(flagDelUser); err != nil {
+				return err
 			}
 		}
 
 		authHandle := auth.GetAuthInstance()
 
 		if flagSyncLDAP {
-			if authHandle.LdapAuth == nil {
-				log.Fatal("cannot sync: LDAP authentication is not configured")
+			if err := syncLDAP(authHandle); err != nil {
+				return err
 			}
-
-			if err := authHandle.LdapAuth.Sync(); err != nil {
-				log.Fatalf("LDAP sync failed: %v", err)
-			}
-			log.Info("LDAP sync successfull")
 		}
 
 		if flagGenJWT != "" {
-			ur := repository.GetUserRepository()
-			user, err := ur.GetUser(flagGenJWT)
-			if err != nil {
-				log.Fatalf("could not get user from JWT: %v", err)
+			if err := generateJWT(authHandle, flagGenJWT); err != nil {
+				return err
 			}
-
-			if !user.HasRole(schema.RoleApi) {
-				log.Warnf("user '%s' does not have the API role", user.Username)
-			}
-
-			jwt, err := authHandle.JwtAuth.ProvideJWT(user)
-			if err != nil {
-				log.Fatalf("failed to provide JWT to user '%s': %v", user.Username, err)
-			}
-
-			fmt.Printf("MAIN > JWT for '%s': %s\n", user.Username, jwt)
 		}
-
-	} else if flagNewUser != "" || flagDelUser != "" {
-		log.Fatal("arguments --add-user and --del-user can only be used if authentication is enabled")
 	}
 
-	if err := archive.Init(config.Keys.Archive, config.Keys.DisableArchive); err != nil {
-		log.Fatalf("failed to initialize archive: %s", err.Error())
+	return nil
+}
+
+// checkDefaultSecurityKeys warns if default JWT keys are detected
+func checkDefaultSecurityKeys() {
+	// Default JWT public key from init.go
+	defaultJWTPublic := "kzfYrYy+TzpanWZHJ5qSdMj5uKUWgq74BWhQG6copP0="
+
+	if os.Getenv("JWT_PUBLIC_KEY") == defaultJWTPublic {
+		cclog.Warn("Using default JWT keys - not recommended for production environments")
+	}
+}
+
+func addUser(userSpec string) error {
+	parts := strings.SplitN(userSpec, ":", 3)
+	if len(parts) != 3 || len(parts[0]) == 0 {
+		return fmt.Errorf("invalid user format, want: <username>:[admin,support,manager,api,user]:<password>, have: %s", userSpec)
 	}
 
-	if err := metricdata.Init(); err != nil {
-		log.Fatalf("failed to initialize metricdata repository: %s", err.Error())
+	ur := repository.GetUserRepository()
+	if err := ur.AddUser(&schema.User{
+		Username: parts[0],
+		Projects: make([]string, 0),
+		Password: parts[2],
+		Roles:    strings.Split(parts[1], ","),
+	}); err != nil {
+		return fmt.Errorf("adding user '%s' with roles '%s': %w", parts[0], parts[1], err)
 	}
 
+	cclog.Infof("Add User: Added new user '%s' with roles '%s'", parts[0], parts[1])
+	return nil
+}
+
+func delUser(username string) error {
+	ur := repository.GetUserRepository()
+	if err := ur.DelUser(username); err != nil {
+		return fmt.Errorf("deleting user '%s': %w", username, err)
+	}
+	cclog.Infof("Delete User: Deleted user '%s' from DB", username)
+	return nil
+}
+
+func syncLDAP(authHandle *auth.Authentication) error {
+	if authHandle.LdapAuth == nil {
+		return fmt.Errorf("LDAP authentication is not configured")
+	}
+
+	if err := authHandle.LdapAuth.Sync(); err != nil {
+		return fmt.Errorf("synchronizing LDAP: %w", err)
+	}
+
+	cclog.Print("Sync LDAP: LDAP synchronization successfull.")
+	return nil
+}
+
+func generateJWT(authHandle *auth.Authentication, username string) error {
+	ur := repository.GetUserRepository()
+	user, err := ur.GetUser(username)
+	if err != nil {
+		return fmt.Errorf("getting user '%s': %w", username, err)
+	}
+
+	if !user.HasRole(schema.RoleAPI) {
+		cclog.Warnf("JWT: User '%s' does not have the role 'api'. REST API endpoints will return error!\n", user.Username)
+	}
+
+	jwt, err := authHandle.JwtAuth.ProvideJWT(user)
+	if err != nil {
+		return fmt.Errorf("generating JWT for user '%s': %w", user.Username, err)
+	}
+
+	cclog.Printf("JWT: Successfully generated JWT for user '%s': %s\n", user.Username, jwt)
+	return nil
+}
+
+func initSubsystems() error {
+	// Initialize nats client
+	natsConfig := ccconf.GetPackageConfig("nats")
+	if err := nats.Init(natsConfig); err != nil {
+		cclog.Warnf("initializing (optional) nats client: %s", err.Error())
+	}
+	nats.Connect()
+
+	// Initialize job archive
+	archiveCfg := ccconf.GetPackageConfig("archive")
+	if archiveCfg == nil {
+		cclog.Debug("Archive configuration not found, using default archive configuration")
+		archiveCfg = json.RawMessage(defaultArchiveConfig)
+	}
+	if err := archive.Init(archiveCfg); err != nil {
+		return fmt.Errorf("initializing archive: %w", err)
+	}
+
+	// Handle database re-initialization
 	if flagReinitDB {
 		if err := importer.InitDB(); err != nil {
-			log.Fatalf("failed to re-initialize repository DB: %s", err.Error())
+			return fmt.Errorf("re-initializing repository DB: %w", err)
 		}
+		cclog.Print("Init DB: Successfully re-initialized repository DB.")
 	}
 
+	// Handle job import
 	if flagImportJob != "" {
 		if err := importer.HandleImportFlag(flagImportJob); err != nil {
-			log.Fatalf("job import failed: %s", err.Error())
+			return fmt.Errorf("importing job: %w", err)
+		}
+		cclog.Infof("Import Job: Imported Job '%s' into DB", flagImportJob)
+	}
+
+	// Initialize taggers
+	if config.Keys.EnableJobTaggers {
+		tagger.Init()
+	}
+
+	// Apply tags if requested
+	if flagApplyTags {
+		tagger.Init()
+
+		if err := tagger.RunTaggers(); err != nil {
+			return fmt.Errorf("running job taggers: %w", err)
 		}
 	}
 
-	if !flagServer {
-		return
-	}
-
-	archiver.Start(repository.GetJobRepository())
-	taskManager.Start()
-	serverInit()
+	return nil
+}
 
+func runServer(ctx context.Context) error {
 	var wg sync.WaitGroup
 
-	wg.Add(1)
-	go func() {
-		defer wg.Done()
-		serverStart()
-	}()
+	// Initialize metric store if configuration is provided
+	haveMetricstore := false
+	mscfg := ccconf.GetPackageConfig("metric-store")
+	if mscfg != nil {
+		metrics := metricstore.BuildMetricList()
+		metricstore.Init(mscfg, metrics, &wg)
 
+		// Inject repository as NodeProvider to break import cycle
+		ms := metricstore.GetMemoryStore()
+		jobRepo := repository.GetJobRepository()
+		ms.SetNodeProvider(jobRepo)
+		metricstore.MetricStoreHandle = &metricstore.InternalMetricStore{}
+		haveMetricstore = true
+	} else {
+		metricstore.MetricStoreHandle = nil
+		cclog.Debug("missing internal metricstore configuration")
+	}
+
+	// Initialize external metric stores if configuration is provided
+	mscfg = ccconf.GetPackageConfig("metric-store-external")
+	if mscfg != nil {
+		err := metricdispatch.Init(mscfg)
+
+		if err != nil {
+			cclog.Debugf("error while initializing external metricdispatch: %v", err)
+		} else {
+			haveMetricstore = true
+		}
+	}
+
+	if !haveMetricstore {
+		return fmt.Errorf("missing metricstore configuration")
+	}
+
+	// Start archiver and task manager
+	archiver.Start(repository.GetJobRepository(), ctx)
+	taskmanager.Start(ccconf.GetPackageConfig("cron"), ccconf.GetPackageConfig("archive"))
+
+	// Initialize web UI
+	cfg := ccconf.GetPackageConfig("ui")
+	if err := web.Init(cfg); err != nil {
+		return fmt.Errorf("initializing web UI: %w", err)
+	}
+
+	// Initialize HTTP server
+	srv, err := NewServer(version, commit, date)
+	if err != nil {
+		return fmt.Errorf("creating server: %w", err)
+	}
+
+	// Channel to collect errors from server
+	errChan := make(chan error, 1)
+
+	// Start HTTP server
+	wg.Go(func() {
+		if err := srv.Start(ctx); err != nil {
+			errChan <- err
+		}
+	})
+
+	// Handle shutdown signals
 	wg.Add(1)
 	sigs := make(chan os.Signal, 1)
 	signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
 	go func() {
 		defer wg.Done()
-		<-sigs
-		runtimeEnv.SystemdNotifiy(false, "Shutting down ...")
+		select {
+		case <-sigs:
+			cclog.Info("Shutdown signal received")
+		case <-ctx.Done():
+		}
 
-		serverShutdown()
-
-		taskManager.Shutdown()
+		runtime.SystemdNotify(false, "Shutting down ...")
+		srv.Shutdown(ctx)
+		util.FsWatcherShutdown()
+		taskmanager.Shutdown()
 	}()
 
-	if os.Getenv("GOGC") == "" {
-		debug.SetGCPercent(25)
+	// Set GC percent if not configured
+	if os.Getenv(envGOGC) == "" {
+		// trigger GC when heap grows 15% above the previous live set
+		debug.SetGCPercent(15)
+	}
+	runtime.SystemdNotify(true, "running")
+
+	waitDone := make(chan struct{})
+	go func() {
+		wg.Wait()
+		close(waitDone)
+	}()
+
+	go func() {
+		<-waitDone
+		close(errChan)
+	}()
+
+	// Wait for either:
+	// 1. An error from server startup
+	// 2. Completion of all goroutines (normal shutdown or crash)
+	select {
+	case err := <-errChan:
+		// errChan will be closed when waitDone is closed, which happens
+		// when all goroutines complete (either from normal shutdown or error)
+		if err != nil {
+			return err
+		}
+	case <-time.After(100 * time.Millisecond):
+		// Give the server 100ms to start and report any immediate startup errors
+		// After that, just wait for normal shutdown completion
+		select {
+		case err := <-errChan:
+			if err != nil {
+				return err
+			}
+		case <-waitDone:
+			// Normal shutdown completed
+		}
+	}
+
+	cclog.Print("Graceful shutdown completed!")
+	return nil
+}
+
+func run() error {
+	cliInit()
+
+	if flagVersion {
+		printVersion()
+		return nil
+	}
+
+	// Initialize logger
+	cclog.Init(flagLogLevel, flagLogDateTime)
+
+	// Handle init flag
+	if flagInit {
+		initEnv()
+		cclog.Exit("Successfully setup environment!\n" +
+			"Please review config.json and .env and adjust it to your needs.\n" +
+			"Add your job-archive at ./var/job-archive.")
+	}
+
+	// Initialize gops agent
+	if err := initGops(); err != nil {
+		return err
+	}
+
+	// Initialize subsystems in dependency order:
+	// 1. Load environment variables from .env file (contains sensitive configuration)
+	// 2. Load configuration from config.json (may reference environment variables)
+	// 3. Handle database migration commands if requested
+	// 4. Initialize database connection (requires config for connection string)
+	// 5. Handle user commands if requested (requires database and authentication config)
+	// 6. Initialize subsystems like archive and metrics (require config and database)
+
+	// Load environment and configuration
+	if err := loadEnvironment(); err != nil {
+		return err
+	}
+
+	if err := initConfiguration(); err != nil {
+		return err
+	}
+
+	// Handle database migration (migrate, revert, force)
+	if err := handleDatabaseCommands(); err != nil {
+		return err
+	}
+
+	// Initialize database
+	if err := initDatabase(); err != nil {
+		return err
+	}
+
+	// Optimize database if requested
+	if flagOptimizeDB {
+		db := repository.GetConnection()
+		cclog.Print("Running VACUUM to reclaim space and defragment database...")
+		if _, err := db.DB.Exec("VACUUM"); err != nil {
+			return fmt.Errorf("VACUUM failed: %w", err)
+		}
+		cclog.Print("Running ANALYZE to update query planner statistics...")
+		if _, err := db.DB.Exec("ANALYZE"); err != nil {
+			return fmt.Errorf("ANALYZE failed: %w", err)
+		}
+		cclog.Exitf("OptimizeDB Success: Database '%s' optimized (VACUUM + ANALYZE).\n", config.Keys.DB)
+	}
+
+	// Handle user commands (add, delete, sync, JWT)
+	if err := handleUserCommands(); err != nil {
+		return err
+	}
+
+	// Initialize subsystems (archive, metrics, taggers)
+	if err := initSubsystems(); err != nil {
+		return err
+	}
+
+	// Exit if start server is not requested
+	if !flagServer {
+		cclog.Exit("No errors, server flag not set. Exiting cc-backend.")
+	}
+
+	// Run server with context
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	return runServer(ctx)
+}
+
+func main() {
+	if err := run(); err != nil {
+		cclog.Error(err.Error())
+		os.Exit(1)
 	}
-	runtimeEnv.SystemdNotifiy(true, "running")
-	wg.Wait()
-	log.Print("Graceful shutdown completed!")
 }

cmd/cc-backend/server.go

@@ -1,7 +1,11 @@
 // Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved.
+// All rights reserved. This file is part of cc-backend.
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
+
+// Package main provides the entry point for the ClusterCockpit backend server.
+// This file contains HTTP server setup, routing configuration, and
+// authentication middleware integration.
 package main
 
 import (
@@ -10,14 +14,15 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"io"
 	"net"
 	"net/http"
 	"os"
 	"strings"
 	"time"
 
+	"github.com/99designs/gqlgen/graphql"
 	"github.com/99designs/gqlgen/graphql/handler"
+	"github.com/99designs/gqlgen/graphql/handler/transport"
 	"github.com/99designs/gqlgen/graphql/playground"
 	"github.com/ClusterCockpit/cc-backend/internal/api"
 	"github.com/ClusterCockpit/cc-backend/internal/archiver"
@@ -26,20 +31,32 @@ import (
 	"github.com/ClusterCockpit/cc-backend/internal/graph"
 	"github.com/ClusterCockpit/cc-backend/internal/graph/generated"
 	"github.com/ClusterCockpit/cc-backend/internal/routerConfig"
-	"github.com/ClusterCockpit/cc-backend/pkg/log"
-	"github.com/ClusterCockpit/cc-backend/pkg/runtimeEnv"
+	"github.com/ClusterCockpit/cc-backend/pkg/metricstore"
 	"github.com/ClusterCockpit/cc-backend/web"
-	"github.com/gorilla/handlers"
-	"github.com/gorilla/mux"
+	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
+	"github.com/ClusterCockpit/cc-lib/v2/nats"
+	"github.com/ClusterCockpit/cc-lib/v2/runtime"
+	"github.com/go-chi/chi/v5"
+	"github.com/go-chi/chi/v5/middleware"
+	"github.com/go-chi/cors"
 	httpSwagger "github.com/swaggo/http-swagger"
 )
 
-var (
-	router    *mux.Router
-	server    *http.Server
-	apiHandle *api.RestApi
+var buildInfo web.Build
+
+// Environment variable names
+const (
+	envDebug = "DEBUG"
 )
 
+// Server encapsulates the HTTP server state and dependencies
+type Server struct {
+	router        chi.Router
+	server        *http.Server
+	restAPIHandle *api.RestAPI
+	natsAPIHandle *api.NatsAPI
+}
+
 func onFailureResponse(rw http.ResponseWriter, r *http.Request, err error) {
 	rw.Header().Add("Content-Type", "application/json")
 	rw.WriteHeader(http.StatusUnauthorized)
@@ -49,22 +66,45 @@ func onFailureResponse(rw http.ResponseWriter, r *http.Request, err error) {
 	})
 }
 
-func serverInit() {
+// NewServer creates and initializes a new Server instance
+func NewServer(version, commit, buildDate string) (*Server, error) {
+	buildInfo = web.Build{Version: version, Hash: commit, Buildtime: buildDate}
+
+	s := &Server{
+		router: chi.NewRouter(),
+	}
+
+	if err := s.init(); err != nil {
+		return nil, err
+	}
+
+	return s, nil
+}
+
+func (s *Server) init() error {
 	// Setup the http.Handler/Router used by the server
 	graph.Init()
 	resolver := graph.GetResolverInstance()
-	graphQLEndpoint := handler.NewDefaultServer(
+	graphQLServer := handler.New(
 		generated.NewExecutableSchema(generated.Config{Resolvers: resolver}))
 
-	if os.Getenv("DEBUG") != "1" {
+	graphQLServer.AddTransport(transport.POST{})
+
+	// Inject a per-request stats cache so that grouped statistics queries
+	// sharing the same (filter, groupBy) pair are executed only once.
+	graphQLServer.AroundOperations(func(ctx context.Context, next graphql.OperationHandler) graphql.ResponseHandler {
+		return next(graph.WithStatsGroupCache(ctx))
+	})
+
+	if os.Getenv(envDebug) != "1" {
 		// Having this handler means that a error message is returned via GraphQL instead of the connection simply beeing closed.
 		// The problem with this is that then, no more stacktrace is printed to stderr.
-		graphQLEndpoint.SetRecoverFunc(func(ctx context.Context, err interface{}) error {
+		graphQLServer.SetRecoverFunc(func(ctx context.Context, err any) error {
 			switch e := err.(type) {
 			case string:
 				return fmt.Errorf("MAIN > Panic: %s", e)
 			case error:
-				return fmt.Errorf("MAIN > Panic caused by: %w", e)
+				return fmt.Errorf("MAIN > Panic caused by: %s", e.Error())
 			}
 
 			return errors.New("MAIN > Internal server error (panic)")
@@ -73,72 +113,70 @@ func serverInit() {
 
 	authHandle := auth.GetAuthInstance()
 
-	apiHandle = api.New()
+	// Middleware must be defined before routes in chi
+	s.router.Use(func(next http.Handler) http.Handler {
+		return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
+			start := time.Now()
+			ww := middleware.NewWrapResponseWriter(rw, r.ProtoMajor)
+			next.ServeHTTP(ww, r)
+			cclog.Debugf("%s %s (%d, %.02fkb, %dms)",
+				r.Method, r.URL.RequestURI(),
+				ww.Status(), float32(ww.BytesWritten())/1024,
+				time.Since(start).Milliseconds())
+		})
+	})
+	s.router.Use(middleware.Compress(5))
+	s.router.Use(middleware.Recoverer)
+	s.router.Use(cors.Handler(cors.Options{
+		AllowCredentials: true,
+		AllowedHeaders:   []string{"X-Requested-With", "Content-Type", "Authorization", "Origin"},
+		AllowedMethods:   []string{"GET", "POST", "HEAD", "OPTIONS"},
+		AllowedOrigins:   []string{"*"},
+	}))
 
-	router = mux.NewRouter()
-	buildInfo := web.Build{Version: version, Hash: commit, Buildtime: date}
+	s.restAPIHandle = api.New()
 
-	info := map[string]interface{}{}
+	info := map[string]any{}
 	info["hasOpenIDConnect"] = false
 
-	if config.Keys.OpenIDConfig != nil {
+	if auth.Keys.OpenIDConfig != nil {
 		openIDConnect := auth.NewOIDC(authHandle)
-		openIDConnect.RegisterEndpoints(router)
+		openIDConnect.RegisterEndpoints(s.router)
 		info["hasOpenIDConnect"] = true
 	}
 
-	router.HandleFunc("/login", func(rw http.ResponseWriter, r *http.Request) {
+	s.router.Get("/login", func(rw http.ResponseWriter, r *http.Request) {
 		rw.Header().Add("Content-Type", "text/html; charset=utf-8")
-		log.Debugf("##%v##", info)
+		cclog.Debugf("##%v##", info)
 		web.RenderTemplate(rw, "login.tmpl", &web.Page{Title: "Login", Build: buildInfo, Infos: info})
-	}).Methods(http.MethodGet)
-	router.HandleFunc("/imprint", func(rw http.ResponseWriter, r *http.Request) {
+	})
+	s.router.HandleFunc("/imprint", func(rw http.ResponseWriter, r *http.Request) {
 		rw.Header().Add("Content-Type", "text/html; charset=utf-8")
 		web.RenderTemplate(rw, "imprint.tmpl", &web.Page{Title: "Imprint", Build: buildInfo})
 	})
-	router.HandleFunc("/privacy", func(rw http.ResponseWriter, r *http.Request) {
+	s.router.HandleFunc("/privacy", func(rw http.ResponseWriter, r *http.Request) {
 		rw.Header().Add("Content-Type", "text/html; charset=utf-8")
 		web.RenderTemplate(rw, "privacy.tmpl", &web.Page{Title: "Privacy", Build: buildInfo})
 	})
 
-	secured := router.PathPrefix("/").Subrouter()
-	securedapi := router.PathPrefix("/api").Subrouter()
-	userapi := router.PathPrefix("/userapi").Subrouter()
-	configapi := router.PathPrefix("/config").Subrouter()
-	frontendapi := router.PathPrefix("/frontend").Subrouter()
-
 	if !config.Keys.DisableAuthentication {
-		router.Handle("/login", authHandle.Login(
-			// On success: Handled within Login()
-			// On failure:
-			func(rw http.ResponseWriter, r *http.Request, err error) {
-				rw.Header().Add("Content-Type", "text/html; charset=utf-8")
-				rw.WriteHeader(http.StatusUnauthorized)
-				web.RenderTemplate(rw, "login.tmpl", &web.Page{
-					Title:   "Login failed - ClusterCockpit",
-					MsgType: "alert-warning",
-					Message: err.Error(),
-					Build:   buildInfo,
-					Infos:   info,
-				})
-			})).Methods(http.MethodPost)
+		// Create login failure handler (used by both /login and /jwt-login)
+		loginFailureHandler := func(rw http.ResponseWriter, r *http.Request, err error) {
+			rw.Header().Add("Content-Type", "text/html; charset=utf-8")
+			rw.WriteHeader(http.StatusUnauthorized)
+			web.RenderTemplate(rw, "login.tmpl", &web.Page{
+				Title:   "Login failed - ClusterCockpit",
+				MsgType: "alert-warning",
+				Message: err.Error(),
+				Build:   buildInfo,
+				Infos:   info,
+			})
+		}
 
-		router.Handle("/jwt-login", authHandle.Login(
-			// On success: Handled within Login()
-			// On failure:
-			func(rw http.ResponseWriter, r *http.Request, err error) {
-				rw.Header().Add("Content-Type", "text/html; charset=utf-8")
-				rw.WriteHeader(http.StatusUnauthorized)
-				web.RenderTemplate(rw, "login.tmpl", &web.Page{
-					Title:   "Login failed - ClusterCockpit",
-					MsgType: "alert-warning",
-					Message: err.Error(),
-					Build:   buildInfo,
-					Infos:   info,
-				})
-			}))
+		s.router.Post("/login", authHandle.Login(loginFailureHandler).ServeHTTP)
+		s.router.HandleFunc("/jwt-login", authHandle.Login(loginFailureHandler).ServeHTTP)
 
-		router.Handle("/logout", authHandle.Logout(
+		s.router.Post("/logout", authHandle.Logout(
 			http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
 				rw.Header().Add("Content-Type", "text/html; charset=utf-8")
 				rw.WriteHeader(http.StatusOK)
@@ -149,139 +187,196 @@ func serverInit() {
 					Build:   buildInfo,
 					Infos:   info,
 				})
-			}))).Methods(http.MethodPost)
-
-		secured.Use(func(next http.Handler) http.Handler {
-			return authHandle.Auth(
-				// On success;
-				next,
-
-				// On failure:
-				func(rw http.ResponseWriter, r *http.Request, err error) {
-					rw.WriteHeader(http.StatusUnauthorized)
-					web.RenderTemplate(rw, "login.tmpl", &web.Page{
-						Title:    "Authentication failed - ClusterCockpit",
-						MsgType:  "alert-danger",
-						Message:  err.Error(),
-						Build:    buildInfo,
-						Infos:    info,
-						Redirect: r.RequestURI,
-					})
-				})
-		})
-
-		securedapi.Use(func(next http.Handler) http.Handler {
-			return authHandle.AuthApi(
-				// On success;
-				next,
-				// On failure: JSON Response
-				onFailureResponse)
-		})
-
-		userapi.Use(func(next http.Handler) http.Handler {
-			return authHandle.AuthUserApi(
-				// On success;
-				next,
-				// On failure: JSON Response
-				onFailureResponse)
-		})
-
-		configapi.Use(func(next http.Handler) http.Handler {
-			return authHandle.AuthConfigApi(
-				// On success;
-				next,
-				// On failure: JSON Response
-				onFailureResponse)
-		})
-
-		frontendapi.Use(func(next http.Handler) http.Handler {
-			return authHandle.AuthFrontendApi(
-				// On success;
-				next,
-				// On failure: JSON Response
-				onFailureResponse)
-		})
+			})).ServeHTTP)
 	}
 
 	if flagDev {
-		router.Handle("/playground", playground.Handler("GraphQL playground", "/query"))
-		router.PathPrefix("/swagger/").Handler(httpSwagger.Handler(
-			httpSwagger.URL("http://" + config.Keys.Addr + "/swagger/doc.json"))).Methods(http.MethodGet)
+		s.router.Handle("/playground", playground.Handler("GraphQL playground", "/query"))
+		s.router.Get("/swagger/*", httpSwagger.Handler(
+			httpSwagger.URL("http://"+config.Keys.Addr+"/swagger/doc.json")))
 	}
-	secured.Handle("/query", graphQLEndpoint)
 
-	// Send a searchId and then reply with a redirect to a user, or directly send query to job table for jobid and project.
-	secured.HandleFunc("/search", func(rw http.ResponseWriter, r *http.Request) {
-		routerConfig.HandleSearchBar(rw, r, buildInfo)
+	// Secured routes (require authentication)
+	s.router.Group(func(secured chi.Router) {
+		if !config.Keys.DisableAuthentication {
+			secured.Use(func(next http.Handler) http.Handler {
+				return authHandle.Auth(
+					next,
+					func(rw http.ResponseWriter, r *http.Request, err error) {
+						rw.WriteHeader(http.StatusUnauthorized)
+						web.RenderTemplate(rw, "login.tmpl", &web.Page{
+							Title:    "Authentication failed - ClusterCockpit",
+							MsgType:  "alert-danger",
+							Message:  err.Error(),
+							Build:    buildInfo,
+							Infos:    info,
+							Redirect: r.RequestURI,
+						})
+					})
+			})
+		}
+
+		secured.Handle("/query", graphQLServer)
+
+		secured.HandleFunc("/search", func(rw http.ResponseWriter, r *http.Request) {
+			routerConfig.HandleSearchBar(rw, r, buildInfo)
+		})
+
+		routerConfig.SetupRoutes(secured, buildInfo)
 	})
 
-	// Mount all /monitoring/... and /api/... routes.
-	routerConfig.SetupRoutes(secured, buildInfo)
-	apiHandle.MountApiRoutes(securedapi)
-	apiHandle.MountUserApiRoutes(userapi)
-	apiHandle.MountConfigApiRoutes(configapi)
-	apiHandle.MountFrontendApiRoutes(frontendapi)
+	// API routes (JWT token auth)
+	s.router.Route("/api", func(apiRouter chi.Router) {
+		// Main API routes with API auth
+		apiRouter.Group(func(securedapi chi.Router) {
+			if !config.Keys.DisableAuthentication {
+				securedapi.Use(func(next http.Handler) http.Handler {
+					return authHandle.AuthAPI(next, onFailureResponse)
+				})
+			}
+			s.restAPIHandle.MountAPIRoutes(securedapi)
+		})
+
+		// Metric store API routes with separate auth
+		apiRouter.Group(func(metricstoreapi chi.Router) {
+			if !config.Keys.DisableAuthentication {
+				metricstoreapi.Use(func(next http.Handler) http.Handler {
+					return authHandle.AuthMetricStoreAPI(next, onFailureResponse)
+				})
+			}
+			s.restAPIHandle.MountMetricStoreAPIRoutes(metricstoreapi)
+		})
+	})
+
+	// User API routes
+	s.router.Route("/userapi", func(userapi chi.Router) {
+		if !config.Keys.DisableAuthentication {
+			userapi.Use(func(next http.Handler) http.Handler {
+				return authHandle.AuthUserAPI(next, onFailureResponse)
+			})
+		}
+		s.restAPIHandle.MountUserAPIRoutes(userapi)
+	})
+
+	// Config API routes (uses Group with full paths to avoid shadowing
+	// the /config page route that is registered in the secured group)
+	s.router.Group(func(configapi chi.Router) {
+		if !config.Keys.DisableAuthentication {
+			configapi.Use(func(next http.Handler) http.Handler {
+				return authHandle.AuthConfigAPI(next, onFailureResponse)
+			})
+		}
+		s.restAPIHandle.MountConfigAPIRoutes(configapi)
+	})
+
+	// Frontend API routes
+	s.router.Route("/frontend", func(frontendapi chi.Router) {
+		if !config.Keys.DisableAuthentication {
+			frontendapi.Use(func(next http.Handler) http.Handler {
+				return authHandle.AuthFrontendAPI(next, onFailureResponse)
+			})
+		}
+		s.restAPIHandle.MountFrontendAPIRoutes(frontendapi)
+	})
+
+	if config.Keys.APISubjects != nil {
+		s.natsAPIHandle = api.NewNatsAPI()
+		if err := s.natsAPIHandle.StartSubscriptions(); err != nil {
+			return fmt.Errorf("starting NATS subscriptions: %w", err)
+		}
+	}
+
+	// 404 handler for pages and API routes
+	notFoundHandler := func(rw http.ResponseWriter, r *http.Request) {
+		if strings.HasPrefix(r.URL.Path, "/api/") || strings.HasPrefix(r.URL.Path, "/userapi/") ||
+			strings.HasPrefix(r.URL.Path, "/frontend/") || strings.HasPrefix(r.URL.Path, "/config/") {
+			rw.Header().Set("Content-Type", "application/json")
+			rw.WriteHeader(http.StatusNotFound)
+			json.NewEncoder(rw).Encode(map[string]string{
+				"status": "Resource not found",
+				"error":  "the requested endpoint does not exist",
+			})
+			return
+		}
+		rw.Header().Set("Content-Type", "text/html; charset=utf-8")
+		rw.WriteHeader(http.StatusNotFound)
+		web.RenderTemplate(rw, "404.tmpl", &web.Page{
+			Title: "Page Not Found",
+			Build: buildInfo,
+		})
+	}
+
+	// Set NotFound on the router so chi uses it for all unmatched routes,
+	// including those under subrouters like /api, /userapi, /frontend, etc.
+	s.router.NotFound(notFoundHandler)
 
 	if config.Keys.EmbedStaticFiles {
 		if i, err := os.Stat("./var/img"); err == nil {
 			if i.IsDir() {
-				log.Info("Use local directory for static images")
-				router.PathPrefix("/img/").Handler(http.StripPrefix("/img/", http.FileServer(http.Dir("./var/img"))))
+				cclog.Info("Use local directory for static images")
+				s.router.Handle("/img/*", http.StripPrefix("/img/", http.FileServer(http.Dir("./var/img"))))
 			}
 		}
-		router.PathPrefix("/").Handler(web.ServeFiles())
+		fileServer := http.StripPrefix("/", web.ServeFiles())
+		s.router.Handle("/*", http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
+			if web.StaticFileExists(r.URL.Path) {
+				fileServer.ServeHTTP(rw, r)
+				return
+			}
+			notFoundHandler(rw, r)
+		}))
 	} else {
-		router.PathPrefix("/").Handler(http.FileServer(http.Dir(config.Keys.StaticFiles)))
+		staticDir := http.Dir(config.Keys.StaticFiles)
+		fileServer := http.FileServer(staticDir)
+		s.router.Handle("/*", http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
+			f, err := staticDir.Open(r.URL.Path)
+			if err == nil {
+				f.Close()
+				fileServer.ServeHTTP(rw, r)
+				return
+			}
+			notFoundHandler(rw, r)
+		}))
 	}
 
-	router.Use(handlers.CompressHandler)
-	router.Use(handlers.RecoveryHandler(handlers.PrintRecoveryStack(true)))
-	router.Use(handlers.CORS(
-		handlers.AllowCredentials(),
-		handlers.AllowedHeaders([]string{"X-Requested-With", "Content-Type", "Authorization", "Origin"}),
-		handlers.AllowedMethods([]string{"GET", "POST", "HEAD", "OPTIONS"}),
-		handlers.AllowedOrigins([]string{"*"})))
+	return nil
 }
 
-func serverStart() {
-	handler := handlers.CustomLoggingHandler(io.Discard, router, func(_ io.Writer, params handlers.LogFormatterParams) {
-		if strings.HasPrefix(params.Request.RequestURI, "/api/") {
-			log.Debugf("%s %s (%d, %.02fkb, %dms)",
-				params.Request.Method, params.URL.RequestURI(),
-				params.StatusCode, float32(params.Size)/1024,
-				time.Since(params.TimeStamp).Milliseconds())
-		} else {
-			log.Debugf("%s %s (%d, %.02fkb, %dms)",
-				params.Request.Method, params.URL.RequestURI(),
-				params.StatusCode, float32(params.Size)/1024,
-				time.Since(params.TimeStamp).Milliseconds())
-		}
-	})
+// Server timeout defaults (in seconds)
+const (
+	defaultReadTimeout  = 20
+	defaultWriteTimeout = 20
+)
 
-	server = &http.Server{
-		ReadTimeout:  20 * time.Second,
-		WriteTimeout: 20 * time.Second,
-		Handler:      handler,
+func (s *Server) Start(ctx context.Context) error {
+	// Use configurable timeouts with defaults
+	readTimeout := time.Duration(defaultReadTimeout) * time.Second
+	writeTimeout := time.Duration(defaultWriteTimeout) * time.Second
+
+	s.server = &http.Server{
+		ReadTimeout:  readTimeout,
+		WriteTimeout: writeTimeout,
+		Handler:      s.router,
 		Addr:         config.Keys.Addr,
 	}
 
 	// Start http or https server
 	listener, err := net.Listen("tcp", config.Keys.Addr)
 	if err != nil {
-		log.Fatalf("starting http listener failed: %v", err)
+		return fmt.Errorf("starting listener on '%s': %w", config.Keys.Addr, err)
 	}
 
-	if !strings.HasSuffix(config.Keys.Addr, ":80") && config.Keys.RedirectHttpTo != "" {
+	if !strings.HasSuffix(config.Keys.Addr, ":80") && config.Keys.RedirectHTTPTo != "" {
 		go func() {
-			http.ListenAndServe(":80", http.RedirectHandler(config.Keys.RedirectHttpTo, http.StatusMovedPermanently))
+			http.ListenAndServe(":80", http.RedirectHandler(config.Keys.RedirectHTTPTo, http.StatusMovedPermanently))
 		}()
 	}
 
-	if config.Keys.HttpsCertFile != "" && config.Keys.HttpsKeyFile != "" {
+	if config.Keys.HTTPSCertFile != "" && config.Keys.HTTPSKeyFile != "" {
 		cert, err := tls.LoadX509KeyPair(
-			config.Keys.HttpsCertFile, config.Keys.HttpsKeyFile)
+			config.Keys.HTTPSCertFile, config.Keys.HTTPSKeyFile)
 		if err != nil {
-			log.Fatalf("loading X509 keypair failed: %v", err)
+			return fmt.Errorf("loading X509 keypair (check 'https-cert-file' and 'https-key-file' in config.json): %w", err)
 		}
 		listener = tls.NewListener(listener, &tls.Config{
 			Certificates: []tls.Certificate{cert},
@@ -292,27 +387,58 @@ func serverStart() {
 			MinVersion:               tls.VersionTLS12,
 			PreferServerCipherSuites: true,
 		})
-		fmt.Printf("HTTPS server listening at %s...", config.Keys.Addr)
+		cclog.Infof("HTTPS server listening at %s...", config.Keys.Addr)
 	} else {
-		fmt.Printf("HTTP server listening at %s...", config.Keys.Addr)
+		cclog.Infof("HTTP server listening at %s...", config.Keys.Addr)
 	}
 	//
 	// Because this program will want to bind to a privileged port (like 80), the listener must
 	// be established first, then the user can be changed, and after that,
 	// the actual http server can be started.
-	if err := runtimeEnv.DropPrivileges(config.Keys.Group, config.Keys.User); err != nil {
-		log.Fatalf("error while preparing server start: %s", err.Error())
+	if err := runtime.DropPrivileges(config.Keys.Group, config.Keys.User); err != nil {
+		return fmt.Errorf("dropping privileges: %w", err)
 	}
 
-	if err = server.Serve(listener); err != nil && err != http.ErrServerClosed {
-		log.Fatalf("starting server failed: %v", err)
+	// Handle context cancellation for graceful shutdown
+	go func() {
+		<-ctx.Done()
+		shutdownCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+		defer cancel()
+		if err := s.server.Shutdown(shutdownCtx); err != nil {
+			cclog.Errorf("Server shutdown error: %v", err)
+		}
+	}()
+
+	if err = s.server.Serve(listener); err != nil && err != http.ErrServerClosed {
+		return fmt.Errorf("server failed: %w", err)
 	}
+	return nil
 }
 
-func serverShutdown() {
+func (s *Server) Shutdown(ctx context.Context) {
+	// Create a shutdown context with timeout
+	shutdownCtx, cancel := context.WithTimeout(ctx, 30*time.Second)
+	defer cancel()
+
+	nc := nats.GetClient()
+	if nc != nil {
+		nc.Close()
+	}
+
 	// First shut down the server gracefully (waiting for all ongoing requests)
-	server.Shutdown(context.Background())
+	if err := s.server.Shutdown(shutdownCtx); err != nil {
+		cclog.Errorf("Server shutdown error: %v", err)
+	}
 
-	// Then, wait for any async archivings still pending...
-	archiver.WaitForArchiving()
+	// Archive all the metric store data
+	ms := metricstore.GetMemoryStore()
+
+	if ms != nil {
+		metricstore.Shutdown()
+	}
+
+	// Shutdown archiver with 10 second timeout for fast shutdown
+	if err := archiver.Shutdown(10 * time.Second); err != nil {
+		cclog.Warnf("Archiver shutdown: %v", err)
+	}
 }

configs/config-demo.json

@@ -1,67 +1,26 @@
 {
-  "addr": "127.0.0.1:8080",
-  "short-running-jobs-duration": 300,
-  "archive": {
-    "kind": "file",
-    "path": "./var/job-archive"
+  "main": {
+    "addr": "127.0.0.1:8080"
   },
-  "jwts": {
-    "max-age": "2000h"
+  "cron": {
+    "commit-job-worker": "1m",
+    "duration-worker": "3m",
+    "footprint-worker": "5m"
   },
-  "enable-resampling": {
-    "trigger": 30,
-    "resolutions": [
-      600,
-      300,
-      120,
-      60
-    ]
-  },
-  "emission-constant": 317,
-  "clusters": [
-    {
-      "name": "fritz",
-      "metricDataRepository": {
-        "kind": "cc-metric-store",
-        "url": "http://localhost:8082",
-        "token": ""
-      },
-      "filterRanges": {
-        "numNodes": {
-          "from": 1,
-          "to": 64
-        },
-        "duration": {
-          "from": 0,
-          "to": 86400
-        },
-        "startTime": {
-          "from": "2022-01-01T00:00:00Z",
-          "to": null
-        }
-      }
-    },
-    {
-      "name": "alex",
-      "metricDataRepository": {
-        "kind": "cc-metric-store",
-        "url": "http://localhost:8082",
-        "token": ""
-      },
-      "filterRanges": {
-        "numNodes": {
-          "from": 1,
-          "to": 64
-        },
-        "duration": {
-          "from": 0,
-          "to": 86400
-        },
-        "startTime": {
-          "from": "2022-01-01T00:00:00Z",
-          "to": null
-        }
-      }
+  "auth": {
+    "jwts": {
+      "max-age": "2000h"
     }
-  ]
+  },
+  "metric-store-external": [
+    {
+      "scope": "fritz",
+      "url": "http://0.0.0.0:8082",
+      "token": "eyJhbGciOiJFZERTQSIsInR5cCI6IkpXVCJ9.eyJleHAiOjE3NzU3Nzg4NDQsImlhdCI6MTc2ODU3ODg0NCwicm9sZXMiOlsiYWRtaW4iLCJhcGkiXSwic3ViIjoiZGVtbyJ9._SDEW9WaUVXSBFmWqGhyIZXLoqoDU8F1hkfh4cXKIqF4yw7w50IUpfUBtwUFUOnoviFKoi563f6RAMC7XxeLDA"
+    }
+  ],
+  "metric-store": {
+    "retention-in-memory": "24h",
+    "memory-cap": 100
+  }
 }

configs/config-mariadb.json

@@ -1,69 +0,0 @@
-{
-  "addr": "127.0.0.1:8080",
-  "short-running-jobs-duration": 300,
-  "archive": {
-    "kind": "file",
-    "path": "./var/job-archive"
-  },
-  "jwts": {
-    "max-age": "2000h"
-  },
-  "db-driver": "mysql",
-  "db": "clustercockpit:demo@tcp(127.0.0.1:3306)/clustercockpit",
-  "enable-resampling": {
-    "trigger": 30,
-    "resolutions": [
-      600,
-      300,
-      120,
-      60
-    ]
-  },
-  "emission-constant": 317,
-  "clusters": [
-    {
-      "name": "fritz",
-      "metricDataRepository": {
-        "kind": "cc-metric-store",
-        "url": "http://localhost:8082",
-        "token": ""
-      },
-      "filterRanges": {
-        "numNodes": {
-          "from": 1,
-          "to": 64
-        },
-        "duration": {
-          "from": 0,
-          "to": 86400
-        },
-        "startTime": {
-          "from": "2022-01-01T00:00:00Z",
-          "to": null
-        }
-      }
-    },
-    {
-      "name": "alex",
-      "metricDataRepository": {
-        "kind": "cc-metric-store",
-        "url": "http://localhost:8082",
-        "token": ""
-      },
-      "filterRanges": {
-        "numNodes": {
-          "from": 1,
-          "to": 64
-        },
-        "duration": {
-          "from": 0,
-          "to": 86400
-        },
-        "startTime": {
-          "from": "2022-01-01T00:00:00Z",
-          "to": null
-        }
-      }
-    }
-  ]
-}

configs/config.json

@@ -1,50 +1,97 @@
 {
+  "main": {
     "addr": "0.0.0.0:443",
-    "ldap": {
-        "url": "ldaps://test",
-        "user_base": "ou=people,ou=hpc,dc=test,dc=de",
-        "search_dn": "cn=hpcmonitoring,ou=roadm,ou=profile,ou=hpc,dc=test,dc=de",
-        "user_bind": "uid={username},ou=people,ou=hpc,dc=test,dc=de",
-        "user_filter": "(&(objectclass=posixAccount))"
-    },
     "https-cert-file": "/etc/letsencrypt/live/url/fullchain.pem",
     "https-key-file": "/etc/letsencrypt/live/url/privkey.pem",
     "user": "clustercockpit",
     "group": "clustercockpit",
-    "archive": {
-        "kind": "file",
-        "path": "./var/job-archive"
+    "api-allowed-ips": ["*"],
+    "short-running-jobs-duration": 300,
+    "enable-job-taggers": true,
+    "nodestate-retention": {
+      "policy": "move",
+      "target-kind": "file",
+      "target-path": "./var/nodestate-archive"
     },
-    "validate": true,
-    "clusters": [
-        {
-            "name": "test",
-            "metricDataRepository": {
-                "kind": "cc-metric-store",
-                "url": "http://localhost:8082",
-                "token": "eyJhbGciOiJF-E-pQBQ"
-            },
-            "filterRanges": {
-                "numNodes": {
-                    "from": 1,
-                    "to": 64
-                },
-                "duration": {
-                    "from": 0,
-                    "to": 86400
-                },
-                "startTime": {
-                    "from": "2022-01-01T00:00:00Z",
-                    "to": null
-                }
-            }
-        }
-    ],
+    "resampling": {
+      "minimum-points": 600,
+      "trigger": 180,
+      "resolutions": [240, 60]
+    },
+    "api-subjects": {
+      "subject-job-event": "cc.job.event",
+      "subject-node-state": "cc.node.state"
+    }
+  },
+  "nats": {
+    "address": "nats://0.0.0.0:4222",
+    "username": "root",
+    "password": "root"
+  },
+  "auth": {
     "jwts": {
-        "cookieName": "",
-        "validateUser": false,
-        "max-age": "2000h",
-        "trustedIssuer": ""
+      "max-age": "2000h"
+    }
+  },
+  "cron": {
+    "commit-job-worker": "1m",
+    "duration-worker": "5m",
+    "footprint-worker": "10m"
+  },
+  "archive": {
+    "kind": "s3",
+    "endpoint": "http://x.x.x.x",
+    "bucket": "jobarchive",
+    "access-key": "xx",
+    "secret-key": "xx",
+    "retention": {
+      "policy": "move",
+      "age": 365,
+      "location": "./var/archive"
+    }
+  },
+  "metric-store-external": [
+    {
+      "scope": "*",
+      "url": "http://x.x.x.x:8082",
+      "token": "MySecret"
     },
-    "short-running-jobs-duration": 300
+    {
+      "scope": "fritz",
+      "url": "http://x.x.x.x:8084",
+      "token": "MySecret"
+    },
+    {
+      "scope": "fritz-spr1tb",
+      "url": "http://x.x.x.x:8083",
+      "token": "MySecret"
+    },
+    {
+      "scope": "alex",
+      "url": "http://x.x.x.x:8084",
+      "token": "MySecret"
+    }
+  ],
+  "metric-store": {
+    "checkpoints": {
+      "directory": "./var/checkpoints"
+    },
+    "memory-cap": 100,
+    "retention-in-memory": "24h",
+    "cleanup": {
+      "mode": "archive",
+      "directory": "./var/archive"
+    },
+    "nats-subscriptions": [
+      {
+        "subscribe-to": "hpc-nats",
+        "cluster-tag": "fritz"
+      },
+      {
+        "subscribe-to": "hpc-nats",
+        "cluster-tag": "alex"
+      }
+    ]
+  },
+  "ui-file": "ui-config.json"
 }

configs/default_metrics.json

@@ -1,12 +0,0 @@
-{
-  "clusters": [
-    {
-      "name": "fritz",
-      "default_metrics": "cpu_load, flops_any, core_power, lustre_open, mem_used, mem_bw, net_bytes_in"
-    },
-    {
-      "name": "alex",
-      "default_metrics": "flops_any, mem_bw, mem_used, vectorization_ratio"
-    }
-  ]
-}

configs/startJobPayload.json

@@ -0,0 +1,22 @@
+{
+  "cluster": "fritz",
+  "jobId": 123000,
+  "jobState": "running",
+  "numAcc": 0,
+  "numHwthreads": 72,
+  "numNodes": 1,
+  "partition": "main",
+  "requestedMemory": 128000,
+  "resources": [{ "hostname": "f0726" }],
+  "startTime": 1649723812,
+  "subCluster": "main",
+  "submitTime": 1649723812,
+  "user": "k106eb10",
+  "project": "k106eb",
+  "walltime": 86400,
+  "metaData": {
+    "slurmInfo": "JobId=398759\nJobName=myJob\nUserId=dummyUser\nGroupId=dummyGroup\nAccount=dummyAccount\nQOS=normal Requeue=False Restarts=0 BatchFlag=True\nTimeLimit=1439'\nSubmitTime=2023-02-09T14:10:18\nPartition=singlenode\nNodeList=xx\nNumNodes=xx NumCPUs=72 NumTasks=72 CPUs/Task=1\nNTasksPerNode:Socket:Core=0:None:None\nTRES_req=cpu=72,mem=250000M,node=1,billing=72\nTRES_alloc=cpu=72,node=1,billing=72\nCommand=myCmd\nWorkDir=myDir\nStdErr=\nStdOut=\n",
+    "jobScript": "#!/bin/bash  -l\n#SBATCH --job-name=dummy_job\n#SBATCH --time=23:59:00\n#SBATCH --partition=singlenode\n#SBATCH --ntasks=72\n#SBATCH --hint=multithread\n#SBATCH --chdir=/home/atuin/k106eb/dummy/\n#SBATCH --export=NONE\nunset SLURM_EXPORT_ENV\n\n#This is a dummy job script\n./mybinary\n",
+    "jobName": "ams_pipeline"
+  }
+}

configs/stopJobPayload.json

@@ -0,0 +1,7 @@
+{
+  "cluster": "fritz",
+  "jobId": 123000,
+  "jobState": "completed",
+  "startTime": 1649723812,
+  "stopTime": 1649763839
+}

configs/tagger/README.md

@@ -0,0 +1,419 @@
+# Job Tagging Configuration
+
+ClusterCockpit provides automatic job tagging functionality to classify and
+categorize jobs based on configurable rules. The tagging system consists of two
+main components:
+
+1. **Application Detection** - Identifies which application a job is running
+2. **Job Classification** - Analyzes job performance characteristics and applies classification tags
+
+## Directory Structure
+
+```
+configs/tagger/
+├── apps/              # Application detection patterns
+│   ├── vasp.txt
+│   ├── gromacs.txt
+│   └── ...
+└── jobclasses/        # Job classification rules
+    ├── parameters.json
+    ├── lowUtilization.json
+    ├── highload.json
+    └── ...
+```
+
+## Activating Tagger Rules
+
+### Step 1: Copy Configuration Files
+
+To activate tagging, review, adapt, and copy the configuration files from
+`configs/tagger/` to `var/tagger/`:
+
+```bash
+# From the cc-backend root directory
+mkdir -p var/tagger
+cp -r configs/tagger/apps var/tagger/
+cp -r configs/tagger/jobclasses var/tagger/
+```
+
+### Step 2: Enable Tagging in Configuration
+
+Add or set the following configuration key in the `main` section of your `config.json`:
+
+```json
+{
+  "enable-job-taggers": true
+}
+```
+
+**Important**: Automatic tagging is disabled by default. You must explicitly
+enable it by setting `enable-job-taggers: true` in the main configuration file.
+
+### Step 3: Restart cc-backend
+
+The tagger system automatically loads configuration from `./var/tagger/` at
+startup. After copying the files and enabling the feature, restart cc-backend:
+
+```bash
+./cc-backend -server
+```
+
+### Step 4: Verify Configuration Loaded
+
+Check the logs for messages indicating successful configuration loading:
+
+```
+[INFO] Setup file watch for ./var/tagger/apps
+[INFO] Setup file watch for ./var/tagger/jobclasses
+```
+
+## How Tagging Works
+
+### Automatic Tagging
+
+When `enable-job-taggers` is set to `true` in the configuration, tags are
+automatically applied when:
+
+- **Job Start**: Application detection runs immediately when a job starts
+- **Job Stop**: Job classification runs when a job completes
+
+The system analyzes job metadata and metrics to determine appropriate tags.
+
+**Note**: Automatic tagging only works for jobs that start or stop after the
+feature is enabled. Existing jobs are not automatically retagged.
+
+### Manual Tagging (Retroactive)
+
+To apply tags to existing jobs in the database, use the `-apply-tags` command
+line option:
+
+```bash
+./cc-backend -apply-tags
+```
+
+This processes all jobs in the database and applies current tagging rules. This
+is useful when:
+
+- You have existing jobs that were created before tagging was enabled
+- You've added new tagging rules and want to apply them to historical data
+- You've modified existing rules and want to re-evaluate all jobs
+
+### Hot Reload
+
+The tagger system watches the configuration directories for changes. You can
+modify or add rules without restarting `cc-backend`:
+
+- Changes to `var/tagger/apps/*` are detected automatically
+- Changes to `var/tagger/jobclasses/*` are detected automatically
+
+## Application Detection
+
+Application detection identifies which software a job is running by matching
+patterns in the job script.
+
+### Configuration Format
+
+Application patterns are stored in text files under `var/tagger/apps/`. Each
+file contains one or more regular expression patterns (one per line) that match
+against the job script.
+
+**Example: `apps/vasp.txt`**
+
+```
+vasp
+VASP
+```
+
+### How It Works
+
+1. When a job starts, the system retrieves the job script from metadata
+2. Each line in the app files is treated as a regex pattern
+3. Patterns are matched case-insensitively against the lowercased job script
+4. If a match is found, a tag of type `app` with the filename (without extension) is applied
+5. Only the first matching application is tagged
+
+### Adding New Applications
+
+1. Create a new file in `var/tagger/apps/` (e.g., `tensorflow.txt`)
+2. Add regex patterns, one per line:
+
+   ```
+   tensorflow
+   tf\.keras
+   import tensorflow
+   ```
+
+3. The file is automatically detected and loaded
+
+**Note**: The tag name will be the filename without the `.txt` extension (e.g., `tensorflow`).
+
+## Job Classification
+
+Job classification analyzes completed jobs based on their metrics and properties
+to identify performance issues or characteristics.
+
+### Configuration Format
+
+Job classification rules are defined in JSON files under
+`var/tagger/jobclasses/`. Each rule file defines:
+
+- **Metrics required**: Which job metrics to analyze
+- **Requirements**: Pre-conditions that must be met
+- **Variables**: Computed values used in the rule
+- **Rule expression**: Boolean expression that determines if the rule matches
+- **Hint template**: Message displayed when the rule matches
+
+### Parameters File
+
+`jobclasses/parameters.json` defines shared threshold values used across multiple rules:
+
+```json
+{
+  "lowcpuload_threshold_factor": 0.9,
+  "highmemoryusage_threshold_factor": 0.9,
+  "job_min_duration_seconds": 600.0,
+  "sampling_interval_seconds": 30.0
+}
+```
+
+### Rule File Structure
+
+**Example: `jobclasses/lowUtilization.json`**
+
+```json
+{
+  "name": "Low resource utilization",
+  "tag": "lowutilization",
+  "parameters": ["job_min_duration_seconds"],
+  "metrics": ["flops_any", "mem_bw"],
+  "requirements": [
+    "job.shared == \"none\"",
+    "job.duration > job_min_duration_seconds"
+  ],
+  "variables": [
+    {
+      "name": "mem_bw_perc",
+      "expr": "1.0 - (mem_bw.avg / mem_bw.limits.peak)"
+    }
+  ],
+  "rule": "flops_any.avg < flops_any.limits.alert",
+  "hint": "Average flop rate {{.flops_any.avg}} falls below threshold {{.flops_any.limits.alert}}"
+}
+```
+
+#### Field Descriptions
+
+| Field          | Description                                                                   |
+| -------------- | ----------------------------------------------------------------------------- |
+| `name`         | Human-readable description of the rule                                        |
+| `tag`          | Tag identifier applied when the rule matches                                  |
+| `parameters`   | List of parameter names from `parameters.json` to include in rule environment |
+| `metrics`      | List of metrics required for evaluation (must be present in job data)         |
+| `requirements` | Boolean expressions that must all be true for the rule to be evaluated        |
+| `variables`    | Named expressions computed before evaluating the main rule                    |
+| `rule`         | Boolean expression that determines if the job matches this classification     |
+| `hint`         | Go template string for generating a user-visible message                      |
+
+### Expression Environment
+
+Expressions in `requirements`, `variables`, and `rule` have access to:
+
+**Job Properties:**
+
+- `job.shared` - Shared node allocation type
+- `job.duration` - Job runtime in seconds
+- `job.numCores` - Number of CPU cores
+- `job.numNodes` - Number of nodes
+- `job.jobState` - Job completion state
+- `job.numAcc` - Number of accelerators
+- `job.smt` - SMT setting
+
+**Metric Statistics (for each metric in `metrics`):**
+
+- `<metric>.min` - Minimum value
+- `<metric>.max` - Maximum value
+- `<metric>.avg` - Average value
+- `<metric>.limits.peak` - Peak limit from cluster config
+- `<metric>.limits.normal` - Normal threshold
+- `<metric>.limits.caution` - Caution threshold
+- `<metric>.limits.alert` - Alert threshold
+
+**Parameters:**
+
+- All parameters listed in the `parameters` field
+
+**Variables:**
+
+- All variables defined in the `variables` array
+
+### Expression Language
+
+Rules use the [expr](https://github.com/expr-lang/expr) language for expressions. Supported operations:
+
+- **Arithmetic**: `+`, `-`, `*`, `/`, `%`, `^`
+- **Comparison**: `==`, `!=`, `<`, `<=`, `>`, `>=`
+- **Logical**: `&&`, `||`, `!`
+- **Functions**: Standard math functions (see expr documentation)
+
+### Hint Templates
+
+Hints use Go's `text/template` syntax. Variables from the evaluation environment are accessible:
+
+```
+{{.flops_any.avg}}          # Access metric average
+{{.job.duration}}            # Access job property
+{{.my_variable}}             # Access computed variable
+```
+
+### Adding New Classification Rules
+
+1. Create a new JSON file in `var/tagger/jobclasses/` (e.g., `memoryLeak.json`)
+2. Define the rule structure:
+
+   ```json
+   {
+     "name": "Memory Leak Detection",
+     "tag": "memory_leak",
+     "parameters": ["memory_leak_slope_threshold"],
+     "metrics": ["mem_used"],
+     "requirements": ["job.duration > 3600"],
+     "variables": [
+       {
+         "name": "mem_growth",
+         "expr": "(mem_used.max - mem_used.min) / job.duration"
+       }
+     ],
+     "rule": "mem_growth > memory_leak_slope_threshold",
+     "hint": "Memory usage grew by {{.mem_growth}} per second"
+   }
+   ```
+
+3. Add any new parameters to `parameters.json`
+4. The file is automatically detected and loaded
+
+## Configuration Paths
+
+The tagger system reads from these paths (relative to cc-backend working directory):
+
+- **Application patterns**: `./var/tagger/apps/`
+- **Job classification rules**: `./var/tagger/jobclasses/`
+
+These paths are defined as constants in the source code and cannot be changed without recompiling.
+
+## Troubleshooting
+
+### Tags Not Applied
+
+1. **Check tagging is enabled**: Verify `enable-job-taggers: true` is set in `config.json`
+
+2. **Check configuration exists**:
+
+   ```bash
+   ls -la var/tagger/apps
+   ls -la var/tagger/jobclasses
+   ```
+
+3. **Check logs for errors**:
+
+   ```bash
+   ./cc-backend -server -loglevel debug
+   ```
+
+4. **Verify file permissions**: Ensure cc-backend can read the configuration files
+
+5. **For existing jobs**: Use `./cc-backend -apply-tags` to retroactively tag jobs
+
+### Rules Not Matching
+
+1. **Enable debug logging**: Set `loglevel: debug` to see detailed rule evaluation
+2. **Check requirements**: Ensure all requirements in the rule are satisfied
+3. **Verify metrics exist**: Classification rules require job metrics to be available
+4. **Check metric names**: Ensure metric names match those in your cluster configuration
+
+### File Watch Not Working
+
+If changes to configuration files aren't detected:
+
+1. Restart cc-backend to reload all configuration
+2. Check filesystem supports file watching (network filesystems may not)
+3. Check logs for file watch setup messages
+
+## Best Practices
+
+1. **Start Simple**: Begin with basic rules and refine based on results
+2. **Use Requirements**: Filter out irrelevant jobs early with requirements
+3. **Test Incrementally**: Add one rule at a time and verify behavior
+4. **Document Rules**: Use descriptive names and clear hint messages
+5. **Share Parameters**: Define common thresholds in `parameters.json` for consistency
+6. **Version Control**: Keep your `var/tagger/` configuration in version control
+7. **Backup Before Changes**: Test new rules on a copy before deploying to production
+
+## Examples
+
+### Simple Application Detection
+
+**File: `var/tagger/apps/python.txt`**
+
+```
+python
+python3
+\.py
+```
+
+This detects jobs running Python scripts.
+
+### Complex Classification Rule
+
+**File: `var/tagger/jobclasses/cpuImbalance.json`**
+
+```json
+{
+  "name": "CPU Load Imbalance",
+  "tag": "cpu_imbalance",
+  "parameters": ["core_load_imbalance_threshold_factor"],
+  "metrics": ["cpu_load"],
+  "requirements": ["job.numCores > 1", "job.duration > 600"],
+  "variables": [
+    {
+      "name": "load_variance",
+      "expr": "(cpu_load.max - cpu_load.min) / cpu_load.avg"
+    }
+  ],
+  "rule": "load_variance > core_load_imbalance_threshold_factor",
+  "hint": "CPU load varies by {{printf \"%.1f%%\" (load_variance * 100)}} across cores"
+}
+```
+
+This detects jobs where CPU load is unevenly distributed across cores.
+
+## Reference
+
+### Configuration Options
+
+**Main Configuration (`config.json`)**:
+
+- `enable-job-taggers` (boolean, default: `false`) - Enables automatic job tagging system
+  - Must be set to `true` to activate automatic tagging on job start/stop events
+  - Does not affect the `-apply-tags` command line option
+
+**Command Line Options**:
+
+- `-apply-tags` - Apply all tagging rules to existing jobs in the database
+  - Works independently of `enable-job-taggers` configuration
+  - Useful for retroactively tagging jobs or re-evaluating with updated rules
+
+### Default Configuration Location
+
+The example configurations are provided in:
+
+- `configs/tagger/apps/` - Example application patterns (16 applications)
+- `configs/tagger/jobclasses/` - Example classification rules (3 rules)
+
+Copy these to `var/tagger/` and customize for your environment.
+
+### Tag Types
+
+- `app` - Application tags (e.g., "vasp", "gromacs")
+- `jobClass` - Classification tags (e.g., "lowutilization", "highload")
+
+Tags can be queried and filtered in the ClusterCockpit UI and API.

configs/tagger/apps/alf.txt

@@ -0,0 +1 @@
+alf

configs/tagger/apps/caracal.txt

@@ -0,0 +1,6 @@
+calc_rate
+qmdffgen
+dynamic
+evbopt
+black_box
+poly_qmdff

configs/tagger/apps/chroma.txt

@@ -0,0 +1,3 @@
+chroma
+qdp
+qmp

configs/tagger/apps/cp2k.txt

@@ -0,0 +1 @@
+cp2k

configs/tagger/apps/cpmd.txt

@@ -0,0 +1 @@
+cpmd

configs/tagger/apps/flame.txt

@@ -0,0 +1 @@
+flame

configs/tagger/apps/gromacs.txt

@@ -0,0 +1,3 @@
+gromacs
+gmx
+mdrun

configs/tagger/apps/julia.txt

@@ -0,0 +1 @@
+julia

configs/tagger/apps/lammps.txt

@@ -0,0 +1 @@
+\blmp\s+

configs/tagger/apps/matlab.txt

@@ -0,0 +1 @@
+matlab

configs/tagger/apps/openfoam.txt

@@ -0,0 +1 @@
+openfoam

configs/tagger/apps/orca.txt

@@ -0,0 +1 @@
+orca

configs/tagger/apps/python.txt

@@ -0,0 +1,4 @@
+python
+pip
+anaconda
+conda

configs/tagger/apps/starccm.txt

@@ -0,0 +1,2 @@
+starccm+
+-podkey

configs/tagger/apps/turbomole.txt

@@ -0,0 +1,10 @@
+dscf
+grad
+ridft
+rdgrad
+ricc2
+statpt
+aoforce
+escf
+egrad
+odft

configs/tagger/apps/vasp.txt

@@ -0,0 +1,3 @@
+vasp_gam
+vasp_ncl
+vasp_std

configs/tagger/jobclasses/highMemoryUsage.json

@@ -0,0 +1,21 @@
+{
+  "name": "High memory usage",
+  "tag": "highmemory",
+  "parameters": [
+    "highmemoryusage_threshold_factor",
+    "job_min_duration_seconds"
+  ],
+  "metrics": ["mem_used"],
+  "requirements": [
+    "job.shared == \"none\"",
+    "job.duration > job_min_duration_seconds"
+  ],
+  "variables": [
+    {
+      "name": "memory_usage_pct",
+      "expr": "mem_used.max / mem_used.limits.peak * 100.0"
+    }
+  ],
+  "rule": "mem_used.max > mem_used.limits.alert",
+  "hint": "This job used high memory: peak memory usage {{.mem_used.max}} GB ({{.memory_usage_pct}}% of {{.mem_used.limits.peak}} GB node capacity), exceeding the {{.highmemoryusage_threshold_factor}} utilization threshold. Risk of out-of-memory conditions."
+}

configs/tagger/jobclasses/highload.json

@@ -0,0 +1,21 @@
+{
+  "name": "Excessive CPU load",
+  "tag": "excessiveload",
+  "parameters": [
+    "excessivecpuload_threshold_factor",
+    "job_min_duration_seconds"
+  ],
+  "metrics": ["cpu_load"],
+  "requirements": [
+    "job.shared == \"none\"",
+    "job.duration > job_min_duration_seconds"
+  ],
+  "variables": [
+    {
+      "name": "load_threshold",
+      "expr": "cpu_load.limits.peak * excessivecpuload_threshold_factor"
+    }
+  ],
+  "rule": "cpu_load.avg > load_threshold",
+  "hint": "This job was detected as having excessive CPU load: average cpu load {{.cpu_load.avg}} exceeds the oversubscription threshold {{.load_threshold}} ({{.excessivecpuload_threshold_factor}} \u00d7 {{.cpu_load.limits.peak}} peak cores), indicating CPU contention."
+}

configs/tagger/jobclasses/lowUtilization.json

@@ -0,0 +1,22 @@
+{
+  "name": "Low resource utilization",
+  "tag": "lowutilization",
+  "parameters": ["job_min_duration_seconds"],
+  "metrics": ["flops_any", "mem_bw"],
+  "requirements": [
+    "job.shared == \"none\"",
+    "job.duration > job_min_duration_seconds"
+  ],
+  "variables": [
+    {
+      "name": "mem_bw_pct",
+      "expr": "mem_bw.avg / mem_bw.limits.peak * 100.0"
+    },
+    {
+      "name": "flops_any_pct",
+      "expr": "flops_any.avg / flops_any.limits.peak * 100.0"
+    }
+  ],
+  "rule": "flops_any.avg < flops_any.limits.alert && mem_bw.avg < mem_bw.limits.alert",
+  "hint": "This job shows low resource utilization: FLOP rate {{.flops_any.avg}} GF/s ({{.flops_any_pct}}% of peak) and memory bandwidth {{.mem_bw.avg}} GB/s ({{.mem_bw_pct}}% of peak) are both below their alert thresholds."
+}

configs/tagger/jobclasses/lowload.json

@@ -0,0 +1,18 @@
+{
+  "name": "Low CPU load",
+  "tag": "lowload",
+  "parameters": ["lowcpuload_threshold_factor", "job_min_duration_seconds"],
+  "metrics": ["cpu_load"],
+  "requirements": [
+    "job.shared == \"none\"",
+    "job.duration > job_min_duration_seconds"
+  ],
+  "variables": [
+    {
+      "name": "load_threshold",
+      "expr": "cpu_load.limits.peak * lowcpuload_threshold_factor"
+    }
+  ],
+  "rule": "cpu_load.avg < load_threshold",
+  "hint": "This job was detected as low CPU load: average cpu load {{.cpu_load.avg}} is below the threshold {{.load_threshold}} ({{.lowcpuload_threshold_factor}})."
+}

configs/tagger/jobclasses/memoryBound.json

@@ -0,0 +1,22 @@
+{
+  "name": "Memory bandwidth bound",
+  "tag": "memorybound",
+  "parameters": ["membound_bw_threshold_factor", "job_min_duration_seconds"],
+  "metrics": ["mem_bw"],
+  "requirements": [
+    "job.shared == \"none\"",
+    "job.duration > job_min_duration_seconds"
+  ],
+  "variables": [
+    {
+      "name": "mem_bw_threshold",
+      "expr": "mem_bw.limits.peak * membound_bw_threshold_factor"
+    },
+    {
+      "name": "mem_bw_pct",
+      "expr": "mem_bw.avg / mem_bw.limits.peak * 100.0"
+    }
+  ],
+  "rule": "mem_bw.avg > mem_bw_threshold",
+  "hint": "This job is memory bandwidth bound: memory bandwidth {{.mem_bw.avg}} GB/s ({{.mem_bw_pct}}% of peak) is within {{.membound_bw_threshold_factor}} of peak bandwidth. Consider improving data reuse or compute intensity."
+}

configs/tagger/jobclasses/parameters.json

@@ -0,0 +1,15 @@
+{
+  "lowcpuload_threshold_factor": 0.85,
+  "excessivecpuload_threshold_factor": 1.2,
+  "highmemoryusage_threshold_factor": 0.9,
+  "node_load_imbalance_threshold_factor": 0.1,
+  "core_load_imbalance_threshold_factor": 0.1,
+  "high_memory_load_threshold_factor": 0.9,
+  "lowgpuload_threshold_factor": 0.7,
+  "membound_bw_threshold_factor": 0.8,
+  "memory_leak_slope_threshold": 0.1,
+  "job_min_duration_seconds": 600.0,
+  "sampling_interval_seconds": 30.0,
+  "cpu_load_pre_cutoff_samples": 11.0,
+  "cpu_load_core_pre_cutoff_samples": 6.0
+}

configs/uiConfig.json

@@ -0,0 +1,45 @@
+{
+  "job-list": {
+    "use-paging": false,
+    "show-footprint":false
+  },
+  "job-view": {
+    "show-polar-plot": true,
+    "show-footprint": true,
+    "show-roofline": true,
+    "show-stat-table": true
+  },
+  "metric-config": {
+    "job-list-metrics": ["mem_bw", "flops_dp"],
+    "job-view-plot-metrics": ["mem_bw", "flops_dp"],
+    "job-view-table-metrics": ["mem_bw", "flops_dp"],
+    "clusters": [
+      {
+        "name": "test",
+        "sub-clusters": [
+          {
+            "name": "one",
+            "job-list-metrics": ["mem_used", "flops_sp"]
+          }
+        ]
+      }
+    ]
+  },
+  "node-list": {
+    "use-paging": true
+  },
+  "plot-configuration": {
+    "plots-per-row": 3,
+    "color-background": true,
+    "line-width": 3,
+    "color-scheme": [
+	"#00bfff",
+	"#0000ff",
+	"#ff00ff",
+	"#ff0000",
+	"#ff8000",
+	"#ffff00",
+	"#80ff00"
+    ]
+  }
+}

go.mod

@@ -1,90 +1,123 @@
 module github.com/ClusterCockpit/cc-backend
 
-go 1.23.5
+go 1.25.0
 
-require (
-	github.com/99designs/gqlgen v0.17.66
-	github.com/ClusterCockpit/cc-units v0.4.0
-	github.com/Masterminds/squirrel v1.5.4
-	github.com/coreos/go-oidc/v3 v3.12.0
-	github.com/go-co-op/gocron/v2 v2.16.0
-	github.com/go-ldap/ldap/v3 v3.4.10
-	github.com/go-sql-driver/mysql v1.9.0
-	github.com/golang-jwt/jwt/v5 v5.2.1
-	github.com/golang-migrate/migrate/v4 v4.18.2
-	github.com/google/gops v0.3.28
-	github.com/gorilla/handlers v1.5.2
-	github.com/gorilla/mux v1.8.1
-	github.com/gorilla/sessions v1.4.0
-	github.com/influxdata/influxdb-client-go/v2 v2.14.0
-	github.com/jmoiron/sqlx v1.4.0
-	github.com/mattn/go-sqlite3 v1.14.24
-	github.com/prometheus/client_golang v1.21.0
-	github.com/prometheus/common v0.62.0
-	github.com/qustavo/sqlhooks/v2 v2.1.0
-	github.com/santhosh-tekuri/jsonschema/v5 v5.3.1
-	github.com/swaggo/http-swagger v1.3.4
-	github.com/swaggo/swag v1.16.4
-	github.com/vektah/gqlparser/v2 v2.5.22
-	golang.org/x/crypto v0.35.0
-	golang.org/x/exp v0.0.0-20250218142911-aa4b98e5adaa
-	golang.org/x/oauth2 v0.27.0
-	golang.org/x/time v0.5.0
+tool (
+	github.com/99designs/gqlgen
+	github.com/swaggo/swag/cmd/swag
 )
 
 require (
-	filippo.io/edwards25519 v1.1.0 // indirect
-	github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect
+	github.com/99designs/gqlgen v0.17.88
+	github.com/ClusterCockpit/cc-lib/v2 v2.8.2
+	github.com/ClusterCockpit/cc-line-protocol/v2 v2.4.0
+	github.com/Masterminds/squirrel v1.5.4
+	github.com/aws/aws-sdk-go-v2 v1.41.3
+	github.com/aws/aws-sdk-go-v2/config v1.32.11
+	github.com/aws/aws-sdk-go-v2/credentials v1.19.11
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.97.0
+	github.com/coreos/go-oidc/v3 v3.17.0
+	github.com/expr-lang/expr v1.17.8
+	github.com/go-chi/chi/v5 v5.2.5
+	github.com/go-chi/cors v1.2.2
+	github.com/go-co-op/gocron/v2 v2.19.1
+	github.com/go-ldap/ldap/v3 v3.4.12
+	github.com/golang-jwt/jwt/v5 v5.3.1
+	github.com/golang-migrate/migrate/v4 v4.19.1
+	github.com/google/gops v0.3.29
+	github.com/gorilla/sessions v1.4.0
+	github.com/jmoiron/sqlx v1.4.0
+	github.com/joho/godotenv v1.5.1
+	github.com/mattn/go-sqlite3 v1.14.34
+	github.com/parquet-go/parquet-go v0.29.0
+	github.com/qustavo/sqlhooks/v2 v2.1.0
+	github.com/santhosh-tekuri/jsonschema/v5 v5.3.1
+	github.com/stretchr/testify v1.11.1
+	github.com/swaggo/http-swagger v1.3.4
+	github.com/swaggo/swag v1.16.6
+	github.com/vektah/gqlparser/v2 v2.5.32
+	golang.org/x/crypto v0.49.0
+	golang.org/x/oauth2 v0.36.0
+	golang.org/x/sync v0.20.0
+	golang.org/x/time v0.15.0
+)
+
+require (
+	github.com/Azure/go-ntlmssp v0.1.0 // indirect
 	github.com/KyleBanks/depth v1.2.1 // indirect
 	github.com/agnivade/levenshtein v1.2.1 // indirect
+	github.com/andybalholm/brotli v1.2.0 // indirect
 	github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect
-	github.com/beorn7/perks v1.0.1 // indirect
-	github.com/cespare/xxhash/v2 v2.3.0 // indirect
-	github.com/cpuguy83/go-md2man/v2 v2.0.6 // indirect
-	github.com/felixge/httpsnoop v1.0.4 // indirect
-	github.com/go-asn1-ber/asn1-ber v1.5.7 // indirect
-	github.com/go-jose/go-jose/v4 v4.0.5 // indirect
-	github.com/go-openapi/jsonpointer v0.21.0 // indirect
-	github.com/go-openapi/jsonreference v0.21.0 // indirect
-	github.com/go-openapi/spec v0.21.0 // indirect
-	github.com/go-openapi/swag v0.23.0 // indirect
-	github.com/go-viper/mapstructure/v2 v2.2.1 // indirect
+	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.6 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.19 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.19 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.19 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.5 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.20 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.6 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.11 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.19 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.19 // indirect
+	github.com/aws/aws-sdk-go-v2/service/signin v1.0.7 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.30.12 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.16 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.41.8 // indirect
+	github.com/aws/smithy-go v1.24.2 // indirect
+	github.com/cpuguy83/go-md2man/v2 v2.0.7 // indirect
+	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
+	github.com/fsnotify/fsnotify v1.9.0 // indirect
+	github.com/go-asn1-ber/asn1-ber v1.5.8-0.20250403174932-29230038a667 // indirect
+	github.com/go-jose/go-jose/v4 v4.1.3 // indirect
+	github.com/go-openapi/jsonpointer v0.22.5 // indirect
+	github.com/go-openapi/jsonreference v0.21.5 // indirect
+	github.com/go-openapi/spec v0.22.4 // indirect
+	github.com/go-openapi/swag/conv v0.25.5 // indirect
+	github.com/go-openapi/swag/jsonname v0.25.5 // indirect
+	github.com/go-openapi/swag/jsonutils v0.25.5 // indirect
+	github.com/go-openapi/swag/loading v0.25.5 // indirect
+	github.com/go-openapi/swag/stringutils v0.25.5 // indirect
+	github.com/go-openapi/swag/typeutils v0.25.5 // indirect
+	github.com/go-openapi/swag/yamlutils v0.25.5 // indirect
+	github.com/go-viper/mapstructure/v2 v2.5.0 // indirect
+	github.com/goccy/go-yaml v1.19.2 // indirect
 	github.com/google/uuid v1.6.0 // indirect
 	github.com/gorilla/securecookie v1.1.2 // indirect
 	github.com/gorilla/websocket v1.5.3 // indirect
-	github.com/hashicorp/errwrap v1.1.0 // indirect
-	github.com/hashicorp/go-multierror v1.1.1 // indirect
 	github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
+	github.com/influxdata/influxdb-client-go/v2 v2.14.0 // indirect
 	github.com/influxdata/line-protocol v0.0.0-20210922203350-b1ad95c89adf // indirect
 	github.com/jonboulle/clockwork v0.5.0 // indirect
-	github.com/josharian/intern v1.0.0 // indirect
-	github.com/jpillora/backoff v1.0.0 // indirect
-	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/klauspost/compress v1.18.4 // indirect
+	github.com/kr/pretty v0.3.1 // indirect
 	github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect
 	github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect
-	github.com/mailru/easyjson v0.9.0 // indirect
-	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
-	github.com/modern-go/reflect2 v1.0.2 // indirect
-	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
-	github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect
-	github.com/oapi-codegen/runtime v1.1.1 // indirect
-	github.com/prometheus/client_model v0.6.1 // indirect
-	github.com/prometheus/procfs v0.15.1 // indirect
+	github.com/nats-io/nats.go v1.49.0 // indirect
+	github.com/nats-io/nkeys v0.4.15 // indirect
+	github.com/nats-io/nuid v1.0.1 // indirect
+	github.com/oapi-codegen/runtime v1.2.0 // indirect
+	github.com/parquet-go/bitpack v1.0.0 // indirect
+	github.com/parquet-go/jsonlite v1.4.0 // indirect
+	github.com/pierrec/lz4/v4 v4.1.26 // indirect
+	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/robfig/cron/v3 v3.0.1 // indirect
+	github.com/rogpeppe/go-internal v1.10.0 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
-	github.com/sosodev/duration v1.3.1 // indirect
+	github.com/sosodev/duration v1.4.0 // indirect
+	github.com/stmcginnis/gofish v0.21.4 // indirect
+	github.com/stretchr/objx v0.5.2 // indirect
 	github.com/swaggo/files v1.0.1 // indirect
-	github.com/urfave/cli/v2 v2.27.5 // indirect
-	github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect
-	go.uber.org/atomic v1.11.0 // indirect
-	golang.org/x/mod v0.23.0 // indirect
-	golang.org/x/net v0.35.0 // indirect
-	golang.org/x/sync v0.11.0 // indirect
-	golang.org/x/sys v0.30.0 // indirect
-	golang.org/x/text v0.22.0 // indirect
-	golang.org/x/tools v0.30.0 // indirect
-	google.golang.org/protobuf v1.36.5 // indirect
-	gopkg.in/yaml.v2 v2.4.0 // indirect
+	github.com/twpayne/go-geom v1.6.1 // indirect
+	github.com/urfave/cli/v2 v2.27.7 // indirect
+	github.com/urfave/cli/v3 v3.7.0 // indirect
+	github.com/xrash/smetrics v0.0.0-20250705151800-55b8f293f342 // indirect
+	go.yaml.in/yaml/v2 v2.4.3 // indirect
+	go.yaml.in/yaml/v3 v3.0.4 // indirect
+	golang.org/x/mod v0.34.0 // indirect
+	golang.org/x/net v0.52.0 // indirect
+	golang.org/x/sys v0.42.0 // indirect
+	golang.org/x/text v0.35.0 // indirect
+	golang.org/x/tools v0.43.0 // indirect
+	google.golang.org/protobuf v1.36.11 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
-	sigs.k8s.io/yaml v1.4.0 // indirect
+	sigs.k8s.io/yaml v1.6.0 // indirect
 )

go.sum

@@ -1,128 +1,184 @@
 filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA=
 filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
-github.com/99designs/gqlgen v0.17.66 h1:2/SRc+h3115fCOZeTtsqrB5R5gTGm+8qCAwcrZa+CXA=
-github.com/99designs/gqlgen v0.17.66/go.mod h1:gucrb5jK5pgCKzAGuOMMVU9C8PnReecHEHd2UxLQwCg=
-github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0=
-github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
-github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 h1:mFRzDkZVAjdal+s7s0MwaRv9igoPqLRdzOLzw/8Xvq8=
-github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU=
-github.com/ClusterCockpit/cc-units v0.4.0 h1:zP5DOu99GmErW0tCDf0gcLrlWt42RQ9dpoONEOh4cI0=
-github.com/ClusterCockpit/cc-units v0.4.0/go.mod h1:3S3PAhAayS3pbgcT4q9Vn9VJw22Op51X0YimtG77zBw=
+github.com/99designs/gqlgen v0.17.88 h1:neMQDgehMwT1vYIOx/w5ZYPUU/iMNAJzRO44I5Intoc=
+github.com/99designs/gqlgen v0.17.88/go.mod h1:qeqYFEgOeSKqWedOjogPizimp2iu4E23bdPvl4jTYic=
+github.com/Azure/go-ntlmssp v0.1.0 h1:DjFo6YtWzNqNvQdrwEyr/e4nhU3vRiwenz5QX7sFz+A=
+github.com/Azure/go-ntlmssp v0.1.0/go.mod h1:NYqdhxd/8aAct/s4qSYZEerdPuH1liG2/X9DiVTbhpk=
+github.com/ClusterCockpit/cc-lib/v2 v2.8.2 h1:rCLZk8wz8yq8xBnBEdVKigvA2ngR8dPmHbEFwxxb3jw=
+github.com/ClusterCockpit/cc-lib/v2 v2.8.2/go.mod h1:FwD8vnTIbBM3ngeLNKmCvp9FoSjQZm7xnuaVxEKR23o=
+github.com/ClusterCockpit/cc-line-protocol/v2 v2.4.0 h1:hIzxgTBWcmCIHtoDKDkSCsKCOCOwUC34sFsbD2wcW0Q=
+github.com/ClusterCockpit/cc-line-protocol/v2 v2.4.0/go.mod h1:y42qUu+YFmu5fdNuUAS4VbbIKxVjxCvbVqFdpdh8ahY=
+github.com/DATA-DOG/go-sqlmock v1.5.2 h1:OcvFkGmslmlZibjAjaHm3L//6LiuBgolP7OputlJIzU=
+github.com/DATA-DOG/go-sqlmock v1.5.2/go.mod h1:88MAG/4G7SMwSE3CeA0ZKzrT5CiOU3OJ+JlNzwDqpNU=
 github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc=
 github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE=
 github.com/Masterminds/squirrel v1.5.4 h1:uUcX/aBc8O7Fg9kaISIUsHXdKuqehiXAMQTYX8afzqM=
 github.com/Masterminds/squirrel v1.5.4/go.mod h1:NNaOrjSoIDfDA40n7sr2tPNZRfjzjA400rg+riTZj10=
-github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
-github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
-github.com/PuerkitoBio/goquery v1.9.3 h1:mpJr/ikUA9/GNJB/DBZcGeFDXUtosHRyRrwh7KGdTG0=
-github.com/PuerkitoBio/goquery v1.9.3/go.mod h1:1ndLHPdTz+DyQPICCWYlYQMPl0oXZj0G6D4LCYA6u4U=
+github.com/NVIDIA/go-nvml v0.13.0-1 h1:OLX8Jq3dONuPOQPC7rndB6+iDmDakw0XTYgzMxObkEw=
+github.com/NVIDIA/go-nvml v0.13.0-1/go.mod h1:+KNA7c7gIBH7SKSJ1ntlwkfN80zdx8ovl4hrK3LmPt4=
+github.com/PuerkitoBio/goquery v1.11.0 h1:jZ7pwMQXIITcUXNH83LLk+txlaEy6NVOfTuP43xxfqw=
+github.com/PuerkitoBio/goquery v1.11.0/go.mod h1:wQHgxUOU3JGuj3oD/QFfxUdlzW6xPHfqyHre6VMY4DQ=
 github.com/RaveNoX/go-jsoncommentstrip v1.0.0/go.mod h1:78ihd09MekBnJnxpICcwzCMzGrKSKYe4AqU6PDYYpjk=
 github.com/agnivade/levenshtein v1.2.1 h1:EHBY3UOn1gwdy/VbFwgo4cxecRznFk7fKWN1KOX7eoM=
 github.com/agnivade/levenshtein v1.2.1/go.mod h1:QVVI16kDrtSuwcpd0p1+xMC6Z/VfhtCyDIjcwga4/DU=
-github.com/alexbrainman/sspi v0.0.0-20231016080023-1a75b4708caa h1:LHTHcTQiSGT7VVbI0o4wBRNQIgn917usHWOd6VAffYI=
-github.com/alexbrainman/sspi v0.0.0-20231016080023-1a75b4708caa/go.mod h1:cEWa1LVoE5KvSD9ONXsZrj0z6KqySlCCNKHlLzbqAt4=
+github.com/alecthomas/assert/v2 v2.10.0 h1:jjRCHsj6hBJhkmhznrCzoNpbA3zqy0fYiUcYZP/GkPY=
+github.com/alecthomas/assert/v2 v2.10.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k=
+github.com/alecthomas/repr v0.4.0 h1:GhI2A8MACjfegCPVq9f1FLvIBS+DrQ2KQBFZP1iFzXc=
+github.com/alecthomas/repr v0.4.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4=
+github.com/alexbrainman/sspi v0.0.0-20250919150558-7d374ff0d59e h1:4dAU9FXIyQktpoUAgOJK3OTFc/xug0PCXYCqU0FgDKI=
+github.com/alexbrainman/sspi v0.0.0-20250919150558-7d374ff0d59e/go.mod h1:cEWa1LVoE5KvSD9ONXsZrj0z6KqySlCCNKHlLzbqAt4=
 github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883 h1:bvNMNQO63//z+xNgfBlViaCIJKLlCJ6/fmUseuG0wVQ=
 github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8=
-github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss=
-github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU=
+github.com/andybalholm/brotli v1.2.0 h1:ukwgCxwYrmACq68yiUqwIWnGY0cTPox/M94sVwToPjQ=
+github.com/andybalholm/brotli v1.2.0/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY=
+github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kktS1LM=
+github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA=
+github.com/antithesishq/antithesis-sdk-go v0.5.0-default-no-op h1:Ucf+QxEKMbPogRO5guBNe5cgd9uZgfoJLOYs8WWhtjM=
+github.com/antithesishq/antithesis-sdk-go v0.5.0-default-no-op/go.mod h1:IUpT2DPAKh6i/YhSbt6Gl3v2yvUZjmKncl7U91fup7E=
 github.com/apapsch/go-jsonmerge/v2 v2.0.0 h1:axGnT1gRIfimI7gJifB699GoE/oq+F2MU7Dml6nw9rQ=
 github.com/apapsch/go-jsonmerge/v2 v2.0.0/go.mod h1:lvDnEdqiQrp0O42VQGgmlKpxL1AP2+08jFMw88y4klk=
 github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0 h1:jfIu9sQUG6Ig+0+Ap1h4unLjW6YQJpKZVmUzxsD4E/Q=
 github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0/go.mod h1:t2tdKJDJF9BV14lnkjHmOQgcvEKgtqs5a1N3LNdJhGE=
+github.com/aws/aws-sdk-go-v2 v1.41.3 h1:4kQ/fa22KjDt13QCy1+bYADvdgcxpfH18f0zP542kZA=
+github.com/aws/aws-sdk-go-v2 v1.41.3/go.mod h1:mwsPRE8ceUUpiTgF7QmQIJ7lgsKUPQOUl3o72QBrE1o=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.6 h1:N4lRUXZpZ1KVEUn6hxtco/1d2lgYhNn1fHkkl8WhlyQ=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.6/go.mod h1:lyw7GFp3qENLh7kwzf7iMzAxDn+NzjXEAGjKS2UOKqI=
+github.com/aws/aws-sdk-go-v2/config v1.32.11 h1:ftxI5sgz8jZkckuUHXfC/wMUc8u3fG1vQS0plr2F2Zs=
+github.com/aws/aws-sdk-go-v2/config v1.32.11/go.mod h1:twF11+6ps9aNRKEDimksp923o44w/Thk9+8YIlzWMmo=
+github.com/aws/aws-sdk-go-v2/credentials v1.19.11 h1:NdV8cwCcAXrCWyxArt58BrvZJ9pZ9Fhf9w6Uh5W3Uyc=
+github.com/aws/aws-sdk-go-v2/credentials v1.19.11/go.mod h1:30yY2zqkMPdrvxBqzI9xQCM+WrlrZKSOpSJEsylVU+8=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.19 h1:INUvJxmhdEbVulJYHI061k4TVuS3jzzthNvjqvVvTKM=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.19/go.mod h1:FpZN2QISLdEBWkayloda+sZjVJL+e9Gl0k1SyTgcswU=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.19 h1:/sECfyq2JTifMI2JPyZ4bdRN77zJmr6SrS1eL3augIA=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.19/go.mod h1:dMf8A5oAqr9/oxOfLkC/c2LU/uMcALP0Rgn2BD5LWn0=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.19 h1:AWeJMk33GTBf6J20XJe6qZoRSJo0WfUhsMdUKhoODXE=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.19/go.mod h1:+GWrYoaAsV7/4pNHpwh1kiNLXkKaSoppxQq9lbH8Ejw=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.8.5 h1:clHU5fm//kWS1C2HgtgWxfQbFbx4b6rx+5jzhgX9HrI=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.8.5/go.mod h1:O3h0IK87yXci+kg6flUKzJnWeziQUKciKrLjcatSNcY=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.20 h1:qi3e/dmpdONhj1RyIZdi6DKKpDXS5Lb8ftr3p7cyHJc=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.20/go.mod h1:V1K+TeJVD5JOk3D9e5tsX2KUdL7BlB+FV6cBhdobN8c=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.6 h1:XAq62tBTJP/85lFD5oqOOe7YYgWxY9LvWq8plyDvDVg=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.6/go.mod h1:x0nZssQ3qZSnIcePWLvcoFisRXJzcTVvYpAAdYX8+GI=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.11 h1:BYf7XNsJMzl4mObARUBUib+j2tf0U//JAAtTnYqvqCw=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.11/go.mod h1:aEUS4WrNk/+FxkBZZa7tVgp4pGH+kFGW40Y8rCPqt5g=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.19 h1:X1Tow7suZk9UCJHE1Iw9GMZJJl0dAnKXXP1NaSDHwmw=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.19/go.mod h1:/rARO8psX+4sfjUQXp5LLifjUt8DuATZ31WptNJTyQA=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.19 h1:JnQeStZvPHFHeyky/7LbMlyQjUa+jIBj36OlWm0pzIk=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.19/go.mod h1:HGyasyHvYdFQeJhvDHfH7HXkHh57htcJGKDZ+7z+I24=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.97.0 h1:zyKY4OxzUImu+DigelJI9o49QQv8CjREs5E1CywjtIA=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.97.0/go.mod h1:NF3JcMGOiARAss1ld3WGORCw71+4ExDD2cbbdKS5PpA=
+github.com/aws/aws-sdk-go-v2/service/signin v1.0.7 h1:Y2cAXlClHsXkkOvWZFXATr34b0hxxloeQu/pAZz2row=
+github.com/aws/aws-sdk-go-v2/service/signin v1.0.7/go.mod h1:idzZ7gmDeqeNrSPkdbtMp9qWMgcBwykA7P7Rzh5DXVU=
+github.com/aws/aws-sdk-go-v2/service/sso v1.30.12 h1:iSsvB9EtQ09YrsmIc44Heqlx5ByGErqhPK1ZQLppias=
+github.com/aws/aws-sdk-go-v2/service/sso v1.30.12/go.mod h1:fEWYKTRGoZNl8tZ77i61/ccwOMJdGxwOhWCkp6TXAr0=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.16 h1:EnUdUqRP1CNzt2DkV67tJx6XDN4xlfBFm+bzeNOQVb0=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.16/go.mod h1:Jic/xv0Rq/pFNCh3WwpH4BEqdbSAl+IyHro8LbibHD8=
+github.com/aws/aws-sdk-go-v2/service/sts v1.41.8 h1:XQTQTF75vnug2TXS8m7CVJfC2nniYPZnO1D4Np761Oo=
+github.com/aws/aws-sdk-go-v2/service/sts v1.41.8/go.mod h1:Xgx+PR1NUOjNmQY+tRMnouRp83JRM8pRMw/vCaVhPkI=
+github.com/aws/smithy-go v1.24.2 h1:FzA3bu/nt/vDvmnkg+R8Xl46gmzEDam6mZ1hzmwXFng=
+github.com/aws/smithy-go v1.24.2/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/bmatcuk/doublestar v1.1.1/go.mod h1:UD6OnuiIn0yFxxA2le/rnRU1G4RaI4UvFv1sNto9p6w=
 github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
 github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
-github.com/coreos/go-oidc/v3 v3.12.0 h1:sJk+8G2qq94rDI6ehZ71Bol3oUHy63qNYmkiSjrc/Jo=
-github.com/coreos/go-oidc/v3 v3.12.0/go.mod h1:gE3LgjOgFoHi9a4ce4/tJczr0Ai2/BoDhf0r5lltWI0=
-github.com/cpuguy83/go-md2man/v2 v2.0.6 h1:XJtiaUW6dEEqVuZiMTn1ldk455QWwEIsMIJlo5vtkx0=
-github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
+github.com/coreos/go-oidc/v3 v3.17.0 h1:hWBGaQfbi0iVviX4ibC7bk8OKT5qNr4klBaCHVNvehc=
+github.com/coreos/go-oidc/v3 v3.17.0/go.mod h1:wqPbKFrVnE90vty060SB40FCJ8fTHTxSwyXJqZH+sI8=
+github.com/cpuguy83/go-md2man/v2 v2.0.7 h1:zbFlGlXEAKlwXpmvle3d8Oe3YnkKIK4xSRTd3sHPnBo=
+github.com/cpuguy83/go-md2man/v2 v2.0.7/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
+github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
+github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/dgryski/trifles v0.0.0-20230903005119-f50d829f2e54 h1:SG7nF6SRlWhcT7cNTs5R6Hk4V2lcmLz2NsG2VnInyNo=
 github.com/dgryski/trifles v0.0.0-20230903005119-f50d829f2e54/go.mod h1:if7Fbed8SFyPtHLHbg49SI7NAdJiC5WIA09pe59rfAA=
-github.com/dhui/dktest v0.4.4 h1:+I4s6JRE1yGuqflzwqG+aIaMdgXIorCf5P98JnaAWa8=
-github.com/dhui/dktest v0.4.4/go.mod h1:4+22R4lgsdAXrDyaH4Nqx2JEz2hLp49MqQmm9HLCQhM=
-github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk=
-github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
-github.com/docker/docker v27.2.0+incompatible h1:Rk9nIVdfH3+Vz4cyI/uhbINhEZ/oLmc+CBXmH6fbNk4=
-github.com/docker/docker v27.2.0+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
-github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c=
-github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc=
-github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
-github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
-github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
-github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
-github.com/go-asn1-ber/asn1-ber v1.5.7 h1:DTX+lbVTWaTw1hQ+PbZPlnDZPEIs0SS/GCZAl535dDk=
-github.com/go-asn1-ber/asn1-ber v1.5.7/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
-github.com/go-co-op/gocron/v2 v2.16.0 h1:uqUF6WFZ4enRU45pWFNcn1xpDLc+jBOTKhPQI16Z1xs=
-github.com/go-co-op/gocron/v2 v2.16.0/go.mod h1:opexeOFy5BplhsKdA7bzY9zeYih8I8/WNJ4arTIFPVc=
-github.com/go-jose/go-jose/v4 v4.0.5 h1:M6T8+mKZl/+fNNuFHvGIzDz7BTLQPIounk/b9dw3AaE=
-github.com/go-jose/go-jose/v4 v4.0.5/go.mod h1:s3P1lRrkT8igV8D9OjyL4WRyHvjB6a4JSllnOrmmBOA=
-github.com/go-ldap/ldap/v3 v3.4.10 h1:ot/iwPOhfpNVgB1o+AVXljizWZ9JTp7YF5oeyONmcJU=
-github.com/go-ldap/ldap/v3 v3.4.10/go.mod h1:JXh4Uxgi40P6E9rdsYqpUtbW46D9UTjJ9QSwGRznplY=
-github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=
-github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
-github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
-github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
-github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ=
-github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY=
-github.com/go-openapi/jsonreference v0.21.0 h1:Rs+Y7hSXT83Jacb7kFyjn4ijOuVGSvOdF2+tg1TRrwQ=
-github.com/go-openapi/jsonreference v0.21.0/go.mod h1:LmZmgsrTkVg9LG4EaHeY8cBDslNPMo06cago5JNLkm4=
-github.com/go-openapi/spec v0.21.0 h1:LTVzPc3p/RzRnkQqLRndbAzjY0d0BCL72A6j3CdL9ZY=
-github.com/go-openapi/spec v0.21.0/go.mod h1:78u6VdPw81XU44qEWGhtr982gJ5BWg2c0I5XwVMotYk=
-github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE=
-github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ=
+github.com/expr-lang/expr v1.17.8 h1:W1loDTT+0PQf5YteHSTpju2qfUfNoBt4yw9+wOEU9VM=
+github.com/expr-lang/expr v1.17.8/go.mod h1:8/vRC7+7HBzESEqt5kKpYXxrxkr31SaO8r40VO/1IT4=
+github.com/frankban/quicktest v1.13.0 h1:yNZif1OkDfNoDfb9zZa9aXIpejNR4F23Wely0c+Qdqk=
+github.com/frankban/quicktest v1.13.0/go.mod h1:qLE0fzW0VuyUAJgPU19zByoIr0HtCHN/r/VLSOOIySU=
+github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k=
+github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
+github.com/go-asn1-ber/asn1-ber v1.5.8-0.20250403174932-29230038a667 h1:BP4M0CvQ4S3TGls2FvczZtj5Re/2ZzkV9VwqPHH/3Bo=
+github.com/go-asn1-ber/asn1-ber v1.5.8-0.20250403174932-29230038a667/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
+github.com/go-chi/chi/v5 v5.2.5 h1:Eg4myHZBjyvJmAFjFvWgrqDTXFyOzjj7YIm3L3mu6Ug=
+github.com/go-chi/chi/v5 v5.2.5/go.mod h1:X7Gx4mteadT3eDOMTsXzmI4/rwUpOwBHLpAfupzFJP0=
+github.com/go-chi/cors v1.2.2 h1:Jmey33TE+b+rB7fT8MUy1u0I4L+NARQlK6LhzKPSyQE=
+github.com/go-chi/cors v1.2.2/go.mod h1:sSbTewc+6wYHBBCW7ytsFSn836hqM7JxpglAy2Vzc58=
+github.com/go-co-op/gocron/v2 v2.19.1 h1:B4iLeA0NB/2iO3EKQ7NfKn5KsQgZfjb2fkvoZJU3yBI=
+github.com/go-co-op/gocron/v2 v2.19.1/go.mod h1:5lEiCKk1oVJV39Zg7/YG10OnaVrDAV5GGR6O0663k6U=
+github.com/go-jose/go-jose/v4 v4.1.3 h1:CVLmWDhDVRa6Mi/IgCgaopNosCaHz7zrMeF9MlZRkrs=
+github.com/go-jose/go-jose/v4 v4.1.3/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08=
+github.com/go-ldap/ldap/v3 v3.4.12 h1:1b81mv7MagXZ7+1r7cLTWmyuTqVqdwbtJSjC0DAp9s4=
+github.com/go-ldap/ldap/v3 v3.4.12/go.mod h1:+SPAGcTtOfmGsCb3h1RFiq4xpp4N636G75OEace8lNo=
+github.com/go-openapi/jsonpointer v0.22.5 h1:8on/0Yp4uTb9f4XvTrM2+1CPrV05QPZXu+rvu2o9jcA=
+github.com/go-openapi/jsonpointer v0.22.5/go.mod h1:gyUR3sCvGSWchA2sUBJGluYMbe1zazrYWIkWPjjMUY0=
+github.com/go-openapi/jsonreference v0.21.5 h1:6uCGVXU/aNF13AQNggxfysJ+5ZcU4nEAe+pJyVWRdiE=
+github.com/go-openapi/jsonreference v0.21.5/go.mod h1:u25Bw85sX4E2jzFodh1FOKMTZLcfifd1Q+iKKOUxExw=
+github.com/go-openapi/spec v0.22.4 h1:4pxGjipMKu0FzFiu/DPwN3CTBRlVM2yLf/YTWorYfDQ=
+github.com/go-openapi/spec v0.22.4/go.mod h1:WQ6Ai0VPWMZgMT4XySjlRIE6GP1bGQOtEThn3gcWLtQ=
+github.com/go-openapi/swag v0.19.15 h1:D2NRCBzS9/pEY3gP9Nl8aDqGUcPFrwG2p+CNFrLyrCM=
+github.com/go-openapi/swag/conv v0.25.5 h1:wAXBYEXJjoKwE5+vc9YHhpQOFj2JYBMF2DUi+tGu97g=
+github.com/go-openapi/swag/conv v0.25.5/go.mod h1:CuJ1eWvh1c4ORKx7unQnFGyvBbNlRKbnRyAvDvzWA4k=
+github.com/go-openapi/swag/jsonname v0.25.5 h1:8p150i44rv/Drip4vWI3kGi9+4W9TdI3US3uUYSFhSo=
+github.com/go-openapi/swag/jsonname v0.25.5/go.mod h1:jNqqikyiAK56uS7n8sLkdaNY/uq6+D2m2LANat09pKU=
+github.com/go-openapi/swag/jsonutils v0.25.5 h1:XUZF8awQr75MXeC+/iaw5usY/iM7nXPDwdG3Jbl9vYo=
+github.com/go-openapi/swag/jsonutils v0.25.5/go.mod h1:48FXUaz8YsDAA9s5AnaUvAmry1UcLcNVWUjY42XkrN4=
+github.com/go-openapi/swag/jsonutils/fixtures_test v0.25.5 h1:SX6sE4FrGb4sEnnxbFL/25yZBb5Hcg1inLeErd86Y1U=
+github.com/go-openapi/swag/jsonutils/fixtures_test v0.25.5/go.mod h1:/2KvOTrKWjVA5Xli3DZWdMCZDzz3uV/T7bXwrKWPquo=
+github.com/go-openapi/swag/loading v0.25.5 h1:odQ/umlIZ1ZVRteI6ckSrvP6e2w9UTF5qgNdemJHjuU=
+github.com/go-openapi/swag/loading v0.25.5/go.mod h1:I8A8RaaQ4DApxhPSWLNYWh9NvmX2YKMoB9nwvv6oW6g=
+github.com/go-openapi/swag/stringutils v0.25.5 h1:NVkoDOA8YBgtAR/zvCx5rhJKtZF3IzXcDdwOsYzrB6M=
+github.com/go-openapi/swag/stringutils v0.25.5/go.mod h1:PKK8EZdu4QJq8iezt17HM8RXnLAzY7gW0O1KKarrZII=
+github.com/go-openapi/swag/typeutils v0.25.5 h1:EFJ+PCga2HfHGdo8s8VJXEVbeXRCYwzzr9u4rJk7L7E=
+github.com/go-openapi/swag/typeutils v0.25.5/go.mod h1:itmFmScAYE1bSD8C4rS0W+0InZUBrB2xSPbWt6DLGuc=
+github.com/go-openapi/swag/yamlutils v0.25.5 h1:kASCIS+oIeoc55j28T4o8KwlV2S4ZLPT6G0iq2SSbVQ=
+github.com/go-openapi/swag/yamlutils v0.25.5/go.mod h1:Gek1/SjjfbYvM+Iq4QGwa/2lEXde9n2j4a3wI3pNuOQ=
+github.com/go-openapi/testify/enable/yaml/v2 v2.4.0 h1:7SgOMTvJkM8yWrQlU8Jm18VeDPuAvB/xWrdxFJkoFag=
+github.com/go-openapi/testify/enable/yaml/v2 v2.4.0/go.mod h1:14iV8jyyQlinc9StD7w1xVPW3CO3q1Gj04Jy//Kw4VM=
+github.com/go-openapi/testify/v2 v2.4.0 h1:8nsPrHVCWkQ4p8h1EsRVymA2XABB4OT40gcvAu+voFM=
+github.com/go-openapi/testify/v2 v2.4.0/go.mod h1:HCPmvFFnheKK2BuwSA0TbbdxJ3I16pjwMkYkP4Ywn54=
 github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
+github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y=
 github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg=
-github.com/go-sql-driver/mysql v1.9.0 h1:Y0zIbQXhQKmQgTp44Y1dp3wTXcn804QoTptLZT1vtvo=
-github.com/go-sql-driver/mysql v1.9.0/go.mod h1:pDetrLJeA3oMujJuvXc8RJoasr589B6A9fwzD3QMrqw=
-github.com/go-viper/mapstructure/v2 v2.2.1 h1:ZAaOCxANMuZx5RCeg0mBdEZk7DZasvvZIxtHqx8aGss=
-github.com/go-viper/mapstructure/v2 v2.2.1/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
-github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
-github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
-github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk=
-github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
-github.com/golang-migrate/migrate/v4 v4.18.2 h1:2VSCMz7x7mjyTXx3m2zPokOY82LTRgxK1yQYKo6wWQ8=
-github.com/golang-migrate/migrate/v4 v4.18.2/go.mod h1:2CM6tJvn2kqPXwnXO/d3rAQYiyoIm180VsO8PRX6Rpk=
-github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
-github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
-github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
-github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/go-viper/mapstructure/v2 v2.5.0 h1:vM5IJoUAy3d7zRSVtIwQgBj7BiWtMPfmPEgAXnvj1Ro=
+github.com/go-viper/mapstructure/v2 v2.5.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
+github.com/goccy/go-yaml v1.19.2 h1:PmFC1S6h8ljIz6gMRBopkjP1TVT7xuwrButHID66PoM=
+github.com/goccy/go-yaml v1.19.2/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA=
+github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY=
+github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE=
+github.com/golang-migrate/migrate/v4 v4.19.1 h1:OCyb44lFuQfYXYLx1SCxPZQGU7mcaZ7gH9yH4jSFbBA=
+github.com/golang-migrate/migrate/v4 v4.19.1/go.mod h1:CTcgfjxhaUtsLipnLoQRWCrjYXycRz/g5+RWDuYgPrE=
+github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
+github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
+github.com/google/go-tpm v0.9.7 h1:u89J4tUUeDTlH8xxC3CTW7OHZjbjKoHdQ9W7gCUhtxA=
+github.com/google/go-tpm v0.9.7/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY=
 github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=
 github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
-github.com/google/gops v0.3.28 h1:2Xr57tqKAmQYRAfG12E+yLcoa2Y42UJo2lOrUFL9ark=
-github.com/google/gops v0.3.28/go.mod h1:6f6+Nl8LcHrzJwi8+p0ii+vmBFSlB4f8cOOkTJ7sk4c=
+github.com/google/gops v0.3.29 h1:n98J2qSOK1NJvRjdLDcjgDryjpIBGhbaqph1mXKL0rY=
+github.com/google/gops v0.3.29/go.mod h1:8N3jZftuPazvUwtYY/ncG4iPrjp15ysNKLfq+QQPiwc=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/gorilla/handlers v1.5.2 h1:cLTUSsNkgcwhgRqvCNmdbRWG0A3N4F+M2nWKdScwyEE=
-github.com/gorilla/handlers v1.5.2/go.mod h1:dX+xVpaxdSw+q0Qek8SSsl3dfMk3jNddUkMzo0GtH0w=
 github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY=
 github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ=
-github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4=
 github.com/gorilla/securecookie v1.1.2 h1:YCIWL56dvtr73r6715mJs5ZvhtnY73hBvEF8kXD8ePA=
 github.com/gorilla/securecookie v1.1.2/go.mod h1:NfCASbcHqRSY+3a8tlWJwsQap2VX5pwzwo4h3eOamfo=
-github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM=
 github.com/gorilla/sessions v1.4.0 h1:kpIYOp/oi6MG/p5PgxApU8srsSw9tuFbt46Lt7auzqQ=
 github.com/gorilla/sessions v1.4.0/go.mod h1:FLWm50oby91+hl7p/wRxDth9bWSuk0qVL2emc7lT5ik=
 github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg=
 github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
-github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
-github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
-github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
-github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
-github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
-github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
 github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8=
 github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
 github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
 github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
+github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=
+github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg=
 github.com/influxdata/influxdb-client-go/v2 v2.14.0 h1:AjbBfJuq+QoaXNcrova8smSjwJdUHnwvfjMF71M1iI4=
 github.com/influxdata/influxdb-client-go/v2 v2.14.0/go.mod h1:Ahpm3QXKMJslpXl3IftVLVezreAUtBOTZssDrjZEFHI=
 github.com/influxdata/line-protocol v0.0.0-20210922203350-b1ad95c89adf h1:7JTmneyiNEwVBOHSjoMxiWAqB992atOeepeFYegn5RU=
 github.com/influxdata/line-protocol v0.0.0-20210922203350-b1ad95c89adf/go.mod h1:xaLFMmpvUxqXtVkUJfg9QmT88cDaCJ3ZKgdZ78oO8Qo=
+github.com/influxdata/line-protocol-corpus v0.0.0-20210922080147-aa28ccfb8937 h1:MHJNQ+p99hFATQm6ORoLmpUCF7ovjwEFshs/NHzAbig=
+github.com/influxdata/line-protocol-corpus v0.0.0-20210922080147-aa28ccfb8937/go.mod h1:BKR9c0uHSmRgM/se9JhFHtTT7JTO67X23MtKMHtZcpo=
 github.com/jcmturner/aescts/v2 v2.0.0 h1:9YKLH6ey7H4eDBXW8khjYslgyqG2xZikXP0EQFKrle8=
 github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs=
 github.com/jcmturner/dnsutils/v2 v2.0.0 h1:lltnkeZGL0wILNvrNiVCR6Ro5PGU/SeBvVO/8c/iPbo=
@@ -137,17 +193,13 @@ github.com/jcmturner/rpc/v2 v2.0.3 h1:7FXXj8Ti1IaVFpSAziCZWNzbNuZmnvw/i6CqLNdWfZ
 github.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc=
 github.com/jmoiron/sqlx v1.4.0 h1:1PLqN7S1UYp5t4SrVVnt4nUVNemrDAtxlulVe+Qgm3o=
 github.com/jmoiron/sqlx v1.4.0/go.mod h1:ZrZ7UsYB/weZdl2Bxg6jCRO9c3YHl8r3ahlKmRT4JLY=
+github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
+github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
 github.com/jonboulle/clockwork v0.5.0 h1:Hyh9A8u51kptdkR+cqRpT1EebBwTn1oK9YfGYbdFz6I=
 github.com/jonboulle/clockwork v0.5.0/go.mod h1:3mZlmanh0g2NDKO5TWZVJAfofYk64M7XN3SzBPjZF60=
-github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
-github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
-github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA=
-github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
-github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
-github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
 github.com/juju/gnuflag v0.0.0-20171113085948-2ce1bb71843d/go.mod h1:2PavIy+JPciBPrBUjwbNvtwB6RQlve+hkpll6QSNmOE=
-github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc=
-github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0=
+github.com/klauspost/compress v1.18.4 h1:RPhnKRAQ4Fh8zU2FY/6ZFDwTVTxgJ/EMydqSTzE9a2c=
+github.com/klauspost/compress v1.18.4/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
 github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
@@ -159,193 +211,149 @@ github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0/go.mod h1:vmVJ0l/dxyfGW6Fm
 github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
 github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
 github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
-github.com/mailru/easyjson v0.9.0 h1:PrnmzHw7262yW8sTBwxi1PdJA3Iw/EKBa8psRf7d9a4=
-github.com/mailru/easyjson v0.9.0/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUto+XFtTU=
 github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
 github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
-github.com/mattn/go-sqlite3 v1.14.24 h1:tpSp2G2KyMnnQu99ngJ47EIkWVmliIizyZBfPrBWDRM=
-github.com/mattn/go-sqlite3 v1.14.24/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
-github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0=
-github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo=
-github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0=
-github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y=
-github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
-github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
-github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
-github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
-github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
-github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A=
-github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc=
+github.com/mattn/go-sqlite3 v1.14.34 h1:3NtcvcUnFBPsuRcno8pUtupspG/GM+9nZ88zgJcp6Zk=
+github.com/mattn/go-sqlite3 v1.14.34/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
+github.com/minio/highwayhash v1.0.4-0.20251030100505-070ab1a87a76 h1:KGuD/pM2JpL9FAYvBrnBBeENKZNh6eNtjqytV6TYjnk=
+github.com/minio/highwayhash v1.0.4-0.20251030100505-070ab1a87a76/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
-github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU=
-github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
-github.com/oapi-codegen/runtime v1.1.1 h1:EXLHh0DXIJnWhdRPN2w4MXAzFyE4CskzhNLUmtpMYro=
-github.com/oapi-codegen/runtime v1.1.1/go.mod h1:SK9X900oXmPWilYR5/WKPzt3Kqxn/uS/+lbpREv+eCg=
-github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
-github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
-github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug=
-github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM=
+github.com/nats-io/jwt/v2 v2.8.0 h1:K7uzyz50+yGZDO5o772eRE7atlcSEENpL7P+b74JV1g=
+github.com/nats-io/jwt/v2 v2.8.0/go.mod h1:me11pOkwObtcBNR8AiMrUbtVOUGkqYjMQZ6jnSdVUIA=
+github.com/nats-io/nats-server/v2 v2.12.3 h1:KRv+1n7lddMVgkJPQer+pt36TcO0ENxjilBmeWdjcHs=
+github.com/nats-io/nats-server/v2 v2.12.3/go.mod h1:MQXjG9WjyXKz9koWzUc3jYUMKD8x3CLmTNy91IQQz3Y=
+github.com/nats-io/nats.go v1.49.0 h1:yh/WvY59gXqYpgl33ZI+XoVPKyut/IcEaqtsiuTJpoE=
+github.com/nats-io/nats.go v1.49.0/go.mod h1:fDCn3mN5cY8HooHwE2ukiLb4p4G4ImmzvXyJt+tGwdw=
+github.com/nats-io/nkeys v0.4.15 h1:JACV5jRVO9V856KOapQ7x+EY8Jo3qw1vJt/9Jpwzkk4=
+github.com/nats-io/nkeys v0.4.15/go.mod h1:CpMchTXC9fxA5zrMo4KpySxNjiDVvr8ANOSZdiNfUrs=
+github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
+github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
+github.com/oapi-codegen/runtime v1.2.0 h1:RvKc1CVS1QeKSNzO97FBQbSMZyQ8s6rZd+LpmzwHMP4=
+github.com/oapi-codegen/runtime v1.2.0/go.mod h1:Y7ZhmmlE8ikZOmuHRRndiIm7nf3xcVv+YMweKgG1DT0=
 github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
-github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
-github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
-github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/parquet-go/bitpack v1.0.0 h1:AUqzlKzPPXf2bCdjfj4sTeacrUwsT7NlcYDMUQxPcQA=
+github.com/parquet-go/bitpack v1.0.0/go.mod h1:XnVk9TH+O40eOOmvpAVZ7K2ocQFrQwysLMnc6M/8lgs=
+github.com/parquet-go/jsonlite v1.4.0 h1:RTG7prqfO0HD5egejU8MUDBN8oToMj55cgSV1I0zNW4=
+github.com/parquet-go/jsonlite v1.4.0/go.mod h1:nDjpkpL4EOtqs6NQugUsi0Rleq9sW/OtC1NnZEnxzF0=
+github.com/parquet-go/parquet-go v0.29.0 h1:xXlPtFVR51jpSVzf+cgHnNIcb7Xet+iuvkbe0HIm90Y=
+github.com/parquet-go/parquet-go v0.29.0/go.mod h1:navtkAYr2LGoJVp141oXPlO/sxLvaOe3la2JEoD8+rg=
+github.com/pierrec/lz4/v4 v4.1.26 h1:GrpZw1gZttORinvzBdXPUXATeqlJjqUG/D87TKMnhjY=
+github.com/pierrec/lz4/v4 v4.1.26/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4=
+github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/prometheus/client_golang v1.21.0 h1:DIsaGmiaBkSangBgMtWdNfxbMNdku5IK6iNhrEqWvdA=
-github.com/prometheus/client_golang v1.21.0/go.mod h1:U9NM32ykUErtVBxdvD3zfi+EuFkkaBvMb09mIfe0Zgg=
-github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E=
-github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY=
-github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ2Io=
-github.com/prometheus/common v0.62.0/go.mod h1:vyBcEuLSvWos9B1+CyL7JZ2up+uFzXhkqml0W5zIY1I=
-github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc=
-github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk=
+github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
+github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o=
+github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg=
+github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk=
+github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE=
+github.com/prometheus/common v0.67.5 h1:pIgK94WWlQt1WLwAC5j2ynLaBRDiinoAb86HZHTUGI4=
+github.com/prometheus/common v0.67.5/go.mod h1:SjE/0MzDEEAyrdr5Gqc6G+sXI67maCxzaT3A2+HqjUw=
+github.com/prometheus/procfs v0.20.0 h1:AA7aCvjxwAquZAlonN7888f2u4IN8WVeFgBi4k82M4Q=
+github.com/prometheus/procfs v0.20.0/go.mod h1:o9EMBZGRyvDrSPH1RqdxhojkuXstoe4UlK79eF5TGGo=
 github.com/qustavo/sqlhooks/v2 v2.1.0 h1:54yBemHnGHp/7xgT+pxwmIlMSDNYKx5JW5dfRAiCZi0=
 github.com/qustavo/sqlhooks/v2 v2.1.0/go.mod h1:aMREyKo7fOKTwiLuWPsaHRXEmtqG4yREztO0idF83AU=
 github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
 github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
-github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8=
-github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4=
+github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
+github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
+github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
 github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 h1:lZUw3E0/J3roVtGQ+SCrUrg3ON6NgVqpn3+iol9aGu4=
 github.com/santhosh-tekuri/jsonschema/v5 v5.3.1/go.mod h1:uToXkOrWAZ6/Oc07xWQrPOhJotwFIyu2bBVN41fcDUY=
 github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8=
 github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I=
-github.com/sosodev/duration v1.3.1 h1:qtHBDMQ6lvMQsL15g4aopM4HEfOaYuhWBw3NPTtlqq4=
-github.com/sosodev/duration v1.3.1/go.mod h1:RQIBBX0+fMLc/D9+Jb/fwvVmo0eZvDDEERAikUR6SDg=
+github.com/sosodev/duration v1.4.0 h1:35ed0KiVFriGHHzZZJaZLgmTEEICIyt8Sx0RQfj9IjE=
+github.com/sosodev/duration v1.4.0/go.mod h1:RQIBBX0+fMLc/D9+Jb/fwvVmo0eZvDDEERAikUR6SDg=
 github.com/spkg/bom v0.0.0-20160624110644-59b7046e48ad/go.mod h1:qLr4V1qq6nMqFKkMo8ZTx3f+BZEkzsRUY10Xsm2mwU0=
+github.com/stmcginnis/gofish v0.21.4 h1:daexK8sh31CgeSMkPUNs21HWHHA9ecCPJPyLCTxukCg=
+github.com/stmcginnis/gofish v0.21.4/go.mod h1:PzF5i8ecRG9A2ol8XT64npKUunyraJ+7t0kYMpQAtqU=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
-github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
+github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
+github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
-github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
-github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
-github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
-github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
+github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
 github.com/swaggo/files v1.0.1 h1:J1bVJ4XHZNq0I46UU90611i9/YzdrF7x92oX1ig5IdE=
 github.com/swaggo/files v1.0.1/go.mod h1:0qXmMNH6sXNf+73t65aKeB+ApmgxdnkQzVTAj2uaMUg=
 github.com/swaggo/http-swagger v1.3.4 h1:q7t/XLx0n15H1Q9/tk3Y9L4n210XzJF5WtnDX64a5ww=
 github.com/swaggo/http-swagger v1.3.4/go.mod h1:9dAh0unqMBAlbp1uE2Uc2mQTxNMU/ha4UbucIg1MFkQ=
-github.com/swaggo/swag v1.16.4 h1:clWJtd9LStiG3VeijiCfOVODP6VpHtKdQy9ELFG3s1A=
-github.com/swaggo/swag v1.16.4/go.mod h1:VBsHJRsDvfYvqoiMKnsdwhNV9LEMHgEDZcyVYX0sxPg=
-github.com/urfave/cli/v2 v2.27.5 h1:WoHEJLdsXr6dDWoJgMq/CboDmyY/8HMMH1fTECbih+w=
-github.com/urfave/cli/v2 v2.27.5/go.mod h1:3Sevf16NykTbInEnD0yKkjDAeZDS0A6bzhBH5hrMvTQ=
-github.com/vektah/gqlparser/v2 v2.5.22 h1:yaaeJ0fu+nv1vUMW0Hl+aS1eiv1vMfapBNjpffAda1I=
-github.com/vektah/gqlparser/v2 v2.5.22/go.mod h1:xMl+ta8a5M1Yo1A1Iwt/k7gSpscwSnHZdw7tfhEGfTM=
-github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGCjxCBTO/36wtF6j2nSip77qHd4x4=
-github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM=
+github.com/swaggo/swag v1.16.6 h1:qBNcx53ZaX+M5dxVyTrgQ0PJ/ACK+NzhwcbieTt+9yI=
+github.com/swaggo/swag v1.16.6/go.mod h1:ngP2etMK5a0P3QBizic5MEwpRmluJZPHjXcMoj4Xesg=
+github.com/twpayne/go-geom v1.6.1 h1:iLE+Opv0Ihm/ABIcvQFGIiFBXd76oBIar9drAwHFhR4=
+github.com/twpayne/go-geom v1.6.1/go.mod h1:Kr+Nly6BswFsKM5sd31YaoWS5PeDDH2NftJTK7Gd028=
+github.com/urfave/cli/v2 v2.27.7 h1:bH59vdhbjLv3LAvIu6gd0usJHgoTTPhCFib8qqOwXYU=
+github.com/urfave/cli/v2 v2.27.7/go.mod h1:CyNAG/xg+iAOg0N4MPGZqVmv2rCoP267496AOXUZjA4=
+github.com/urfave/cli/v3 v3.7.0 h1:AGSnbUyjtLiM+WJUb4dzXKldl/gL+F8OwmRDtVr6g2U=
+github.com/urfave/cli/v3 v3.7.0/go.mod h1:ysVLtOEmg2tOy6PknnYVhDoouyC/6N42TMeoMzskhso=
+github.com/vektah/gqlparser/v2 v2.5.32 h1:k9QPJd4sEDTL+qB4ncPLflqTJ3MmjB9SrVzJrawpFSc=
+github.com/vektah/gqlparser/v2 v2.5.32/go.mod h1:c1I28gSOVNzlfc4WuDlqU7voQnsqI6OG2amkBAFmgts=
+github.com/xrash/smetrics v0.0.0-20250705151800-55b8f293f342 h1:FnBeRrxr7OU4VvAzt5X7s6266i6cSVkkFPS0TuXWbIg=
+github.com/xrash/smetrics v0.0.0-20250705151800-55b8f293f342/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM=
+github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU=
+github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
-go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 h1:TT4fX+nBOA/+LUkobKGW1ydGcn+G3vRw9+g5HwCphpk=
-go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0/go.mod h1:L7UH0GbB0p47T4Rri3uHjbpCFYrVrwc1I25QhNPiGK8=
-go.opentelemetry.io/otel v1.29.0 h1:PdomN/Al4q/lN6iBJEN3AwPvUiHPMlt93c8bqTG5Llw=
-go.opentelemetry.io/otel v1.29.0/go.mod h1:N/WtXPs1CNCUEx+Agz5uouwCba+i+bJGFicT8SR4NP8=
-go.opentelemetry.io/otel/metric v1.29.0 h1:vPf/HFWTNkPu1aYeIsc98l4ktOQaL6LeSoeV2g+8YLc=
-go.opentelemetry.io/otel/metric v1.29.0/go.mod h1:auu/QWieFVWx+DmQOUMgj0F8LHWdgalxXqvp7BII/W8=
-go.opentelemetry.io/otel/trace v1.29.0 h1:J/8ZNK4XgR7a21DZUAsbF8pZ5Jcw1VhACmnYt39JTi4=
-go.opentelemetry.io/otel/trace v1.29.0/go.mod h1:eHl3w0sp3paPkYstJOmAimxhiFXPg+MMTlEh3nsQgWQ=
-go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE=
-go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
 go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
 go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
+go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0=
+go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8=
+go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
+go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
-golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58=
-golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
-golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
-golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
-golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
-golang.org/x/crypto v0.35.0 h1:b15kiHdrGCHrP6LvwaQ3c03kgNhhiMgvlhxHQhmg2Xs=
-golang.org/x/crypto v0.35.0/go.mod h1:dy7dXNW32cAb/6/PRuTNsix8T+vJAqvuIy5Bli/x0YQ=
-golang.org/x/exp v0.0.0-20250218142911-aa4b98e5adaa h1:t2QcU6V556bFjYgu4L6C+6VrCPyJZ+eyRsABUPs1mz4=
-golang.org/x/exp v0.0.0-20250218142911-aa4b98e5adaa/go.mod h1:BHOTPb3L19zxehTsLoJXVaTktb06DFgmdW6Wb9s8jqk=
+golang.org/x/crypto v0.49.0 h1:+Ng2ULVvLHnJ/ZFEq4KdcDd/cfjrrjjNSXNzxg0Y4U4=
+golang.org/x/crypto v0.49.0/go.mod h1:ErX4dUh2UM+CFYiXZRTcMpEcN8b/1gxEuv3nODoYtCA=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
-golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
-golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
-golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
-golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
-golang.org/x/mod v0.23.0 h1:Zb7khfcRGKk+kqfxFaP5tZqCnDZMjC5VtUBs87Hr6QM=
-golang.org/x/mod v0.23.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY=
+golang.org/x/mod v0.34.0 h1:xIHgNUUnW6sYkcM5Jleh05DvLOtwc6RitGHbDk4akRI=
+golang.org/x/mod v0.34.0/go.mod h1:ykgH52iCZe79kzLLMhyCUzhMci+nQj+0XkbXpNYtVjY=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
-golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
-golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
-golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
-golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
-golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
-golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
-golang.org/x/net v0.35.0 h1:T5GQRQb2y08kTAByq9L4/bz8cipCdA8FbRTXewonqY8=
-golang.org/x/net v0.35.0/go.mod h1:EglIi67kWsHKlRzzVMUD93VMSWGFOMSZgxFjparz1Qk=
-golang.org/x/oauth2 v0.27.0 h1:da9Vo7/tDv5RH/7nZDz1eMGS/q1Vv1N/7FCrBhI9I3M=
-golang.org/x/oauth2 v0.27.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT8=
+golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0=
+golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw=
+golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs=
+golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
-golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
-golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
-golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
-golang.org/x/sync v0.11.0 h1:GGz8+XQP4FvTTrjZPzNKTMFtSXH80RAzG+5ghFPgK9w=
-golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
+golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc=
-golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
+golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo=
+golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
-golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
-golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU=
-golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
-golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
-golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
-golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
-golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
-golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
-golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
-golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
-golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM=
-golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY=
-golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=
-golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
+golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8=
+golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA=
+golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
+golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
-golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
-golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
-golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
-golang.org/x/tools v0.30.0 h1:BgcpHewrV5AUp2G9MebG4XPFI1E2W41zU1SaqVA9vJY=
-golang.org/x/tools v0.30.0/go.mod h1:c347cR/OJfw5TI+GfX7RUPNMdDRRbjvYTS0jPyvsVtY=
+golang.org/x/tools v0.43.0 h1:12BdW9CeB3Z+J/I/wj34VMl8X+fEXBxVR90JeMX5E7s=
+golang.org/x/tools v0.43.0/go.mod h1:uHkMso649BX2cZK6+RpuIPXS3ho2hZo4FVwfoy1vIk0=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM=
-google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE=
+google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=
+google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
-gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
-gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E=
-sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY=
+sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs=
+sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4=

gqlgen.yml

@@ -32,6 +32,7 @@ resolver:
 autobind:
   - "github.com/99designs/gqlgen/graphql/introspection"
   - "github.com/ClusterCockpit/cc-backend/internal/graph/model"
+  - "github.com/ClusterCockpit/cc-backend/internal/config"
 
 # This section declares type mapping between the GraphQL and go type systems
 #
@@ -51,61 +52,51 @@ models:
       - github.com/99designs/gqlgen/graphql.Int64
       - github.com/99designs/gqlgen/graphql.Int32
   Job:
-    model: "github.com/ClusterCockpit/cc-backend/pkg/schema.Job"
+    model: "github.com/ClusterCockpit/cc-lib/v2/schema.Job"
     fields:
       tags:
         resolver: true
       metaData:
         resolver: true
   Cluster:
-    model: "github.com/ClusterCockpit/cc-backend/pkg/schema.Cluster"
+    model: "github.com/ClusterCockpit/cc-lib/v2/schema.Cluster"
     fields:
       partitions:
         resolver: true
-  NullableFloat:
-    { model: "github.com/ClusterCockpit/cc-backend/pkg/schema.Float" }
-  MetricScope:
-    { model: "github.com/ClusterCockpit/cc-backend/pkg/schema.MetricScope" }
-  MetricValue:
-    { model: "github.com/ClusterCockpit/cc-backend/pkg/schema.MetricValue" }
+  # Node:
+  #   model: "github.com/ClusterCockpit/cc-lib/v2/schema.Node"
+  #   fields:
+  #     metaData:
+  #       resolver: true
+  NullableFloat: { model: "github.com/ClusterCockpit/cc-lib/v2/schema.Float" }
+  MetricScope: { model: "github.com/ClusterCockpit/cc-lib/v2/schema.MetricScope" }
+  MetricValue: { model: "github.com/ClusterCockpit/cc-lib/v2/schema.MetricValue" }
   JobStatistics:
-    { model: "github.com/ClusterCockpit/cc-backend/pkg/schema.JobStatistics" }
+    { model: "github.com/ClusterCockpit/cc-lib/v2/schema.JobStatistics" }
   GlobalMetricListItem:
-    {
-      model: "github.com/ClusterCockpit/cc-backend/pkg/schema.GlobalMetricListItem",
-    }
+    { model: "github.com/ClusterCockpit/cc-lib/v2/schema.GlobalMetricListItem" }
   ClusterSupport:
-    { model: "github.com/ClusterCockpit/cc-backend/pkg/schema.ClusterSupport" }
-  Tag: { model: "github.com/ClusterCockpit/cc-backend/pkg/schema.Tag" }
-  Resource:
-    { model: "github.com/ClusterCockpit/cc-backend/pkg/schema.Resource" }
-  JobState:
-    { model: "github.com/ClusterCockpit/cc-backend/pkg/schema.JobState" }
-  TimeRange:
-    { model: "github.com/ClusterCockpit/cc-backend/pkg/schema.TimeRange" }
-  IntRange:
-    { model: "github.com/ClusterCockpit/cc-backend/pkg/schema.IntRange" }
-  JobMetric:
-    { model: "github.com/ClusterCockpit/cc-backend/pkg/schema.JobMetric" }
-  Series: { model: "github.com/ClusterCockpit/cc-backend/pkg/schema.Series" }
+    { model: "github.com/ClusterCockpit/cc-lib/v2/schema.ClusterSupport" }
+  Tag: { model: "github.com/ClusterCockpit/cc-lib/v2/schema.Tag" }
+  Resource: { model: "github.com/ClusterCockpit/cc-lib/v2/schema.Resource" }
+  JobState: { model: "github.com/ClusterCockpit/cc-lib/v2/schema.JobState" }
+  Node: { model: "github.com/ClusterCockpit/cc-lib/v2/schema.Node" }
+  SchedulerState:
+    { model: "github.com/ClusterCockpit/cc-lib/v2/schema.SchedulerState" }
+  HealthState:
+    { model: "github.com/ClusterCockpit/cc-lib/v2/schema.MonitoringState" }
+  JobMetric: { model: "github.com/ClusterCockpit/cc-lib/v2/schema.JobMetric" }
+  Series: { model: "github.com/ClusterCockpit/cc-lib/v2/schema.Series" }
   MetricStatistics:
-    {
-      model: "github.com/ClusterCockpit/cc-backend/pkg/schema.MetricStatistics",
-    }
+    { model: "github.com/ClusterCockpit/cc-lib/v2/schema.MetricStatistics" }
   MetricConfig:
-    { model: "github.com/ClusterCockpit/cc-backend/pkg/schema.MetricConfig" }
+    { model: "github.com/ClusterCockpit/cc-lib/v2/schema.MetricConfig" }
   SubClusterConfig:
-    {
-      model: "github.com/ClusterCockpit/cc-backend/pkg/schema.SubClusterConfig",
-    }
-  Accelerator:
-    { model: "github.com/ClusterCockpit/cc-backend/pkg/schema.Accelerator" }
-  Topology:
-    { model: "github.com/ClusterCockpit/cc-backend/pkg/schema.Topology" }
+    { model: "github.com/ClusterCockpit/cc-lib/v2/schema.SubClusterConfig" }
+  Accelerator: { model: "github.com/ClusterCockpit/cc-lib/v2/schema.Accelerator" }
+  Topology: { model: "github.com/ClusterCockpit/cc-lib/v2/schema.Topology" }
   FilterRanges:
-    { model: "github.com/ClusterCockpit/cc-backend/pkg/schema.FilterRanges" }
-  SubCluster:
-    { model: "github.com/ClusterCockpit/cc-backend/pkg/schema.SubCluster" }
-  StatsSeries:
-    { model: "github.com/ClusterCockpit/cc-backend/pkg/schema.StatsSeries" }
-  Unit: { model: "github.com/ClusterCockpit/cc-backend/pkg/schema.Unit" }
+    { model: "github.com/ClusterCockpit/cc-lib/v2/schema.FilterRanges" }
+  SubCluster: { model: "github.com/ClusterCockpit/cc-lib/v2/schema.SubCluster" }
+  StatsSeries: { model: "github.com/ClusterCockpit/cc-lib/v2/schema.StatsSeries" }
+  Unit: { model: "github.com/ClusterCockpit/cc-lib/v2/schema.Unit" }

init/clustercockpit.service

@@ -3,7 +3,7 @@ Description=ClusterCockpit Web Server
 Documentation=https://github.com/ClusterCockpit/cc-backend
 Wants=network-online.target
 After=network-online.target
-After=mariadb.service mysql.service
+# Database is file-based SQLite - no service dependency required
 
 [Service]
 WorkingDirectory=/opt/monitoring/cc-backend
@@ -12,7 +12,7 @@ NotifyAccess=all
 Restart=on-failure
 RestartSec=30
 TimeoutStopSec=100
-ExecStart=/opt/monitoring/cc-backend/cc-backend --config ./config.json
+ExecStart=/opt/monitoring/cc-backend/cc-backend --config ./config.json --server
 
 [Install]
 WantedBy=multi-user.target

internal/api/api_test.go

@@ -1,5 +1,5 @@
 // Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved.
+// All rights reserved. This file is part of cc-backend.
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 package api_test
@@ -23,41 +23,47 @@ import (
 	"github.com/ClusterCockpit/cc-backend/internal/auth"
 	"github.com/ClusterCockpit/cc-backend/internal/config"
 	"github.com/ClusterCockpit/cc-backend/internal/graph"
-	"github.com/ClusterCockpit/cc-backend/internal/metricDataDispatcher"
-	"github.com/ClusterCockpit/cc-backend/internal/metricdata"
+	"github.com/ClusterCockpit/cc-backend/internal/metricdispatch"
 	"github.com/ClusterCockpit/cc-backend/internal/repository"
 	"github.com/ClusterCockpit/cc-backend/pkg/archive"
-	"github.com/ClusterCockpit/cc-backend/pkg/log"
-	"github.com/ClusterCockpit/cc-backend/pkg/schema"
-	"github.com/gorilla/mux"
+	"github.com/ClusterCockpit/cc-backend/pkg/metricstore"
+	ccconf "github.com/ClusterCockpit/cc-lib/v2/ccConfig"
+	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
+	"github.com/ClusterCockpit/cc-lib/v2/schema"
+	"github.com/go-chi/chi/v5"
 
 	_ "github.com/mattn/go-sqlite3"
 )
 
-func setup(t *testing.T) *api.RestApi {
+func setup(t *testing.T) *api.RestAPI {
+	repository.ResetConnection()
+
 	const testconfig = `{
-	"addr":            "0.0.0.0:8080",
-	"validate": false,
-	"archive": {
-		"kind": "file",
-		"path": "./var/job-archive"
-	},
-    "jwts": {
-        "max-age": "2m"
+	"main": {
+	   "addr":            "0.0.0.0:8080",
+	   "validate": false,
+     "api-allowed-ips": [
+       "*"
+      ]
+  },
+  "metric-store": {
+    "checkpoints": {
+      "interval": "12h"
     },
-	"clusters": [
-	{
-	   "name": "testcluster",
-	   "metricDataRepository": {"kind": "test", "url": "bla:8081"},
-	   "filterRanges": {
-		"numNodes": { "from": 1, "to": 64 },
-		"duration": { "from": 0, "to": 86400 },
-		"startTime": { "from": "2022-01-01T00:00:00Z", "to": null }
-	   }
-	}
-	]
+    "retention-in-memory": "48h",
+    "memory-cap": 100
+  },
+	"archive": {
+		 "kind": "file",
+		 "path": "./var/job-archive"
+	},
+	"auth": {
+     "jwts": {
+     "max-age": "2m"
+  }
+  }
 }`
-	const testclusterJson = `{
+	const testclusterJSON = `{
         "name": "testcluster",
 		"subClusters": [
 			{
@@ -113,58 +119,73 @@ func setup(t *testing.T) *api.RestApi {
 		]
 	}`
 
-	log.Init("info", true)
+	cclog.Init("info", true)
 	tmpdir := t.TempDir()
 	jobarchive := filepath.Join(tmpdir, "job-archive")
-	if err := os.Mkdir(jobarchive, 0777); err != nil {
+	if err := os.Mkdir(jobarchive, 0o777); err != nil {
 		t.Fatal(err)
 	}
 
-	if err := os.WriteFile(filepath.Join(jobarchive, "version.txt"), []byte(fmt.Sprintf("%d", 2)), 0666); err != nil {
+	if err := os.WriteFile(filepath.Join(jobarchive, "version.txt"), fmt.Appendf(nil, "%d", 3), 0o666); err != nil {
 		t.Fatal(err)
 	}
 
-	if err := os.Mkdir(filepath.Join(jobarchive, "testcluster"), 0777); err != nil {
+	if err := os.Mkdir(filepath.Join(jobarchive, "testcluster"), 0o777); err != nil {
 		t.Fatal(err)
 	}
 
-	if err := os.WriteFile(filepath.Join(jobarchive, "testcluster", "cluster.json"), []byte(testclusterJson), 0666); err != nil {
+	if err := os.WriteFile(filepath.Join(jobarchive, "testcluster", "cluster.json"), []byte(testclusterJSON), 0o666); err != nil {
 		t.Fatal(err)
 	}
 
 	dbfilepath := filepath.Join(tmpdir, "test.db")
-	err := repository.MigrateDB("sqlite3", dbfilepath)
+	err := repository.MigrateDB(dbfilepath)
 	if err != nil {
 		t.Fatal(err)
 	}
 
 	cfgFilePath := filepath.Join(tmpdir, "config.json")
-	if err := os.WriteFile(cfgFilePath, []byte(testconfig), 0666); err != nil {
+	if err := os.WriteFile(cfgFilePath, []byte(testconfig), 0o666); err != nil {
 		t.Fatal(err)
 	}
 
-	config.Init(cfgFilePath)
+	ccconf.Init(cfgFilePath)
+	metricstore.MetricStoreHandle = &metricstore.InternalMetricStore{}
+
+	// Load and check main configuration
+	if cfg := ccconf.GetPackageConfig("main"); cfg != nil {
+		config.Init(cfg)
+	} else {
+		cclog.Abort("Main configuration must be present")
+	}
 	archiveCfg := fmt.Sprintf("{\"kind\": \"file\",\"path\": \"%s\"}", jobarchive)
 
-	repository.Connect("sqlite3", dbfilepath)
+	repository.Connect(dbfilepath)
 
-	if err := archive.Init(json.RawMessage(archiveCfg), config.Keys.DisableArchive); err != nil {
+	if err := archive.Init(json.RawMessage(archiveCfg)); err != nil {
 		t.Fatal(err)
 	}
 
-	if err := metricdata.Init(); err != nil {
-		t.Fatal(err)
+	// metricstore initialization removed - it's initialized via callback in tests
+
+	archiver.Start(repository.GetJobRepository(), context.Background())
+
+	if cfg := ccconf.GetPackageConfig("auth"); cfg != nil {
+		auth.Init(&cfg)
+	} else {
+		cclog.Warn("Authentication disabled due to missing configuration")
+		auth.Init(nil)
 	}
 
-	archiver.Start(repository.GetJobRepository())
-	auth.Init()
 	graph.Init()
 
 	return api.New()
 }
 
 func cleanup() {
-	// TODO: Clear all caches, reset all modules, etc...
+	if err := archiver.Shutdown(5 * time.Second); err != nil {
+		cclog.Warnf("Archiver shutdown timeout in tests: %v", err)
+	}
 }
 
 /*
@@ -191,21 +212,19 @@ func TestRestApi(t *testing.T) {
 		},
 	}
 
-	metricdata.TestLoadDataCallback = func(job *schema.Job, metrics []string, scopes []schema.MetricScope, ctx context.Context, resolution int) (schema.JobData, error) {
+	metricstore.TestLoadDataCallback = func(job *schema.Job, metrics []string, scopes []schema.MetricScope, ctx context.Context, resolution int) (schema.JobData, error) {
 		return testData, nil
 	}
 
-	r := mux.NewRouter()
-	r.PathPrefix("/api").Subrouter()
-	r.StrictSlash(true)
-	restapi.MountApiRoutes(r)
+	r := chi.NewRouter()
+	restapi.MountAPIRoutes(r)
 
-	var TestJobId int64 = 123
-	var TestClusterName string = "testcluster"
+	var TestJobID int64 = 123
+	TestClusterName := "testcluster"
 	var TestStartTime int64 = 123456789
 
 	const startJobBody string = `{
-        "jobId":            123,
+    "jobId":            123,
 		"user":             "testuser",
 		"project":          "testproj",
 		"cluster":          "testcluster",
@@ -215,10 +234,9 @@ func TestRestApi(t *testing.T) {
 		"numNodes":         1,
 		"numHwthreads":     8,
 		"numAcc":           0,
-		"exclusive":        1,
+		"shared":           "none",
 		"monitoringStatus": 1,
 		"smt":              1,
-		"tags":             [{ "type": "testTagType", "name": "testTagName", "scope": "testuser" }],
 		"resources": [
 			{
 				"hostname": "host123",
@@ -249,16 +267,17 @@ func TestRestApi(t *testing.T) {
 		if response.StatusCode != http.StatusCreated {
 			t.Fatal(response.Status, recorder.Body.String())
 		}
-		resolver := graph.GetResolverInstance()
-		job, err := restapi.JobRepository.Find(&TestJobId, &TestClusterName, &TestStartTime)
+		// resolver := graph.GetResolverInstance()
+		restapi.JobRepository.SyncJobs()
+		job, err := restapi.JobRepository.Find(&TestJobID, &TestClusterName, &TestStartTime)
 		if err != nil {
 			t.Fatal(err)
 		}
 
-		job.Tags, err = resolver.Job().Tags(ctx, job)
-		if err != nil {
-			t.Fatal(err)
-		}
+		// job.Tags, err = resolver.Job().Tags(ctx, job)
+		// if err != nil {
+		// 	t.Fatal(err)
+		// }
 
 		if job.JobID != 123 ||
 			job.User != "testuser" ||
@@ -267,21 +286,20 @@ func TestRestApi(t *testing.T) {
 			job.SubCluster != "sc1" ||
 			job.Partition != "default" ||
 			job.Walltime != 3600 ||
-			job.ArrayJobId != 0 ||
+			job.ArrayJobID != 0 ||
 			job.NumNodes != 1 ||
 			job.NumHWThreads != 8 ||
 			job.NumAcc != 0 ||
-			job.Exclusive != 1 ||
 			job.MonitoringStatus != 1 ||
 			job.SMT != 1 ||
 			!reflect.DeepEqual(job.Resources, []*schema.Resource{{Hostname: "host123", HWThreads: []int{0, 1, 2, 3, 4, 5, 6, 7}}}) ||
-			job.StartTime.Unix() != 123456789 {
+			job.StartTime != 123456789 {
 			t.Fatalf("unexpected job properties: %#v", job)
 		}
 
-		if len(job.Tags) != 1 || job.Tags[0].Type != "testTagType" || job.Tags[0].Name != "testTagName" || job.Tags[0].Scope != "testuser" {
-			t.Fatalf("unexpected tags: %#v", job.Tags)
-		}
+		// if len(job.Tags) != 1 || job.Tags[0].Type != "testTagType" || job.Tags[0].Name != "testTagName" || job.Tags[0].Scope != "testuser" {
+		// 	t.Fatalf("unexpected tags: %#v", job.Tags)
+		// }
 	}); !ok {
 		return
 	}
@@ -308,8 +326,8 @@ func TestRestApi(t *testing.T) {
 			t.Fatal(response.Status, recorder.Body.String())
 		}
 
-		archiver.WaitForArchiving()
-		job, err := restapi.JobRepository.Find(&TestJobId, &TestClusterName, &TestStartTime)
+		// Archiving happens asynchronously, will be completed in cleanup
+		job, err := restapi.JobRepository.Find(&TestJobID, &TestClusterName, &TestStartTime)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -337,7 +355,7 @@ func TestRestApi(t *testing.T) {
 	}
 
 	t.Run("CheckArchive", func(t *testing.T) {
-		data, err := metricDataDispatcher.LoadData(stoppedJob, []string{"load_one"}, []schema.MetricScope{schema.MetricScopeNode}, context.Background(), 60)
+		data, err := metricdispatch.LoadData(stoppedJob, []string{"load_one"}, []schema.MetricScope{schema.MetricScopeNode}, context.Background(), 60)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -349,7 +367,7 @@ func TestRestApi(t *testing.T) {
 
 	t.Run("CheckDoubleStart", func(t *testing.T) {
 		// Starting a job with the same jobId and cluster should only be allowed if the startTime is far appart!
-		body := strings.Replace(startJobBody, `"startTime": 123456789`, `"startTime": 123456790`, -1)
+		body := strings.ReplaceAll(startJobBody, `"startTime": 123456789`, `"startTime": 123456790`)
 
 		req := httptest.NewRequest(http.MethodPost, "/jobs/start_job/", bytes.NewBuffer([]byte(body)))
 		recorder := httptest.NewRecorder()
@@ -371,7 +389,7 @@ func TestRestApi(t *testing.T) {
 		"partition":        "default",
 		"walltime":         3600,
 		"numNodes":         1,
-		"exclusive":        1,
+		"shared":        	"none",
 		"monitoringStatus": 1,
 		"smt":              1,
 		"resources": [
@@ -399,6 +417,7 @@ func TestRestApi(t *testing.T) {
 	}
 
 	time.Sleep(1 * time.Second)
+	restapi.JobRepository.SyncJobs()
 
 	const stopJobBodyFailed string = `{
     "jobId":     12345,
@@ -420,7 +439,7 @@ func TestRestApi(t *testing.T) {
 			t.Fatal(response.Status, recorder.Body.String())
 		}
 
-		archiver.WaitForArchiving()
+		// Archiving happens asynchronously, will be completed in cleanup
 		jobid, cluster := int64(12345), "testcluster"
 		job, err := restapi.JobRepository.Find(&jobid, &cluster, nil)
 		if err != nil {
@@ -434,4 +453,198 @@ func TestRestApi(t *testing.T) {
 	if !ok {
 		t.Fatal("subtest failed")
 	}
+
+	t.Run("GetUsedNodesNoRunning", func(t *testing.T) {
+		contextUserValue := &schema.User{
+			Username:   "testuser",
+			Projects:   make([]string, 0),
+			Roles:      []string{"api"},
+			AuthType:   0,
+			AuthSource: 2,
+		}
+
+		req := httptest.NewRequest(http.MethodGet, "/jobs/used_nodes?ts=123456790", nil)
+		recorder := httptest.NewRecorder()
+
+		ctx := context.WithValue(req.Context(), contextUserKey, contextUserValue)
+
+		r.ServeHTTP(recorder, req.WithContext(ctx))
+		response := recorder.Result()
+		if response.StatusCode != http.StatusOK {
+			t.Fatal(response.Status, recorder.Body.String())
+		}
+
+		var result api.GetUsedNodesAPIResponse
+		if err := json.NewDecoder(response.Body).Decode(&result); err != nil {
+			t.Fatal(err)
+		}
+
+		if result.UsedNodes == nil {
+			t.Fatal("expected usedNodes to be non-nil")
+		}
+
+		if len(result.UsedNodes) != 0 {
+			t.Fatalf("expected no used nodes for stopped jobs, got: %v", result.UsedNodes)
+		}
+	})
+}
+
+// TestStopJobWithReusedJobId verifies that stopping a recently started job works
+// even when an older job with the same jobId exists in the job table (e.g. with
+// state "failed"). This is a regression test for the bug where Find() on the job
+// table would match the old job instead of the new one still in job_cache.
+func TestStopJobWithReusedJobId(t *testing.T) {
+	restapi := setup(t)
+	t.Cleanup(cleanup)
+
+	testData := schema.JobData{
+		"load_one": map[schema.MetricScope]*schema.JobMetric{
+			schema.MetricScopeNode: {
+				Unit:     schema.Unit{Base: "load"},
+				Timestep: 60,
+				Series: []schema.Series{
+					{
+						Hostname:   "host123",
+						Statistics: schema.MetricStatistics{Min: 0.1, Avg: 0.2, Max: 0.3},
+						Data:       []schema.Float{0.1, 0.1, 0.1, 0.2, 0.2, 0.2, 0.3, 0.3, 0.3},
+					},
+				},
+			},
+		},
+	}
+
+	metricstore.TestLoadDataCallback = func(job *schema.Job, metrics []string, scopes []schema.MetricScope, ctx context.Context, resolution int) (schema.JobData, error) {
+		return testData, nil
+	}
+
+	r := chi.NewRouter()
+	restapi.MountAPIRoutes(r)
+
+	const contextUserKey repository.ContextKey = "user"
+	contextUserValue := &schema.User{
+		Username:   "testuser",
+		Projects:   make([]string, 0),
+		Roles:      []string{"user"},
+		AuthType:   0,
+		AuthSource: 2,
+	}
+
+	// Step 1: Start the first job (jobId=999)
+	const startJobBody1 string = `{
+		"jobId":            999,
+		"user":             "testuser",
+		"project":          "testproj",
+		"cluster":          "testcluster",
+		"partition":        "default",
+		"walltime":         3600,
+		"numNodes":         1,
+		"numHwthreads":     8,
+		"numAcc":           0,
+		"shared":           "none",
+		"monitoringStatus": 1,
+		"smt":              1,
+		"resources": [{"hostname": "host123", "hwthreads": [0, 1, 2, 3, 4, 5, 6, 7]}],
+		"startTime":        200000000
+	}`
+
+	if ok := t.Run("StartFirstJob", func(t *testing.T) {
+		req := httptest.NewRequest(http.MethodPost, "/jobs/start_job/", bytes.NewBuffer([]byte(startJobBody1)))
+		recorder := httptest.NewRecorder()
+		ctx := context.WithValue(req.Context(), contextUserKey, contextUserValue)
+		r.ServeHTTP(recorder, req.WithContext(ctx))
+		if recorder.Result().StatusCode != http.StatusCreated {
+			t.Fatal(recorder.Result().Status, recorder.Body.String())
+		}
+	}); !ok {
+		return
+	}
+
+	// Step 2: Sync to move job from cache to job table, then stop it as "failed"
+	time.Sleep(1 * time.Second)
+	restapi.JobRepository.SyncJobs()
+
+	const stopJobBody1 string = `{
+		"jobId":     999,
+		"startTime": 200000000,
+		"cluster":   "testcluster",
+		"jobState":  "failed",
+		"stopTime":  200001000
+	}`
+
+	if ok := t.Run("StopFirstJobAsFailed", func(t *testing.T) {
+		req := httptest.NewRequest(http.MethodPost, "/jobs/stop_job/", bytes.NewBuffer([]byte(stopJobBody1)))
+		recorder := httptest.NewRecorder()
+		ctx := context.WithValue(req.Context(), contextUserKey, contextUserValue)
+		r.ServeHTTP(recorder, req.WithContext(ctx))
+		if recorder.Result().StatusCode != http.StatusOK {
+			t.Fatal(recorder.Result().Status, recorder.Body.String())
+		}
+
+		jobid, cluster := int64(999), "testcluster"
+		job, err := restapi.JobRepository.Find(&jobid, &cluster, nil)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if job.State != schema.JobStateFailed {
+			t.Fatalf("expected first job to be failed, got: %s", job.State)
+		}
+	}); !ok {
+		return
+	}
+
+	// Wait for archiving to complete
+	time.Sleep(1 * time.Second)
+
+	// Step 3: Start a NEW job with the same jobId=999 but different startTime.
+	// This job will sit in job_cache (not yet synced).
+	const startJobBody2 string = `{
+		"jobId":            999,
+		"user":             "testuser",
+		"project":          "testproj",
+		"cluster":          "testcluster",
+		"partition":        "default",
+		"walltime":         3600,
+		"numNodes":         1,
+		"numHwthreads":     8,
+		"numAcc":           0,
+		"shared":           "none",
+		"monitoringStatus": 1,
+		"smt":              1,
+		"resources": [{"hostname": "host123", "hwthreads": [0, 1, 2, 3, 4, 5, 6, 7]}],
+		"startTime":        300000000
+	}`
+
+	if ok := t.Run("StartSecondJob", func(t *testing.T) {
+		req := httptest.NewRequest(http.MethodPost, "/jobs/start_job/", bytes.NewBuffer([]byte(startJobBody2)))
+		recorder := httptest.NewRecorder()
+		ctx := context.WithValue(req.Context(), contextUserKey, contextUserValue)
+		r.ServeHTTP(recorder, req.WithContext(ctx))
+		if recorder.Result().StatusCode != http.StatusCreated {
+			t.Fatal(recorder.Result().Status, recorder.Body.String())
+		}
+	}); !ok {
+		return
+	}
+
+	// Step 4: Stop the second job WITHOUT syncing first.
+	// Before the fix, this would fail because Find() on the job table would
+	// match the old failed job (jobId=999) and reject with "already stopped".
+	const stopJobBody2 string = `{
+		"jobId":     999,
+		"startTime": 300000000,
+		"cluster":   "testcluster",
+		"jobState":  "completed",
+		"stopTime":  300001000
+	}`
+
+	t.Run("StopSecondJobBeforeSync", func(t *testing.T) {
+		req := httptest.NewRequest(http.MethodPost, "/jobs/stop_job/", bytes.NewBuffer([]byte(stopJobBody2)))
+		recorder := httptest.NewRecorder()
+		ctx := context.WithValue(req.Context(), contextUserKey, contextUserValue)
+		r.ServeHTTP(recorder, req.WithContext(ctx))
+		if recorder.Result().StatusCode != http.StatusOK {
+			t.Fatalf("expected stop to succeed for cached job, got: %s %s",
+				recorder.Result().Status, recorder.Body.String())
+		}
+	})
 }

internal/api/cluster.go

@@ -0,0 +1,71 @@
+// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
+// All rights reserved. This file is part of cc-backend.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package api
+
+import (
+	"bufio"
+	"encoding/json"
+	"fmt"
+	"net/http"
+
+	"github.com/ClusterCockpit/cc-backend/internal/repository"
+	"github.com/ClusterCockpit/cc-backend/pkg/archive"
+	"github.com/ClusterCockpit/cc-lib/v2/schema"
+)
+
+// GetClustersAPIResponse model
+type GetClustersAPIResponse struct {
+	Clusters []*schema.Cluster `json:"clusters"` // Array of clusters
+}
+
+// getClusters godoc
+// @summary     Lists all cluster configs
+// @tags Cluster query
+// @description Get a list of all cluster configs. Specific cluster can be requested using query parameter.
+// @produce     json
+// @param       cluster        query    string            false "Job Cluster"
+// @success     200            {object} api.GetClustersAPIResponse  "Array of clusters"
+// @failure     400            {object} api.ErrorResponse       "Bad Request"
+// @failure     401            {object} api.ErrorResponse       "Unauthorized"
+// @failure     403            {object} api.ErrorResponse       "Forbidden"
+// @failure     500            {object} api.ErrorResponse       "Internal Server Error"
+// @security    ApiKeyAuth
+// @router      /api/clusters/ [get]
+func (api *RestAPI) getClusters(rw http.ResponseWriter, r *http.Request) {
+	if user := repository.GetUserFromContext(r.Context()); user != nil &&
+		!user.HasRole(schema.RoleAPI) {
+
+		handleError(fmt.Errorf("missing role: %v", schema.GetRoleString(schema.RoleAPI)), http.StatusForbidden, rw)
+		return
+	}
+
+	rw.Header().Add("Content-Type", "application/json")
+	bw := bufio.NewWriter(rw)
+	defer bw.Flush()
+
+	var clusters []*schema.Cluster
+
+	if r.URL.Query().Has("cluster") {
+		name := r.URL.Query().Get("cluster")
+		cluster := archive.GetCluster(name)
+		if cluster == nil {
+			handleError(fmt.Errorf("unknown cluster: %s", name), http.StatusBadRequest, rw)
+			return
+		}
+		clusters = append(clusters, cluster)
+	} else {
+		clusters = archive.Clusters
+	}
+
+	payload := GetClustersAPIResponse{
+		Clusters: clusters,
+	}
+
+	if err := json.NewEncoder(bw).Encode(payload); err != nil {
+		handleError(err, http.StatusInternalServerError, rw)
+		return
+	}
+}

internal/api/log.go

@@ -0,0 +1,165 @@
+// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
+// All rights reserved. This file is part of cc-backend.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+package api
+
+import (
+	"bufio"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"os/exec"
+	"regexp"
+	"strconv"
+	"strings"
+
+	"github.com/ClusterCockpit/cc-backend/internal/config"
+	"github.com/ClusterCockpit/cc-backend/internal/repository"
+	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
+	"github.com/ClusterCockpit/cc-lib/v2/schema"
+)
+
+type LogEntry struct {
+	Timestamp string `json:"timestamp"`
+	Priority  int    `json:"priority"`
+	Message   string `json:"message"`
+	Unit      string `json:"unit"`
+}
+
+var safePattern = regexp.MustCompile(`^[a-zA-Z0-9 :\-\.]+$`)
+
+func (api *RestAPI) getJournalLog(rw http.ResponseWriter, r *http.Request) {
+	user := repository.GetUserFromContext(r.Context())
+	if !user.HasRole(schema.RoleAdmin) {
+		handleError(fmt.Errorf("only admins are allowed to view logs"), http.StatusForbidden, rw)
+		return
+	}
+
+	since := r.URL.Query().Get("since")
+	if since == "" {
+		since = "1 hour ago"
+	}
+	if !safePattern.MatchString(since) {
+		handleError(fmt.Errorf("invalid 'since' parameter"), http.StatusBadRequest, rw)
+		return
+	}
+
+	lines := 200
+	if l := r.URL.Query().Get("lines"); l != "" {
+		n, err := strconv.Atoi(l)
+		if err != nil || n < 1 {
+			handleError(fmt.Errorf("invalid 'lines' parameter"), http.StatusBadRequest, rw)
+			return
+		}
+		if n > 1000 {
+			n = 1000
+		}
+		lines = n
+	}
+
+	unit := config.Keys.SystemdUnit
+	if unit == "" {
+		unit = "clustercockpit.service"
+	}
+
+	args := []string{
+		"--output=json",
+		"--no-pager",
+		"-n", fmt.Sprintf("%d", lines),
+		"--since", since,
+		"-u", unit,
+	}
+
+	if level := r.URL.Query().Get("level"); level != "" {
+		n, err := strconv.Atoi(level)
+		if err != nil || n < 0 || n > 7 {
+			handleError(fmt.Errorf("invalid 'level' parameter (must be 0-7)"), http.StatusBadRequest, rw)
+			return
+		}
+		args = append(args, "--priority", fmt.Sprintf("%d", n))
+	}
+
+	if search := r.URL.Query().Get("search"); search != "" {
+		if !safePattern.MatchString(search) {
+			handleError(fmt.Errorf("invalid 'search' parameter"), http.StatusBadRequest, rw)
+			return
+		}
+		args = append(args, "--grep", search)
+	}
+
+	cclog.Debugf("calling journalctl with %s", strings.Join(args, " "))
+	cmd := exec.CommandContext(r.Context(), "journalctl", args...)
+	stdout, err := cmd.StdoutPipe()
+	if err != nil {
+		handleError(fmt.Errorf("failed to create pipe: %w", err), http.StatusInternalServerError, rw)
+		return
+	}
+
+	if err := cmd.Start(); err != nil {
+		handleError(fmt.Errorf("failed to start journalctl: %w", err), http.StatusInternalServerError, rw)
+		return
+	}
+
+	entries := make([]LogEntry, 0, lines)
+	scanner := bufio.NewScanner(stdout)
+	for scanner.Scan() {
+		var raw map[string]any
+		if err := json.Unmarshal(scanner.Bytes(), &raw); err != nil {
+			cclog.Debugf("error unmarshal log output: %v", err)
+			continue
+		}
+
+		priority := 6 // default info
+		if p, ok := raw["PRIORITY"]; ok {
+			switch v := p.(type) {
+			case string:
+				if n, err := strconv.Atoi(v); err == nil {
+					priority = n
+				}
+			case float64:
+				priority = int(v)
+			}
+		}
+
+		msg := ""
+		if m, ok := raw["MESSAGE"]; ok {
+			if s, ok := m.(string); ok {
+				msg = s
+			}
+		}
+
+		ts := ""
+		if t, ok := raw["__REALTIME_TIMESTAMP"]; ok {
+			if s, ok := t.(string); ok {
+				ts = s
+			}
+		}
+
+		unitName := ""
+		if u, ok := raw["_SYSTEMD_UNIT"]; ok {
+			if s, ok := u.(string); ok {
+				unitName = s
+			}
+		}
+
+		entries = append(entries, LogEntry{
+			Timestamp: ts,
+			Priority:  priority,
+			Message:   msg,
+			Unit:      unitName,
+		})
+	}
+
+	if err := cmd.Wait(); err != nil {
+		// journalctl returns exit code 1 when --grep matches nothing
+		if len(entries) == 0 {
+			cclog.Debugf("journalctl exited with: %v", err)
+		}
+	}
+
+	rw.Header().Set("Content-Type", "application/json")
+	if err := json.NewEncoder(rw).Encode(entries); err != nil {
+		cclog.Errorf("Failed to encode log entries: %v", err)
+	}
+}

internal/api/metricstore.go

@@ -0,0 +1,151 @@
+// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
+// All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package api
+
+import (
+	"bufio"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/http"
+	"strconv"
+	"strings"
+
+	"github.com/ClusterCockpit/cc-backend/pkg/metricstore"
+	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
+
+	"github.com/ClusterCockpit/cc-line-protocol/v2/lineprotocol"
+)
+
+// handleFree godoc
+// @summary
+// @tags free
+// @description This endpoint allows the users to free the Buffers from the
+// metric store. This endpoint offers the users to remove then systematically
+// and also allows then to prune the data under node, if they do not want to
+// remove the whole node.
+// @produce     json
+// @param       to        query    string        false  "up to timestamp"
+// @success     200            {string} string  "ok"
+// @failure     400            {object} api.ErrorResponse       "Bad Request"
+// @failure     401            {object} api.ErrorResponse       "Unauthorized"
+// @failure     403            {object} api.ErrorResponse       "Forbidden"
+// @failure     500            {object} api.ErrorResponse       "Internal Server Error"
+// @security    ApiKeyAuth
+// @router      /free/ [post]
+func freeMetrics(rw http.ResponseWriter, r *http.Request) {
+	rawTo := r.URL.Query().Get("to")
+	if rawTo == "" {
+		handleError(errors.New("'to' is a required query parameter"), http.StatusBadRequest, rw)
+		return
+	}
+
+	to, err := strconv.ParseInt(rawTo, 10, 64)
+	if err != nil {
+		handleError(err, http.StatusInternalServerError, rw)
+		return
+	}
+
+	bodyDec := json.NewDecoder(r.Body)
+	var selectors [][]string
+	err = bodyDec.Decode(&selectors)
+	if err != nil {
+		http.Error(rw, err.Error(), http.StatusBadRequest)
+		return
+	}
+
+	ms := metricstore.GetMemoryStore()
+	n := 0
+	for _, sel := range selectors {
+		bn, err := ms.Free(sel, to)
+		if err != nil {
+			handleError(err, http.StatusInternalServerError, rw)
+			return
+		}
+
+		n += bn
+	}
+
+	rw.WriteHeader(http.StatusOK)
+	fmt.Fprintf(rw, "buffers freed: %d\n", n)
+}
+
+// handleWrite godoc
+// @summary Receive metrics in InfluxDB line-protocol
+// @tags write
+// @description Write data to the in-memory store in the InfluxDB line-protocol using [this format](https://github.com/ClusterCockpit/cc-specifications/blob/master/metrics/lineprotocol_alternative.md)
+
+// @accept      plain
+// @produce     json
+// @param       cluster        query string false "If the lines in the body do not have a cluster tag, use this value instead."
+// @success     200            {string} string  "ok"
+// @failure     400            {object} api.ErrorResponse       "Bad Request"
+// @failure     401            {object} api.ErrorResponse       "Unauthorized"
+// @failure     403            {object} api.ErrorResponse       "Forbidden"
+// @failure     500            {object} api.ErrorResponse       "Internal Server Error"
+// @security    ApiKeyAuth
+// @router      /write/ [post]
+func writeMetrics(rw http.ResponseWriter, r *http.Request) {
+	rw.Header().Add("Content-Type", "application/json")
+
+	// Extract the "cluster" query parameter without allocating a url.Values map.
+	cluster := queryParam(r.URL.RawQuery, "cluster")
+
+	// Stream directly from the request body instead of copying it into a
+	// temporary buffer via io.ReadAll. The line-protocol decoder supports
+	// io.Reader natively, so this avoids the largest heap allocation.
+	ms := metricstore.GetMemoryStore()
+	dec := lineprotocol.NewDecoder(r.Body)
+	if err := metricstore.DecodeLine(dec, ms, cluster); err != nil {
+		cclog.Errorf("/api/write error: %s", err.Error())
+		handleError(err, http.StatusBadRequest, rw)
+		return
+	}
+	rw.WriteHeader(http.StatusOK)
+}
+
+// queryParam extracts a single query-parameter value from a raw query string
+// without allocating a url.Values map. Returns "" if the key is not present.
+func queryParam(raw, key string) string {
+	for raw != "" {
+		var kv string
+		kv, raw, _ = strings.Cut(raw, "&")
+		k, v, _ := strings.Cut(kv, "=")
+		if k == key {
+			return v
+		}
+	}
+	return ""
+}
+
+// handleDebug godoc
+// @summary Debug endpoint
+// @tags debug
+// @description This endpoint allows the users to print the content of
+// nodes/clusters/metrics to review the state of the data.
+// @produce     json
+// @param       selector        query    string            false "Selector"
+// @success     200            {string} string  "Debug dump"
+// @failure     400            {object} api.ErrorResponse       "Bad Request"
+// @failure     401            {object} api.ErrorResponse       "Unauthorized"
+// @failure     403            {object} api.ErrorResponse       "Forbidden"
+// @failure     500            {object} api.ErrorResponse       "Internal Server Error"
+// @security    ApiKeyAuth
+// @router      /debug/ [post]
+func debugMetrics(rw http.ResponseWriter, r *http.Request) {
+	raw := r.URL.Query().Get("selector")
+	rw.Header().Add("Content-Type", "application/json")
+	selector := []string{}
+	if len(raw) != 0 {
+		selector = strings.Split(raw, ":")
+	}
+
+	ms := metricstore.GetMemoryStore()
+	if err := ms.DebugDump(bufio.NewWriter(rw), selector); err != nil {
+		handleError(err, http.StatusBadRequest, rw)
+		return
+	}
+}

internal/api/nats.go

@@ -0,0 +1,400 @@
+// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
+// All rights reserved. This file is part of cc-backend.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package api
+
+import (
+	"database/sql"
+	"encoding/json"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/ClusterCockpit/cc-backend/internal/archiver"
+	"github.com/ClusterCockpit/cc-backend/internal/config"
+	"github.com/ClusterCockpit/cc-backend/internal/importer"
+	"github.com/ClusterCockpit/cc-backend/internal/repository"
+	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
+	lp "github.com/ClusterCockpit/cc-lib/v2/ccMessage"
+	"github.com/ClusterCockpit/cc-lib/v2/nats"
+	"github.com/ClusterCockpit/cc-lib/v2/receivers"
+	"github.com/ClusterCockpit/cc-lib/v2/schema"
+	influx "github.com/ClusterCockpit/cc-line-protocol/v2/lineprotocol"
+)
+
+// NatsAPI provides NATS subscription-based handlers for Job and Node operations.
+// It mirrors the functionality of the REST API but uses NATS messaging with
+// InfluxDB line protocol as the message format.
+//
+// # Message Format
+//
+// All NATS messages use InfluxDB line protocol format (https://docs.influxdata.com/influxdb/v2.0/reference/syntax/line-protocol/)
+// with the following structure:
+//
+//	measurement,tag1=value1,tag2=value2 field1=value1,field2=value2 timestamp
+//
+// # Job Events
+//
+// Job start/stop events use the "job" measurement with a "function" tag to distinguish operations:
+//
+//	job,function=start_job event="{...JSON payload...}" <timestamp>
+//	job,function=stop_job event="{...JSON payload...}" <timestamp>
+//
+// The JSON payload in the "event" field follows the schema.Job or StopJobAPIRequest structure.
+//
+// Example job start message:
+//
+//	job,function=start_job event="{\"jobId\":1001,\"user\":\"testuser\",\"cluster\":\"testcluster\",...}" 1234567890000000000
+//
+// # Node State Events
+//
+// Node state updates use the "nodestate" measurement with cluster information:
+//
+//	nodestate event="{...JSON payload...}" <timestamp>
+//
+// The JSON payload follows the UpdateNodeStatesRequest structure.
+//
+// Example node state message:
+//
+//	nodestate event="{\"cluster\":\"testcluster\",\"nodes\":[{\"hostname\":\"node01\",\"states\":[\"idle\"]}]}" 1234567890000000000
+type NatsAPI struct {
+	JobRepository *repository.JobRepository
+	// RepositoryMutex protects job creation operations from race conditions
+	// when checking for duplicate jobs during startJob calls.
+	RepositoryMutex sync.Mutex
+}
+
+// NewNatsAPI creates a new NatsAPI instance with default dependencies.
+func NewNatsAPI() *NatsAPI {
+	return &NatsAPI{
+		JobRepository: repository.GetJobRepository(),
+	}
+}
+
+// StartSubscriptions registers all NATS subscriptions for Job and Node APIs.
+// Returns an error if the NATS client is not available or subscription fails.
+func (api *NatsAPI) StartSubscriptions() error {
+	client := nats.GetClient()
+	if client == nil {
+		cclog.Warn("NATS client not available, skipping API subscriptions")
+		return nil
+	}
+
+	if config.Keys.APISubjects != nil {
+
+		s := config.Keys.APISubjects
+
+		if err := client.Subscribe(s.SubjectJobEvent, api.handleJobEvent); err != nil {
+			return err
+		}
+
+		if err := client.Subscribe(s.SubjectNodeState, api.handleNodeState); err != nil {
+			return err
+		}
+
+		cclog.Info("NATS API subscriptions started")
+	}
+	return nil
+}
+
+// processJobEvent routes job event messages to the appropriate handler based on the "function" tag.
+// Validates that required tags and fields are present before processing.
+func (api *NatsAPI) processJobEvent(msg lp.CCMessage) {
+	function, ok := msg.GetTag("function")
+	if !ok {
+		cclog.Errorf("Job event is missing required tag 'function': measurement=%s", msg.Name())
+		return
+	}
+
+	switch function {
+	case "start_job":
+		v, ok := msg.GetEventValue()
+		if !ok {
+			cclog.Errorf("Job start event is missing event field with JSON payload")
+			return
+		}
+		api.handleStartJob(v)
+
+	case "stop_job":
+		v, ok := msg.GetEventValue()
+		if !ok {
+			cclog.Errorf("Job stop event is missing event field with JSON payload")
+			return
+		}
+		api.handleStopJob(v)
+
+	default:
+		cclog.Warnf("Unknown job event function '%s', expected 'start_job' or 'stop_job'", function)
+	}
+}
+
+// handleJobEvent processes job-related messages received via NATS using InfluxDB line protocol.
+// The message must be in line protocol format with measurement="job" and include:
+//   - tag "function" with value "start_job" or "stop_job"
+//   - field "event" containing JSON payload (schema.Job or StopJobAPIRequest)
+//
+// Example: job,function=start_job event="{\"jobId\":1001,...}" 1234567890000000000
+func (api *NatsAPI) handleJobEvent(subject string, data []byte) {
+	if len(data) == 0 {
+		cclog.Warnf("NATS %s: received empty message", subject)
+		return
+	}
+
+	d := influx.NewDecoderWithBytes(data)
+
+	for d.Next() {
+		m, err := receivers.DecodeInfluxMessage(d)
+		if err != nil {
+			cclog.Errorf("NATS %s: failed to decode InfluxDB line protocol message: %v", subject, err)
+			return
+		}
+
+		if !m.IsEvent() {
+			cclog.Debugf("NATS %s: received non-event message, skipping", subject)
+			continue
+		}
+
+		if m.Name() == "job" {
+			api.processJobEvent(m)
+		} else {
+			cclog.Debugf("NATS %s: unexpected measurement name '%s', expected 'job'", subject, m.Name())
+		}
+	}
+}
+
+// handleStartJob processes job start messages received via NATS.
+// The payload parameter contains JSON following the schema.Job structure.
+// Jobs are validated, checked for duplicates, and inserted into the database.
+func (api *NatsAPI) handleStartJob(payload string) {
+	if payload == "" {
+		cclog.Error("NATS start job: payload is empty")
+		return
+	}
+	req := schema.Job{
+		Shared:           "none",
+		MonitoringStatus: schema.MonitoringStatusRunningOrArchiving,
+	}
+
+	dec := json.NewDecoder(strings.NewReader(payload))
+	dec.DisallowUnknownFields()
+	if err := dec.Decode(&req); err != nil {
+		cclog.Errorf("NATS start job: parsing request failed: %v", err)
+		return
+	}
+
+	cclog.Debugf("NATS start job: %s", req.GoString())
+	req.State = schema.JobStateRunning
+
+	if err := importer.SanityChecks(&req); err != nil {
+		cclog.Errorf("NATS start job: sanity check failed: %v", err)
+		return
+	}
+
+	var unlockOnce sync.Once
+	api.RepositoryMutex.Lock()
+	defer unlockOnce.Do(api.RepositoryMutex.Unlock)
+
+	jobs, err := api.JobRepository.FindAll(&req.JobID, &req.Cluster, nil)
+	if err != nil && err != sql.ErrNoRows {
+		cclog.Errorf("NATS start job: checking for duplicate failed: %v", err)
+		return
+	}
+	if err == nil {
+		for _, job := range jobs {
+			if (req.StartTime - job.StartTime) < secondsPerDay {
+				cclog.Errorf("NATS start job: job with jobId %d, cluster %s already exists (dbid: %d)",
+					req.JobID, req.Cluster, job.ID)
+				return
+			}
+		}
+	}
+
+	// When tags are present, insert directly into the job table so that the
+	// returned ID can be used with AddTagOrCreate (which queries the job table).
+	var id int64
+	if len(req.Tags) > 0 {
+		id, err = api.JobRepository.StartDirect(&req)
+	} else {
+		id, err = api.JobRepository.Start(&req)
+	}
+	if err != nil {
+		cclog.Errorf("NATS start job: insert into database failed: %v", err)
+		return
+	}
+	unlockOnce.Do(api.RepositoryMutex.Unlock)
+
+	for _, tag := range req.Tags {
+		if _, err := api.JobRepository.AddTagOrCreate(nil, id, tag.Type, tag.Name, tag.Scope); err != nil {
+			cclog.Errorf("NATS start job: adding tag to new job %d failed: %v", id, err)
+			return
+		}
+	}
+
+	cclog.Infof("NATS: new job (id: %d): cluster=%s, jobId=%d, user=%s, startTime=%d",
+		id, req.Cluster, req.JobID, req.User, req.StartTime)
+}
+
+// handleStopJob processes job stop messages received via NATS.
+// The payload parameter contains JSON following the StopJobAPIRequest structure.
+// The job is marked as stopped in the database and archiving is triggered if monitoring is enabled.
+func (api *NatsAPI) handleStopJob(payload string) {
+	if payload == "" {
+		cclog.Error("NATS stop job: payload is empty")
+		return
+	}
+	var req StopJobAPIRequest
+
+	dec := json.NewDecoder(strings.NewReader(payload))
+	dec.DisallowUnknownFields()
+	if err := dec.Decode(&req); err != nil {
+		cclog.Errorf("NATS job stop: parsing request failed: %v", err)
+		return
+	}
+
+	if req.JobID == nil {
+		cclog.Errorf("NATS job stop: the field 'jobId' is required")
+		return
+	}
+
+	isCached := false
+	job, err := api.JobRepository.FindCached(req.JobID, req.Cluster, req.StartTime)
+	if err != nil {
+		// Not in cache, try main job table
+		job, err = api.JobRepository.Find(req.JobID, req.Cluster, req.StartTime)
+		if err != nil {
+			cclog.Errorf("NATS job stop: finding job failed: %v", err)
+			return
+		}
+	} else {
+		isCached = true
+	}
+
+	if job.State != schema.JobStateRunning {
+		cclog.Errorf("NATS job stop: jobId %d (id %d) on %s: job has already been stopped (state is: %s)",
+			job.JobID, job.ID, job.Cluster, job.State)
+		return
+	}
+
+	if job.StartTime > req.StopTime {
+		cclog.Errorf("NATS job stop: jobId %d (id %d) on %s: stopTime %d must be >= startTime %d",
+			job.JobID, job.ID, job.Cluster, req.StopTime, job.StartTime)
+		return
+	}
+
+	if req.State != "" && !req.State.Valid() {
+		cclog.Errorf("NATS job stop: jobId %d (id %d) on %s: invalid job state: %#v",
+			job.JobID, job.ID, job.Cluster, req.State)
+		return
+	} else if req.State == "" {
+		req.State = schema.JobStateCompleted
+	}
+
+	job.Duration = int32(req.StopTime - job.StartTime)
+	job.State = req.State
+	api.JobRepository.Mutex.Lock()
+	defer api.JobRepository.Mutex.Unlock()
+
+	// If the job is still in job_cache, transfer it to the job table first
+	if isCached {
+		newID, err := api.JobRepository.TransferCachedJobToMain(*job.ID)
+		if err != nil {
+			cclog.Errorf("NATS job stop: jobId %d (id %d) on %s: transferring cached job failed: %v",
+				job.JobID, *job.ID, job.Cluster, err)
+			return
+		}
+		cclog.Infof("NATS: transferred cached job to main table: old id %d -> new id %d (jobId=%d)", *job.ID, newID, job.JobID)
+		job.ID = &newID
+	}
+
+	if err := api.JobRepository.Stop(*job.ID, job.Duration, job.State, job.MonitoringStatus); err != nil {
+		cclog.Errorf("NATS job stop: jobId %d (id %d) on %s: marking job as '%s' failed: %v",
+			job.JobID, *job.ID, job.Cluster, job.State, err)
+		return
+	}
+
+	cclog.Infof("NATS: archiving job (dbid: %d): cluster=%s, jobId=%d, user=%s, startTime=%d, duration=%d, state=%s",
+		*job.ID, job.Cluster, job.JobID, job.User, job.StartTime, job.Duration, job.State)
+
+	if job.MonitoringStatus == schema.MonitoringStatusDisabled {
+		return
+	}
+
+	archiver.TriggerArchiving(job)
+}
+
+// processNodestateEvent extracts and processes node state data from the InfluxDB message.
+// Updates node states in the repository for all nodes in the payload.
+func (api *NatsAPI) processNodestateEvent(msg lp.CCMessage) {
+	v, ok := msg.GetEventValue()
+	if !ok {
+		cclog.Errorf("Nodestate event is missing event field with JSON payload")
+		return
+	}
+
+	var req UpdateNodeStatesRequest
+
+	dec := json.NewDecoder(strings.NewReader(v))
+	dec.DisallowUnknownFields()
+	if err := dec.Decode(&req); err != nil {
+		cclog.Errorf("NATS nodestate: parsing request failed: %v", err)
+		return
+	}
+
+	repo := repository.GetNodeRepository()
+	requestReceived := time.Now().Unix()
+
+	for _, node := range req.Nodes {
+		state := determineState(node.States)
+		nodeState := schema.NodeStateDB{
+			TimeStamp:       requestReceived,
+			NodeState:       state,
+			CpusAllocated:   node.CpusAllocated,
+			MemoryAllocated: node.MemoryAllocated,
+			GpusAllocated:   node.GpusAllocated,
+			HealthState:     schema.MonitoringStateFull,
+			JobsRunning:     node.JobsRunning,
+		}
+
+		if err := repo.UpdateNodeState(node.Hostname, req.Cluster, &nodeState); err != nil {
+			cclog.Errorf("NATS nodestate: updating node state for %s on %s failed: %v",
+				node.Hostname, req.Cluster, err)
+		}
+	}
+
+	cclog.Debugf("NATS nodestate: updated %d node states for cluster %s", len(req.Nodes), req.Cluster)
+}
+
+// handleNodeState processes node state update messages received via NATS using InfluxDB line protocol.
+// The message must be in line protocol format with measurement="nodestate" and include:
+//   - field "event" containing JSON payload (UpdateNodeStatesRequest)
+//
+// Example: nodestate event="{\"cluster\":\"testcluster\",\"nodes\":[...]}" 1234567890000000000
+func (api *NatsAPI) handleNodeState(subject string, data []byte) {
+	if len(data) == 0 {
+		cclog.Warnf("NATS %s: received empty message", subject)
+		return
+	}
+
+	d := influx.NewDecoderWithBytes(data)
+
+	for d.Next() {
+		m, err := receivers.DecodeInfluxMessage(d)
+		if err != nil {
+			cclog.Errorf("NATS %s: failed to decode InfluxDB line protocol message: %v", subject, err)
+			return
+		}
+
+		if !m.IsEvent() {
+			cclog.Warnf("NATS %s: received non-event message, skipping", subject)
+			continue
+		}
+
+		if m.Name() == "nodestate" {
+			api.processNodestateEvent(m)
+		} else {
+			cclog.Warnf("NATS %s: unexpected measurement name '%s', expected 'nodestate'", subject, m.Name())
+		}
+	}
+}

internal/api/nats_test.go

@@ -0,0 +1,947 @@
+// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
+// All rights reserved. This file is part of cc-backend.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+package api
+
+import (
+	"context"
+	"database/sql"
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/ClusterCockpit/cc-backend/internal/archiver"
+	"github.com/ClusterCockpit/cc-backend/internal/auth"
+	"github.com/ClusterCockpit/cc-backend/internal/config"
+	"github.com/ClusterCockpit/cc-backend/internal/graph"
+	"github.com/ClusterCockpit/cc-backend/internal/repository"
+	"github.com/ClusterCockpit/cc-backend/pkg/archive"
+	"github.com/ClusterCockpit/cc-backend/pkg/metricstore"
+	ccconf "github.com/ClusterCockpit/cc-lib/v2/ccConfig"
+	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
+	lp "github.com/ClusterCockpit/cc-lib/v2/ccMessage"
+	"github.com/ClusterCockpit/cc-lib/v2/schema"
+
+	_ "github.com/mattn/go-sqlite3"
+)
+
+func setupNatsTest(t *testing.T) *NatsAPI {
+	repository.ResetConnection()
+
+	const testconfig = `{
+		"main": {
+	"addr":            "0.0.0.0:8080",
+	"validate": false,
+  "api-allowed-ips": [
+    "*"
+  ]
+	},
+	"archive": {
+		"kind": "file",
+		"path": "./var/job-archive"
+	},
+	"auth": {
+  "jwts": {
+      "max-age": "2m"
+  }
+	}
+}`
+	const testclusterJSON = `{
+        "name": "testcluster",
+		"subClusters": [
+			{
+				"name": "sc1",
+				"nodes": "host123,host124,host125",
+				"processorType": "Intel Core i7-4770",
+				"socketsPerNode": 1,
+				"coresPerSocket": 4,
+				"threadsPerCore": 2,
+                "flopRateScalar": {
+                  "unit": {
+                    "prefix": "G",
+                    "base": "F/s"
+                  },
+                  "value": 14
+                },
+                "flopRateSimd": {
+                  "unit": {
+                    "prefix": "G",
+                    "base": "F/s"
+                  },
+                  "value": 112
+                },
+                "memoryBandwidth": {
+                  "unit": {
+                    "prefix": "G",
+                    "base": "B/s"
+                  },
+                  "value": 24
+                },
+                "numberOfNodes": 70,
+				"topology": {
+					"node": [0, 1, 2, 3, 4, 5, 6, 7],
+					"socket": [[0, 1, 2, 3, 4, 5, 6, 7]],
+					"memoryDomain": [[0, 1, 2, 3, 4, 5, 6, 7]],
+					"die": [[0, 1, 2, 3, 4, 5, 6, 7]],
+					"core": [[0], [1], [2], [3], [4], [5], [6], [7]]
+				}
+			}
+		],
+		"metricConfig": [
+			{
+				"name": "load_one",
+			    "unit": { "base": ""},
+				"scope": "node",
+				"timestep": 60,
+                "aggregation": "avg",
+				"peak": 8,
+				"normal": 0,
+				"caution": 0,
+				"alert": 0
+			}
+		]
+	}`
+
+	cclog.Init("info", true)
+	tmpdir := t.TempDir()
+	jobarchive := filepath.Join(tmpdir, "job-archive")
+	if err := os.Mkdir(jobarchive, 0o777); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := os.WriteFile(filepath.Join(jobarchive, "version.txt"), fmt.Appendf(nil, "%d", 3), 0o666); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := os.Mkdir(filepath.Join(jobarchive, "testcluster"), 0o777); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := os.WriteFile(filepath.Join(jobarchive, "testcluster", "cluster.json"), []byte(testclusterJSON), 0o666); err != nil {
+		t.Fatal(err)
+	}
+
+	dbfilepath := filepath.Join(tmpdir, "test.db")
+	err := repository.MigrateDB(dbfilepath)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	cfgFilePath := filepath.Join(tmpdir, "config.json")
+	if err := os.WriteFile(cfgFilePath, []byte(testconfig), 0o666); err != nil {
+		t.Fatal(err)
+	}
+
+	ccconf.Init(cfgFilePath)
+
+	// Load and check main configuration
+	if cfg := ccconf.GetPackageConfig("main"); cfg != nil {
+		config.Init(cfg)
+	} else {
+		cclog.Abort("Main configuration must be present")
+	}
+	archiveCfg := fmt.Sprintf("{\"kind\": \"file\",\"path\": \"%s\"}", jobarchive)
+
+	repository.Connect(dbfilepath)
+
+	if err := archive.Init(json.RawMessage(archiveCfg)); err != nil {
+		t.Fatal(err)
+	}
+
+	// metricstore initialization removed - it's initialized via callback in tests
+
+	archiver.Start(repository.GetJobRepository(), context.Background())
+
+	if cfg := ccconf.GetPackageConfig("auth"); cfg != nil {
+		auth.Init(&cfg)
+	} else {
+		cclog.Warn("Authentication disabled due to missing configuration")
+		auth.Init(nil)
+	}
+
+	graph.Init()
+
+	return NewNatsAPI()
+}
+
+func cleanupNatsTest() {
+	if err := archiver.Shutdown(5 * time.Second); err != nil {
+		cclog.Warnf("Archiver shutdown timeout in tests: %v", err)
+	}
+}
+
+func TestNatsHandleStartJob(t *testing.T) {
+	natsAPI := setupNatsTest(t)
+	t.Cleanup(cleanupNatsTest)
+
+	tests := []struct {
+		name          string
+		payload       string
+		expectError   bool
+		validateJob   func(t *testing.T, job *schema.Job)
+		shouldFindJob bool
+	}{
+		{
+			name: "valid job start",
+			payload: `{
+				"jobId": 1001,
+				"user": "testuser1",
+				"project": "testproj1",
+				"cluster": "testcluster",
+				"partition": "main",
+				"walltime": 7200,
+				"numNodes": 1,
+				"numHwthreads": 8,
+				"numAcc": 0,
+				"shared": "none",
+				"monitoringStatus": 1,
+				"smt": 1,
+				"resources": [
+					{
+						"hostname": "host123",
+						"hwthreads": [0, 1, 2, 3, 4, 5, 6, 7]
+					}
+				],
+				"startTime": 1234567890
+			}`,
+			expectError:   false,
+			shouldFindJob: true,
+			validateJob: func(t *testing.T, job *schema.Job) {
+				if job.JobID != 1001 {
+					t.Errorf("expected JobID 1001, got %d", job.JobID)
+				}
+				if job.User != "testuser1" {
+					t.Errorf("expected user testuser1, got %s", job.User)
+				}
+				if job.State != schema.JobStateRunning {
+					t.Errorf("expected state running, got %s", job.State)
+				}
+			},
+		},
+		{
+			name: "invalid JSON",
+			payload: `{
+				"jobId": "not a number",
+				"user": "testuser2"
+			}`,
+			expectError:   true,
+			shouldFindJob: false,
+		},
+		{
+			name: "missing required fields",
+			payload: `{
+				"jobId": 1002
+			}`,
+			expectError:   true,
+			shouldFindJob: false,
+		},
+		{
+			name: "job with unknown fields (should fail due to DisallowUnknownFields)",
+			payload: `{
+				"jobId": 1003,
+				"user": "testuser3",
+				"project": "testproj3",
+				"cluster": "testcluster",
+				"partition": "main",
+				"walltime": 3600,
+				"numNodes": 1,
+				"numHwthreads": 8,
+				"unknownField": "should cause error",
+				"startTime": 1234567900
+			}`,
+			expectError:   true,
+			shouldFindJob: false,
+		},
+		{
+			name: "job with tags",
+			payload: `{
+				"jobId": 1004,
+				"user": "testuser4",
+				"project": "testproj4",
+				"cluster": "testcluster",
+				"partition": "main",
+				"walltime": 3600,
+				"numNodes": 1,
+				"numHwthreads": 8,
+				"numAcc": 0,
+				"shared": "none",
+				"monitoringStatus": 1,
+				"smt": 1,
+				"resources": [
+					{
+						"hostname": "host123",
+						"hwthreads": [0, 1, 2, 3]
+					}
+				],
+				"tags": [
+					{
+						"type": "test",
+						"name": "testtag",
+						"scope": "testuser4"
+					}
+				],
+				"startTime": 1234567910
+			}`,
+			expectError:   false,
+			shouldFindJob: true,
+			validateJob: func(t *testing.T, job *schema.Job) {
+				if job.JobID != 1004 {
+					t.Errorf("expected JobID 1004, got %d", job.JobID)
+				}
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			natsAPI.handleStartJob(tt.payload)
+			natsAPI.JobRepository.SyncJobs()
+
+			// Allow some time for async operations
+			time.Sleep(100 * time.Millisecond)
+
+			if tt.shouldFindJob {
+				// Extract jobId from payload
+				var payloadMap map[string]any
+				json.Unmarshal([]byte(tt.payload), &payloadMap)
+				jobID := int64(payloadMap["jobId"].(float64))
+				cluster := payloadMap["cluster"].(string)
+				startTime := int64(payloadMap["startTime"].(float64))
+
+				job, err := natsAPI.JobRepository.Find(&jobID, &cluster, &startTime)
+				if err != nil {
+					if !tt.expectError {
+						t.Fatalf("expected to find job, but got error: %v", err)
+					}
+					return
+				}
+
+				if tt.validateJob != nil {
+					tt.validateJob(t, job)
+				}
+			}
+		})
+	}
+}
+
+func TestNatsHandleStopJob(t *testing.T) {
+	natsAPI := setupNatsTest(t)
+	t.Cleanup(cleanupNatsTest)
+
+	// First, create a running job
+	startPayload := `{
+		"jobId": 2001,
+		"user": "testuser",
+		"project": "testproj",
+		"cluster": "testcluster",
+		"partition": "main",
+		"walltime": 3600,
+		"numNodes": 1,
+		"numHwthreads": 8,
+		"numAcc": 0,
+		"shared": "none",
+		"monitoringStatus": 1,
+		"smt": 1,
+		"resources": [
+			{
+				"hostname": "host123",
+				"hwthreads": [0, 1, 2, 3, 4, 5, 6, 7]
+			}
+		],
+		"startTime": 1234567890
+	}`
+
+	natsAPI.handleStartJob(startPayload)
+	natsAPI.JobRepository.SyncJobs()
+	time.Sleep(100 * time.Millisecond)
+
+	tests := []struct {
+		name         string
+		payload      string
+		expectError  bool
+		validateJob  func(t *testing.T, job *schema.Job)
+		setupJobFunc func() // Optional: create specific test job
+	}{
+		{
+			name: "valid job stop - completed",
+			payload: `{
+				"jobId": 2001,
+				"cluster": "testcluster",
+				"startTime": 1234567890,
+				"jobState": "completed",
+				"stopTime": 1234571490
+			}`,
+			expectError: false,
+			validateJob: func(t *testing.T, job *schema.Job) {
+				if job.State != schema.JobStateCompleted {
+					t.Errorf("expected state completed, got %s", job.State)
+				}
+				expectedDuration := int32(1234571490 - 1234567890)
+				if job.Duration != expectedDuration {
+					t.Errorf("expected duration %d, got %d", expectedDuration, job.Duration)
+				}
+			},
+		},
+		{
+			name: "valid job stop - failed",
+			setupJobFunc: func() {
+				startPayloadFailed := `{
+					"jobId": 2002,
+					"user": "testuser",
+					"project": "testproj",
+					"cluster": "testcluster",
+					"partition": "main",
+					"walltime": 3600,
+					"numNodes": 1,
+					"numHwthreads": 8,
+					"numAcc": 0,
+					"shared": "none",
+					"monitoringStatus": 1,
+					"smt": 1,
+					"resources": [
+						{
+							"hostname": "host123",
+							"hwthreads": [0, 1, 2, 3]
+						}
+					],
+					"startTime": 1234567900
+				}`
+				natsAPI.handleStartJob(startPayloadFailed)
+				natsAPI.JobRepository.SyncJobs()
+				time.Sleep(100 * time.Millisecond)
+			},
+			payload: `{
+				"jobId": 2002,
+				"cluster": "testcluster",
+				"startTime": 1234567900,
+				"jobState": "failed",
+				"stopTime": 1234569900
+			}`,
+			expectError: false,
+			validateJob: func(t *testing.T, job *schema.Job) {
+				if job.State != schema.JobStateFailed {
+					t.Errorf("expected state failed, got %s", job.State)
+				}
+			},
+		},
+		{
+			name: "invalid JSON",
+			payload: `{
+				"jobId": "not a number"
+			}`,
+			expectError: true,
+		},
+		{
+			name: "missing jobId",
+			payload: `{
+				"cluster": "testcluster",
+				"jobState": "completed",
+				"stopTime": 1234571490
+			}`,
+			expectError: true,
+		},
+		{
+			name: "invalid job state",
+			setupJobFunc: func() {
+				startPayloadInvalid := `{
+					"jobId": 2003,
+					"user": "testuser",
+					"project": "testproj",
+					"cluster": "testcluster",
+					"partition": "main",
+					"walltime": 3600,
+					"numNodes": 1,
+					"numHwthreads": 8,
+					"numAcc": 0,
+					"shared": "none",
+					"monitoringStatus": 1,
+					"smt": 1,
+					"resources": [
+						{
+							"hostname": "host123",
+							"hwthreads": [0, 1]
+						}
+					],
+					"startTime": 1234567910
+				}`
+				natsAPI.handleStartJob(startPayloadInvalid)
+				natsAPI.JobRepository.SyncJobs()
+				time.Sleep(100 * time.Millisecond)
+			},
+			payload: `{
+				"jobId": 2003,
+				"cluster": "testcluster",
+				"startTime": 1234567910,
+				"jobState": "invalid_state",
+				"stopTime": 1234571510
+			}`,
+			expectError: true,
+		},
+		{
+			name: "stopTime before startTime",
+			setupJobFunc: func() {
+				startPayloadTime := `{
+					"jobId": 2004,
+					"user": "testuser",
+					"project": "testproj",
+					"cluster": "testcluster",
+					"partition": "main",
+					"walltime": 3600,
+					"numNodes": 1,
+					"numHwthreads": 8,
+					"numAcc": 0,
+					"shared": "none",
+					"monitoringStatus": 1,
+					"smt": 1,
+					"resources": [
+						{
+							"hostname": "host123",
+							"hwthreads": [0]
+						}
+					],
+					"startTime": 1234567920
+				}`
+				natsAPI.handleStartJob(startPayloadTime)
+				natsAPI.JobRepository.SyncJobs()
+				time.Sleep(100 * time.Millisecond)
+			},
+			payload: `{
+				"jobId": 2004,
+				"cluster": "testcluster",
+				"startTime": 1234567920,
+				"jobState": "completed",
+				"stopTime": 1234567900
+			}`,
+			expectError: true,
+		},
+		{
+			name: "job not found",
+			payload: `{
+				"jobId": 99999,
+				"cluster": "testcluster",
+				"startTime": 1234567890,
+				"jobState": "completed",
+				"stopTime": 1234571490
+			}`,
+			expectError: true,
+		},
+	}
+
+	testData := schema.JobData{
+		"load_one": map[schema.MetricScope]*schema.JobMetric{
+			schema.MetricScopeNode: {
+				Unit:     schema.Unit{Base: "load"},
+				Timestep: 60,
+				Series: []schema.Series{
+					{
+						Hostname:   "host123",
+						Statistics: schema.MetricStatistics{Min: 0.1, Avg: 0.2, Max: 0.3},
+						Data:       []schema.Float{0.1, 0.1, 0.1, 0.2, 0.2, 0.2, 0.3, 0.3, 0.3},
+					},
+				},
+			},
+		},
+	}
+
+	metricstore.TestLoadDataCallback = func(job *schema.Job, metrics []string, scopes []schema.MetricScope, ctx context.Context, resolution int) (schema.JobData, error) {
+		return testData, nil
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if tt.setupJobFunc != nil {
+				tt.setupJobFunc()
+			}
+
+			natsAPI.handleStopJob(tt.payload)
+
+			// Allow some time for async operations
+			time.Sleep(100 * time.Millisecond)
+
+			if !tt.expectError && tt.validateJob != nil {
+				// Extract job details from payload
+				var payloadMap map[string]any
+				json.Unmarshal([]byte(tt.payload), &payloadMap)
+				jobID := int64(payloadMap["jobId"].(float64))
+				cluster := payloadMap["cluster"].(string)
+
+				var startTime *int64
+				if st, ok := payloadMap["startTime"]; ok {
+					t := int64(st.(float64))
+					startTime = &t
+				}
+
+				job, err := natsAPI.JobRepository.Find(&jobID, &cluster, startTime)
+				if err != nil {
+					t.Fatalf("expected to find job, but got error: %v", err)
+				}
+
+				tt.validateJob(t, job)
+			}
+		})
+	}
+}
+
+func TestNatsHandleNodeState(t *testing.T) {
+	natsAPI := setupNatsTest(t)
+	t.Cleanup(cleanupNatsTest)
+
+	tests := []struct {
+		name        string
+		data        []byte
+		expectError bool
+		validateFn  func(t *testing.T)
+	}{
+		{
+			name:        "valid node state update",
+			data:        []byte(`nodestate event="{\"cluster\":\"testcluster\",\"nodes\":[{\"hostname\":\"host123\",\"states\":[\"allocated\"],\"cpusAllocated\":8,\"memoryAllocated\":16384,\"gpusAllocated\":0,\"jobsRunning\":1}]}" 1234567890000000000`),
+			expectError: false,
+			validateFn: func(t *testing.T) {
+				// In a full test, we would verify the node state was updated in the database
+				// For now, just ensure no error occurred
+			},
+		},
+		{
+			name:        "multiple nodes",
+			data:        []byte(`nodestate event="{\"cluster\":\"testcluster\",\"nodes\":[{\"hostname\":\"host123\",\"states\":[\"idle\"],\"cpusAllocated\":0,\"memoryAllocated\":0,\"gpusAllocated\":0,\"jobsRunning\":0},{\"hostname\":\"host124\",\"states\":[\"allocated\"],\"cpusAllocated\":4,\"memoryAllocated\":8192,\"gpusAllocated\":1,\"jobsRunning\":1}]}" 1234567890000000000`),
+			expectError: false,
+		},
+		{
+			name:        "invalid JSON in event field",
+			data:        []byte(`nodestate event="{\"cluster\":\"testcluster\",\"nodes\":\"not an array\"}" 1234567890000000000`),
+			expectError: true,
+		},
+		{
+			name:        "empty nodes array",
+			data:        []byte(`nodestate event="{\"cluster\":\"testcluster\",\"nodes\":[]}" 1234567890000000000`),
+			expectError: false, // Empty array should not cause error
+		},
+		{
+			name:        "invalid line protocol format",
+			data:        []byte(`invalid line protocol format`),
+			expectError: true,
+		},
+		{
+			name:        "empty data",
+			data:        []byte(``),
+			expectError: false, // Should be handled gracefully with warning
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			natsAPI.handleNodeState("test.subject", tt.data)
+
+			// Allow some time for async operations
+			time.Sleep(50 * time.Millisecond)
+
+			if tt.validateFn != nil {
+				tt.validateFn(t)
+			}
+		})
+	}
+}
+
+func TestNatsProcessJobEvent(t *testing.T) {
+	natsAPI := setupNatsTest(t)
+	t.Cleanup(cleanupNatsTest)
+
+	msgStartJob, err := lp.NewMessage(
+		"job",
+		map[string]string{"function": "start_job"},
+		nil,
+		map[string]any{
+			"event": `{
+				"jobId": 3001,
+				"user": "testuser",
+				"project": "testproj",
+				"cluster": "testcluster",
+				"partition": "main",
+				"walltime": 3600,
+				"numNodes": 1,
+				"numHwthreads": 8,
+				"numAcc": 0,
+				"shared": "none",
+				"monitoringStatus": 1,
+				"smt": 1,
+				"resources": [
+					{
+						"hostname": "host123",
+						"hwthreads": [0, 1, 2, 3]
+					}
+				],
+				"startTime": 1234567890
+			}`,
+		},
+		time.Now(),
+	)
+	if err != nil {
+		t.Fatalf("failed to create test message: %v", err)
+	}
+
+	msgMissingTag, err := lp.NewMessage(
+		"job",
+		map[string]string{},
+		nil,
+		map[string]any{
+			"event": `{}`,
+		},
+		time.Now(),
+	)
+	if err != nil {
+		t.Fatalf("failed to create test message: %v", err)
+	}
+
+	msgUnknownFunc, err := lp.NewMessage(
+		"job",
+		map[string]string{"function": "unknown_function"},
+		nil,
+		map[string]any{
+			"event": `{}`,
+		},
+		time.Now(),
+	)
+	if err != nil {
+		t.Fatalf("failed to create test message: %v", err)
+	}
+
+	tests := []struct {
+		name        string
+		message     lp.CCMessage
+		expectError bool
+	}{
+		{
+			name:        "start_job function",
+			message:     msgStartJob,
+			expectError: false,
+		},
+		{
+			name:        "missing function tag",
+			message:     msgMissingTag,
+			expectError: true,
+		},
+		{
+			name:        "unknown function",
+			message:     msgUnknownFunc,
+			expectError: false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			natsAPI.processJobEvent(tt.message)
+			time.Sleep(50 * time.Millisecond)
+		})
+	}
+}
+
+func TestNatsHandleJobEvent(t *testing.T) {
+	natsAPI := setupNatsTest(t)
+	t.Cleanup(cleanupNatsTest)
+
+	tests := []struct {
+		name        string
+		data        []byte
+		expectError bool
+	}{
+		{
+			name:        "valid influx line protocol",
+			data:        []byte(`job,function=start_job event="{\"jobId\":4001,\"user\":\"testuser\",\"project\":\"testproj\",\"cluster\":\"testcluster\",\"partition\":\"main\",\"walltime\":3600,\"numNodes\":1,\"numHwthreads\":8,\"numAcc\":0,\"shared\":\"none\",\"monitoringStatus\":1,\"smt\":1,\"resources\":[{\"hostname\":\"host123\",\"hwthreads\":[0,1,2,3]}],\"startTime\":1234567890}" 1234567890000000000`),
+			expectError: false,
+		},
+		{
+			name:        "invalid influx line protocol",
+			data:        []byte(`invalid line protocol format`),
+			expectError: true,
+		},
+		{
+			name:        "empty data",
+			data:        []byte(``),
+			expectError: false, // Decoder should handle empty input gracefully
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			// HandleJobEvent doesn't return errors, it logs them
+			// We're just ensuring it doesn't panic
+			natsAPI.handleJobEvent("test.subject", tt.data)
+			time.Sleep(50 * time.Millisecond)
+		})
+	}
+}
+
+func TestNatsHandleJobEventEdgeCases(t *testing.T) {
+	natsAPI := setupNatsTest(t)
+	t.Cleanup(cleanupNatsTest)
+
+	tests := []struct {
+		name        string
+		data        []byte
+		expectError bool
+		description string
+	}{
+		{
+			name:        "non-event message (metric data)",
+			data:        []byte(`job,function=start_job value=123.45 1234567890000000000`),
+			expectError: false,
+			description: "Should skip non-event messages gracefully",
+		},
+		{
+			name:        "wrong measurement name",
+			data:        []byte(`wrongmeasurement,function=start_job event="{}" 1234567890000000000`),
+			expectError: false,
+			description: "Should warn about unexpected measurement but not fail",
+		},
+		{
+			name:        "missing event field",
+			data:        []byte(`job,function=start_job other_field="value" 1234567890000000000`),
+			expectError: true,
+			description: "Should error when event field is missing",
+		},
+		{
+			name:        "multiple measurements in one message",
+			data:        []byte("job,function=start_job event=\"{}\" 1234567890000000000\njob,function=stop_job event=\"{}\" 1234567890000000000"),
+			expectError: false,
+			description: "Should process multiple lines",
+		},
+		{
+			name:        "escaped quotes in JSON payload",
+			data:        []byte(`job,function=start_job event="{\"jobId\":6001,\"user\":\"test\\\"user\",\"cluster\":\"test\"}" 1234567890000000000`),
+			expectError: true,
+			description: "Should handle escaped quotes (though JSON parsing may fail)",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			natsAPI.handleJobEvent("test.subject", tt.data)
+			time.Sleep(50 * time.Millisecond)
+		})
+	}
+}
+
+func TestNatsHandleNodeStateEdgeCases(t *testing.T) {
+	natsAPI := setupNatsTest(t)
+	t.Cleanup(cleanupNatsTest)
+
+	tests := []struct {
+		name        string
+		data        []byte
+		expectError bool
+		description string
+	}{
+		{
+			name:        "missing cluster field in JSON",
+			data:        []byte(`nodestate event="{\"nodes\":[]}" 1234567890000000000`),
+			expectError: true,
+			description: "Should fail when cluster is missing",
+		},
+		{
+			name:        "malformed JSON with unescaped quotes",
+			data:        []byte(`nodestate event="{\"cluster\":\"test"cluster\",\"nodes\":[]}" 1234567890000000000`),
+			expectError: true,
+			description: "Should fail on malformed JSON",
+		},
+		{
+			name:        "unicode characters in hostname",
+			data:        []byte(`nodestate event="{\"cluster\":\"testcluster\",\"nodes\":[{\"hostname\":\"host-ñ123\",\"states\":[\"idle\"],\"cpusAllocated\":0,\"memoryAllocated\":0,\"gpusAllocated\":0,\"jobsRunning\":0}]}" 1234567890000000000`),
+			expectError: false,
+			description: "Should handle unicode characters",
+		},
+		{
+			name:        "very large node count",
+			data:        []byte(`nodestate event="{\"cluster\":\"testcluster\",\"nodes\":[{\"hostname\":\"node1\",\"states\":[\"idle\"],\"cpusAllocated\":0,\"memoryAllocated\":0,\"gpusAllocated\":0,\"jobsRunning\":0},{\"hostname\":\"node2\",\"states\":[\"idle\"],\"cpusAllocated\":0,\"memoryAllocated\":0,\"gpusAllocated\":0,\"jobsRunning\":0},{\"hostname\":\"node3\",\"states\":[\"idle\"],\"cpusAllocated\":0,\"memoryAllocated\":0,\"gpusAllocated\":0,\"jobsRunning\":0}]}" 1234567890000000000`),
+			expectError: false,
+			description: "Should handle multiple nodes efficiently",
+		},
+		{
+			name:        "timestamp in past",
+			data:        []byte(`nodestate event="{\"cluster\":\"testcluster\",\"nodes\":[]}" 1000000000000000000`),
+			expectError: false,
+			description: "Should accept any valid timestamp",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			natsAPI.handleNodeState("test.subject", tt.data)
+			time.Sleep(50 * time.Millisecond)
+		})
+	}
+}
+
+func TestNatsHandleStartJobDuplicatePrevention(t *testing.T) {
+	natsAPI := setupNatsTest(t)
+	t.Cleanup(cleanupNatsTest)
+
+	// Start a job
+	payload := `{
+		"jobId": 5001,
+		"user": "testuser",
+		"project": "testproj",
+		"cluster": "testcluster",
+		"partition": "main",
+		"walltime": 3600,
+		"numNodes": 1,
+		"numHwthreads": 8,
+		"numAcc": 0,
+		"shared": "none",
+		"monitoringStatus": 1,
+		"smt": 1,
+		"resources": [
+			{
+				"hostname": "host123",
+				"hwthreads": [0, 1, 2, 3]
+			}
+		],
+		"startTime": 1234567890
+	}`
+
+	natsAPI.handleStartJob(payload)
+	natsAPI.JobRepository.SyncJobs()
+	time.Sleep(100 * time.Millisecond)
+
+	// Try to start the same job again (within 24 hours)
+	duplicatePayload := `{
+		"jobId": 5001,
+		"user": "testuser",
+		"project": "testproj",
+		"cluster": "testcluster",
+		"partition": "main",
+		"walltime": 3600,
+		"numNodes": 1,
+		"numHwthreads": 8,
+		"numAcc": 0,
+		"shared": "none",
+		"monitoringStatus": 1,
+		"smt": 1,
+		"resources": [
+			{
+				"hostname": "host123",
+				"hwthreads": [0, 1, 2, 3]
+			}
+		],
+		"startTime": 1234567900
+	}`
+
+	natsAPI.handleStartJob(duplicatePayload)
+	natsAPI.JobRepository.SyncJobs()
+	time.Sleep(100 * time.Millisecond)
+
+	// Verify only one job exists
+	jobID := int64(5001)
+	cluster := "testcluster"
+	jobs, err := natsAPI.JobRepository.FindAll(&jobID, &cluster, nil)
+	if err != nil && err != sql.ErrNoRows {
+		t.Fatalf("unexpected error: %v", err)
+	}
+
+	if len(jobs) != 1 {
+		t.Errorf("expected 1 job, got %d", len(jobs))
+	}
+}

internal/api/node.go

@@ -0,0 +1,145 @@
+// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
+// All rights reserved. This file is part of cc-backend.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package api
+
+import (
+	"fmt"
+	"maps"
+	"net/http"
+	"strings"
+	"time"
+
+	"github.com/ClusterCockpit/cc-backend/internal/metricdispatch"
+	"github.com/ClusterCockpit/cc-backend/internal/repository"
+	"github.com/ClusterCockpit/cc-backend/pkg/archive"
+	"github.com/ClusterCockpit/cc-backend/pkg/metricstore"
+	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
+	"github.com/ClusterCockpit/cc-lib/v2/schema"
+)
+
+type UpdateNodeStatesRequest struct {
+	Nodes   []schema.NodePayload `json:"nodes"`
+	Cluster string               `json:"cluster" example:"fritz"`
+}
+
+// metricListToNames converts a map of metric configurations to a list of metric names
+func metricListToNames(metricList map[string]*schema.Metric) []string {
+	names := make([]string, 0, len(metricList))
+	for name := range metricList {
+		names = append(names, name)
+	}
+	return names
+}
+
+// this routine assumes that only one of them exists per node
+func determineState(states []string) schema.SchedulerState {
+	for _, state := range states {
+		switch strings.ToLower(state) {
+		case "allocated":
+			return schema.NodeStateAllocated
+		case "reserved":
+			return schema.NodeStateReserved
+		case "idle":
+			return schema.NodeStateIdle
+		case "down":
+			return schema.NodeStateDown
+		case "mixed":
+			return schema.NodeStateMixed
+		}
+	}
+
+	return schema.NodeStateUnknown
+}
+
+// updateNodeStates godoc
+// @summary     Deliver updated Slurm node states
+// @tags Nodestates
+// @description Returns a JSON-encoded list of users.
+// @description Required query-parameter defines if all users or only users with additional special roles are returned.
+// @produce     json
+// @param       request body UpdateNodeStatesRequest true "Request body containing nodes and their states"
+// @success     200     {object} api.DefaultAPIResponse "Success message"
+// @failure     400     {object} api.ErrorResponse      "Bad Request"
+// @failure     401     {object} api.ErrorResponse      "Unauthorized"
+// @failure     403     {object} api.ErrorResponse      "Forbidden"
+// @failure     500     {object} api.ErrorResponse      "Internal Server Error"
+// @security    ApiKeyAuth
+// @router      /api/nodestats/ [post]
+func (api *RestAPI) updateNodeStates(rw http.ResponseWriter, r *http.Request) {
+	// Parse request body
+	req := UpdateNodeStatesRequest{}
+	if err := decode(r.Body, &req); err != nil {
+		handleError(fmt.Errorf("parsing request body failed: %w", err),
+			http.StatusBadRequest, rw)
+		return
+	}
+	requestReceived := time.Now().Unix()
+	repo := repository.GetNodeRepository()
+
+	m := make(map[string][]string)
+	metricNames := make(map[string][]string)
+	healthResults := make(map[string]metricstore.HealthCheckResult)
+
+	startMs := time.Now()
+
+	// Step 1: Build nodeList and metricList per subcluster
+	for _, node := range req.Nodes {
+		if sc, err := archive.GetSubClusterByNode(req.Cluster, node.Hostname); err == nil {
+			m[sc] = append(m[sc], node.Hostname)
+		}
+	}
+
+	for sc := range m {
+		if sc != "" {
+			metricList := archive.GetMetricConfigSubCluster(req.Cluster, sc)
+			metricNames[sc] = metricListToNames(metricList)
+		}
+	}
+
+	// Step 2: Determine which metric store to query and perform health check
+	healthRepo, err := metricdispatch.GetHealthCheckRepo(req.Cluster)
+	if err != nil {
+		cclog.Warnf("updateNodeStates: no metric store for cluster %s, skipping health check: %v", req.Cluster, err)
+	} else {
+		for sc, nl := range m {
+			if sc != "" {
+				if results, err := healthRepo.HealthCheck(req.Cluster, nl, metricNames[sc]); err == nil {
+					maps.Copy(healthResults, results)
+				}
+			}
+		}
+	}
+
+	cclog.Debugf("Timer updateNodeStates, MemStore HealthCheck: %s", time.Since(startMs))
+	startDB := time.Now()
+
+	for _, node := range req.Nodes {
+		state := determineState(node.States)
+		healthState := schema.MonitoringStateFailed
+		var healthMetrics string
+		if result, ok := healthResults[node.Hostname]; ok {
+			healthState = result.State
+			healthMetrics = result.HealthMetrics
+		}
+		nodeState := schema.NodeStateDB{
+			TimeStamp:       requestReceived,
+			NodeState:       state,
+			CpusAllocated:   node.CpusAllocated,
+			MemoryAllocated: node.MemoryAllocated,
+			GpusAllocated:   node.GpusAllocated,
+			HealthState:     healthState,
+			HealthMetrics:   healthMetrics,
+			JobsRunning:     node.JobsRunning,
+		}
+
+		if err := repo.UpdateNodeState(node.Hostname, req.Cluster, &nodeState); err != nil {
+			cclog.Errorf("updateNodeStates: updating node state for %s on %s failed: %v",
+				node.Hostname, req.Cluster, err)
+		}
+	}
+
+	cclog.Debugf("Timer updateNodeStates, SQLite Inserts: %s", time.Since(startDB))
+}

internal/api/user.go

@@ -0,0 +1,221 @@
+// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
+// All rights reserved. This file is part of cc-backend.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package api
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+
+	"github.com/ClusterCockpit/cc-backend/internal/repository"
+	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
+	"github.com/ClusterCockpit/cc-lib/v2/schema"
+	"github.com/go-chi/chi/v5"
+)
+
+type APIReturnedUser struct {
+	Username string   `json:"username"`
+	Name     string   `json:"name"`
+	Roles    []string `json:"roles"`
+	Email    string   `json:"email"`
+	Projects []string `json:"projects"`
+}
+
+// getUsers godoc
+// @summary     Returns a list of users
+// @tags User
+// @description Returns a JSON-encoded list of users.
+// @description Required query-parameter defines if all users or only users with additional special roles are returned.
+// @produce     json
+// @param       not-just-user query bool true "If returned list should contain all users or only users with additional special roles"
+// @success     200     {array} api.APIReturnedUser "List of users returned successfully"
+// @failure     400     {string} string             "Bad Request"
+// @failure     401     {string} string             "Unauthorized"
+// @failure     403     {string} string             "Forbidden"
+// @failure     500     {string} string             "Internal Server Error"
+// @security    ApiKeyAuth
+// @router      /api/users/ [get]
+func (api *RestAPI) getUsers(rw http.ResponseWriter, r *http.Request) {
+	// SecuredCheck() only worked with TokenAuth: Removed
+
+	if user := repository.GetUserFromContext(r.Context()); !user.HasRole(schema.RoleAdmin) {
+		handleError(fmt.Errorf("only admins are allowed to fetch a list of users"), http.StatusForbidden, rw)
+		return
+	}
+
+	users, err := repository.GetUserRepository().ListUsers(r.URL.Query().Get("not-just-user") == "true")
+	if err != nil {
+		handleError(fmt.Errorf("listing users failed: %w", err), http.StatusInternalServerError, rw)
+		return
+	}
+
+	rw.Header().Set("Content-Type", "application/json")
+	if err := json.NewEncoder(rw).Encode(users); err != nil {
+		cclog.Errorf("Failed to encode users response: %v", err)
+	}
+}
+
+// updateUser godoc
+// @summary     Update user roles and projects
+// @tags User
+// @description Allows admins to add/remove roles and projects for a user
+// @produce     plain
+// @param       id          path   string  true  "Username"
+// @param       add-role    formData string false "Role to add"
+// @param       remove-role formData string false "Role to remove"
+// @param       add-project formData string false "Project to add"
+// @param       remove-project formData string false "Project to remove"
+// @success     200     {string} string "Success message"
+// @failure     403     {object} api.ErrorResponse "Forbidden"
+// @failure     422     {object} api.ErrorResponse "Unprocessable Entity"
+// @security    ApiKeyAuth
+// @router      /api/user/{id} [post]
+func (api *RestAPI) updateUser(rw http.ResponseWriter, r *http.Request) {
+	// SecuredCheck() only worked with TokenAuth: Removed
+
+	if user := repository.GetUserFromContext(r.Context()); !user.HasRole(schema.RoleAdmin) {
+		handleError(fmt.Errorf("only admins are allowed to update a user"), http.StatusForbidden, rw)
+		return
+	}
+
+	// Get Values
+	newrole := r.FormValue("add-role")
+	delrole := r.FormValue("remove-role")
+	newproj := r.FormValue("add-project")
+	delproj := r.FormValue("remove-project")
+
+	rw.Header().Set("Content-Type", "application/json")
+
+	// Handle role updates
+	if newrole != "" {
+		if err := repository.GetUserRepository().AddRole(r.Context(), chi.URLParam(r, "id"), newrole); err != nil {
+			handleError(fmt.Errorf("adding role failed: %w", err), http.StatusUnprocessableEntity, rw)
+			return
+		}
+		if err := json.NewEncoder(rw).Encode(DefaultAPIResponse{Message: "Add Role Success"}); err != nil {
+			cclog.Errorf("Failed to encode response: %v", err)
+		}
+	} else if delrole != "" {
+		if err := repository.GetUserRepository().RemoveRole(r.Context(), chi.URLParam(r, "id"), delrole); err != nil {
+			handleError(fmt.Errorf("removing role failed: %w", err), http.StatusUnprocessableEntity, rw)
+			return
+		}
+		if err := json.NewEncoder(rw).Encode(DefaultAPIResponse{Message: "Remove Role Success"}); err != nil {
+			cclog.Errorf("Failed to encode response: %v", err)
+		}
+	} else if newproj != "" {
+		if err := repository.GetUserRepository().AddProject(r.Context(), chi.URLParam(r, "id"), newproj); err != nil {
+			handleError(fmt.Errorf("adding project failed: %w", err), http.StatusUnprocessableEntity, rw)
+			return
+		}
+		if err := json.NewEncoder(rw).Encode(DefaultAPIResponse{Message: "Add Project Success"}); err != nil {
+			cclog.Errorf("Failed to encode response: %v", err)
+		}
+	} else if delproj != "" {
+		if err := repository.GetUserRepository().RemoveProject(r.Context(), chi.URLParam(r, "id"), delproj); err != nil {
+			handleError(fmt.Errorf("removing project failed: %w", err), http.StatusUnprocessableEntity, rw)
+			return
+		}
+		if err := json.NewEncoder(rw).Encode(DefaultAPIResponse{Message: "Remove Project Success"}); err != nil {
+			cclog.Errorf("Failed to encode response: %v", err)
+		}
+	} else {
+		handleError(fmt.Errorf("no operation specified: must provide add-role, remove-role, add-project, or remove-project"), http.StatusBadRequest, rw)
+	}
+}
+
+// createUser godoc
+// @summary     Create a new user
+// @tags User
+// @description Creates a new user with specified credentials and role
+// @produce     plain
+// @param       username formData string true  "Username"
+// @param       password formData string false "Password (not required for API users)"
+// @param       role     formData string true  "User role"
+// @param       name     formData string false "Full name"
+// @param       email    formData string false "Email address"
+// @param       project  formData string false "Project (required for managers)"
+// @success     200     {string} string "Success message"
+// @failure     400     {object} api.ErrorResponse "Bad Request"
+// @failure     403     {object} api.ErrorResponse "Forbidden"
+// @failure     422     {object} api.ErrorResponse "Unprocessable Entity"
+// @security    ApiKeyAuth
+// @router      /api/users/ [post]
+func (api *RestAPI) createUser(rw http.ResponseWriter, r *http.Request) {
+	// SecuredCheck() only worked with TokenAuth: Removed
+
+	rw.Header().Set("Content-Type", "text/plain")
+	me := repository.GetUserFromContext(r.Context())
+	if !me.HasRole(schema.RoleAdmin) {
+		handleError(fmt.Errorf("only admins are allowed to create new users"), http.StatusForbidden, rw)
+		return
+	}
+
+	username, password, role, name, email, project := r.FormValue("username"),
+		r.FormValue("password"), r.FormValue("role"), r.FormValue("name"),
+		r.FormValue("email"), r.FormValue("project")
+
+	// Validate username length
+	if len(username) == 0 || len(username) > 100 {
+		handleError(fmt.Errorf("username must be between 1 and 100 characters"), http.StatusBadRequest, rw)
+		return
+	}
+
+	if len(password) == 0 && role != schema.GetRoleString(schema.RoleAPI) {
+		handleError(fmt.Errorf("only API users are allowed to have a blank password (login will be impossible)"), http.StatusBadRequest, rw)
+		return
+	}
+
+	if len(project) != 0 && role != schema.GetRoleString(schema.RoleManager) {
+		handleError(fmt.Errorf("only managers require a project (can be changed later)"), http.StatusBadRequest, rw)
+		return
+	} else if len(project) == 0 && role == schema.GetRoleString(schema.RoleManager) {
+		handleError(fmt.Errorf("managers require a project to manage (can be changed later)"), http.StatusBadRequest, rw)
+		return
+	}
+
+	if err := repository.GetUserRepository().AddUser(&schema.User{
+		Username: username,
+		Name:     name,
+		Password: password,
+		Email:    email,
+		Projects: []string{project},
+		Roles:    []string{role},
+	}); err != nil {
+		handleError(fmt.Errorf("adding user failed: %w", err), http.StatusUnprocessableEntity, rw)
+		return
+	}
+
+	fmt.Fprintf(rw, "User %v successfully created!\n", username)
+}
+
+// deleteUser godoc
+// @summary     Delete a user
+// @tags User
+// @description Deletes a user from the system
+// @produce     plain
+// @param       username formData string true "Username to delete"
+// @success     200     {string} string "Success"
+// @failure     403     {object} api.ErrorResponse "Forbidden"
+// @failure     422     {object} api.ErrorResponse "Unprocessable Entity"
+// @security    ApiKeyAuth
+// @router      /api/users/ [delete]
+func (api *RestAPI) deleteUser(rw http.ResponseWriter, r *http.Request) {
+	// SecuredCheck() only worked with TokenAuth: Removed
+
+	if user := repository.GetUserFromContext(r.Context()); !user.HasRole(schema.RoleAdmin) {
+		handleError(fmt.Errorf("only admins are allowed to delete a user"), http.StatusForbidden, rw)
+		return
+	}
+
+	username := r.FormValue("username")
+	if err := repository.GetUserRepository().DelUser(username); err != nil {
+		handleError(fmt.Errorf("deleting user failed: %w", err), http.StatusUnprocessableEntity, rw)
+		return
+	}
+
+	rw.WriteHeader(http.StatusOK)
+}

internal/archiver/README.md

@@ -0,0 +1,189 @@
+# Archiver Package
+
+The `archiver` package provides asynchronous job archiving functionality for ClusterCockpit. When jobs complete, their metric data is archived from the metric store to a persistent archive backend (filesystem, S3, SQLite, etc.).
+
+## Architecture
+
+### Producer-Consumer Pattern
+
+```
+┌──────────────┐     TriggerArchiving()      ┌───────────────┐
+│  API Handler │  ───────────────────────▶   │ archiveChannel│
+│ (Job Stop)   │                             │  (buffer: 128)│
+└──────────────┘                             └───────┬───────┘
+                                                     │
+                   ┌─────────────────────────────────┘
+                   │
+                   ▼
+         ┌──────────────────────┐
+         │  archivingWorker()   │
+         │   (goroutine)        │
+         └──────────┬───────────┘
+                    │
+                    ▼
+         1. Fetch job metadata
+         2. Load metric data
+         3. Calculate statistics
+         4. Archive to backend
+         5. Update database
+         6. Call hooks
+```
+
+### Components
+
+- **archiveChannel**: Buffered channel (128 jobs) for async communication
+- **archivePending**: WaitGroup tracking in-flight archiving operations
+- **archivingWorker**: Background goroutine processing archiving requests
+- **shutdownCtx**: Context for graceful cancellation during shutdown
+
+## Usage
+
+### Initialization
+
+```go
+// Start archiver with context for shutdown control
+ctx, cancel := context.WithCancel(context.Background())
+defer cancel()
+
+archiver.Start(jobRepository, ctx)
+```
+
+### Archiving a Job
+
+```go
+// Called automatically when a job completes
+archiver.TriggerArchiving(job)
+```
+
+The function returns immediately. Actual archiving happens in the background.
+
+### Graceful Shutdown
+
+```go
+// Shutdown with 10 second timeout
+if err := archiver.Shutdown(10 * time.Second); err != nil {
+    log.Printf("Archiver shutdown timeout: %v", err)
+}
+```
+
+**Shutdown process:**
+1. Closes channel (rejects new jobs)
+2. Waits for pending jobs (up to timeout)
+3. Cancels context if timeout exceeded
+4. Waits for worker to exit cleanly
+
+## Configuration
+
+### Channel Buffer Size
+
+The archiving channel has a buffer of 128 jobs. If more than 128 jobs are queued simultaneously, `TriggerArchiving()` will block until space is available.
+
+To adjust:
+```go
+// In archiveWorker.go Start() function
+archiveChannel = make(chan *schema.Job, 256) // Increase buffer
+```
+
+### Scope Selection
+
+Archive data scopes are automatically selected based on job size:
+
+- **Node scope**: Always included
+- **Core scope**: Included for jobs with ≤8 nodes (reduces data volume for large jobs)
+- **Accelerator scope**: Included if job used accelerators (`NumAcc > 0`)
+
+To adjust the node threshold:
+```go
+// In archiver.go ArchiveJob() function
+if job.NumNodes <= 16 { // Change from 8 to 16
+    scopes = append(scopes, schema.MetricScopeCore)
+}
+```
+
+### Resolution
+
+Data is archived at the highest available resolution (typically 60s intervals). To change:
+
+```go
+// In archiver.go ArchiveJob() function
+jobData, err := metricdispatch.LoadData(job, allMetrics, scopes, ctx, 300)
+// 0 = highest resolution
+// 300 = 5-minute resolution
+```
+
+## Error Handling
+
+### Automatic Retry
+
+The archiver does **not** automatically retry failed archiving operations. If archiving fails:
+
+1. Error is logged
+2. Job is marked as `MonitoringStatusArchivingFailed` in database
+3. Worker continues processing other jobs
+
+### Manual Retry
+
+To re-archive failed jobs, query for jobs with `MonitoringStatusArchivingFailed` and call `TriggerArchiving()` again.
+
+## Performance Considerations
+
+### Single Worker Thread
+
+The archiver uses a single worker goroutine. For high-throughput systems:
+
+- Large channel buffer (128) prevents blocking
+- Archiving is typically I/O bound (writing to storage)
+- Single worker prevents overwhelming storage backend
+
+### Shutdown Timeout
+
+Recommended timeout values:
+- **Development**: 5-10 seconds
+- **Production**: 10-30 seconds
+- **High-load**: 30-60 seconds
+
+Choose based on:
+- Average archiving time per job
+- Storage backend latency
+- Acceptable shutdown delay
+
+## Monitoring
+
+### Logging
+
+The archiver logs:
+- **Info**: Startup, shutdown, successful completions
+- **Debug**: Individual job archiving times
+- **Error**: Archiving failures with job ID and reason
+- **Warn**: Shutdown timeout exceeded
+
+### Metrics
+
+Monitor these signals for archiver health:
+- Jobs with `MonitoringStatusArchivingFailed`
+- Time from job stop to successful archive
+- Shutdown timeout occurrences
+
+## Thread Safety
+
+All exported functions are safe for concurrent use:
+- `Start()` - Safe to call once
+- `TriggerArchiving()` - Safe from multiple goroutines
+- `Shutdown()` - Safe to call once
+
+Internal state is protected by:
+- Channel synchronization (`archiveChannel`)
+- WaitGroup for pending count (`archivePending`)
+- Context for cancellation (`shutdownCtx`)
+
+## Files
+
+- **archiveWorker.go**: Worker lifecycle, channel management, shutdown logic
+- **archiver.go**: Core archiving logic, metric loading, statistics calculation
+
+## Dependencies
+
+- `internal/repository`: Database operations for job metadata
+- `internal/metricdispatch`: Loading metric data from various backends
+- `pkg/archive`: Archive backend abstraction (filesystem, S3, SQLite)
+- `cc-lib/schema`: Job and metric data structures

internal/archiver/archiveWorker.go

@@ -1,17 +1,61 @@
 // Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved.
+// All rights reserved. This file is part of cc-backend.
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
+
+// Package archiver provides asynchronous job archiving functionality for ClusterCockpit.
+//
+// The archiver runs a background worker goroutine that processes job archiving requests
+// from a buffered channel. When jobs complete, their metric data is archived from the
+// metric store to the configured archive backend (filesystem, S3, etc.).
+//
+// # Architecture
+//
+// The archiver uses a producer-consumer pattern:
+//   - Producer: TriggerArchiving() sends jobs to archiveChannel
+//   - Consumer: archivingWorker() processes jobs from the channel
+//   - Coordination: sync.WaitGroup tracks pending archive operations
+//
+// # Lifecycle
+//
+//  1. Start(repo, ctx) - Initialize worker with context for cancellation
+//  2. TriggerArchiving(job) - Queue job for archiving (called when job stops)
+//  3. archivingWorker() - Background goroutine processes jobs
+//  4. Shutdown(timeout) - Graceful shutdown with timeout
+//
+// # Graceful Shutdown
+//
+// The archiver supports graceful shutdown with configurable timeout:
+//   - Closes channel to reject new jobs
+//   - Waits for pending jobs to complete (up to timeout)
+//   - Cancels context if timeout exceeded
+//   - Ensures worker goroutine exits cleanly
+//
+// # Example Usage
+//
+//	// Initialize archiver
+//	ctx, cancel := context.WithCancel(context.Background())
+//	defer cancel()
+//	archiver.Start(jobRepository, ctx)
+//
+//	// Trigger archiving when job completes
+//	archiver.TriggerArchiving(job)
+//
+//	// Graceful shutdown with 10 second timeout
+//	if err := archiver.Shutdown(10 * time.Second); err != nil {
+//	    log.Printf("Archiver shutdown timeout: %v", err)
+//	}
 package archiver
 
 import (
 	"context"
+	"fmt"
 	"sync"
 	"time"
 
 	"github.com/ClusterCockpit/cc-backend/internal/repository"
-	"github.com/ClusterCockpit/cc-backend/pkg/log"
-	"github.com/ClusterCockpit/cc-backend/pkg/schema"
+	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
+	"github.com/ClusterCockpit/cc-lib/v2/schema"
 	sq "github.com/Masterminds/squirrel"
 )
 
@@ -19,76 +63,188 @@ var (
 	archivePending sync.WaitGroup
 	archiveChannel chan *schema.Job
 	jobRepo        *repository.JobRepository
+	shutdownCtx    context.Context
+	shutdownCancel context.CancelFunc
+	workerDone     chan struct{}
 )
 
-func Start(r *repository.JobRepository) {
+// Start initializes the archiver and starts the background worker goroutine.
+//
+// The archiver processes job archiving requests asynchronously via a buffered channel.
+// Jobs are sent to the channel using TriggerArchiving() and processed by the worker.
+//
+// Parameters:
+//   - r: JobRepository instance for database operations
+//   - ctx: Context for cancellation (shutdown signal propagation)
+//
+// The worker goroutine will run until:
+//   - ctx is cancelled (via parent shutdown)
+//   - archiveChannel is closed (via Shutdown())
+//
+// Must be called before TriggerArchiving(). Safe to call only once.
+func Start(r *repository.JobRepository, ctx context.Context) {
+	shutdownCtx, shutdownCancel = context.WithCancel(ctx)
 	archiveChannel = make(chan *schema.Job, 128)
+	workerDone = make(chan struct{})
 	jobRepo = r
 
 	go archivingWorker()
 }
 
-// Archiving worker thread
+// archivingWorker is the background goroutine that processes job archiving requests.
+//
+// The worker loop:
+//  1. Blocks waiting for jobs on archiveChannel or shutdown signal
+//  2. Fetches job metadata from repository
+//  3. Archives job data to configured backend (calls ArchiveJob)
+//  4. Updates job footprint and energy metrics in database
+//  5. Marks job as successfully archived
+//  6. Calls job stop hooks
+//
+// The worker exits when:
+//   - shutdownCtx is cancelled (timeout during shutdown)
+//   - archiveChannel is closed (normal shutdown)
+//
+// Errors during archiving are logged and the job is marked as failed,
+// but the worker continues processing other jobs.
 func archivingWorker() {
+	defer close(workerDone)
+
 	for {
 		select {
+		case <-shutdownCtx.Done():
+			cclog.Info("Archive worker received shutdown signal")
+			return
+
 		case job, ok := <-archiveChannel:
 			if !ok {
-				break
+				cclog.Info("Archive channel closed, worker exiting")
+				return
 			}
+
 			start := time.Now()
 			// not using meta data, called to load JobMeta into Cache?
 			// will fail if job meta not in repository
 			if _, err := jobRepo.FetchMetadata(job); err != nil {
-				log.Errorf("archiving job (dbid: %d) failed at check metadata step: %s", job.ID, err.Error())
-				jobRepo.UpdateMonitoringStatus(job.ID, schema.MonitoringStatusArchivingFailed)
+				cclog.Errorf("archiving job (dbid: %d) failed at check metadata step: %s", *job.ID, err.Error())
+				jobRepo.UpdateMonitoringStatus(*job.ID, schema.MonitoringStatusArchivingFailed)
+				archivePending.Done()
 				continue
 			}
 
 			// ArchiveJob will fetch all the data from a MetricDataRepository and push into configured archive backend
-			// TODO: Maybe use context with cancel/timeout here
-			jobMeta, err := ArchiveJob(job, context.Background())
+			// Use shutdown context to allow cancellation
+			jobMeta, err := ArchiveJob(job, shutdownCtx)
 			if err != nil {
-				log.Errorf("archiving job (dbid: %d) failed at archiving job step: %s", job.ID, err.Error())
-				jobRepo.UpdateMonitoringStatus(job.ID, schema.MonitoringStatusArchivingFailed)
+				cclog.Errorf("archiving job (dbid: %d) failed at archiving job step: %s", *job.ID, err.Error())
+				jobRepo.UpdateMonitoringStatus(*job.ID, schema.MonitoringStatusArchivingFailed)
+				archivePending.Done()
 				continue
 			}
 
 			stmt := sq.Update("job").Where("job.id = ?", job.ID)
 
 			if stmt, err = jobRepo.UpdateFootprint(stmt, jobMeta); err != nil {
-				log.Errorf("archiving job (dbid: %d) failed at update Footprint step: %s", job.ID, err.Error())
+				cclog.Errorf("archiving job (dbid: %d) failed at update Footprint step: %s", *job.ID, err.Error())
+				archivePending.Done()
 				continue
 			}
 			if stmt, err = jobRepo.UpdateEnergy(stmt, jobMeta); err != nil {
-				log.Errorf("archiving job (dbid: %d) failed at update Energy step: %s", job.ID, err.Error())
+				cclog.Errorf("archiving job (dbid: %d) failed at update Energy step: %s", *job.ID, err.Error())
+				archivePending.Done()
 				continue
 			}
 			// Update the jobs database entry one last time:
 			stmt = jobRepo.MarkArchived(stmt, schema.MonitoringStatusArchivingSuccessful)
 			if err := jobRepo.Execute(stmt); err != nil {
-				log.Errorf("archiving job (dbid: %d) failed at db execute: %s", job.ID, err.Error())
+				cclog.Errorf("archiving job (dbid: %d) failed at db execute: %s", *job.ID, err.Error())
+				archivePending.Done()
 				continue
 			}
-			log.Debugf("archiving job %d took %s", job.JobID, time.Since(start))
-			log.Printf("archiving job (dbid: %d) successful", job.ID)
+			cclog.Debugf("archiving job %d took %s", job.JobID, time.Since(start))
+			cclog.Infof("archiving job (dbid: %d) successful", *job.ID)
+
+			repository.CallJobStopHooks(job)
 			archivePending.Done()
 		}
 	}
 }
 
-// Trigger async archiving
+// TriggerArchiving queues a job for asynchronous archiving.
+//
+// This function should be called when a job completes (stops) to archive its
+// metric data from the metric store to the configured archive backend.
+//
+// The function:
+//  1. Increments the pending job counter (WaitGroup)
+//  2. Sends the job to the archiving channel (buffered, capacity 128)
+//  3. Returns immediately (non-blocking unless channel is full)
+//
+// The actual archiving is performed asynchronously by the worker goroutine.
+// Upon completion, the worker will decrement the pending counter.
+//
+// Panics if Start() has not been called first.
 func TriggerArchiving(job *schema.Job) {
 	if archiveChannel == nil {
-		log.Fatal("Cannot archive without archiving channel. Did you Start the archiver?")
+		cclog.Fatal("Cannot archive without archiving channel. Did you Start the archiver?")
 	}
 
 	archivePending.Add(1)
 	archiveChannel <- job
 }
 
-// Wait for background thread to finish pending archiving operations
-func WaitForArchiving() {
-	// close channel and wait for worker to process remaining jobs
-	archivePending.Wait()
+// Shutdown performs a graceful shutdown of the archiver with a configurable timeout.
+//
+// The shutdown process:
+//  1. Closes archiveChannel - no new jobs will be accepted
+//  2. Waits for pending jobs to complete (up to timeout duration)
+//  3. If timeout is exceeded:
+//     - Cancels shutdownCtx to interrupt ongoing ArchiveJob operations
+//     - Returns error indicating timeout
+//  4. Waits for worker goroutine to exit cleanly
+//
+// Parameters:
+//   - timeout: Maximum duration to wait for pending jobs to complete
+//     (recommended: 10-30 seconds for production)
+//
+// Returns:
+//   - nil if all jobs completed within timeout
+//   - error if timeout was exceeded (some jobs may not have been archived)
+//
+// Jobs that don't complete within the timeout will be marked as failed.
+// The function always ensures the worker goroutine exits before returning.
+//
+// Example:
+//
+//	if err := archiver.Shutdown(10 * time.Second); err != nil {
+//	    log.Printf("Some jobs did not complete: %v", err)
+//	}
+func Shutdown(timeout time.Duration) error {
+	cclog.Info("Initiating archiver shutdown...")
+
+	// Close channel to signal no more jobs will be accepted
+	close(archiveChannel)
+
+	// Create a channel to signal when all jobs are done
+	done := make(chan struct{})
+	go func() {
+		archivePending.Wait()
+		close(done)
+	}()
+
+	// Wait for jobs to complete or timeout
+	select {
+	case <-done:
+		cclog.Info("All archive jobs completed successfully")
+		// Wait for worker to exit
+		<-workerDone
+		return nil
+	case <-time.After(timeout):
+		cclog.Warn("Archiver shutdown timeout exceeded, cancelling remaining operations")
+		// Cancel any ongoing operations
+		shutdownCancel()
+		// Wait for worker to exit
+		<-workerDone
+		return fmt.Errorf("archiver shutdown timeout after %v", timeout)
+	}
 }

internal/archiver/archiver.go

@@ -1,22 +1,47 @@
 // Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved.
+// All rights reserved. This file is part of cc-backend.
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
+
 package archiver
 
 import (
 	"context"
 	"math"
 
-	"github.com/ClusterCockpit/cc-backend/internal/config"
-	"github.com/ClusterCockpit/cc-backend/internal/metricDataDispatcher"
+	"github.com/ClusterCockpit/cc-backend/internal/metricdispatch"
 	"github.com/ClusterCockpit/cc-backend/pkg/archive"
-	"github.com/ClusterCockpit/cc-backend/pkg/log"
-	"github.com/ClusterCockpit/cc-backend/pkg/schema"
+	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
+	"github.com/ClusterCockpit/cc-lib/v2/schema"
 )
 
-// Writes a running job to the job-archive
-func ArchiveJob(job *schema.Job, ctx context.Context) (*schema.JobMeta, error) {
+// ArchiveJob archives a completed job's metric data to the configured archive backend.
+//
+// This function performs the following operations:
+//  1. Loads all metric data for the job from the metric data repository
+//  2. Calculates job-level statistics (avg, min, max) for each metric
+//  3. Stores the job metadata and metric data to the archive backend
+//
+// Metric data is retrieved at the highest available resolution (typically 60s)
+// for the following scopes:
+//   - Node scope (always)
+//   - Core scope (for jobs with ≤8 nodes, to reduce data volume)
+//   - Accelerator scope (if job used accelerators)
+//
+// The function respects context cancellation. If ctx is cancelled (e.g., during
+// shutdown timeout), the operation will be interrupted and return an error.
+//
+// Parameters:
+//   - job: The job to archive (must be a completed job)
+//   - ctx: Context for cancellation and timeout control
+//
+// Returns:
+//   - *schema.Job with populated Statistics field
+//   - error if data loading or archiving fails
+//
+// If config.Keys.DisableArchive is true, only job statistics are calculated
+// and returned (no data is written to archive backend).
+func ArchiveJob(job *schema.Job, ctx context.Context) (*schema.Job, error) {
 	allMetrics := make([]string, 0)
 	metricConfigs := archive.GetCluster(job.Cluster).MetricConfig
 	for _, mc := range metricConfigs {
@@ -34,17 +59,13 @@ func ArchiveJob(job *schema.Job, ctx context.Context) (*schema.JobMeta, error) {
 		scopes = append(scopes, schema.MetricScopeAccelerator)
 	}
 
-	jobData, err := metricDataDispatcher.LoadData(job, allMetrics, scopes, ctx, 0) // 0 Resulotion-Value retrieves highest res (60s)
+	jobData, err := metricdispatch.LoadData(job, allMetrics, scopes, ctx, 0) // 0 Resulotion-Value retrieves highest res (60s)
 	if err != nil {
-		log.Error("Error wile loading job data for archiving")
+		cclog.Error("Error wile loading job data for archiving")
 		return nil, err
 	}
 
-	jobMeta := &schema.JobMeta{
-		BaseJob:    job.BaseJob,
-		StartTime:  job.StartTime.Unix(),
-		Statistics: make(map[string]schema.JobStatistics),
-	}
+	job.Statistics = make(map[string]schema.JobStatistics)
 
 	for metric, data := range jobData {
 		avg, min, max := 0.0, math.MaxFloat32, -math.MaxFloat32
@@ -61,7 +82,7 @@ func ArchiveJob(job *schema.Job, ctx context.Context) (*schema.JobMeta, error) {
 		}
 
 		// Round AVG Result to 2 Digits
-		jobMeta.Statistics[metric] = schema.JobStatistics{
+		job.Statistics[metric] = schema.JobStatistics{
 			Unit: schema.Unit{
 				Prefix: archive.GetMetricConfig(job.Cluster, metric).Unit.Prefix,
 				Base:   archive.GetMetricConfig(job.Cluster, metric).Unit.Base,
@@ -72,12 +93,5 @@ func ArchiveJob(job *schema.Job, ctx context.Context) (*schema.JobMeta, error) {
 		}
 	}
 
-	// If the file based archive is disabled,
-	// only return the JobMeta structure as the
-	// statistics in there are needed.
-	if config.Keys.DisableArchive {
-		return jobMeta, nil
-	}
-
-	return jobMeta, archive.GetHandle().ImportJob(jobMeta, &jobData)
+	return job, archive.GetHandle().ImportJob(job, &jobData)
 }

internal/auth/auth.go

@@ -1,15 +1,20 @@
 // Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved.
+// All rights reserved. This file is part of cc-backend.
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
+
+// Package auth implements various authentication methods
 package auth
 
 import (
+	"bytes"
 	"context"
 	"crypto/rand"
 	"database/sql"
 	"encoding/base64"
+	"encoding/json"
 	"errors"
+	"fmt"
 	"net"
 	"net/http"
 	"os"
@@ -20,13 +25,25 @@ import (
 
 	"github.com/ClusterCockpit/cc-backend/internal/config"
 	"github.com/ClusterCockpit/cc-backend/internal/repository"
-	"github.com/ClusterCockpit/cc-backend/pkg/log"
-	"github.com/ClusterCockpit/cc-backend/pkg/schema"
+	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
+	"github.com/ClusterCockpit/cc-lib/v2/schema"
+	"github.com/ClusterCockpit/cc-lib/v2/util"
 	"github.com/gorilla/sessions"
 )
 
+// Authenticator is the interface for all authentication methods.
+// Each authenticator determines if it can handle a login request (CanLogin)
+// and performs the actual authentication (Login).
 type Authenticator interface {
+	// CanLogin determines if this authenticator can handle the login request.
+	// It returns the user object if available and a boolean indicating if this
+	// authenticator should attempt the login. This method should not perform
+	// expensive operations or actual authentication.
 	CanLogin(user *schema.User, username string, rw http.ResponseWriter, r *http.Request) (*schema.User, bool)
+
+	// Login performs the actually authentication for the user.
+	// It returns the authenticated user or an error if authentication fails.
+	// The user parameter may be nil if the user doesn't exist in the database yet.
 	Login(user *schema.User, rw http.ResponseWriter, r *http.Request) (*schema.User, error)
 }
 
@@ -35,19 +52,70 @@ var (
 	authInstance *Authentication
 )
 
-var ipUserLimiters sync.Map
-
-func getIPUserLimiter(ip, username string) *rate.Limiter {
-	key := ip + ":" + username
-	limiter, ok := ipUserLimiters.Load(key)
-	if !ok {
-		newLimiter := rate.NewLimiter(rate.Every(time.Hour/10), 10)
-		ipUserLimiters.Store(key, newLimiter)
-		return newLimiter
-	}
-	return limiter.(*rate.Limiter)
+// rateLimiterEntry tracks a rate limiter and its last use time for cleanup
+type rateLimiterEntry struct {
+	limiter  *rate.Limiter
+	lastUsed time.Time
 }
 
+var ipUserLimiters sync.Map
+
+// getIPUserLimiter returns a rate limiter for the given IP and username combination.
+// Rate limiters are created on demand and track 5 attempts per 15 minutes.
+func getIPUserLimiter(ip, username string) *rate.Limiter {
+	key := ip + ":" + username
+	now := time.Now()
+
+	if entry, ok := ipUserLimiters.Load(key); ok {
+		rle := entry.(*rateLimiterEntry)
+		rle.lastUsed = now
+		return rle.limiter
+	}
+
+	// More aggressive rate limiting: 5 attempts per 15 minutes
+	newLimiter := rate.NewLimiter(rate.Every(15*time.Minute/5), 5)
+	ipUserLimiters.Store(key, &rateLimiterEntry{
+		limiter:  newLimiter,
+		lastUsed: now,
+	})
+	return newLimiter
+}
+
+// cleanupOldRateLimiters removes rate limiters that haven't been used recently
+func cleanupOldRateLimiters(olderThan time.Time) {
+	ipUserLimiters.Range(func(key, value any) bool {
+		entry := value.(*rateLimiterEntry)
+		if entry.lastUsed.Before(olderThan) {
+			ipUserLimiters.Delete(key)
+			cclog.Debugf("Cleaned up rate limiter for %v", key)
+		}
+		return true
+	})
+}
+
+// startRateLimiterCleanup starts a background goroutine to clean up old rate limiters
+func startRateLimiterCleanup() {
+	go func() {
+		ticker := time.NewTicker(1 * time.Hour)
+		defer ticker.Stop()
+		for range ticker.C {
+			// Clean up limiters not used in the last 24 hours
+			cleanupOldRateLimiters(time.Now().Add(-24 * time.Hour))
+		}
+	}()
+}
+
+// AuthConfig contains configuration for all authentication methods
+type AuthConfig struct {
+	LdapConfig   *LdapConfig    `json:"ldap"`
+	JwtConfig    *JWTAuthConfig `json:"jwts"`
+	OpenIDConfig *OpenIDConfig  `json:"oidc"`
+}
+
+// Keys holds the global authentication configuration
+var Keys AuthConfig
+
+// Authentication manages all authentication methods and session handling
 type Authentication struct {
 	sessionStore   *sessions.CookieStore
 	LdapAuth       *LdapAuthenticator
@@ -63,7 +131,7 @@ func (auth *Authentication) AuthViaSession(
 ) (*schema.User, error) {
 	session, err := auth.sessionStore.Get(r, "session")
 	if err != nil {
-		log.Error("Error while getting session store")
+		cclog.Error("Error while getting session store")
 		return nil, err
 	}
 
@@ -71,10 +139,31 @@ func (auth *Authentication) AuthViaSession(
 		return nil, nil
 	}
 
-	// TODO: Check if session keys exist
-	username, _ := session.Values["username"].(string)
-	projects, _ := session.Values["projects"].([]string)
-	roles, _ := session.Values["roles"].([]string)
+	// Validate session data with proper type checking
+	username, ok := session.Values["username"].(string)
+	if !ok || username == "" {
+		cclog.Warn("Invalid session: missing or invalid username")
+		// Invalidate the corrupted session
+		session.Options.MaxAge = -1
+		_ = auth.sessionStore.Save(r, rw, session)
+		return nil, errors.New("invalid session data")
+	}
+
+	projects, ok := session.Values["projects"].([]string)
+	if !ok {
+		cclog.Warn("Invalid session: projects not found or invalid type, using empty list")
+		projects = []string{}
+	}
+
+	roles, ok := session.Values["roles"].([]string)
+	if !ok || len(roles) == 0 {
+		cclog.Warn("Invalid session: missing or invalid roles")
+		// Invalidate the corrupted session
+		session.Options.MaxAge = -1
+		_ = auth.sessionStore.Save(r, rw, session)
+		return nil, errors.New("invalid session data")
+	}
+
 	return &schema.User{
 		Username:   username,
 		Projects:   projects,
@@ -84,22 +173,25 @@ func (auth *Authentication) AuthViaSession(
 	}, nil
 }
 
-func Init() {
+func Init(authCfg *json.RawMessage) {
 	initOnce.Do(func() {
 		authInstance = &Authentication{}
 
+		// Start background cleanup of rate limiters
+		startRateLimiterCleanup()
+
 		sessKey := os.Getenv("SESSION_KEY")
 		if sessKey == "" {
-			log.Warn("environment variable 'SESSION_KEY' not set (will use non-persistent random key)")
+			cclog.Warn("environment variable 'SESSION_KEY' not set (will use non-persistent random key)")
 			bytes := make([]byte, 32)
 			if _, err := rand.Read(bytes); err != nil {
-				log.Fatal("Error while initializing authentication -> failed to generate random bytes for session key")
+				cclog.Fatal("Error while initializing authentication -> failed to generate random bytes for session key")
 			}
 			authInstance.sessionStore = sessions.NewCookieStore(bytes)
 		} else {
 			bytes, err := base64.StdEncoding.DecodeString(sessKey)
 			if err != nil {
-				log.Fatal("Error while initializing authentication -> decoding session key failed")
+				cclog.Fatal("Error while initializing authentication -> decoding session key failed")
 			}
 			authInstance.sessionStore = sessions.NewCookieStore(bytes)
 		}
@@ -108,44 +200,55 @@ func Init() {
 			authInstance.SessionMaxAge = d
 		}
 
-		if config.Keys.LdapConfig != nil {
+		if authCfg == nil {
+			return
+		}
+
+		config.Validate(configSchema, *authCfg)
+		dec := json.NewDecoder(bytes.NewReader(*authCfg))
+		dec.DisallowUnknownFields()
+		if err := dec.Decode(&Keys); err != nil {
+			cclog.Errorf("error while decoding ldap config: %v", err)
+		}
+
+		if Keys.LdapConfig != nil {
 			ldapAuth := &LdapAuthenticator{}
 			if err := ldapAuth.Init(); err != nil {
-				log.Warn("Error while initializing authentication -> ldapAuth init failed")
+				cclog.Warn("Error while initializing authentication -> ldapAuth init failed")
 			} else {
 				authInstance.LdapAuth = ldapAuth
 				authInstance.authenticators = append(authInstance.authenticators, authInstance.LdapAuth)
 			}
 		} else {
-			log.Info("Missing LDAP configuration: No LDAP support!")
+			cclog.Info("Missing LDAP configuration: No LDAP support!")
 		}
 
-		if config.Keys.JwtConfig != nil {
+		if Keys.JwtConfig != nil {
 			authInstance.JwtAuth = &JWTAuthenticator{}
 			if err := authInstance.JwtAuth.Init(); err != nil {
-				log.Fatal("Error while initializing authentication -> jwtAuth init failed")
+				cclog.Fatal("Error while initializing authentication -> jwtAuth init failed")
 			}
 
 			jwtSessionAuth := &JWTSessionAuthenticator{}
 			if err := jwtSessionAuth.Init(); err != nil {
-				log.Info("jwtSessionAuth init failed: No JWT login support!")
+				cclog.Info("jwtSessionAuth init failed: No JWT login support!")
 			} else {
 				authInstance.authenticators = append(authInstance.authenticators, jwtSessionAuth)
 			}
 
 			jwtCookieSessionAuth := &JWTCookieSessionAuthenticator{}
 			if err := jwtCookieSessionAuth.Init(); err != nil {
-				log.Info("jwtCookieSessionAuth init failed: No JWT cookie login support!")
+				cclog.Info("jwtCookieSessionAuth init failed: No JWT cookie login support!")
 			} else {
 				authInstance.authenticators = append(authInstance.authenticators, jwtCookieSessionAuth)
 			}
 		} else {
-			log.Info("Missing JWT configuration: No JWT token support!")
+			cclog.Info("Missing JWT configuration: No JWT token support!")
 		}
 
 		authInstance.LocalAuth = &LocalAuthenticator{}
 		if err := authInstance.LocalAuth.Init(); err != nil {
-			log.Fatal("Error while initializing authentication -> localAuth init failed")
+			cclog.Fatal("Error while initializing authentication -> localAuth init failed")
 		}
 		authInstance.authenticators = append(authInstance.authenticators, authInstance.LocalAuth)
 	})
@@ -153,50 +256,53 @@ func Init() {
 
 func GetAuthInstance() *Authentication {
 	if authInstance == nil {
-		log.Fatal("Authentication module not initialized!")
+		cclog.Fatal("Authentication module not initialized!")
 	}
 
 	return authInstance
 }
 
-func handleTokenUser(tokenUser *schema.User) {
+// handleUserSync syncs or updates a user in the database based on configuration.
+// This is used for LDAP, JWT and OIDC authentications when syncUserOnLogin or updateUserOnLogin is enabled.
+func handleUserSync(user *schema.User, syncUserOnLogin, updateUserOnLogin bool) {
 	r := repository.GetUserRepository()
-	dbUser, err := r.GetUser(tokenUser.Username)
+	dbUser, err := r.GetUser(user.Username)
 
 	if err != nil && err != sql.ErrNoRows {
-		log.Errorf("Error while loading user '%s': %v", tokenUser.Username, err)
-	} else if err == sql.ErrNoRows && config.Keys.JwtConfig.SyncUserOnLogin { // Adds New User
-		if err := r.AddUser(tokenUser); err != nil {
-			log.Errorf("Error while adding user '%s' to DB: %v", tokenUser.Username, err)
+		cclog.Errorf("Error while loading user '%s': %v", user.Username, err)
+		return
+	}
+
+	if err == sql.ErrNoRows && syncUserOnLogin { // Add new user
+		if err := r.AddUser(user); err != nil {
+			cclog.Errorf("Error while adding user '%s' to DB: %v", user.Username, err)
 		}
-	} else if err == nil && config.Keys.JwtConfig.UpdateUserOnLogin { // Update Existing User
-		if err := r.UpdateUser(dbUser, tokenUser); err != nil {
-			log.Errorf("Error while updating user '%s' to DB: %v", dbUser.Username, err)
+	} else if err == nil && updateUserOnLogin { // Update existing user
+		if err := r.UpdateUser(dbUser, user); err != nil {
+			cclog.Errorf("Error while updating user '%s' in DB: %v", dbUser.Username, err)
 		}
 	}
 }
 
-func handleOIDCUser(OIDCUser *schema.User) {
-	r := repository.GetUserRepository()
-	dbUser, err := r.GetUser(OIDCUser.Username)
+// handleTokenUser syncs JWT token user with database
+func handleTokenUser(tokenUser *schema.User) {
+	handleUserSync(tokenUser, Keys.JwtConfig.SyncUserOnLogin, Keys.JwtConfig.UpdateUserOnLogin)
+}
 
-	if err != nil && err != sql.ErrNoRows {
-		log.Errorf("Error while loading user '%s': %v", OIDCUser.Username, err)
-	} else if err == sql.ErrNoRows && config.Keys.OpenIDConfig.SyncUserOnLogin { // Adds New User
-		if err := r.AddUser(OIDCUser); err != nil {
-			log.Errorf("Error while adding user '%s' to DB: %v", OIDCUser.Username, err)
-		}
-	} else if err == nil && config.Keys.OpenIDConfig.UpdateUserOnLogin { // Update Existing User
-		if err := r.UpdateUser(dbUser, OIDCUser); err != nil {
-			log.Errorf("Error while updating user '%s' to DB: %v", dbUser.Username, err)
-		}
-	}
+// handleOIDCUser syncs OIDC user with database
+func handleOIDCUser(OIDCUser *schema.User) {
+	handleUserSync(OIDCUser, Keys.OpenIDConfig.SyncUserOnLogin, Keys.OpenIDConfig.UpdateUserOnLogin)
+}
+
+// handleLdapUser syncs LDAP user with database
+func handleLdapUser(ldapUser *schema.User) {
+	handleUserSync(ldapUser, Keys.LdapConfig.SyncUserOnLogin, Keys.LdapConfig.UpdateUserOnLogin)
 }
 
 func (auth *Authentication) SaveSession(rw http.ResponseWriter, r *http.Request, user *schema.User) error {
 	session, err := auth.sessionStore.New(r, "session")
 	if err != nil {
-		log.Errorf("session creation failed: %s", err.Error())
+		cclog.Errorf("session creation failed: %s", err.Error())
 		http.Error(rw, err.Error(), http.StatusInternalServerError)
 		return err
 	}
@@ -204,7 +310,13 @@ func (auth *Authentication) SaveSession(rw http.ResponseWriter, r *http.Request,
 	if auth.SessionMaxAge != 0 {
 		session.Options.MaxAge = int(auth.SessionMaxAge.Seconds())
 	}
-	if config.Keys.HttpsCertFile == "" && config.Keys.HttpsKeyFile == "" {
+	if r.TLS == nil && r.Header.Get("X-Forwarded-Proto") != "https" {
+		// If neither TLS or an encrypted reverse proxy are used, do not mark cookies as secure.
+		cclog.Warn("Authenticating with unencrypted request. Session cookies will not have Secure flag set (insecure for production)")
+		if r.Header.Get("X-Forwarded-Proto") == "" {
+			// This warning will not be printed if e.g. X-Forwarded-Proto == http
+			cclog.Warn("If you are using a reverse proxy, make sure X-Forwarded-Proto is set")
+		}
 		session.Options.Secure = false
 	}
 	session.Options.SameSite = http.SameSiteStrictMode
@@ -212,7 +324,7 @@ func (auth *Authentication) SaveSession(rw http.ResponseWriter, r *http.Request,
 	session.Values["projects"] = user.Projects
 	session.Values["roles"] = user.Roles
 	if err := auth.sessionStore.Save(r, rw, session); err != nil {
-		log.Warnf("session save failed: %s", err.Error())
+		cclog.Warnf("session save failed: %s", err.Error())
 		http.Error(rw, err.Error(), http.StatusInternalServerError)
 		return err
 	}
@@ -233,9 +345,9 @@ func (auth *Authentication) Login(
 
 		limiter := getIPUserLimiter(ip, username)
 		if !limiter.Allow() {
-				log.Warnf("AUTH/RATE > Too many login attempts for combination IP: %s, Username: %s", ip, username)
-				onfailure(rw, r, errors.New("Too many login attempts, try again in a few minutes."))
-				return
+			cclog.Warnf("AUTH/RATE > Too many login attempts for combination IP: %s, Username: %s", ip, username)
+			onfailure(rw, r, errors.New("too many login attempts, try again in a few minutes"))
+			return
 		}
 
 		var dbUser *schema.User
@@ -243,7 +355,7 @@ func (auth *Authentication) Login(
 			var err error
 			dbUser, err = repository.GetUserRepository().GetUser(username)
 			if err != nil && err != sql.ErrNoRows {
-				log.Errorf("Error while loading user '%v'", username)
+				cclog.Errorf("Error while loading user '%v'", username)
 			}
 		}
 
@@ -253,12 +365,12 @@ func (auth *Authentication) Login(
 			if user, ok = authenticator.CanLogin(dbUser, username, rw, r); !ok {
 				continue
 			} else {
-				log.Debugf("Can login with user %v", user)
+				cclog.Debugf("Can login with user %v", user)
 			}
 
 			user, err := authenticator.Login(user, rw, r)
 			if err != nil {
-				log.Warnf("user login failed: %s", err.Error())
+				cclog.Warnf("user login failed: %s", err.Error())
 				onfailure(rw, r, err)
 				return
 			}
@@ -267,7 +379,7 @@ func (auth *Authentication) Login(
 				return
 			}
 
-			log.Infof("login successfull: user: %#v (roles: %v, projects: %v)", user.Username, user.Roles, user.Projects)
+			cclog.Infof("login successfull: user: %#v (roles: %v, projects: %v)", user.Username, user.Roles, user.Projects)
 			ctx := context.WithValue(r.Context(), repository.ContextUserKey, user)
 
 			if r.FormValue("redirect") != "" {
@@ -279,7 +391,7 @@ func (auth *Authentication) Login(
 			return
 		}
 
-		log.Debugf("login failed: no authenticator applied")
+		cclog.Debugf("login failed: no authenticator applied")
 		onfailure(rw, r, errors.New("no authenticator applied"))
 	})
 }
@@ -291,14 +403,14 @@ func (auth *Authentication) Auth(
 	return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
 		user, err := auth.JwtAuth.AuthViaJWT(rw, r)
 		if err != nil {
-			log.Infof("auth -> authentication failed: %s", err.Error())
+			cclog.Infof("auth -> authentication failed: %s", err.Error())
 			http.Error(rw, err.Error(), http.StatusUnauthorized)
 			return
 		}
 		if user == nil {
 			user, err = auth.AuthViaSession(rw, r)
 			if err != nil {
-				log.Infof("auth -> authentication failed: %s", err.Error())
+				cclog.Infof("auth -> authentication failed: %s", err.Error())
 				http.Error(rw, err.Error(), http.StatusUnauthorized)
 				return
 			}
@@ -309,89 +421,134 @@ func (auth *Authentication) Auth(
 			return
 		}
 
-		log.Info("auth -> authentication failed")
+		cclog.Info("auth -> authentication failed")
 		onfailure(rw, r, errors.New("unauthorized (please login first)"))
 	})
 }
 
-func (auth *Authentication) AuthApi(
+func (auth *Authentication) AuthAPI(
 	onsuccess http.Handler,
 	onfailure func(rw http.ResponseWriter, r *http.Request, authErr error),
 ) http.Handler {
 	return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
 		user, err := auth.JwtAuth.AuthViaJWT(rw, r)
 		if err != nil {
-			log.Infof("auth api -> authentication failed: %s", err.Error())
+			cclog.Infof("auth api -> authentication failed: %s", err.Error())
 			onfailure(rw, r, err)
 			return
 		}
+
+		ipErr := securedCheck(user, r)
+		if ipErr != nil {
+			cclog.Infof("auth api -> secured check failed: %s", ipErr.Error())
+			onfailure(rw, r, ipErr)
+			return
+		}
+
 		if user != nil {
 			switch {
 			case len(user.Roles) == 1:
-				if user.HasRole(schema.RoleApi) {
+				if user.HasRole(schema.RoleAPI) {
 					ctx := context.WithValue(r.Context(), repository.ContextUserKey, user)
 					onsuccess.ServeHTTP(rw, r.WithContext(ctx))
 					return
 				}
 			case len(user.Roles) >= 2:
-				if user.HasAllRoles([]schema.Role{schema.RoleAdmin, schema.RoleApi}) {
+				if user.HasAllRoles([]schema.Role{schema.RoleAdmin, schema.RoleAPI}) {
 					ctx := context.WithValue(r.Context(), repository.ContextUserKey, user)
 					onsuccess.ServeHTTP(rw, r.WithContext(ctx))
 					return
 				}
 			default:
-				log.Info("auth api -> authentication failed: missing role")
+				cclog.Info("auth api -> authentication failed: missing role")
 				onfailure(rw, r, errors.New("unauthorized"))
 			}
 		}
-		log.Info("auth api -> authentication failed: no auth")
+		cclog.Info("auth api -> authentication failed: no auth")
 		onfailure(rw, r, errors.New("unauthorized"))
 	})
 }
 
-func (auth *Authentication) AuthUserApi(
+func (auth *Authentication) AuthUserAPI(
 	onsuccess http.Handler,
 	onfailure func(rw http.ResponseWriter, r *http.Request, authErr error),
 ) http.Handler {
 	return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
 		user, err := auth.JwtAuth.AuthViaJWT(rw, r)
 		if err != nil {
-			log.Infof("auth user api -> authentication failed: %s", err.Error())
+			cclog.Infof("auth user api -> authentication failed: %s", err.Error())
 			onfailure(rw, r, err)
 			return
 		}
+
 		if user != nil {
 			switch {
 			case len(user.Roles) == 1:
-				if user.HasRole(schema.RoleApi) {
+				if user.HasRole(schema.RoleAPI) {
 					ctx := context.WithValue(r.Context(), repository.ContextUserKey, user)
 					onsuccess.ServeHTTP(rw, r.WithContext(ctx))
 					return
 				}
 			case len(user.Roles) >= 2:
-				if user.HasRole(schema.RoleApi) && user.HasAnyRole([]schema.Role{schema.RoleUser, schema.RoleManager, schema.RoleAdmin}) {
+				if user.HasRole(schema.RoleAPI) && user.HasAnyRole([]schema.Role{schema.RoleUser, schema.RoleManager, schema.RoleSupport, schema.RoleAdmin}) {
 					ctx := context.WithValue(r.Context(), repository.ContextUserKey, user)
 					onsuccess.ServeHTTP(rw, r.WithContext(ctx))
 					return
 				}
 			default:
-				log.Info("auth user api -> authentication failed: missing role")
+				cclog.Info("auth user api -> authentication failed: missing role")
 				onfailure(rw, r, errors.New("unauthorized"))
 			}
 		}
-		log.Info("auth user api -> authentication failed: no auth")
+		cclog.Info("auth user api -> authentication failed: no auth")
 		onfailure(rw, r, errors.New("unauthorized"))
 	})
 }
 
-func (auth *Authentication) AuthConfigApi(
+func (auth *Authentication) AuthMetricStoreAPI(
+	onsuccess http.Handler,
+	onfailure func(rw http.ResponseWriter, r *http.Request, authErr error),
+) http.Handler {
+	return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
+		user, err := auth.JwtAuth.AuthViaJWT(rw, r)
+		if err != nil {
+			cclog.Infof("auth metricstore api -> authentication failed: %s", err.Error())
+			onfailure(rw, r, err)
+			return
+		}
+
+		if user != nil {
+			switch {
+			case len(user.Roles) == 1:
+				if user.HasRole(schema.RoleAPI) {
+					ctx := context.WithValue(r.Context(), repository.ContextUserKey, user)
+					onsuccess.ServeHTTP(rw, r.WithContext(ctx))
+					return
+				}
+			case len(user.Roles) >= 2:
+				if user.HasRole(schema.RoleAPI) && user.HasAnyRole([]schema.Role{schema.RoleUser, schema.RoleManager, schema.RoleAdmin}) {
+					ctx := context.WithValue(r.Context(), repository.ContextUserKey, user)
+					onsuccess.ServeHTTP(rw, r.WithContext(ctx))
+					return
+				}
+			default:
+				cclog.Info("auth metricstore api -> authentication failed: missing role")
+				onfailure(rw, r, errors.New("unauthorized"))
+			}
+		}
+		cclog.Info("auth metricstore api -> authentication failed: no auth")
+		onfailure(rw, r, errors.New("unauthorized"))
+	})
+}
+
+func (auth *Authentication) AuthConfigAPI(
 	onsuccess http.Handler,
 	onfailure func(rw http.ResponseWriter, r *http.Request, authErr error),
 ) http.Handler {
 	return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
 		user, err := auth.AuthViaSession(rw, r)
 		if err != nil {
-			log.Infof("auth config api -> authentication failed: %s", err.Error())
+			cclog.Infof("auth config api -> authentication failed: %s", err.Error())
 			onfailure(rw, r, err)
 			return
 		}
@@ -400,19 +557,19 @@ func (auth *Authentication) AuthConfigApi(
 			onsuccess.ServeHTTP(rw, r.WithContext(ctx))
 			return
 		}
-		log.Info("auth config api -> authentication failed: no auth")
+		cclog.Info("auth config api -> authentication failed: no auth")
 		onfailure(rw, r, errors.New("unauthorized"))
 	})
 }
 
-func (auth *Authentication) AuthFrontendApi(
+func (auth *Authentication) AuthFrontendAPI(
 	onsuccess http.Handler,
 	onfailure func(rw http.ResponseWriter, r *http.Request, authErr error),
 ) http.Handler {
 	return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
 		user, err := auth.AuthViaSession(rw, r)
 		if err != nil {
-			log.Infof("auth frontend api -> authentication failed: %s", err.Error())
+			cclog.Infof("auth frontend api -> authentication failed: %s", err.Error())
 			onfailure(rw, r, err)
 			return
 		}
@@ -421,7 +578,7 @@ func (auth *Authentication) AuthFrontendApi(
 			onsuccess.ServeHTTP(rw, r.WithContext(ctx))
 			return
 		}
-		log.Info("auth frontend api -> authentication failed: no auth")
+		cclog.Info("auth frontend api -> authentication failed: no auth")
 		onfailure(rw, r, errors.New("unauthorized"))
 	})
 }
@@ -445,3 +602,42 @@ func (auth *Authentication) Logout(onsuccess http.Handler) http.Handler {
 		onsuccess.ServeHTTP(rw, r)
 	})
 }
+
+// Helper Moved To MiddleWare Auth Handlers
+func securedCheck(user *schema.User, r *http.Request) error {
+	if user == nil {
+		return fmt.Errorf("no user for secured check")
+	}
+
+	// extract IP address for checking
+	IPAddress := r.Header.Get("X-Real-Ip")
+	if IPAddress == "" {
+		IPAddress = r.Header.Get("X-Forwarded-For")
+	}
+	if IPAddress == "" {
+		IPAddress = r.RemoteAddr
+	}
+
+	// Handle both IPv4 and IPv6 addresses properly
+	// For IPv6, this will strip the port and brackets
+	// For IPv4, this will strip the port
+	if host, _, err := net.SplitHostPort(IPAddress); err == nil {
+		IPAddress = host
+	}
+	// If SplitHostPort fails, IPAddress is already just a host (no port)
+
+	// If nothing declared in config: Continue
+	if len(config.Keys.APIAllowedIPs) == 0 {
+		return nil
+	}
+	// If wildcard declared in config: Continue
+	if config.Keys.APIAllowedIPs[0] == "*" {
+		return nil
+	}
+	// check if IP is allowed
+	if !util.Contains(config.Keys.APIAllowedIPs, IPAddress) {
+		return fmt.Errorf("unknown ip: %v", IPAddress)
+	}
+
+	return nil
+}

internal/auth/auth_test.go

@@ -0,0 +1,176 @@
+// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
+// All rights reserved. This file is part of cc-backend.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package auth
+
+import (
+	"net"
+	"testing"
+	"time"
+)
+
+// TestGetIPUserLimiter tests the rate limiter creation and retrieval
+func TestGetIPUserLimiter(t *testing.T) {
+	ip := "192.168.1.1"
+	username := "testuser"
+
+	// Get limiter for the first time
+	limiter1 := getIPUserLimiter(ip, username)
+	if limiter1 == nil {
+		t.Fatal("Expected limiter to be created")
+	}
+
+	// Get the same limiter again
+	limiter2 := getIPUserLimiter(ip, username)
+	if limiter1 != limiter2 {
+		t.Error("Expected to get the same limiter instance")
+	}
+
+	// Get a different limiter for different user
+	limiter3 := getIPUserLimiter(ip, "otheruser")
+	if limiter1 == limiter3 {
+		t.Error("Expected different limiter for different user")
+	}
+
+	// Get a different limiter for different IP
+	limiter4 := getIPUserLimiter("192.168.1.2", username)
+	if limiter1 == limiter4 {
+		t.Error("Expected different limiter for different IP")
+	}
+}
+
+// TestRateLimiterBehavior tests that rate limiting works correctly
+func TestRateLimiterBehavior(t *testing.T) {
+	ip := "10.0.0.1"
+	username := "ratelimituser"
+
+	limiter := getIPUserLimiter(ip, username)
+
+	// Should allow first 5 attempts
+	for i := range 5 {
+		if !limiter.Allow() {
+			t.Errorf("Request %d should be allowed within rate limit", i+1)
+		}
+	}
+
+	// 6th attempt should be blocked
+	if limiter.Allow() {
+		t.Error("Request 6 should be blocked by rate limiter")
+	}
+}
+
+// TestCleanupOldRateLimiters tests the cleanup function
+func TestCleanupOldRateLimiters(t *testing.T) {
+	// Clear all existing limiters first to avoid interference from other tests
+	cleanupOldRateLimiters(time.Now().Add(24 * time.Hour))
+
+	// Create some new rate limiters
+	limiter1 := getIPUserLimiter("1.1.1.1", "user1")
+	limiter2 := getIPUserLimiter("2.2.2.2", "user2")
+
+	if limiter1 == nil || limiter2 == nil {
+		t.Fatal("Failed to create test limiters")
+	}
+
+	// Cleanup limiters older than 1 second from now (should keep both)
+	time.Sleep(10 * time.Millisecond) // Small delay to ensure timestamp difference
+	cleanupOldRateLimiters(time.Now().Add(-1 * time.Second))
+
+	// Verify they still exist (should get same instance)
+	if getIPUserLimiter("1.1.1.1", "user1") != limiter1 {
+		t.Error("Limiter 1 was incorrectly cleaned up")
+	}
+	if getIPUserLimiter("2.2.2.2", "user2") != limiter2 {
+		t.Error("Limiter 2 was incorrectly cleaned up")
+	}
+
+	// Cleanup limiters older than 1 hour from now (should remove both)
+	cleanupOldRateLimiters(time.Now().Add(2 * time.Hour))
+
+	// Getting them again should create new instances
+	newLimiter1 := getIPUserLimiter("1.1.1.1", "user1")
+	if newLimiter1 == limiter1 {
+		t.Error("Old limiter should have been cleaned up")
+	}
+}
+
+// TestIPv4Extraction tests extracting IPv4 addresses
+func TestIPv4Extraction(t *testing.T) {
+	tests := []struct {
+		name     string
+		input    string
+		expected string
+	}{
+		{"IPv4 with port", "192.168.1.1:8080", "192.168.1.1"},
+		{"IPv4 without port", "192.168.1.1", "192.168.1.1"},
+		{"Localhost with port", "127.0.0.1:3000", "127.0.0.1"},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			result := tt.input
+			if host, _, err := net.SplitHostPort(result); err == nil {
+				result = host
+			}
+
+			if result != tt.expected {
+				t.Errorf("Expected %s, got %s", tt.expected, result)
+			}
+		})
+	}
+}
+
+// TestIPv6Extraction tests extracting IPv6 addresses
+func TestIPv6Extraction(t *testing.T) {
+	tests := []struct {
+		name     string
+		input    string
+		expected string
+	}{
+		{"IPv6 with port", "[2001:db8::1]:8080", "2001:db8::1"},
+		{"IPv6 localhost with port", "[::1]:3000", "::1"},
+		{"IPv6 without port", "2001:db8::1", "2001:db8::1"},
+		{"IPv6 localhost", "::1", "::1"},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			result := tt.input
+			if host, _, err := net.SplitHostPort(result); err == nil {
+				result = host
+			}
+
+			if result != tt.expected {
+				t.Errorf("Expected %s, got %s", tt.expected, result)
+			}
+		})
+	}
+}
+
+// TestIPExtractionEdgeCases tests edge cases for IP extraction
+func TestIPExtractionEdgeCases(t *testing.T) {
+	tests := []struct {
+		name     string
+		input    string
+		expected string
+	}{
+		{"Hostname without port", "example.com", "example.com"},
+		{"Empty string", "", ""},
+		{"Just port", ":8080", ""},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			result := tt.input
+			if host, _, err := net.SplitHostPort(result); err == nil {
+				result = host
+			}
+
+			if result != tt.expected {
+				t.Errorf("Expected %s, got %s", tt.expected, result)
+			}
+		})
+	}
+}

internal/auth/jwt.go

@@ -1,7 +1,8 @@
 // Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved.
+// All rights reserved. This file is part of cc-backend.
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
+
 package auth
 
 import (
@@ -13,13 +14,33 @@ import (
 	"strings"
 	"time"
 
-	"github.com/ClusterCockpit/cc-backend/internal/config"
-	"github.com/ClusterCockpit/cc-backend/internal/repository"
-	"github.com/ClusterCockpit/cc-backend/pkg/log"
-	"github.com/ClusterCockpit/cc-backend/pkg/schema"
+	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
+	"github.com/ClusterCockpit/cc-lib/v2/schema"
 	"github.com/golang-jwt/jwt/v5"
 )
 
+type JWTAuthConfig struct {
+	// Specifies for how long a JWT token shall be valid
+	// as a string parsable by time.ParseDuration().
+	MaxAge string `json:"max-age"`
+
+	// Specifies which cookie should be checked for a JWT token (if no authorization header is present)
+	CookieName string `json:"cookie-name"`
+
+	// Deny login for users not in database (but defined in JWT).
+	// Ignore user roles defined in JWTs ('roles' claim), get them from db.
+	ValidateUser bool `json:"validate-user"`
+
+	// Specifies which issuer should be accepted when validating external JWTs ('iss' claim)
+	TrustedIssuer string `json:"trusted-issuer"`
+
+	// Should an non-existent user be added to the DB based on the information in the token
+	SyncUserOnLogin bool `json:"sync-user-on-login"`
+
+	// Should an existent user be updated in the DB based on the information in the token
+	UpdateUserOnLogin bool `json:"update-user-on-login"`
+}
+
 type JWTAuthenticator struct {
 	publicKey  ed25519.PublicKey
 	privateKey ed25519.PrivateKey
@@ -28,17 +49,17 @@ type JWTAuthenticator struct {
 func (ja *JWTAuthenticator) Init() error {
 	pubKey, privKey := os.Getenv("JWT_PUBLIC_KEY"), os.Getenv("JWT_PRIVATE_KEY")
 	if pubKey == "" || privKey == "" {
-		log.Warn("environment variables 'JWT_PUBLIC_KEY' or 'JWT_PRIVATE_KEY' not set (token based authentication will not work)")
+		cclog.Warn("environment variables 'JWT_PUBLIC_KEY' or 'JWT_PRIVATE_KEY' not set (token based authentication will not work)")
 	} else {
 		bytes, err := base64.StdEncoding.DecodeString(pubKey)
 		if err != nil {
-			log.Warn("Could not decode JWT public key")
+			cclog.Warn("Could not decode JWT public key")
 			return err
 		}
 		ja.publicKey = ed25519.PublicKey(bytes)
 		bytes, err = base64.StdEncoding.DecodeString(privKey)
 		if err != nil {
-			log.Warn("Could not decode JWT private key")
+			cclog.Warn("Could not decode JWT private key")
 			return err
 		}
 		ja.privateKey = ed25519.PrivateKey(bytes)
@@ -62,7 +83,7 @@ func (ja *JWTAuthenticator) AuthViaJWT(
 		return nil, nil
 	}
 
-	token, err := jwt.Parse(rawtoken, func(t *jwt.Token) (interface{}, error) {
+	token, err := jwt.Parse(rawtoken, func(t *jwt.Token) (any, error) {
 		if t.Method != jwt.SigningMethodEdDSA {
 			return nil, errors.New("only Ed25519/EdDSA supported")
 		}
@@ -70,51 +91,34 @@ func (ja *JWTAuthenticator) AuthViaJWT(
 		return ja.publicKey, nil
 	})
 	if err != nil {
-		log.Warn("Error while parsing JWT token")
+		cclog.Warn("Error while parsing JWT token")
 		return nil, err
 	}
 	if !token.Valid {
-		log.Warn("jwt token claims are not valid")
+		cclog.Warn("jwt token claims are not valid")
 		return nil, errors.New("jwt token claims are not valid")
 	}
 
 	// Token is valid, extract payload
 	claims := token.Claims.(jwt.MapClaims)
-	sub, _ := claims["sub"].(string)
 
-	var roles []string
-
-	// Validate user + roles from JWT against database?
-	if config.Keys.JwtConfig.ValidateUser {
-		ur := repository.GetUserRepository()
-		user, err := ur.GetUser(sub)
-		// Deny any logins for unknown usernames
-		if err != nil {
-			log.Warn("Could not find user from JWT in internal database.")
-			return nil, errors.New("unknown user")
-		}
-		// Take user roles from database instead of trusting the JWT
-		roles = user.Roles
-	} else {
-		// Extract roles from JWT (if present)
-		if rawroles, ok := claims["roles"].([]interface{}); ok {
-			for _, rr := range rawroles {
-				if r, ok := rr.(string); ok {
-					roles = append(roles, r)
-				}
-			}
-		}
+	// Use shared helper to get user from JWT claims
+	var user *schema.User
+	user, err = getUserFromJWT(claims, Keys.JwtConfig.ValidateUser, schema.AuthToken, -1)
+	if err != nil {
+		return nil, err
 	}
 
-	return &schema.User{
-		Username:   sub,
-		Roles:      roles,
-		AuthType:   schema.AuthToken,
-		AuthSource: -1,
-	}, nil
+	// If not validating user, we only get roles from JWT (no projects for this auth method)
+	if !Keys.JwtConfig.ValidateUser {
+		user.Roles = extractRolesFromClaims(claims, false)
+		user.Projects = nil // Standard JWT auth doesn't include projects
+	}
+
+	return user, nil
 }
 
-// Generate a new JWT that can be used for authentication
+// ProvideJWT generates a new JWT that can be used for authentication
 func (ja *JWTAuthenticator) ProvideJWT(user *schema.User) (string, error) {
 	if ja.privateKey == nil {
 		return "", errors.New("environment variable 'JWT_PRIVATE_KEY' not set")
@@ -126,8 +130,8 @@ func (ja *JWTAuthenticator) ProvideJWT(user *schema.User) (string, error) {
 		"roles": user.Roles,
 		"iat":   now.Unix(),
 	}
-	if config.Keys.JwtConfig.MaxAge != "" {
-		d, err := time.ParseDuration(config.Keys.JwtConfig.MaxAge)
+	if Keys.JwtConfig.MaxAge != "" {
+		d, err := time.ParseDuration(Keys.JwtConfig.MaxAge)
 		if err != nil {
 			return "", errors.New("cannot parse max-age config key")
 		}

internal/auth/jwtCookieSession.go

@@ -1,22 +1,19 @@
 // Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved.
+// All rights reserved. This file is part of cc-backend.
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
+
 package auth
 
 import (
 	"crypto/ed25519"
-	"database/sql"
 	"encoding/base64"
 	"errors"
-	"fmt"
 	"net/http"
 	"os"
 
-	"github.com/ClusterCockpit/cc-backend/internal/config"
-	"github.com/ClusterCockpit/cc-backend/internal/repository"
-	"github.com/ClusterCockpit/cc-backend/pkg/log"
-	"github.com/ClusterCockpit/cc-backend/pkg/schema"
+	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
+	"github.com/ClusterCockpit/cc-lib/v2/schema"
 	"github.com/golang-jwt/jwt/v5"
 )
 
@@ -31,18 +28,18 @@ var _ Authenticator = (*JWTCookieSessionAuthenticator)(nil)
 func (ja *JWTCookieSessionAuthenticator) Init() error {
 	pubKey, privKey := os.Getenv("JWT_PUBLIC_KEY"), os.Getenv("JWT_PRIVATE_KEY")
 	if pubKey == "" || privKey == "" {
-		log.Warn("environment variables 'JWT_PUBLIC_KEY' or 'JWT_PRIVATE_KEY' not set (token based authentication will not work)")
+		cclog.Warn("environment variables 'JWT_PUBLIC_KEY' or 'JWT_PRIVATE_KEY' not set (token based authentication will not work)")
 		return errors.New("environment variables 'JWT_PUBLIC_KEY' or 'JWT_PRIVATE_KEY' not set (token based authentication will not work)")
 	} else {
 		bytes, err := base64.StdEncoding.DecodeString(pubKey)
 		if err != nil {
-			log.Warn("Could not decode JWT public key")
+			cclog.Warn("Could not decode JWT public key")
 			return err
 		}
 		ja.publicKey = ed25519.PublicKey(bytes)
 		bytes, err = base64.StdEncoding.DecodeString(privKey)
 		if err != nil {
-			log.Warn("Could not decode JWT private key")
+			cclog.Warn("Could not decode JWT private key")
 			return err
 		}
 		ja.privateKey = ed25519.PrivateKey(bytes)
@@ -53,36 +50,35 @@ func (ja *JWTCookieSessionAuthenticator) Init() error {
 	if keyFound && pubKeyCrossLogin != "" {
 		bytes, err := base64.StdEncoding.DecodeString(pubKeyCrossLogin)
 		if err != nil {
-			log.Warn("Could not decode cross login JWT public key")
+			cclog.Warn("Could not decode cross login JWT public key")
 			return err
 		}
 		ja.publicKeyCrossLogin = ed25519.PublicKey(bytes)
 	} else {
 		ja.publicKeyCrossLogin = nil
-		log.Debug("environment variable 'CROSS_LOGIN_JWT_PUBLIC_KEY' not set (cross login token based authentication will not work)")
+		cclog.Debug("environment variable 'CROSS_LOGIN_JWT_PUBLIC_KEY' not set (cross login token based authentication will not work)")
 		return errors.New("environment variable 'CROSS_LOGIN_JWT_PUBLIC_KEY' not set (cross login token based authentication will not work)")
 	}
 
-	jc := config.Keys.JwtConfig
 	// Warn if other necessary settings are not configured
-	if jc != nil {
-		if jc.CookieName == "" {
-			log.Info("cookieName for JWTs not configured (cross login via JWT cookie will fail)")
+	if Keys.JwtConfig != nil {
+		if Keys.JwtConfig.CookieName == "" {
+			cclog.Info("cookieName for JWTs not configured (cross login via JWT cookie will fail)")
 			return errors.New("cookieName for JWTs not configured (cross login via JWT cookie will fail)")
 		}
-		if !jc.ValidateUser {
-			log.Info("forceJWTValidationViaDatabase not set to true: CC will accept users and roles defined in JWTs regardless of its own database!")
+		if !Keys.JwtConfig.ValidateUser {
+			cclog.Info("forceJWTValidationViaDatabase not set to true: CC will accept users and roles defined in JWTs regardless of its own database!")
 		}
-		if jc.TrustedIssuer == "" {
-			log.Info("trustedExternalIssuer for JWTs not configured (cross login via JWT cookie will fail)")
+		if Keys.JwtConfig.TrustedIssuer == "" {
+			cclog.Info("trustedExternalIssuer for JWTs not configured (cross login via JWT cookie will fail)")
 			return errors.New("trustedExternalIssuer for JWTs not configured (cross login via JWT cookie will fail)")
 		}
 	} else {
-		log.Warn("config for JWTs not configured (cross login via JWT cookie will fail)")
+		cclog.Warn("config for JWTs not configured (cross login via JWT cookie will fail)")
 		return errors.New("config for JWTs not configured (cross login via JWT cookie will fail)")
 	}
 
-	log.Info("JWT Cookie Session authenticator successfully registered")
+	cclog.Info("JWT Cookie Session authenticator successfully registered")
 	return nil
 }
 
@@ -92,7 +88,7 @@ func (ja *JWTCookieSessionAuthenticator) CanLogin(
 	rw http.ResponseWriter,
 	r *http.Request,
 ) (*schema.User, bool) {
-	jc := config.Keys.JwtConfig
+	jc := Keys.JwtConfig
 	cookieName := ""
 	if jc.CookieName != "" {
 		cookieName = jc.CookieName
@@ -115,7 +111,7 @@ func (ja *JWTCookieSessionAuthenticator) Login(
 	rw http.ResponseWriter,
 	r *http.Request,
 ) (*schema.User, error) {
-	jc := config.Keys.JwtConfig
+	jc := Keys.JwtConfig
 	jwtCookie, err := r.Cookie(jc.CookieName)
 	var rawtoken string
 
@@ -123,7 +119,7 @@ func (ja *JWTCookieSessionAuthenticator) Login(
 		rawtoken = jwtCookie.Value
 	}
 
-	token, err := jwt.Parse(rawtoken, func(t *jwt.Token) (interface{}, error) {
+	token, err := jwt.Parse(rawtoken, func(t *jwt.Token) (any, error) {
 		if t.Method != jwt.SigningMethodEdDSA {
 			return nil, errors.New("only Ed25519/EdDSA supported")
 		}
@@ -140,67 +136,26 @@ func (ja *JWTCookieSessionAuthenticator) Login(
 		return ja.publicKey, nil
 	})
 	if err != nil {
-		log.Warn("JWT cookie session: error while parsing token")
+		cclog.Warn("JWT cookie session: error while parsing token")
 		return nil, err
 	}
 
 	if !token.Valid {
-		log.Warn("jwt token claims are not valid")
+		cclog.Warn("jwt token claims are not valid")
 		return nil, errors.New("jwt token claims are not valid")
 	}
 
 	claims := token.Claims.(jwt.MapClaims)
-	sub, _ := claims["sub"].(string)
 
-	var roles []string
-	projects := make([]string, 0)
+	// Use shared helper to get user from JWT claims
+	user, err = getUserFromJWT(claims, jc.ValidateUser, schema.AuthSession, schema.AuthViaToken)
+	if err != nil {
+		return nil, err
+	}
 
-	if jc.ValidateUser {
-		var err error
-		user, err = repository.GetUserRepository().GetUser(sub)
-		if err != nil && err != sql.ErrNoRows {
-			log.Errorf("Error while loading user '%v'", sub)
-		}
-
-		// Deny any logins for unknown usernames
-		if user == nil {
-			log.Warn("Could not find user from JWT in internal database.")
-			return nil, errors.New("unknown user")
-		}
-	} else {
-		var name string
-		if wrap, ok := claims["name"].(map[string]interface{}); ok {
-			if vals, ok := wrap["values"].([]interface{}); ok {
-				if len(vals) != 0 {
-					name = fmt.Sprintf("%v", vals[0])
-
-					for i := 1; i < len(vals); i++ {
-						name += fmt.Sprintf(" %v", vals[i])
-					}
-				}
-			}
-		}
-
-		// Extract roles from JWT (if present)
-		if rawroles, ok := claims["roles"].([]interface{}); ok {
-			for _, rr := range rawroles {
-				if r, ok := rr.(string); ok {
-					roles = append(roles, r)
-				}
-			}
-		}
-		user = &schema.User{
-			Username:   sub,
-			Name:       name,
-			Roles:      roles,
-			Projects:   projects,
-			AuthType:   schema.AuthSession,
-			AuthSource: schema.AuthViaToken,
-		}
-
-		if jc.SyncUserOnLogin || jc.UpdateUserOnLogin {
-			handleTokenUser(user)
-		}
+	// Sync or update user if configured
+	if !jc.ValidateUser && (jc.SyncUserOnLogin || jc.UpdateUserOnLogin) {
+		handleTokenUser(user)
 	}
 
 	// (Ask browser to) Delete JWT cookie

internal/auth/jwtHelpers.go

@@ -0,0 +1,138 @@
+// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
+// All rights reserved. This file is part of cc-backend.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package auth
+
+import (
+	"database/sql"
+	"errors"
+	"fmt"
+	"strings"
+
+	"github.com/ClusterCockpit/cc-backend/internal/repository"
+	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
+	"github.com/ClusterCockpit/cc-lib/v2/schema"
+	"github.com/golang-jwt/jwt/v5"
+)
+
+// extractStringFromClaims extracts a string value from JWT claims
+func extractStringFromClaims(claims jwt.MapClaims, key string) string {
+	if val, ok := claims[key].(string); ok {
+		return val
+	}
+	return ""
+}
+
+// extractRolesFromClaims extracts roles from JWT claims
+// If validateRoles is true, only valid roles are returned
+func extractRolesFromClaims(claims jwt.MapClaims, validateRoles bool) []string {
+	var roles []string
+
+	if rawroles, ok := claims["roles"].([]any); ok {
+		for _, rr := range rawroles {
+			if r, ok := rr.(string); ok {
+				if validateRoles {
+					if schema.IsValidRole(r) {
+						roles = append(roles, r)
+					}
+				} else {
+					roles = append(roles, r)
+				}
+			}
+		}
+	}
+
+	return roles
+}
+
+// extractProjectsFromClaims extracts projects from JWT claims
+func extractProjectsFromClaims(claims jwt.MapClaims) []string {
+	projects := make([]string, 0)
+
+	if rawprojs, ok := claims["projects"].([]any); ok {
+		for _, pp := range rawprojs {
+			if p, ok := pp.(string); ok {
+				projects = append(projects, p)
+			}
+		}
+	} else if rawprojs, ok := claims["projects"]; ok {
+		if projSlice, ok := rawprojs.([]string); ok {
+			projects = append(projects, projSlice...)
+		}
+	}
+
+	return projects
+}
+
+// extractNameFromClaims extracts name from JWT claims
+// Handles both simple string and complex nested structure
+func extractNameFromClaims(claims jwt.MapClaims) string {
+	// Try simple string first
+	if name, ok := claims["name"].(string); ok {
+		return name
+	}
+
+	// Try nested structure: {name: {values: [...]}}
+	if wrap, ok := claims["name"].(map[string]any); ok {
+		if vals, ok := wrap["values"].([]any); ok {
+			if len(vals) == 0 {
+				return ""
+			}
+
+			var name strings.Builder
+			name.WriteString(fmt.Sprintf("%v", vals[0]))
+			for i := 1; i < len(vals); i++ {
+				name.WriteString(fmt.Sprintf(" %v", vals[i]))
+			}
+			return name.String()
+		}
+	}
+
+	return ""
+}
+
+// getUserFromJWT creates or retrieves a user based on JWT claims
+// If validateUser is true, the user must exist in the database
+// Otherwise, a new user object is created from claims
+// authSource should be a schema.AuthSource constant (like schema.AuthViaToken)
+func getUserFromJWT(claims jwt.MapClaims, validateUser bool, authType schema.AuthType, authSource schema.AuthSource) (*schema.User, error) {
+	sub := extractStringFromClaims(claims, "sub")
+	if sub == "" {
+		return nil, errors.New("missing 'sub' claim in JWT")
+	}
+
+	if validateUser {
+		// Validate user against database
+		ur := repository.GetUserRepository()
+		user, err := ur.GetUser(sub)
+		if err != nil && err != sql.ErrNoRows {
+			cclog.Errorf("Error while loading user '%v': %v", sub, err)
+			return nil, fmt.Errorf("database error: %w", err)
+		}
+
+		// Deny any logins for unknown usernames
+		if user == nil || err == sql.ErrNoRows {
+			cclog.Warn("Could not find user from JWT in internal database.")
+			return nil, errors.New("unknown user")
+		}
+
+		// Return database user (with database roles)
+		return user, nil
+	}
+
+	// Create user from JWT claims
+	name := extractNameFromClaims(claims)
+	roles := extractRolesFromClaims(claims, true) // Validate roles
+	projects := extractProjectsFromClaims(claims)
+
+	return &schema.User{
+		Username:   sub,
+		Name:       name,
+		Roles:      roles,
+		Projects:   projects,
+		AuthType:   authType,
+		AuthSource: authSource,
+	}, nil
+}

internal/auth/jwtHelpers_test.go

@@ -0,0 +1,280 @@
+// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
+// All rights reserved. This file is part of cc-backend.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package auth
+
+import (
+	"testing"
+
+	"github.com/ClusterCockpit/cc-lib/v2/schema"
+	"github.com/golang-jwt/jwt/v5"
+)
+
+// TestExtractStringFromClaims tests extracting string values from JWT claims
+func TestExtractStringFromClaims(t *testing.T) {
+	claims := jwt.MapClaims{
+		"sub":   "testuser",
+		"email": "test@example.com",
+		"age":   25, // not a string
+	}
+
+	tests := []struct {
+		name     string
+		key      string
+		expected string
+	}{
+		{"Existing string", "sub", "testuser"},
+		{"Another string", "email", "test@example.com"},
+		{"Non-existent key", "missing", ""},
+		{"Non-string value", "age", ""},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			result := extractStringFromClaims(claims, tt.key)
+			if result != tt.expected {
+				t.Errorf("Expected %s, got %s", tt.expected, result)
+			}
+		})
+	}
+}
+
+// TestExtractRolesFromClaims tests role extraction and validation
+func TestExtractRolesFromClaims(t *testing.T) {
+	tests := []struct {
+		name          string
+		claims        jwt.MapClaims
+		validateRoles bool
+		expected      []string
+	}{
+		{
+			name: "Valid roles without validation",
+			claims: jwt.MapClaims{
+				"roles": []any{"admin", "user", "invalid_role"},
+			},
+			validateRoles: false,
+			expected:      []string{"admin", "user", "invalid_role"},
+		},
+		{
+			name: "Valid roles with validation",
+			claims: jwt.MapClaims{
+				"roles": []any{"admin", "user", "api"},
+			},
+			validateRoles: true,
+			expected:      []string{"admin", "user", "api"},
+		},
+		{
+			name: "Invalid roles with validation",
+			claims: jwt.MapClaims{
+				"roles": []any{"invalid_role", "fake_role"},
+			},
+			validateRoles: true,
+			expected:      []string{}, // Should filter out invalid roles
+		},
+		{
+			name:          "No roles claim",
+			claims:        jwt.MapClaims{},
+			validateRoles: false,
+			expected:      []string{},
+		},
+		{
+			name: "Non-array roles",
+			claims: jwt.MapClaims{
+				"roles": "admin",
+			},
+			validateRoles: false,
+			expected:      []string{},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			result := extractRolesFromClaims(tt.claims, tt.validateRoles)
+
+			if len(result) != len(tt.expected) {
+				t.Errorf("Expected %d roles, got %d", len(tt.expected), len(result))
+				return
+			}
+
+			for i, role := range result {
+				if i >= len(tt.expected) || role != tt.expected[i] {
+					t.Errorf("Expected role %s at position %d, got %s", tt.expected[i], i, role)
+				}
+			}
+		})
+	}
+}
+
+// TestExtractProjectsFromClaims tests project extraction from claims
+func TestExtractProjectsFromClaims(t *testing.T) {
+	tests := []struct {
+		name     string
+		claims   jwt.MapClaims
+		expected []string
+	}{
+		{
+			name: "Projects as array of interfaces",
+			claims: jwt.MapClaims{
+				"projects": []any{"project1", "project2", "project3"},
+			},
+			expected: []string{"project1", "project2", "project3"},
+		},
+		{
+			name: "Projects as string array",
+			claims: jwt.MapClaims{
+				"projects": []string{"projectA", "projectB"},
+			},
+			expected: []string{"projectA", "projectB"},
+		},
+		{
+			name:     "No projects claim",
+			claims:   jwt.MapClaims{},
+			expected: []string{},
+		},
+		{
+			name: "Mixed types in projects array",
+			claims: jwt.MapClaims{
+				"projects": []any{"project1", 123, "project2"},
+			},
+			expected: []string{"project1", "project2"}, // Should skip non-strings
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			result := extractProjectsFromClaims(tt.claims)
+
+			if len(result) != len(tt.expected) {
+				t.Errorf("Expected %d projects, got %d", len(tt.expected), len(result))
+				return
+			}
+
+			for i, project := range result {
+				if i >= len(tt.expected) || project != tt.expected[i] {
+					t.Errorf("Expected project %s at position %d, got %s", tt.expected[i], i, project)
+				}
+			}
+		})
+	}
+}
+
+// TestExtractNameFromClaims tests name extraction from various formats
+func TestExtractNameFromClaims(t *testing.T) {
+	tests := []struct {
+		name     string
+		claims   jwt.MapClaims
+		expected string
+	}{
+		{
+			name: "Simple string name",
+			claims: jwt.MapClaims{
+				"name": "John Doe",
+			},
+			expected: "John Doe",
+		},
+		{
+			name: "Nested name structure",
+			claims: jwt.MapClaims{
+				"name": map[string]any{
+					"values": []any{"John", "Doe"},
+				},
+			},
+			expected: "John Doe",
+		},
+		{
+			name: "Nested name with single value",
+			claims: jwt.MapClaims{
+				"name": map[string]any{
+					"values": []any{"Alice"},
+				},
+			},
+			expected: "Alice",
+		},
+		{
+			name:     "No name claim",
+			claims:   jwt.MapClaims{},
+			expected: "",
+		},
+		{
+			name: "Empty nested values",
+			claims: jwt.MapClaims{
+				"name": map[string]any{
+					"values": []any{},
+				},
+			},
+			expected: "",
+		},
+		{
+			name: "Nested with non-string values",
+			claims: jwt.MapClaims{
+				"name": map[string]any{
+					"values": []any{123, "Smith"},
+				},
+			},
+			expected: "123 Smith", // Should convert to string
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			result := extractNameFromClaims(tt.claims)
+			if result != tt.expected {
+				t.Errorf("Expected '%s', got '%s'", tt.expected, result)
+			}
+		})
+	}
+}
+
+// TestGetUserFromJWT_NoValidation tests getUserFromJWT without database validation
+func TestGetUserFromJWT_NoValidation(t *testing.T) {
+	claims := jwt.MapClaims{
+		"sub":      "testuser",
+		"name":     "Test User",
+		"roles":    []any{"user", "admin"},
+		"projects": []any{"project1", "project2"},
+	}
+
+	user, err := getUserFromJWT(claims, false, schema.AuthToken, -1)
+	if err != nil {
+		t.Fatalf("Unexpected error: %v", err)
+	}
+
+	if user.Username != "testuser" {
+		t.Errorf("Expected username 'testuser', got '%s'", user.Username)
+	}
+
+	if user.Name != "Test User" {
+		t.Errorf("Expected name 'Test User', got '%s'", user.Name)
+	}
+
+	if len(user.Roles) != 2 {
+		t.Errorf("Expected 2 roles, got %d", len(user.Roles))
+	}
+
+	if len(user.Projects) != 2 {
+		t.Errorf("Expected 2 projects, got %d", len(user.Projects))
+	}
+
+	if user.AuthType != schema.AuthToken {
+		t.Errorf("Expected AuthType %v, got %v", schema.AuthToken, user.AuthType)
+	}
+}
+
+// TestGetUserFromJWT_MissingSub tests error when sub claim is missing
+func TestGetUserFromJWT_MissingSub(t *testing.T) {
+	claims := jwt.MapClaims{
+		"name": "Test User",
+	}
+
+	_, err := getUserFromJWT(claims, false, schema.AuthToken, -1)
+
+	if err == nil {
+		t.Error("Expected error for missing sub claim")
+	}
+
+	if err.Error() != "missing 'sub' claim in JWT" {
+		t.Errorf("Expected specific error message, got: %v", err)
+	}
+}

internal/auth/jwtSession.go

@@ -1,11 +1,11 @@
 // Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved.
+// All rights reserved. This file is part of cc-backend.
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
+
 package auth
 
 import (
-	"database/sql"
 	"encoding/base64"
 	"errors"
 	"fmt"
@@ -13,10 +13,8 @@ import (
 	"os"
 	"strings"
 
-	"github.com/ClusterCockpit/cc-backend/internal/config"
-	"github.com/ClusterCockpit/cc-backend/internal/repository"
-	"github.com/ClusterCockpit/cc-backend/pkg/log"
-	"github.com/ClusterCockpit/cc-backend/pkg/schema"
+	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
+	"github.com/ClusterCockpit/cc-lib/v2/schema"
 	"github.com/golang-jwt/jwt/v5"
 )
 
@@ -30,13 +28,13 @@ func (ja *JWTSessionAuthenticator) Init() error {
 	if pubKey := os.Getenv("CROSS_LOGIN_JWT_HS512_KEY"); pubKey != "" {
 		bytes, err := base64.StdEncoding.DecodeString(pubKey)
 		if err != nil {
-			log.Warn("Could not decode cross login JWT HS512 key")
+			cclog.Warn("Could not decode cross login JWT HS512 key")
 			return err
 		}
 		ja.loginTokenKey = bytes
 	}
 
-	log.Info("JWT Session authenticator successfully registered")
+	cclog.Info("JWT Session authenticator successfully registered")
 	return nil
 }
 
@@ -60,87 +58,33 @@ func (ja *JWTSessionAuthenticator) Login(
 		rawtoken = r.URL.Query().Get("login-token")
 	}
 
-	token, err := jwt.Parse(rawtoken, func(t *jwt.Token) (interface{}, error) {
+	token, err := jwt.Parse(rawtoken, func(t *jwt.Token) (any, error) {
 		if t.Method == jwt.SigningMethodHS256 || t.Method == jwt.SigningMethodHS512 {
 			return ja.loginTokenKey, nil
 		}
 		return nil, fmt.Errorf("unkown signing method for login token: %s (known: HS256, HS512, EdDSA)", t.Method.Alg())
 	})
 	if err != nil {
-		log.Warn("Error while parsing jwt token")
+		cclog.Warn("Error while parsing jwt token")
 		return nil, err
 	}
 
 	if !token.Valid {
-		log.Warn("jwt token claims are not valid")
+		cclog.Warn("jwt token claims are not valid")
 		return nil, errors.New("jwt token claims are not valid")
 	}
 
 	claims := token.Claims.(jwt.MapClaims)
-	sub, _ := claims["sub"].(string)
 
-	var roles []string
-	projects := make([]string, 0)
+	// Use shared helper to get user from JWT claims
+	user, err = getUserFromJWT(claims, Keys.JwtConfig.ValidateUser, schema.AuthSession, schema.AuthViaToken)
+	if err != nil {
+		return nil, err
+	}
 
-	if config.Keys.JwtConfig.ValidateUser {
-		var err error
-		user, err = repository.GetUserRepository().GetUser(sub)
-		if err != nil && err != sql.ErrNoRows {
-			log.Errorf("Error while loading user '%v'", sub)
-		}
-
-		// Deny any logins for unknown usernames
-		if user == nil {
-			log.Warn("Could not find user from JWT in internal database.")
-			return nil, errors.New("unknown user")
-		}
-	} else {
-		var name string
-		if wrap, ok := claims["name"].(map[string]interface{}); ok {
-			if vals, ok := wrap["values"].([]interface{}); ok {
-				if len(vals) != 0 {
-					name = fmt.Sprintf("%v", vals[0])
-
-					for i := 1; i < len(vals); i++ {
-						name += fmt.Sprintf(" %v", vals[i])
-					}
-				}
-			}
-		}
-
-		// Extract roles from JWT (if present)
-		if rawroles, ok := claims["roles"].([]interface{}); ok {
-			for _, rr := range rawroles {
-				if r, ok := rr.(string); ok {
-					if schema.IsValidRole(r) {
-						roles = append(roles, r)
-					}
-				}
-			}
-		}
-
-		if rawprojs, ok := claims["projects"].([]interface{}); ok {
-			for _, pp := range rawprojs {
-				if p, ok := pp.(string); ok {
-					projects = append(projects, p)
-				}
-			}
-		} else if rawprojs, ok := claims["projects"]; ok {
-			projects = append(projects, rawprojs.([]string)...)
-		}
-
-		user = &schema.User{
-			Username:   sub,
-			Name:       name,
-			Roles:      roles,
-			Projects:   projects,
-			AuthType:   schema.AuthSession,
-			AuthSource: schema.AuthViaToken,
-		}
-
-		if config.Keys.JwtConfig.SyncUserOnLogin || config.Keys.JwtConfig.UpdateUserOnLogin {
-			handleTokenUser(user)
-		}
+	// Sync or update user if configured
+	if !Keys.JwtConfig.ValidateUser && (Keys.JwtConfig.SyncUserOnLogin || Keys.JwtConfig.UpdateUserOnLogin) {
+		handleTokenUser(user)
 	}
 
 	return user, nil

internal/auth/ldap.go

@@ -1,26 +1,44 @@
 // Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved.
+// All rights reserved. This file is part of cc-backend.
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
+
 package auth
 
 import (
-	"errors"
 	"fmt"
+	"net"
 	"net/http"
 	"os"
 	"strings"
+	"time"
 
-	"github.com/ClusterCockpit/cc-backend/internal/config"
 	"github.com/ClusterCockpit/cc-backend/internal/repository"
-	"github.com/ClusterCockpit/cc-backend/pkg/log"
-	"github.com/ClusterCockpit/cc-backend/pkg/schema"
+	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
+	"github.com/ClusterCockpit/cc-lib/v2/schema"
 	"github.com/go-ldap/ldap/v3"
 )
 
+type LdapConfig struct {
+	URL             string `json:"url"`
+	UserBase        string `json:"user-base"`
+	SearchDN        string `json:"search-dn"`
+	UserBind        string `json:"user-bind"`
+	UserFilter      string `json:"user-filter"`
+	UserAttr        string `json:"username-attr"`
+	UIDAttr         string `json:"uid-attr"`
+	SyncInterval    string `json:"sync-interval"` // Parsed using time.ParseDuration.
+	SyncDelOldUsers bool   `json:"sync-del-old-users"`
+
+	// Should a non-existent user be added to the DB if user exists in ldap directory
+	SyncUserOnLogin   bool `json:"sync-user-on-login"`
+	UpdateUserOnLogin bool `json:"update-user-on-login"`
+}
+
 type LdapAuthenticator struct {
 	syncPassword string
 	UserAttr     string
+	UIDAttr      string
 }
 
 var _ Authenticator = (*LdapAuthenticator)(nil)
@@ -28,17 +46,21 @@ var _ Authenticator = (*LdapAuthenticator)(nil)
 func (la *LdapAuthenticator) Init() error {
 	la.syncPassword = os.Getenv("LDAP_ADMIN_PASSWORD")
 	if la.syncPassword == "" {
-		log.Warn("environment variable 'LDAP_ADMIN_PASSWORD' not set (ldap sync will not work)")
+		cclog.Warn("environment variable 'LDAP_ADMIN_PASSWORD' not set (ldap sync will not work)")
 	}
 
-	lc := config.Keys.LdapConfig
-
-	if lc.UserAttr != "" {
-		la.UserAttr = lc.UserAttr
+	if Keys.LdapConfig.UserAttr != "" {
+		la.UserAttr = Keys.LdapConfig.UserAttr
 	} else {
 		la.UserAttr = "gecos"
 	}
 
+	if Keys.LdapConfig.UIDAttr != "" {
+		la.UIDAttr = Keys.LdapConfig.UIDAttr
+	} else {
+		la.UIDAttr = "uid"
+	}
+
 	return nil
 }
 
@@ -48,60 +70,50 @@ func (la *LdapAuthenticator) CanLogin(
 	rw http.ResponseWriter,
 	r *http.Request,
 ) (*schema.User, bool) {
-	lc := config.Keys.LdapConfig
+	lc := Keys.LdapConfig
 
 	if user != nil {
 		if user.AuthSource == schema.AuthViaLDAP {
 			return user, true
 		}
-	} else {
-		if lc.SyncUserOnLogin {
-			l, err := la.getLdapConnection(true)
-			if err != nil {
-				log.Error("LDAP connection error")
-			}
-			defer l.Close()
-
-			// Search for the given username
-			searchRequest := ldap.NewSearchRequest(
-				lc.UserBase,
-				ldap.ScopeWholeSubtree, ldap.NeverDerefAliases, 0, 0, false,
-				fmt.Sprintf("(&%s(uid=%s))", lc.UserFilter, username),
-				[]string{"dn", "uid", la.UserAttr}, nil)
-
-			sr, err := l.Search(searchRequest)
-			if err != nil {
-				log.Warn(err)
-				return nil, false
-			}
-
-			if len(sr.Entries) != 1 {
-				log.Warn("LDAP: User does not exist or too many entries returned")
-				return nil, false
-			}
-
-			entry := sr.Entries[0]
-			name := entry.GetAttributeValue(la.UserAttr)
-			var roles []string
-			roles = append(roles, schema.GetRoleString(schema.RoleUser))
-			projects := make([]string, 0)
-
-			user = &schema.User{
-				Username:   username,
-				Name:       name,
-				Roles:      roles,
-				Projects:   projects,
-				AuthType:   schema.AuthSession,
-				AuthSource: schema.AuthViaLDAP,
-			}
-
-			if err := repository.GetUserRepository().AddUser(user); err != nil {
-				log.Errorf("User '%s' LDAP: Insert into DB failed", username)
-				return nil, false
-			}
-
-			return user, true
+	} else if lc.SyncUserOnLogin {
+		l, err := la.getLdapConnection(true)
+		if err != nil {
+			cclog.Error("LDAP connection error")
+			return nil, false
 		}
+		defer l.Close()
+
+		// Search for the given username
+		searchRequest := ldap.NewSearchRequest(
+			lc.UserBase,
+			ldap.ScopeWholeSubtree, ldap.NeverDerefAliases, 0, 0, false,
+			fmt.Sprintf("(&%s(%s=%s))", lc.UserFilter, la.UIDAttr, ldap.EscapeFilter(username)),
+			[]string{"dn", la.UIDAttr, la.UserAttr}, nil)
+
+		sr, err := l.Search(searchRequest)
+		if err != nil {
+			cclog.Warn(err)
+			return nil, false
+		}
+
+		if len(sr.Entries) != 1 {
+			cclog.Warn("LDAP: User does not exist or too many entries returned")
+			return nil, false
+		}
+
+		entry := sr.Entries[0]
+		user = &schema.User{
+			Username:   username,
+			Name:       entry.GetAttributeValue(la.UserAttr),
+			Roles:      []string{schema.GetRoleString(schema.RoleUser)},
+			Projects:   make([]string, 0),
+			AuthType:   schema.AuthSession,
+			AuthSource: schema.AuthViaLDAP,
+		}
+
+		handleLdapUser(user)
+		return user, true
 	}
 
 	return nil, false
@@ -114,14 +126,14 @@ func (la *LdapAuthenticator) Login(
 ) (*schema.User, error) {
 	l, err := la.getLdapConnection(false)
 	if err != nil {
-		log.Warn("Error while getting ldap connection")
+		cclog.Warn("Error while getting ldap connection")
 		return nil, err
 	}
 	defer l.Close()
 
-	userDn := strings.Replace(config.Keys.LdapConfig.UserBind, "{username}", user.Username, -1)
+	userDn := strings.ReplaceAll(Keys.LdapConfig.UserBind, "{username}", ldap.EscapeDN(user.Username))
 	if err := l.Bind(userDn, r.FormValue("password")); err != nil {
-		log.Errorf("AUTH/LDAP > Authentication for user %s failed: %v",
+		cclog.Errorf("AUTH/LDAP > Authentication for user %s failed: %v",
 			user.Username, err)
 		return nil, fmt.Errorf("Authentication failed")
 	}
@@ -130,11 +142,11 @@ func (la *LdapAuthenticator) Login(
 }
 
 func (la *LdapAuthenticator) Sync() error {
-	const IN_DB int = 1
-	const IN_LDAP int = 2
-	const IN_BOTH int = 3
+	const InDB int = 1
+	const InLdap int = 2
+	const InBoth int = 3
 	ur := repository.GetUserRepository()
-	lc := config.Keys.LdapConfig
+	lc := Keys.LdapConfig
 
 	users := map[string]int{}
 	usernames, err := ur.GetLdapUsernames()
@@ -143,12 +155,12 @@ func (la *LdapAuthenticator) Sync() error {
 	}
 
 	for _, username := range usernames {
-		users[username] = IN_DB
+		users[username] = InDB
 	}
 
 	l, err := la.getLdapConnection(true)
 	if err != nil {
-		log.Error("LDAP connection error")
+		cclog.Error("LDAP connection error")
 		return err
 	}
 	defer l.Close()
@@ -157,50 +169,49 @@ func (la *LdapAuthenticator) Sync() error {
 		lc.UserBase,
 		ldap.ScopeWholeSubtree, ldap.NeverDerefAliases, 0, 0, false,
 		lc.UserFilter,
-		[]string{"dn", "uid", la.UserAttr}, nil))
+		[]string{"dn", la.UIDAttr, la.UserAttr}, nil))
 	if err != nil {
-		log.Warn("LDAP search error")
+		cclog.Warn("LDAP search error")
 		return err
 	}
 
 	newnames := map[string]string{}
 	for _, entry := range ldapResults.Entries {
-		username := entry.GetAttributeValue("uid")
+		username := entry.GetAttributeValue(la.UIDAttr)
 		if username == "" {
-			return errors.New("no attribute 'uid'")
+			return fmt.Errorf("no attribute '%s'", la.UIDAttr)
 		}
 
 		_, ok := users[username]
 		if !ok {
-			users[username] = IN_LDAP
+			users[username] = InLdap
 			newnames[username] = entry.GetAttributeValue(la.UserAttr)
 		} else {
-			users[username] = IN_BOTH
+			users[username] = InBoth
 		}
 	}
 
 	for username, where := range users {
-		if where == IN_DB && lc.SyncDelOldUsers {
-			ur.DelUser(username)
-			log.Debugf("sync: remove %v (does not show up in LDAP anymore)", username)
-		} else if where == IN_LDAP {
+		if where == InDB && lc.SyncDelOldUsers {
+			if err := ur.DelUser(username); err != nil {
+				cclog.Errorf("User '%s' LDAP: Delete from DB failed: %v", username, err)
+				return err
+			}
+			cclog.Debugf("sync: remove %v (does not show up in LDAP anymore)", username)
+		} else if where == InLdap {
 			name := newnames[username]
 
-			var roles []string
-			roles = append(roles, schema.GetRoleString(schema.RoleUser))
-			projects := make([]string, 0)
-
 			user := &schema.User{
 				Username:   username,
 				Name:       name,
-				Roles:      roles,
-				Projects:   projects,
+				Roles:      []string{schema.GetRoleString(schema.RoleUser)},
+				Projects:   make([]string, 0),
 				AuthSource: schema.AuthViaLDAP,
 			}
 
-			log.Debugf("sync: add %v (name: %v, roles: [user], ldap: true)", username, name)
+			cclog.Debugf("sync: add %v (name: %v, roles: [user], ldap: true)", username, name)
 			if err := ur.AddUser(user); err != nil {
-				log.Errorf("User '%s' LDAP: Insert into DB failed", username)
+				cclog.Errorf("User '%s' LDAP: Insert into DB failed", username)
 				return err
 			}
 		}
@@ -210,17 +221,19 @@ func (la *LdapAuthenticator) Sync() error {
 }
 
 func (la *LdapAuthenticator) getLdapConnection(admin bool) (*ldap.Conn, error) {
-	lc := config.Keys.LdapConfig
-	conn, err := ldap.DialURL(lc.Url)
+	lc := Keys.LdapConfig
+	conn, err := ldap.DialURL(lc.URL,
+		ldap.DialWithDialer(&net.Dialer{Timeout: 10 * time.Second}))
 	if err != nil {
-		log.Warn("LDAP URL dial failed")
+		cclog.Warn("LDAP URL dial failed")
 		return nil, err
 	}
+	conn.SetTimeout(30 * time.Second)
 
 	if admin {
 		if err := conn.Bind(lc.SearchDN, la.syncPassword); err != nil {
 			conn.Close()
-			log.Warn("LDAP connection bind failed")
+			cclog.Warn("LDAP connection bind failed")
 			return nil, err
 		}
 	}

internal/auth/local.go

@@ -1,15 +1,16 @@
 // Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved.
+// All rights reserved. This file is part of cc-backend.
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
+
 package auth
 
 import (
 	"fmt"
 	"net/http"
 
-	"github.com/ClusterCockpit/cc-backend/pkg/log"
-	"github.com/ClusterCockpit/cc-backend/pkg/schema"
+	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
+	"github.com/ClusterCockpit/cc-lib/v2/schema"
 	"golang.org/x/crypto/bcrypt"
 )
 
@@ -27,19 +28,19 @@ func (la *LocalAuthenticator) CanLogin(
 	user *schema.User,
 	username string,
 	rw http.ResponseWriter,
-	r *http.Request) (*schema.User, bool) {
-
+	r *http.Request,
+) (*schema.User, bool) {
 	return user, user != nil && user.AuthSource == schema.AuthViaLocalPassword
 }
 
 func (la *LocalAuthenticator) Login(
 	user *schema.User,
 	rw http.ResponseWriter,
-	r *http.Request) (*schema.User, error) {
-
+	r *http.Request,
+) (*schema.User, error) {
 	if e := bcrypt.CompareHashAndPassword([]byte(user.Password),
 		[]byte(r.FormValue("password"))); e != nil {
-		log.Errorf("AUTH/LOCAL > Authentication for user %s failed!", user.Username)
+		cclog.Errorf("AUTH/LOCAL > Authentication for user %s failed!", user.Username)
 		return nil, fmt.Errorf("Authentication failed")
 	}
 

internal/auth/oidc.go

@@ -1,27 +1,34 @@
 // Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved.
+// All rights reserved. This file is part of cc-backend.
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
+
 package auth
 
 import (
 	"context"
 	"crypto/rand"
 	"encoding/base64"
+	"fmt"
 	"io"
 	"net/http"
 	"os"
 	"time"
 
-	"github.com/ClusterCockpit/cc-backend/internal/config"
 	"github.com/ClusterCockpit/cc-backend/internal/repository"
-	"github.com/ClusterCockpit/cc-backend/pkg/log"
-	"github.com/ClusterCockpit/cc-backend/pkg/schema"
+	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
+	"github.com/ClusterCockpit/cc-lib/v2/schema"
 	"github.com/coreos/go-oidc/v3/oidc"
-	"github.com/gorilla/mux"
+	"github.com/go-chi/chi/v5"
 	"golang.org/x/oauth2"
 )
 
+type OpenIDConfig struct {
+	Provider          string `json:"provider"`
+	SyncUserOnLogin   bool   `json:"sync-user-on-login"`
+	UpdateUserOnLogin bool   `json:"update-user-on-login"`
+}
+
 type OIDC struct {
 	client         *oauth2.Config
 	provider       *oidc.Provider
@@ -44,30 +51,35 @@ func setCallbackCookie(w http.ResponseWriter, r *http.Request, name, value strin
 		MaxAge:   int(time.Hour.Seconds()),
 		Secure:   r.TLS != nil,
 		HttpOnly: true,
+		SameSite: http.SameSiteLaxMode,
 	}
 	http.SetCookie(w, c)
 }
 
+// NewOIDC creates a new OIDC authenticator with the configured provider
 func NewOIDC(a *Authentication) *OIDC {
-	provider, err := oidc.NewProvider(context.Background(), config.Keys.OpenIDConfig.Provider)
+	// Use context with timeout for provider initialization
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	provider, err := oidc.NewProvider(ctx, Keys.OpenIDConfig.Provider)
 	if err != nil {
-		log.Fatal(err)
+		cclog.Fatal(err)
 	}
 	clientID := os.Getenv("OID_CLIENT_ID")
 	if clientID == "" {
-		log.Warn("environment variable 'OID_CLIENT_ID' not set (Open ID connect auth will not work)")
+		cclog.Warn("environment variable 'OID_CLIENT_ID' not set (Open ID connect auth will not work)")
 	}
 	clientSecret := os.Getenv("OID_CLIENT_SECRET")
 	if clientSecret == "" {
-		log.Warn("environment variable 'OID_CLIENT_SECRET' not set (Open ID connect auth will not work)")
+		cclog.Warn("environment variable 'OID_CLIENT_SECRET' not set (Open ID connect auth will not work)")
 	}
 
 	client := &oauth2.Config{
 		ClientID:     clientID,
 		ClientSecret: clientSecret,
 		Endpoint:     provider.Endpoint(),
-		RedirectURL:  "oidc-callback",
-		Scopes:       []string{oidc.ScopeOpenID, "profile", "email"},
+		Scopes:       []string{oidc.ScopeOpenID, "profile"},
 	}
 
 	oa := &OIDC{provider: provider, client: client, clientID: clientID, authentication: a}
@@ -75,7 +87,7 @@ func NewOIDC(a *Authentication) *OIDC {
 	return oa
 }
 
-func (oa *OIDC) RegisterEndpoints(r *mux.Router) {
+func (oa *OIDC) RegisterEndpoints(r chi.Router) {
 	r.HandleFunc("/oidc-login", oa.OAuth2Login)
 	r.HandleFunc("/oidc-callback", oa.OAuth2Callback)
 }
@@ -105,55 +117,99 @@ func (oa *OIDC) OAuth2Callback(rw http.ResponseWriter, r *http.Request) {
 		http.Error(rw, "Code not found", http.StatusBadRequest)
 		return
 	}
-	token, err := oa.client.Exchange(context.Background(), code, oauth2.VerifierOption(codeVerifier))
+	// Exchange authorization code for token with timeout
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	token, err := oa.client.Exchange(ctx, code, oauth2.VerifierOption(codeVerifier))
 	if err != nil {
-		http.Error(rw, "Failed to exchange token: "+err.Error(), http.StatusInternalServerError)
+		cclog.Errorf("token exchange failed: %s", err.Error())
+		http.Error(rw, "Authentication failed during token exchange", http.StatusInternalServerError)
 		return
 	}
 
-	userInfo, err := oa.provider.UserInfo(context.Background(), oauth2.StaticTokenSource(token))
+	// Get user info from OIDC provider with same timeout
+	userInfo, err := oa.provider.UserInfo(ctx, oauth2.StaticTokenSource(token))
 	if err != nil {
-		http.Error(rw, "Failed to get userinfo: "+err.Error(), http.StatusInternalServerError)
+		cclog.Errorf("failed to get userinfo: %s", err.Error())
+		http.Error(rw, "Failed to retrieve user information", http.StatusInternalServerError)
 		return
 	}
 
-	// // Extract the ID Token from OAuth2 token.
-	// rawIDToken, ok := token.Extra("id_token").(string)
-	// if !ok {
-	// 	http.Error(rw, "Cannot access idToken", http.StatusInternalServerError)
-	// }
-	//
-	// verifier := oa.provider.Verifier(&oidc.Config{ClientID: oa.clientID})
-	// // Parse and verify ID Token payload.
-	// idToken, err := verifier.Verify(context.Background(), rawIDToken)
-	// if err != nil {
-	// 	http.Error(rw, "Failed to extract idToken: "+err.Error(), http.StatusInternalServerError)
-	// }
+	// Verify ID token and nonce to prevent replay attacks
+	rawIDToken, ok := token.Extra("id_token").(string)
+	if !ok {
+		http.Error(rw, "ID token not found in response", http.StatusInternalServerError)
+		return
+	}
+
+	nonceCookie, err := r.Cookie("nonce")
+	if err != nil {
+		http.Error(rw, "nonce cookie not found", http.StatusBadRequest)
+		return
+	}
+
+	verifier := oa.provider.Verifier(&oidc.Config{ClientID: oa.clientID})
+	idToken, err := verifier.Verify(ctx, rawIDToken)
+	if err != nil {
+		cclog.Errorf("ID token verification failed: %s", err.Error())
+		http.Error(rw, "ID token verification failed", http.StatusInternalServerError)
+		return
+	}
+
+	if idToken.Nonce != nonceCookie.Value {
+		http.Error(rw, "Nonce mismatch", http.StatusBadRequest)
+		return
+	}
 
 	projects := make([]string, 0)
 
-	// Extract custom claims
+	// Extract custom claims from userinfo
 	var claims struct {
 		Username string `json:"preferred_username"`
 		Name     string `json:"name"`
-		Profile  struct {
+		// Keycloak realm-level roles
+		RealmAccess struct {
+			Roles []string `json:"roles"`
+		} `json:"realm_access"`
+		// Keycloak client-level roles
+		ResourceAccess struct {
 			Client struct {
 				Roles []string `json:"roles"`
 			} `json:"clustercockpit"`
 		} `json:"resource_access"`
 	}
 	if err := userInfo.Claims(&claims); err != nil {
-		http.Error(rw, "Failed to extract Claims: "+err.Error(), http.StatusInternalServerError)
+		cclog.Errorf("failed to extract claims: %s", err.Error())
+		http.Error(rw, "Failed to extract user claims", http.StatusInternalServerError)
+		return
+	}
+
+	if claims.Username == "" {
+		http.Error(rw, "Username claim missing from OIDC provider", http.StatusBadRequest)
+		return
+	}
+
+	// Merge roles from both client-level and realm-level access
+	oidcRoles := append(claims.ResourceAccess.Client.Roles, claims.RealmAccess.Roles...)
+
+	roleSet := make(map[string]bool)
+	for _, r := range oidcRoles {
+		switch r {
+		case "user":
+			roleSet[schema.GetRoleString(schema.RoleUser)] = true
+		case "admin":
+			roleSet[schema.GetRoleString(schema.RoleAdmin)] = true
+		case "manager":
+			roleSet[schema.GetRoleString(schema.RoleManager)] = true
+		case "support":
+			roleSet[schema.GetRoleString(schema.RoleSupport)] = true
+		}
 	}
 
 	var roles []string
-	for _, r := range claims.Profile.Client.Roles {
-		switch r {
-		case "user":
-			roles = append(roles, schema.GetRoleString(schema.RoleUser))
-		case "admin":
-			roles = append(roles, schema.GetRoleString(schema.RoleAdmin))
-		}
+	for role := range roleSet {
+		roles = append(roles, role)
 	}
 
 	if len(roles) == 0 {
@@ -168,14 +224,18 @@ func (oa *OIDC) OAuth2Callback(rw http.ResponseWriter, r *http.Request) {
 		AuthSource: schema.AuthViaOIDC,
 	}
 
-	if config.Keys.OpenIDConfig.SyncUserOnLogin || config.Keys.OpenIDConfig.UpdateUserOnLogin {
+	if Keys.OpenIDConfig.SyncUserOnLogin || Keys.OpenIDConfig.UpdateUserOnLogin {
 		handleOIDCUser(user)
 	}
 
-	oa.authentication.SaveSession(rw, r, user)
-	log.Infof("login successfull: user: %#v (roles: %v, projects: %v)", user.Username, user.Roles, user.Projects)
-	ctx := context.WithValue(r.Context(), repository.ContextUserKey, user)
-	http.RedirectHandler("/", http.StatusTemporaryRedirect).ServeHTTP(rw, r.WithContext(ctx))
+	if err := oa.authentication.SaveSession(rw, r, user); err != nil {
+		cclog.Errorf("session save failed for user %q: %s", user.Username, err.Error())
+		http.Error(rw, "Failed to create session", http.StatusInternalServerError)
+		return
+	}
+	cclog.Infof("login successful: user: %#v (roles: %v, projects: %v)", user.Username, user.Roles, user.Projects)
+	userCtx := context.WithValue(r.Context(), repository.ContextUserKey, user)
+	http.RedirectHandler("/", http.StatusTemporaryRedirect).ServeHTTP(rw, r.WithContext(userCtx))
 }
 
 func (oa *OIDC) OAuth2Login(rw http.ResponseWriter, r *http.Request) {
@@ -190,7 +250,24 @@ func (oa *OIDC) OAuth2Login(rw http.ResponseWriter, r *http.Request) {
 	codeVerifier := oauth2.GenerateVerifier()
 	setCallbackCookie(rw, r, "verifier", codeVerifier)
 
+	// Generate nonce for ID token replay protection
+	nonce, err := randString(16)
+	if err != nil {
+		http.Error(rw, "Internal error", http.StatusInternalServerError)
+		return
+	}
+	setCallbackCookie(rw, r, "nonce", nonce)
+
+	// Build redirect URL from the incoming request
+	scheme := "https"
+	if r.TLS == nil && r.Header.Get("X-Forwarded-Proto") != "https" {
+		scheme = "http"
+	}
+	oa.client.RedirectURL = fmt.Sprintf("%s://%s/oidc-callback", scheme, r.Host)
+
 	// Redirect user to consent page to ask for permission
-	url := oa.client.AuthCodeURL(state, oauth2.AccessTypeOffline, oauth2.S256ChallengeOption(codeVerifier))
+	url := oa.client.AuthCodeURL(state, oauth2.AccessTypeOffline,
+		oauth2.S256ChallengeOption(codeVerifier),
+		oidc.Nonce(nonce))
 	http.Redirect(rw, r, url, http.StatusFound)
 }

internal/auth/schema.go

@@ -0,0 +1,111 @@
+// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
+// All rights reserved. This file is part of cc-backend.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package auth
+
+var configSchema = `
+	{
+    "jwts": {
+      "description": "For JWT token authentication.",
+      "type": "object",
+      "properties": {
+        "max-age": {
+          "description": "Configure how long a token is valid. As string parsable by time.ParseDuration()",
+          "type": "string"
+        },
+        "cookie-name": {
+          "description": "Cookie that should be checked for a JWT token.",
+          "type": "string"
+        },
+        "validate-user": {
+          "description": "Deny login for users not in database (but defined in JWT). Overwrite roles in JWT with database roles.",
+          "type": "boolean"
+        },
+        "trusted-issuer": {
+          "description": "Issuer that should be accepted when validating external JWTs ",
+          "type": "string"
+        },
+        "sync-user-on-login": {
+          "description": "Add non-existent user to DB at login attempt with values provided in JWT.",
+          "type": "boolean"
+        },
+        "update-user-on-login": {
+          "description": "Should an existent user attributes in the DB be updated at login attempt with values provided in JWT.",
+          "type": "boolean"
+        }
+      },
+      "required": ["max-age"]
+    },
+    "oidc": {
+      "type": "object",
+      "properties": {
+        "provider": {
+          "description": "OpenID Connect provider URL.",
+          "type": "string"
+        },
+        "sync-user-on-login": {
+          "description": "Add non-existent user to DB at login attempt with values provided.",
+          "type": "boolean"
+        },
+        "update-user-on-login": {
+          "description": "Should an existent user attributes in the DB be updated at login attempt with values provided.",
+          "type": "boolean"
+        }
+      },
+      "required": ["provider"]
+    },
+    "ldap": {
+      "description": "For LDAP Authentication and user synchronisation.",
+      "type": "object",
+      "properties": {
+        "url": {
+          "description": "URL of LDAP directory server.",
+          "type": "string"
+        },
+        "user-base": {
+          "description": "Base DN of user tree root.",
+          "type": "string"
+        },
+        "search-dn": {
+          "description": "DN for authenticating LDAP admin account with general read rights.",
+          "type": "string"
+        },
+        "user-bind": {
+          "description": "Expression used to authenticate users via LDAP bind. Must contain uid={username}.",
+          "type": "string"
+        },
+        "user-filter": {
+          "description": "Filter to extract users for syncing.",
+          "type": "string"
+        },
+        "username-attr": {
+          "description": "Attribute with full username. Default: gecos",
+          "type": "string"
+        },
+        "sync-interval": {
+          "description": "Interval used for syncing local user table with LDAP directory. Parsed using time.ParseDuration.",
+          "type": "string"
+        },
+        "sync-del-old-users": {
+          "description": "Delete obsolete users in database.",
+          "type": "boolean"
+        },
+        "uid-attr": {
+          "description": "LDAP attribute used as login username. Default: uid",
+          "type": "string"
+        },
+        "sync-user-on-login": {
+          "description": "Add non-existent user to DB at login attempt if user exists in Ldap directory",
+          "type": "boolean"
+        },
+        "update-user-on-login": {
+          "description": "Should an existent user attributes in the DB be updated at login attempt with values from LDAP.",
+          "type": "boolean"
+        }
+      },
+      "required": ["url", "user-base", "search-dn", "user-bind", "user-filter"]
+    },
+  "required": ["jwts"]
+	}`

internal/config/config.go

@@ -1,72 +1,158 @@
 // Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved.
+// All rights reserved. This file is part of cc-backend.
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
+
+// Package config implements the program configuration data structures, validation and parsing
 package config
 
 import (
 	"bytes"
 	"encoding/json"
-	"log"
-	"os"
+	"time"
 
-	"github.com/ClusterCockpit/cc-backend/pkg/schema"
+	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
+	"github.com/ClusterCockpit/cc-lib/v2/resampler"
 )
 
-var Keys schema.ProgramConfig = schema.ProgramConfig{
+type ProgramConfig struct {
+	// Address where the http (or https) server will listen on (for example: 'localhost:80').
+	Addr string `json:"addr"`
+
+	// Addresses from which secured admin API endpoints can be reached, can be wildcard "*"
+	APIAllowedIPs []string `json:"api-allowed-ips"`
+
+	APISubjects *NATSConfig `json:"api-subjects"`
+
+	// Drop root permissions once .env was read and the port was taken.
+	User  string `json:"user"`
+	Group string `json:"group"`
+
+	// Disable authentication (for everything: API, Web-UI, ...)
+	DisableAuthentication bool `json:"disable-authentication"`
+
+	// If `embed-static-files` is true (default), the frontend files are directly
+	// embeded into the go binary and expected to be in web/frontend. Only if
+	// it is false the files in `static-files` are served instead.
+	EmbedStaticFiles bool   `json:"embed-static-files"`
+	StaticFiles      string `json:"static-files"`
+
+	// Path to SQLite database file
+	DB string `json:"db"`
+
+	EnableJobTaggers bool `json:"enable-job-taggers"`
+
+	// Validate json input against schema
+	Validate bool `json:"validate"`
+
+	// If 0 or empty, the session does not expire!
+	SessionMaxAge string `json:"session-max-age"`
+
+	// If both those options are not empty, use HTTPS using those certificates.
+	HTTPSCertFile string `json:"https-cert-file"`
+	HTTPSKeyFile  string `json:"https-key-file"`
+
+	// If not the empty string and `addr` does not end in ":80",
+	// redirect every request incoming at port 80 to that url.
+	RedirectHTTPTo string `json:"redirect-http-to"`
+
+	// Where to store MachineState files
+	MachineStateDir string `json:"machine-state-dir"`
+
+	// If not zero, automatically mark jobs as stopped running X seconds longer than their walltime.
+	StopJobsExceedingWalltime int `json:"stop-jobs-exceeding-walltime"`
+
+	// Defines time X in seconds in which jobs are considered to be "short" and will be filtered in specific views.
+	ShortRunningJobsDuration int `json:"short-running-jobs-duration"`
+
+	// Energy Mix CO2 Emission Constant [g/kWh]
+	// If entered, displays estimated CO2 emission for job based on jobs totalEnergy
+	EmissionConstant int `json:"emission-constant"`
+
+	// If exists, will enable dynamic zoom in frontend metric plots using the configured values
+	EnableResampling *ResampleConfig `json:"resampling"`
+
+	// Systemd unit name for log viewer (default: "clustercockpit")
+	SystemdUnit string `json:"systemd-unit"`
+
+	// Node state retention configuration
+	NodeStateRetention *NodeStateRetention `json:"nodestate-retention"`
+
+	// Database tuning configuration
+	DbConfig *DbConfig `json:"db-config"`
+}
+
+type DbConfig struct {
+	CacheSizeMB               int `json:"cache-size-mb"`
+	SoftHeapLimitMB           int `json:"soft-heap-limit-mb"`
+	MaxOpenConnections        int `json:"max-open-connections"`
+	MaxIdleConnections        int `json:"max-idle-connections"`
+	ConnectionMaxIdleTimeMins int `json:"max-idle-time-minutes"`
+}
+
+type NodeStateRetention struct {
+	Policy             string `json:"policy"`      // "delete" or "move"
+	Age                int    `json:"age"`         // hours, default 24
+	TargetKind         string `json:"target-kind"` // "file" or "s3"
+	TargetPath         string `json:"target-path"`
+	TargetEndpoint     string `json:"target-endpoint"`
+	TargetBucket       string `json:"target-bucket"`
+	TargetAccessKey    string `json:"target-access-key"`
+	TargetSecretKey    string `json:"target-secret-key"`
+	TargetRegion       string `json:"target-region"`
+	TargetUsePathStyle bool   `json:"target-use-path-style"`
+	MaxFileSizeMB      int    `json:"max-file-size-mb"`
+}
+
+type ResampleConfig struct {
+	// Minimum number of points to trigger resampling of data
+	MinimumPoints int `json:"minimum-points"`
+	// Array of resampling target resolutions, in seconds; Example: [600,300,60]
+	Resolutions []int `json:"resolutions"`
+	// Trigger next zoom level at less than this many visible datapoints
+	Trigger int `json:"trigger"`
+}
+
+type NATSConfig struct {
+	SubjectJobEvent  string `json:"subject-job-event"`
+	SubjectNodeState string `json:"subject-node-state"`
+}
+
+type IntRange struct {
+	From int `json:"from"`
+	To   int `json:"to"`
+}
+
+type TimeRange struct {
+	From  *time.Time `json:"from"`
+	To    *time.Time `json:"to"`
+	Range string     `json:"range,omitempty"`
+}
+
+type FilterRanges struct {
+	Duration  *IntRange  `json:"duration"`
+	NumNodes  *IntRange  `json:"num-nodes"`
+	StartTime *TimeRange `json:"start-time"`
+}
+
+var Keys ProgramConfig = ProgramConfig{
 	Addr:                      "localhost:8080",
-	DisableAuthentication:     false,
 	EmbedStaticFiles:          true,
-	DBDriver:                  "sqlite3",
 	DB:                        "./var/job.db",
-	Archive:                   json.RawMessage(`{\"kind\":\"file\",\"path\":\"./var/job-archive\"}`),
-	DisableArchive:            false,
-	Validate:                  false,
 	SessionMaxAge:             "168h",
 	StopJobsExceedingWalltime: 0,
 	ShortRunningJobsDuration:  5 * 60,
-	UiDefaults: map[string]interface{}{
-		"analysis_view_histogramMetrics":         []string{"flops_any", "mem_bw", "mem_used"},
-		"analysis_view_scatterPlotMetrics":       [][]string{{"flops_any", "mem_bw"}, {"flops_any", "cpu_load"}, {"cpu_load", "mem_bw"}},
-		"job_view_nodestats_selectedMetrics":     []string{"flops_any", "mem_bw", "mem_used"},
-		"job_view_selectedMetrics":               []string{"flops_any", "mem_bw", "mem_used"},
-		"job_view_showFootprint":                 true,
-		"job_list_usePaging":                     false,
-		"plot_general_colorBackground":           true,
-		"plot_general_colorscheme":               []string{"#00bfff", "#0000ff", "#ff00ff", "#ff0000", "#ff8000", "#ffff00", "#80ff00"},
-		"plot_general_lineWidth":                 3,
-		"plot_list_jobsPerPage":                  50,
-		"plot_list_selectedMetrics":              []string{"cpu_load", "mem_used", "flops_any", "mem_bw"},
-		"plot_view_plotsPerRow":                  3,
-		"plot_view_showPolarplot":                true,
-		"plot_view_showRoofline":                 true,
-		"plot_view_showStatTable":                true,
-		"system_view_selectedMetric":             "cpu_load",
-		"analysis_view_selectedTopEntity":        "user",
-		"analysis_view_selectedTopCategory":      "totalWalltime",
-		"status_view_selectedTopUserCategory":    "totalJobs",
-		"status_view_selectedTopProjectCategory": "totalJobs",
-	},
 }
 
-func Init(flagConfigFile string) {
-	raw, err := os.ReadFile(flagConfigFile)
-	if err != nil {
-		if !os.IsNotExist(err) {
-			log.Fatalf("CONFIG ERROR: %v", err)
-		}
-	} else {
-		if err := schema.Validate(schema.Config, bytes.NewReader(raw)); err != nil {
-			log.Fatalf("Validate config: %v\n", err)
-		}
-		dec := json.NewDecoder(bytes.NewReader(raw))
-		dec.DisallowUnknownFields()
-		if err := dec.Decode(&Keys); err != nil {
-			log.Fatalf("could not decode: %v", err)
-		}
+func Init(mainConfig json.RawMessage) {
+	Validate(configSchema, mainConfig)
+	dec := json.NewDecoder(bytes.NewReader(mainConfig))
+	dec.DisallowUnknownFields()
+	if err := dec.Decode(&Keys); err != nil {
+		cclog.Abortf("Config Init: Could not decode config file '%s'.\nError: %s\n", mainConfig, err.Error())
+	}
 
-		if Keys.Clusters == nil || len(Keys.Clusters) < 1 {
-			log.Fatal("At least one cluster required in config!")
-		}
+	if Keys.EnableResampling != nil && Keys.EnableResampling.MinimumPoints > 0 {
+		resampler.SetMinimumRequiredPoints(Keys.EnableResampling.MinimumPoints)
 	}
 }

internal/config/config_test.go

@@ -1,16 +1,26 @@
 // Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved.
+// All rights reserved. This file is part of cc-backend.
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
+
 package config
 
 import (
 	"testing"
+
+	ccconf "github.com/ClusterCockpit/cc-lib/v2/ccConfig"
+	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
 )
 
 func TestInit(t *testing.T) {
 	fp := "../../configs/config.json"
-	Init(fp)
+	ccconf.Init(fp)
+	if cfg := ccconf.GetPackageConfig("main"); cfg != nil {
+		Init(cfg)
+	} else {
+		cclog.Abort("Main configuration must be present")
+	}
+
 	if Keys.Addr != "0.0.0.0:443" {
 		t.Errorf("wrong addr\ngot: %s \nwant: 0.0.0.0:443", Keys.Addr)
 	}
@@ -18,7 +28,13 @@ func TestInit(t *testing.T) {
 
 func TestInitMinimal(t *testing.T) {
 	fp := "../../configs/config-demo.json"
-	Init(fp)
+	ccconf.Init(fp)
+	if cfg := ccconf.GetPackageConfig("main"); cfg != nil {
+		Init(cfg)
+	} else {
+		cclog.Abort("Main configuration must be present")
+	}
+
 	if Keys.Addr != "127.0.0.1:8080" {
 		t.Errorf("wrong addr\ngot: %s \nwant: 127.0.0.1:8080", Keys.Addr)
 	}

internal/config/default_metrics.go

@@ -1,3 +1,8 @@
+// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
+// All rights reserved. This file is part of cc-backend.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
 package config
 
 import (
@@ -6,9 +11,11 @@ import (
 	"strings"
 )
 
+// DEPRECATED: SUPERSEDED BY NEW USER CONFIG - userConfig.go / web.go
+
 type DefaultMetricsCluster struct {
 	Name           string `json:"name"`
-	DefaultMetrics string `json:"default_metrics"`
+	DefaultMetrics string `json:"default-metrics"`
 }
 
 type DefaultMetricsConfig struct {

internal/config/schema.go

@@ -0,0 +1,208 @@
+// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
+// All rights reserved. This file is part of cc-backend.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package config
+
+var configSchema = `
+{
+  "type": "object",
+  "properties": {
+    "addr": {
+      "description": "Address where the http (or https) server will listen on (for example: 'localhost:80').",
+      "type": "string"
+    },
+    "api-allowed-ips": {
+      "description": "Addresses from which secured API endpoints can be reached",
+      "type": "array",
+      "items": {
+        "type": "string"
+      }
+    },
+    "user": {
+      "description": "Drop root permissions once .env was read and the port was taken. Only applicable if using privileged port.",
+      "type": "string"
+    },
+    "group": {
+      "description": "Drop root permissions once .env was read and the port was taken. Only applicable if using privileged port.",
+      "type": "string"
+    },
+    "disable-authentication": {
+      "description": "Disable authentication (for everything: API, Web-UI, ...).",
+      "type": "boolean"
+    },
+    "embed-static-files": {
+      "description": "If all files in web/frontend/public should be served from within the binary itself (they are embedded) or not.",
+      "type": "boolean"
+    },
+    "static-files": {
+      "description": "Folder where static assets can be found, if embed-static-files is false.",
+      "type": "string"
+    },
+    "db": {
+      "description": "Path to SQLite database file (e.g., './var/job.db')",
+      "type": "string"
+    },
+    "enable-job-taggers": {
+      "description": "Turn on automatic application and jobclass taggers",
+      "type": "boolean"
+    },
+    "validate": {
+      "description": "Validate all input json documents against json schema.",
+      "type": "boolean"
+    },
+    "session-max-age": {
+      "description": "Specifies for how long a session shall be valid  as a string parsable by time.ParseDuration(). If 0 or empty, the session/token does not expire!",
+      "type": "string"
+    },
+    "https-cert-file": {
+      "description": "Filepath to SSL certificate. If also https-key-file is set use HTTPS using those certificates.",
+      "type": "string"
+    },
+    "https-key-file": {
+      "description": "Filepath to SSL key file. If also https-cert-file is set use HTTPS using those certificates.",
+      "type": "string"
+    },
+    "redirect-http-to": {
+      "description": "If not the empty string and addr does not end in :80, redirect every request incoming at port 80 to that url.",
+      "type": "string"
+    },
+    "stop-jobs-exceeding-walltime": {
+      "description": "If not zero, automatically mark jobs as stopped running X seconds longer than their walltime. Only applies if walltime is set for job.",
+      "type": "integer"
+    },
+    "short-running-jobs-duration": {
+      "description": "Do not show running jobs shorter than X seconds.",
+      "type": "integer"
+    },
+    "emission-constant": {
+      "description": "Energy mix CO2 emission constant [g/kWh]. If set, displays estimated CO2 emission for jobs.",
+      "type": "integer"
+    },
+    "machine-state-dir": {
+      "description": "Where to store MachineState files.",
+      "type": "string"
+    },
+    "systemd-unit": {
+      "description": "Systemd unit name for log viewer (default: 'clustercockpit').",
+      "type": "string"
+    },
+    "resampling": {
+      "description": "Enable dynamic zoom in frontend metric plots.",
+      "type": "object",
+      "properties": {
+        "minimum-points": {
+          "description": "Minimum points to trigger resampling of time-series data.",
+          "type": "integer"
+        },
+        "trigger": {
+          "description": "Trigger next zoom level at less than this many visible datapoints.",
+          "type": "integer"
+        },
+        "resolutions": {
+          "description": "Array of resampling target resolutions, in seconds.",
+          "type": "array",
+          "items": {
+            "type": "integer"
+          }
+        }
+      },
+      "required": ["trigger", "resolutions"]
+    },
+    "api-subjects": {
+      "description": "NATS subjects configuration for subscribing to job and node events.",
+      "type": "object",
+      "properties": {
+        "subject-job-event": {
+          "description": "NATS subject for job events (start_job, stop_job)",
+          "type": "string"
+        },
+        "subject-node-state": {
+          "description": "NATS subject for node state updates",
+          "type": "string"
+        }
+      },
+      "required": ["subject-job-event", "subject-node-state"]
+    },
+    "nodestate-retention": {
+      "description": "Node state retention configuration for cleaning up old node_state rows.",
+      "type": "object",
+      "properties": {
+        "policy": {
+          "description": "Retention policy: 'delete' to remove old rows, 'move' to archive to Parquet then delete.",
+          "type": "string",
+          "enum": ["delete", "move"]
+        },
+        "age": {
+          "description": "Retention age in hours (default: 24).",
+          "type": "integer"
+        },
+        "target-kind": {
+          "description": "Target kind for parquet archiving: 'file' or 's3'.",
+          "type": "string",
+          "enum": ["file", "s3"]
+        },
+        "target-path": {
+          "description": "Filesystem path for parquet file target.",
+          "type": "string"
+        },
+        "target-endpoint": {
+          "description": "S3 endpoint URL.",
+          "type": "string"
+        },
+        "target-bucket": {
+          "description": "S3 bucket name.",
+          "type": "string"
+        },
+        "target-access-key": {
+          "description": "S3 access key.",
+          "type": "string"
+        },
+        "target-secret-key": {
+          "description": "S3 secret key.",
+          "type": "string"
+        },
+        "target-region": {
+          "description": "S3 region.",
+          "type": "string"
+        },
+        "target-use-path-style": {
+          "description": "Use path-style S3 addressing.",
+          "type": "boolean"
+        },
+        "max-file-size-mb": {
+          "description": "Maximum parquet file size in MB (default: 128).",
+          "type": "integer"
+        }
+      },
+      "required": ["policy"]
+    },
+    "db-config": {
+      "description": "SQLite database tuning configuration.",
+      "type": "object",
+      "properties": {
+        "cache-size-mb": {
+          "description": "SQLite page cache size per connection in MB (default: 2048).",
+          "type": "integer"
+        },
+        "soft-heap-limit-mb": {
+          "description": "Process-wide SQLite soft heap limit in MB (default: 16384).",
+          "type": "integer"
+        },
+        "max-open-connections": {
+          "description": "Maximum number of open database connections (default: 4).",
+          "type": "integer"
+        },
+        "max-idle-connections": {
+          "description": "Maximum number of idle database connections (default: 4).",
+          "type": "integer"
+        },
+        "max-idle-time-minutes": {
+          "description": "Maximum idle time for a connection in minutes (default: 10).",
+          "type": "integer"
+        }
+      }
+    }
+  }
+}`

internal/config/validate.go

@@ -0,0 +1,29 @@
+// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
+// All rights reserved. This file is part of cc-backend.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package config
+
+import (
+	"encoding/json"
+
+	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
+	"github.com/santhosh-tekuri/jsonschema/v5"
+)
+
+func Validate(schema string, instance json.RawMessage) {
+	sch, err := jsonschema.CompileString("schema.json", schema)
+	if err != nil {
+		cclog.Fatalf("%#v", err)
+	}
+
+	var v any
+	if err := json.Unmarshal([]byte(instance), &v); err != nil {
+		cclog.Fatal(err)
+	}
+
+	if err = sch.Validate(v); err != nil {
+		cclog.Fatalf("%#v", err)
+	}
+}

internal/graph/model/models.go

@@ -1,5 +1,6 @@
 // Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved.
+// All rights reserved. This file is part of cc-backend.
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
+
 package model

internal/graph/model/models_gen.go

@@ -3,14 +3,28 @@
 package model
 
 import (
+	"bytes"
 	"fmt"
 	"io"
 	"strconv"
 	"time"
 
-	"github.com/ClusterCockpit/cc-backend/pkg/schema"
+	"github.com/ClusterCockpit/cc-backend/internal/config"
+	"github.com/ClusterCockpit/cc-lib/v2/schema"
 )
 
+type ClusterMetricWithName struct {
+	Name     string         `json:"name"`
+	Unit     *schema.Unit   `json:"unit,omitempty"`
+	Timestep int            `json:"timestep"`
+	Data     []schema.Float `json:"data"`
+}
+
+type ClusterMetrics struct {
+	NodeCount int                      `json:"nodeCount"`
+	Metrics   []*ClusterMetricWithName `json:"metrics"`
+}
+
 type Count struct {
 	Name  string `json:"name"`
 	Count int    `json:"count"`
@@ -50,23 +64,26 @@ type IntRangeOutput struct {
 
 type JobFilter struct {
 	Tags            []string          `json:"tags,omitempty"`
+	DbID            []string          `json:"dbId,omitempty"`
 	JobID           *StringInput      `json:"jobId,omitempty"`
 	ArrayJobID      *int              `json:"arrayJobId,omitempty"`
 	User            *StringInput      `json:"user,omitempty"`
 	Project         *StringInput      `json:"project,omitempty"`
 	JobName         *StringInput      `json:"jobName,omitempty"`
 	Cluster         *StringInput      `json:"cluster,omitempty"`
+	SubCluster      *StringInput      `json:"subCluster,omitempty"`
 	Partition       *StringInput      `json:"partition,omitempty"`
-	Duration        *schema.IntRange  `json:"duration,omitempty"`
+	Duration        *config.IntRange  `json:"duration,omitempty"`
 	Energy          *FloatRange       `json:"energy,omitempty"`
 	MinRunningFor   *int              `json:"minRunningFor,omitempty"`
-	NumNodes        *schema.IntRange  `json:"numNodes,omitempty"`
-	NumAccelerators *schema.IntRange  `json:"numAccelerators,omitempty"`
-	NumHWThreads    *schema.IntRange  `json:"numHWThreads,omitempty"`
-	StartTime       *schema.TimeRange `json:"startTime,omitempty"`
+	NumNodes        *config.IntRange  `json:"numNodes,omitempty"`
+	NumAccelerators *config.IntRange  `json:"numAccelerators,omitempty"`
+	NumHWThreads    *config.IntRange  `json:"numHWThreads,omitempty"`
+	StartTime       *config.TimeRange `json:"startTime,omitempty"`
 	State           []schema.JobState `json:"state,omitempty"`
 	MetricStats     []*MetricStatItem `json:"metricStats,omitempty"`
-	Exclusive       *int              `json:"exclusive,omitempty"`
+	Shared          *string           `json:"shared,omitempty"`
+	Schedule        *string           `json:"schedule,omitempty"`
 	Node            *StringInput      `json:"node,omitempty"`
 }
 
@@ -81,11 +98,6 @@ type JobLinkResultList struct {
 	Count     *int       `json:"count,omitempty"`
 }
 
-type JobMetricStatWithName struct {
-	Name  string                   `json:"name"`
-	Stats *schema.MetricStatistics `json:"stats"`
-}
-
 type JobMetricWithName struct {
 	Name   string             `json:"name"`
 	Scope  schema.MetricScope `json:"scope"`
@@ -100,9 +112,23 @@ type JobResultList struct {
 	HasNextPage *bool         `json:"hasNextPage,omitempty"`
 }
 
+type JobStats struct {
+	ID              int           `json:"id"`
+	JobID           string        `json:"jobId"`
+	StartTime       int           `json:"startTime"`
+	Duration        int           `json:"duration"`
+	Cluster         string        `json:"cluster"`
+	SubCluster      string        `json:"subCluster"`
+	NumNodes        int           `json:"numNodes"`
+	NumHWThreads    *int          `json:"numHWThreads,omitempty"`
+	NumAccelerators *int          `json:"numAccelerators,omitempty"`
+	Stats           []*NamedStats `json:"stats"`
+}
+
 type JobsStatistics struct {
 	ID             string               `json:"id"`
 	Name           string               `json:"name"`
+	TotalUsers     int                  `json:"totalUsers"`
 	TotalJobs      int                  `json:"totalJobs"`
 	RunningJobs    int                  `json:"runningJobs"`
 	ShortJobs      int                  `json:"shortJobs"`
@@ -147,12 +173,49 @@ type MetricStatItem struct {
 type Mutation struct {
 }
 
+type NamedStats struct {
+	Name string                   `json:"name"`
+	Data *schema.MetricStatistics `json:"data"`
+}
+
+type NamedStatsWithScope struct {
+	Name  string             `json:"name"`
+	Scope schema.MetricScope `json:"scope"`
+	Stats []*ScopedStats     `json:"stats"`
+}
+
+type NodeFilter struct {
+	Hostname       *StringInput           `json:"hostname,omitempty"`
+	Cluster        *StringInput           `json:"cluster,omitempty"`
+	SubCluster     *StringInput           `json:"subCluster,omitempty"`
+	SchedulerState *schema.SchedulerState `json:"schedulerState,omitempty"`
+	HealthState    *string                `json:"healthState,omitempty"`
+	TimeStart      *int                   `json:"timeStart,omitempty"`
+}
+
 type NodeMetrics struct {
 	Host       string               `json:"host"`
+	State      string               `json:"state"`
 	SubCluster string               `json:"subCluster"`
 	Metrics    []*JobMetricWithName `json:"metrics"`
 }
 
+type NodeStateResultList struct {
+	Items []*schema.Node `json:"items"`
+	Count *int           `json:"count,omitempty"`
+}
+
+type NodeStates struct {
+	State string `json:"state"`
+	Count int    `json:"count"`
+}
+
+type NodeStatesTimed struct {
+	State  string `json:"state"`
+	Counts []int  `json:"counts"`
+	Times  []int  `json:"times"`
+}
+
 type NodesResultList struct {
 	Items       []*NodeMetrics `json:"items"`
 	Offset      *int           `json:"offset,omitempty"`
@@ -173,6 +236,12 @@ type PageRequest struct {
 	Page         int `json:"page"`
 }
 
+type ScopedStats struct {
+	Hostname string                   `json:"hostname"`
+	ID       *string                  `json:"id,omitempty"`
+	Data     *schema.MetricStatistics `json:"data"`
+}
+
 type StringInput struct {
 	Eq         *string  `json:"eq,omitempty"`
 	Neq        *string  `json:"neq,omitempty"`
@@ -203,20 +272,22 @@ type User struct {
 type Aggregate string
 
 const (
-	AggregateUser    Aggregate = "USER"
-	AggregateProject Aggregate = "PROJECT"
-	AggregateCluster Aggregate = "CLUSTER"
+	AggregateUser       Aggregate = "USER"
+	AggregateProject    Aggregate = "PROJECT"
+	AggregateCluster    Aggregate = "CLUSTER"
+	AggregateSubcluster Aggregate = "SUBCLUSTER"
 )
 
 var AllAggregate = []Aggregate{
 	AggregateUser,
 	AggregateProject,
 	AggregateCluster,
+	AggregateSubcluster,
 }
 
 func (e Aggregate) IsValid() bool {
 	switch e {
-	case AggregateUser, AggregateProject, AggregateCluster:
+	case AggregateUser, AggregateProject, AggregateCluster, AggregateSubcluster:
 		return true
 	}
 	return false
@@ -243,11 +314,26 @@ func (e Aggregate) MarshalGQL(w io.Writer) {
 	fmt.Fprint(w, strconv.Quote(e.String()))
 }
 
+func (e *Aggregate) UnmarshalJSON(b []byte) error {
+	s, err := strconv.Unquote(string(b))
+	if err != nil {
+		return err
+	}
+	return e.UnmarshalGQL(s)
+}
+
+func (e Aggregate) MarshalJSON() ([]byte, error) {
+	var buf bytes.Buffer
+	e.MarshalGQL(&buf)
+	return buf.Bytes(), nil
+}
+
 type SortByAggregate string
 
 const (
 	SortByAggregateTotalwalltime  SortByAggregate = "TOTALWALLTIME"
 	SortByAggregateTotaljobs      SortByAggregate = "TOTALJOBS"
+	SortByAggregateTotalusers     SortByAggregate = "TOTALUSERS"
 	SortByAggregateTotalnodes     SortByAggregate = "TOTALNODES"
 	SortByAggregateTotalnodehours SortByAggregate = "TOTALNODEHOURS"
 	SortByAggregateTotalcores     SortByAggregate = "TOTALCORES"
@@ -259,6 +345,7 @@ const (
 var AllSortByAggregate = []SortByAggregate{
 	SortByAggregateTotalwalltime,
 	SortByAggregateTotaljobs,
+	SortByAggregateTotalusers,
 	SortByAggregateTotalnodes,
 	SortByAggregateTotalnodehours,
 	SortByAggregateTotalcores,
@@ -269,7 +356,7 @@ var AllSortByAggregate = []SortByAggregate{
 
 func (e SortByAggregate) IsValid() bool {
 	switch e {
-	case SortByAggregateTotalwalltime, SortByAggregateTotaljobs, SortByAggregateTotalnodes, SortByAggregateTotalnodehours, SortByAggregateTotalcores, SortByAggregateTotalcorehours, SortByAggregateTotalaccs, SortByAggregateTotalacchours:
+	case SortByAggregateTotalwalltime, SortByAggregateTotaljobs, SortByAggregateTotalusers, SortByAggregateTotalnodes, SortByAggregateTotalnodehours, SortByAggregateTotalcores, SortByAggregateTotalcorehours, SortByAggregateTotalaccs, SortByAggregateTotalacchours:
 		return true
 	}
 	return false
@@ -296,6 +383,20 @@ func (e SortByAggregate) MarshalGQL(w io.Writer) {
 	fmt.Fprint(w, strconv.Quote(e.String()))
 }
 
+func (e *SortByAggregate) UnmarshalJSON(b []byte) error {
+	s, err := strconv.Unquote(string(b))
+	if err != nil {
+		return err
+	}
+	return e.UnmarshalGQL(s)
+}
+
+func (e SortByAggregate) MarshalJSON() ([]byte, error) {
+	var buf bytes.Buffer
+	e.MarshalGQL(&buf)
+	return buf.Bytes(), nil
+}
+
 type SortDirectionEnum string
 
 const (
@@ -336,3 +437,17 @@ func (e *SortDirectionEnum) UnmarshalGQL(v any) error {
 func (e SortDirectionEnum) MarshalGQL(w io.Writer) {
 	fmt.Fprint(w, strconv.Quote(e.String()))
 }
+
+func (e *SortDirectionEnum) UnmarshalJSON(b []byte) error {
+	s, err := strconv.Unquote(string(b))
+	if err != nil {
+		return err
+	}
+	return e.UnmarshalGQL(s)
+}
+
+func (e SortDirectionEnum) MarshalJSON() ([]byte, error) {
+	var buf bytes.Buffer
+	e.MarshalGQL(&buf)
+	return buf.Bytes(), nil
+}

internal/graph/resolver.go

@@ -4,7 +4,7 @@ import (
 	"sync"
 
 	"github.com/ClusterCockpit/cc-backend/internal/repository"
-	"github.com/ClusterCockpit/cc-backend/pkg/log"
+	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
 	"github.com/jmoiron/sqlx"
 )
 
@@ -32,7 +32,7 @@ func Init() {
 
 func GetResolverInstance() *Resolver {
 	if resolverInstance == nil {
-		log.Fatal("Authentication module not initialized!")
+		cclog.Fatal("Authentication module not initialized!")
 	}
 
 	return resolverInstance

internal/graph/schema.resolvers.go

@@ -1,13 +1,15 @@
 package graph
 
-// This file will be automatically regenerated based on the schema, any resolver implementations
+// This file will be automatically regenerated based on the schema, any resolver
+// implementations
 // will be copied through when generating and any unknown code will be moved to the end.
-// Code generated by github.com/99designs/gqlgen version v0.17.66
+// Code generated by github.com/99designs/gqlgen version v0.17.88
 
 import (
 	"context"
 	"errors"
 	"fmt"
+	"math"
 	"regexp"
 	"slices"
 	"strconv"
@@ -17,11 +19,12 @@ import (
 	"github.com/ClusterCockpit/cc-backend/internal/config"
 	"github.com/ClusterCockpit/cc-backend/internal/graph/generated"
 	"github.com/ClusterCockpit/cc-backend/internal/graph/model"
-	"github.com/ClusterCockpit/cc-backend/internal/metricDataDispatcher"
+	"github.com/ClusterCockpit/cc-backend/internal/metricdispatch"
 	"github.com/ClusterCockpit/cc-backend/internal/repository"
 	"github.com/ClusterCockpit/cc-backend/pkg/archive"
-	"github.com/ClusterCockpit/cc-backend/pkg/log"
-	"github.com/ClusterCockpit/cc-backend/pkg/schema"
+	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
+	ccunit "github.com/ClusterCockpit/cc-lib/v2/ccUnits"
+	"github.com/ClusterCockpit/cc-lib/v2/schema"
 )
 
 // Partitions is the resolver for the partitions field.
@@ -29,15 +32,21 @@ func (r *clusterResolver) Partitions(ctx context.Context, obj *schema.Cluster) (
 	return r.Repo.Partitions(obj.Name)
 }
 
+// StartTime is the resolver for the startTime field.
+func (r *jobResolver) StartTime(ctx context.Context, obj *schema.Job) (*time.Time, error) {
+	timestamp := time.Unix(obj.StartTime, 0)
+	return &timestamp, nil
+}
+
 // Tags is the resolver for the tags field.
 func (r *jobResolver) Tags(ctx context.Context, obj *schema.Job) ([]*schema.Tag, error) {
-	return r.Repo.GetTags(repository.GetUserFromContext(ctx), &obj.ID)
+	return r.Repo.GetTags(repository.GetUserFromContext(ctx), obj.ID)
 }
 
 // ConcurrentJobs is the resolver for the concurrentJobs field.
 func (r *jobResolver) ConcurrentJobs(ctx context.Context, obj *schema.Job) (*model.JobLinkResultList, error) {
 	// FIXME: Make the hardcoded duration configurable
-	if obj.Exclusive != 1 && obj.Duration > 600 {
+	if obj.Shared != "none" && obj.Duration > 600 {
 		return r.Repo.FindConcurrentJobs(ctx, obj)
 	}
 
@@ -48,7 +57,7 @@ func (r *jobResolver) ConcurrentJobs(ctx context.Context, obj *schema.Job) (*mod
 func (r *jobResolver) Footprint(ctx context.Context, obj *schema.Job) ([]*model.FootprintValue, error) {
 	rawFootprint, err := r.Repo.FetchFootprint(obj)
 	if err != nil {
-		log.Warn("Error while fetching job footprint data")
+		cclog.Warn("Error while fetching job footprint data")
 		return nil, err
 	}
 
@@ -73,21 +82,21 @@ func (r *jobResolver) Footprint(ctx context.Context, obj *schema.Job) ([]*model.
 func (r *jobResolver) EnergyFootprint(ctx context.Context, obj *schema.Job) ([]*model.EnergyFootprintValue, error) {
 	rawEnergyFootprint, err := r.Repo.FetchEnergyFootprint(obj)
 	if err != nil {
-		log.Warn("Error while fetching job energy footprint data")
+		cclog.Warn("Error while fetching job energy footprint data")
 		return nil, err
 	}
 
 	res := []*model.EnergyFootprintValue{}
 	for name, value := range rawEnergyFootprint {
 		// Suboptimal: Nearly hardcoded metric name expectations
-		matchCpu := regexp.MustCompile(`cpu|Cpu|CPU`)
+		matchCPU := regexp.MustCompile(`cpu|Cpu|CPU`)
 		matchAcc := regexp.MustCompile(`acc|Acc|ACC`)
 		matchMem := regexp.MustCompile(`mem|Mem|MEM`)
 		matchCore := regexp.MustCompile(`core|Core|CORE`)
 
 		hwType := ""
 		switch test := name; { // NOtice ';' for var declaration
-		case matchCpu.MatchString(test):
+		case matchCPU.MatchString(test):
 			hwType = "CPU"
 		case matchAcc.MatchString(test):
 			hwType = "Accelerator"
@@ -125,40 +134,75 @@ func (r *metricValueResolver) Name(ctx context.Context, obj *schema.MetricValue)
 
 // CreateTag is the resolver for the createTag field.
 func (r *mutationResolver) CreateTag(ctx context.Context, typeArg string, name string, scope string) (*schema.Tag, error) {
-	id, err := r.Repo.CreateTag(typeArg, name, scope)
-	if err != nil {
-		log.Warn("Error while creating tag")
-		return nil, err
+	user := repository.GetUserFromContext(ctx)
+	if user == nil {
+		return nil, fmt.Errorf("no user in context")
 	}
 
-	return &schema.Tag{ID: id, Type: typeArg, Name: name, Scope: scope}, nil
+	// Test Access: Admins && Admin Tag OR Support/Admin and Global Tag OR Everyone && Private Tag
+	if user.HasRole(schema.RoleAdmin) && scope == "admin" ||
+		user.HasAnyRole([]schema.Role{schema.RoleAdmin, schema.RoleSupport}) && scope == "global" ||
+		user.Username == scope {
+		// Create in DB
+		id, err := r.Repo.CreateTag(typeArg, name, scope)
+		if err != nil {
+			cclog.Warn("Error while creating tag")
+			return nil, err
+		}
+		return &schema.Tag{ID: id, Type: typeArg, Name: name, Scope: scope}, nil
+	} else {
+		cclog.Warnf("Not authorized to create tag with scope: %s", scope)
+		return nil, fmt.Errorf("not authorized to create tag with scope: %s", scope)
+	}
 }
 
 // DeleteTag is the resolver for the deleteTag field.
 func (r *mutationResolver) DeleteTag(ctx context.Context, id string) (string, error) {
+	// This Uses ID string <-> ID string, removeTagFromList uses []string <-> []int
 	panic(fmt.Errorf("not implemented: DeleteTag - deleteTag"))
 }
 
 // AddTagsToJob is the resolver for the addTagsToJob field.
 func (r *mutationResolver) AddTagsToJob(ctx context.Context, job string, tagIds []string) ([]*schema.Tag, error) {
-	// Selectable Tags Pre-Filtered by Scope in Frontend: No backend check required
+	user := repository.GetUserFromContext(ctx)
+	if user == nil {
+		return nil, fmt.Errorf("no user in context")
+	}
+
 	jid, err := strconv.ParseInt(job, 10, 64)
 	if err != nil {
-		log.Warn("Error while adding tag to job")
+		cclog.Warn("Error while adding tag to job")
 		return nil, err
 	}
 
 	tags := []*schema.Tag{}
-	for _, tagId := range tagIds {
-		tid, err := strconv.ParseInt(tagId, 10, 64)
+	for _, tagID := range tagIds {
+		// Get ID
+		tid, err := strconv.ParseInt(tagID, 10, 64)
 		if err != nil {
-			log.Warn("Error while parsing tag id")
+			cclog.Warn("Error while parsing tag id")
 			return nil, err
 		}
 
-		if tags, err = r.Repo.AddTag(repository.GetUserFromContext(ctx), jid, tid); err != nil {
-			log.Warn("Error while adding tag")
-			return nil, err
+		// Test Exists
+		_, _, tscope, exists := r.Repo.TagInfo(tid)
+		if !exists {
+			cclog.Warnf("Tag does not exist (ID): %d", tid)
+			return nil, fmt.Errorf("tag does not exist (ID): %d", tid)
+		}
+
+		// Test Access: Admins && Admin Tag OR Support/Admin and Global Tag OR Everyone && Private Tag
+		if user.HasRole(schema.RoleAdmin) && tscope == "admin" ||
+			user.HasAnyRole([]schema.Role{schema.RoleAdmin, schema.RoleSupport}) && tscope == "global" ||
+			user.Username == tscope {
+			// Add to Job
+			if tags, err = r.Repo.AddTag(user, jid, tid); err != nil {
+				cclog.Warn("Error while adding tag")
+				return nil, err
+			}
+		} else {
+			cclog.Warnf("Not authorized to add tag: %d", tid)
+			return nil, fmt.Errorf("not authorized to add tag: %d", tid)
 		}
 	}
 
@@ -167,40 +211,148 @@ func (r *mutationResolver) AddTagsToJob(ctx context.Context, job string, tagIds
 
 // RemoveTagsFromJob is the resolver for the removeTagsFromJob field.
 func (r *mutationResolver) RemoveTagsFromJob(ctx context.Context, job string, tagIds []string) ([]*schema.Tag, error) {
-	// Removable Tags Pre-Filtered by Scope in Frontend: No backend check required
+	user := repository.GetUserFromContext(ctx)
+	if user == nil {
+		return nil, fmt.Errorf("no user in context")
+	}
+
 	jid, err := strconv.ParseInt(job, 10, 64)
 	if err != nil {
-		log.Warn("Error while parsing job id")
+		cclog.Warn("Error while parsing job id")
 		return nil, err
 	}
 
 	tags := []*schema.Tag{}
-	for _, tagId := range tagIds {
-		tid, err := strconv.ParseInt(tagId, 10, 64)
+	for _, tagID := range tagIds {
+		// Get ID
+		tid, err := strconv.ParseInt(tagID, 10, 64)
 		if err != nil {
-			log.Warn("Error while parsing tag id")
+			cclog.Warn("Error while parsing tag id")
 			return nil, err
 		}
 
-		if tags, err = r.Repo.RemoveTag(repository.GetUserFromContext(ctx), jid, tid); err != nil {
-			log.Warn("Error while removing tag")
-			return nil, err
+		// Test Exists
+		_, _, tscope, exists := r.Repo.TagInfo(tid)
+		if !exists {
+			cclog.Warnf("Tag does not exist (ID): %d", tid)
+			return nil, fmt.Errorf("tag does not exist (ID): %d", tid)
 		}
+
+		// Test Access: Admins && Admin Tag OR Support/Admin and Global Tag OR Everyone && Private Tag
+		if user.HasRole(schema.RoleAdmin) && tscope == "admin" ||
+			user.HasAnyRole([]schema.Role{schema.RoleAdmin, schema.RoleSupport}) && tscope == "global" ||
+			user.Username == tscope {
+			// Remove from Job
+			if tags, err = r.Repo.RemoveTag(user, jid, tid); err != nil {
+				cclog.Warn("Error while removing tag")
+				return nil, err
+			}
+		} else {
+			cclog.Warnf("Not authorized to remove tag: %d", tid)
+			return nil, fmt.Errorf("not authorized to remove tag: %d", tid)
+		}
+
 	}
 
 	return tags, nil
 }
 
+// RemoveTagFromList is the resolver for the removeTagFromList field.
+func (r *mutationResolver) RemoveTagFromList(ctx context.Context, tagIds []string) ([]int, error) {
+	// Needs Contextuser
+	user := repository.GetUserFromContext(ctx)
+	if user == nil {
+		return nil, fmt.Errorf("no user in context")
+	}
+
+	tags := []int{}
+	for _, tagID := range tagIds {
+		// Get ID
+		tid, err := strconv.ParseInt(tagID, 10, 64)
+		if err != nil {
+			cclog.Warn("Error while parsing tag id for removal")
+			return nil, err
+		}
+
+		// Test Exists
+		_, _, tscope, exists := r.Repo.TagInfo(tid)
+		if !exists {
+			cclog.Warnf("Tag does not exist (ID): %d", tid)
+			return nil, fmt.Errorf("tag does not exist (ID): %d", tid)
+		}
+
+		// Test Access: Admins && Admin Tag OR Everyone && Private Tag
+		if user.HasRole(schema.RoleAdmin) && (tscope == "global" || tscope == "admin") || user.Username == tscope {
+			// Remove from DB
+			if err = r.Repo.RemoveTagByID(tid); err != nil {
+				cclog.Warn("Error while removing tag")
+				return nil, err
+			} else {
+				tags = append(tags, int(tid))
+			}
+		} else {
+			cclog.Warnf("Not authorized to remove tag: %d", tid)
+			return nil, fmt.Errorf("not authorized to remove tag: %d", tid)
+		}
+	}
+	return tags, nil
+}
+
 // UpdateConfiguration is the resolver for the updateConfiguration field.
 func (r *mutationResolver) UpdateConfiguration(ctx context.Context, name string, value string) (*string, error) {
 	if err := repository.GetUserCfgRepo().UpdateConfig(name, value, repository.GetUserFromContext(ctx)); err != nil {
-		log.Warn("Error while updating user config")
+		cclog.Warn("Error while updating user config")
 		return nil, err
 	}
 
 	return nil, nil
 }
 
+// ID is the resolver for the id field.
+func (r *nodeResolver) ID(ctx context.Context, obj *schema.Node) (string, error) {
+	panic(fmt.Errorf("not implemented: ID - id"))
+}
+
+// SchedulerState is the resolver for the schedulerState field.
+func (r *nodeResolver) SchedulerState(ctx context.Context, obj *schema.Node) (schema.SchedulerState, error) {
+	if obj.NodeState != "" {
+		return obj.NodeState, nil
+	} else {
+		return "", fmt.Errorf("resolver: no SchedulerState (NodeState) on node object")
+	}
+}
+
+// HealthState is the resolver for the healthState field.
+func (r *nodeResolver) HealthState(ctx context.Context, obj *schema.Node) (string, error) {
+	if obj.HealthState != "" {
+		return string(obj.HealthState), nil
+	} else {
+		return "", fmt.Errorf("resolver: no HealthState (NodeState) on node object")
+	}
+}
+
+// MetaData is the resolver for the metaData field.
+func (r *nodeResolver) MetaData(ctx context.Context, obj *schema.Node) (any, error) {
+	if obj.MetaData != nil {
+		return obj.MetaData, nil
+	} else {
+		cclog.Debug("resolver: no MetaData (NodeState) on node object")
+		emptyMeta := make(map[string]string, 0)
+		return emptyMeta, nil
+	}
+}
+
+// HealthData is the resolver for the healthData field.
+func (r *nodeResolver) HealthData(ctx context.Context, obj *schema.Node) (any, error) {
+	if obj.HealthData != nil {
+		return obj.HealthData, nil
+	} else {
+		cclog.Debug("resolver: no HealthData (NodeState) on node object")
+		emptyHealth := make(map[string][]string, 0)
+		return emptyHealth, nil
+	}
+}
+
 // Clusters is the resolver for the clusters field.
 func (r *queryResolver) Clusters(ctx context.Context) ([]*schema.Cluster, error) {
 	return archive.Clusters, nil
@@ -213,6 +365,14 @@ func (r *queryResolver) Tags(ctx context.Context) ([]*schema.Tag, error) {
 
 // GlobalMetrics is the resolver for the globalMetrics field.
 func (r *queryResolver) GlobalMetrics(ctx context.Context) ([]*schema.GlobalMetricListItem, error) {
+	user := repository.GetUserFromContext(ctx)
+
+	if user != nil {
+		if user.HasRole(schema.RoleUser) || user.HasRole(schema.RoleManager) {
+			return archive.GlobalUserMetricList, nil
+		}
+	}
+
 	return archive.GlobalMetricList, nil
 }
 
@@ -225,7 +385,7 @@ func (r *queryResolver) User(ctx context.Context, username string) (*model.User,
 func (r *queryResolver) AllocatedNodes(ctx context.Context, cluster string) ([]*model.Count, error) {
 	data, err := r.Repo.AllocatedNodes(cluster)
 	if err != nil {
-		log.Warn("Error while fetching allocated nodes")
+		cclog.Warn("Error while fetching allocated nodes")
 		return nil, err
 	}
 
@@ -240,17 +400,91 @@ func (r *queryResolver) AllocatedNodes(ctx context.Context, cluster string) ([]*
 	return counts, nil
 }
 
+// Node is the resolver for the node field.
+func (r *queryResolver) Node(ctx context.Context, id string) (*schema.Node, error) {
+	repo := repository.GetNodeRepository()
+	numericID, err := strconv.ParseInt(id, 10, 64)
+	if err != nil {
+		cclog.Warn("Error while parsing job id")
+		return nil, err
+	}
+	return repo.GetNodeByID(numericID, false)
+}
+
+// Nodes is the resolver for the nodes field.
+func (r *queryResolver) Nodes(ctx context.Context, filter []*model.NodeFilter, order *model.OrderByInput) (*model.NodeStateResultList, error) {
+	repo := repository.GetNodeRepository()
+	nodes, err := repo.QueryNodes(ctx, filter, nil, order) // Ignore Paging, Order Unused
+	count := len(nodes)
+	return &model.NodeStateResultList{Items: nodes, Count: &count}, err
+}
+
+// NodesWithMeta is the resolver for the nodesWithMeta field.
+func (r *queryResolver) NodesWithMeta(ctx context.Context, filter []*model.NodeFilter, order *model.OrderByInput) (*model.NodeStateResultList, error) {
+	// Why Extra Handler? -> graphql.CollectAllFields(ctx) only returns toplevel fields (i.e.: items, count), and not subfields like item.metaData
+	repo := repository.GetNodeRepository()
+	nodes, err := repo.QueryNodesWithMeta(ctx, filter, nil, order) // Ignore Paging, Order Unused
+	count := len(nodes)
+	return &model.NodeStateResultList{Items: nodes, Count: &count}, err
+}
+
+// NodeStates is the resolver for the nodeStates field.
+func (r *queryResolver) NodeStates(ctx context.Context, filter []*model.NodeFilter) ([]*model.NodeStates, error) {
+	repo := repository.GetNodeRepository()
+
+	stateCounts, serr := repo.CountStates(ctx, filter, "node_state")
+	if serr != nil {
+		cclog.Warnf("Error while counting nodeStates: %s", serr.Error())
+		return nil, serr
+	}
+
+	healthCounts, herr := repo.CountStates(ctx, filter, "health_state")
+	if herr != nil {
+		cclog.Warnf("Error while counting healthStates: %s", herr.Error())
+		return nil, herr
+	}
+
+	allCounts := append(stateCounts, healthCounts...)
+
+	return allCounts, nil
+}
+
+// NodeStatesTimed is the resolver for the nodeStatesTimed field.
+func (r *queryResolver) NodeStatesTimed(ctx context.Context, filter []*model.NodeFilter, typeArg string) ([]*model.NodeStatesTimed, error) {
+	repo := repository.GetNodeRepository()
+
+	if typeArg == "node" {
+		stateCounts, serr := repo.CountStatesTimed(ctx, filter, "node_state")
+		if serr != nil {
+			cclog.Warnf("Error while counting nodeStates in time: %s", serr.Error())
+			return nil, serr
+		}
+		return stateCounts, nil
+	}
+
+	if typeArg == "health" {
+		healthCounts, herr := repo.CountStatesTimed(ctx, filter, "health_state")
+		if herr != nil {
+			cclog.Warnf("Error while counting healthStates in time: %s", herr.Error())
+			return nil, herr
+		}
+		return healthCounts, nil
+	}
+
+	return nil, errors.New("unknown Node State Query Type")
+}
+
 // Job is the resolver for the job field.
 func (r *queryResolver) Job(ctx context.Context, id string) (*schema.Job, error) {
-	numericId, err := strconv.ParseInt(id, 10, 64)
+	numericID, err := strconv.ParseInt(id, 10, 64)
 	if err != nil {
-		log.Warn("Error while parsing job id")
+		cclog.Warn("Error while parsing job id")
 		return nil, err
 	}
 
-	job, err := r.Repo.FindById(ctx, numericId)
+	job, err := r.Repo.FindByID(ctx, numericID)
 	if err != nil {
-		log.Warn("Error while finding job by id")
+		cclog.Warn("Error while finding job by id")
 		return nil, err
 	}
 
@@ -277,13 +511,13 @@ func (r *queryResolver) JobMetrics(ctx context.Context, id string, metrics []str
 
 	job, err := r.Query().Job(ctx, id)
 	if err != nil {
-		log.Warn("Error while querying job for metrics")
+		cclog.Warn("Error while querying job for metrics")
 		return nil, err
 	}
 
-	data, err := metricDataDispatcher.LoadData(job, metrics, scopes, ctx, *resolution)
+	data, err := metricdispatch.LoadData(job, metrics, scopes, ctx, *resolution)
 	if err != nil {
-		log.Warn("Error while loading job data")
+		cclog.Warn("Error while loading job data")
 		return nil, err
 	}
 
@@ -301,36 +535,67 @@ func (r *queryResolver) JobMetrics(ctx context.Context, id string, metrics []str
 	return res, err
 }
 
-// JobMetricStats is the resolver for the jobMetricStats field.
-func (r *queryResolver) JobMetricStats(ctx context.Context, id string, metrics []string) ([]*model.JobMetricStatWithName, error) {
-
+// JobStats is the resolver for the jobStats field.
+func (r *queryResolver) JobStats(ctx context.Context, id string, metrics []string) ([]*model.NamedStats, error) {
 	job, err := r.Query().Job(ctx, id)
 	if err != nil {
-		log.Warn("Error while querying job for metrics")
+		cclog.Warnf("Error while querying job %s for metadata", id)
 		return nil, err
 	}
 
-	data, err := metricDataDispatcher.LoadStatData(job, metrics, ctx)
+	data, err := metricdispatch.LoadJobStats(job, metrics, ctx)
 	if err != nil {
-		log.Warn("Error while loading job stat data")
+		cclog.Warnf("Error while loading jobStats data for job id %s", id)
 		return nil, err
 	}
 
-	res := []*model.JobMetricStatWithName{}
+	res := []*model.NamedStats{}
 	for name, md := range data {
-		res = append(res, &model.JobMetricStatWithName{
-			Name:  name,
-			Stats: &md,
+		res = append(res, &model.NamedStats{
+			Name: name,
+			Data: &md,
 		})
 	}
 
 	return res, err
 }
 
-// JobsFootprints is the resolver for the jobsFootprints field.
-func (r *queryResolver) JobsFootprints(ctx context.Context, filter []*model.JobFilter, metrics []string) (*model.Footprints, error) {
-	// NOTE: Legacy Naming! This resolver is for normalized histograms in analysis view only - *Not* related to DB "footprint" column!
-	return r.jobsFootprints(ctx, filter, metrics)
+// ScopedJobStats is the resolver for the scopedJobStats field.
+func (r *queryResolver) ScopedJobStats(ctx context.Context, id string, metrics []string, scopes []schema.MetricScope) ([]*model.NamedStatsWithScope, error) {
+	job, err := r.Query().Job(ctx, id)
+	if err != nil {
+		cclog.Warnf("Error while querying job %s for metadata", id)
+		return nil, err
+	}
+
+	data, err := metricdispatch.LoadScopedJobStats(job, metrics, scopes, ctx)
+	if err != nil {
+		cclog.Warnf("Error while loading scopedJobStats data for job id %s", id)
+		return nil, err
+	}
+
+	res := make([]*model.NamedStatsWithScope, 0)
+	for name, scoped := range data {
+		for scope, stats := range scoped {
+
+			mdlStats := make([]*model.ScopedStats, 0)
+			for _, stat := range stats {
+				mdlStats = append(mdlStats, &model.ScopedStats{
+					Hostname: stat.Hostname,
+					ID:       stat.ID,
+					Data:     stat.Data,
+				})
+			}
+
+			res = append(res, &model.NamedStatsWithScope{
+				Name:  name,
+				Scope: scope,
+				Stats: mdlStats,
+			})
+		}
+	}
+
+	return res, nil
 }
 
 // Jobs is the resolver for the jobs field.
@@ -344,40 +609,38 @@ func (r *queryResolver) Jobs(ctx context.Context, filter []*model.JobFilter, pag
 
 	jobs, err := r.Repo.QueryJobs(ctx, filter, page, order)
 	if err != nil {
-		log.Warn("Error while querying jobs")
+		cclog.Warn("Error while querying jobs")
 		return nil, err
 	}
 
 	count, err := r.Repo.CountJobs(ctx, filter)
 	if err != nil {
-		log.Warn("Error while counting jobs")
+		cclog.Warn("Error while counting jobs")
 		return nil, err
 	}
 
-	if !config.Keys.UiDefaults["job_list_usePaging"].(bool) {
-		hasNextPage := false
-		// page.Page += 1 		   : Simple, but expensive
-		// Example Page 4 @ 10 IpP : Does item 41 exist?
-		// Minimal Page 41 @ 1 IpP : If len(result) is 1, Page 5 @ 10 IpP exists.
+	// Note: Even if App-Default 'config.Keys.UiDefaults["job_list_usePaging"]' is set, always return hasNextPage boolean.
+	// Users can decide in frontend to use continuous scroll, even if app-default is paging!
+	// Skip if page.ItemsPerPage == -1 ("Load All" -> No Next Page required, Status Dashboards)
+	/*
+	  Example Page 4 @ 10 IpP : Does item 41 exist?
+	  Minimal Page 41 @ 1 IpP : If len(result) is 1, Page 5 @ 10 IpP exists.
+	*/
+	hasNextPage := false
+	if page.ItemsPerPage != -1 {
 		nextPage := &model.PageRequest{
 			ItemsPerPage: 1,
 			Page:         ((page.Page * page.ItemsPerPage) + 1),
 		}
-
 		nextJobs, err := r.Repo.QueryJobs(ctx, filter, nextPage, order)
 		if err != nil {
-			log.Warn("Error while querying next jobs")
+			cclog.Warn("Error while querying next jobs")
 			return nil, err
 		}
-
-		if len(nextJobs) == 1 {
-			hasNextPage = true
-		}
-
-		return &model.JobResultList{Items: jobs, Count: &count, HasNextPage: &hasNextPage}, nil
-	} else {
-		return &model.JobResultList{Items: jobs, Count: &count}, nil
+		hasNextPage = len(nextJobs) == 1
 	}
+
+	return &model.JobResultList{Items: jobs, Count: &count, HasNextPage: &hasNextPage}, nil
 }
 
 // JobsStatistics is the resolver for the jobsStatistics field.
@@ -386,34 +649,68 @@ func (r *queryResolver) JobsStatistics(ctx context.Context, filter []*model.JobF
 	var stats []*model.JobsStatistics
 
 	// Top Level Defaults
-	var defaultDurationBins string = "1h"
-	var defaultMetricBins int = 10
+	defaultDurationBins := "1h"
+	defaultMetricBins := 10
 
-	if requireField(ctx, "totalJobs") || requireField(ctx, "totalWalltime") || requireField(ctx, "totalNodes") || requireField(ctx, "totalCores") ||
-		requireField(ctx, "totalAccs") || requireField(ctx, "totalNodeHours") || requireField(ctx, "totalCoreHours") || requireField(ctx, "totalAccHours") {
+	// Build requested fields map for selective column computation
+	statsFields := []string{
+		"totalJobs", "totalUsers", "totalWalltime", "totalNodes", "totalCores",
+		"totalAccs", "totalNodeHours", "totalCoreHours", "totalAccHours", "runningJobs", "shortJobs",
+	}
+	reqFields := make(map[string]bool, len(statsFields))
+	fetchedMainStats := false
+	for _, f := range statsFields {
+		if requireField(ctx, f) {
+			reqFields[f] = true
+			if f != "runningJobs" && f != "shortJobs" {
+				fetchedMainStats = true
+			}
+		}
+	}
+
+	if fetchedMainStats {
 		if groupBy == nil {
-			stats, err = r.Repo.JobsStats(ctx, filter)
+			stats, err = r.Repo.JobsStats(ctx, filter, reqFields)
 		} else {
-			stats, err = r.Repo.JobsStatsGrouped(ctx, filter, page, sortBy, groupBy)
+			startGrouped := time.Now()
+			// Use request-scoped cache: multiple aliases with same (filter, groupBy)
+			// but different sortBy/page hit the DB only once.
+			if cache := getStatsGroupCache(ctx); cache != nil {
+				key := statsCacheKey(filter, groupBy)
+				var allStats []*model.JobsStatistics
+				allStats, err = cache.getOrCompute(key, func() ([]*model.JobsStatistics, error) {
+					return r.Repo.JobsStatsGrouped(ctx, filter, nil, nil, groupBy, nil)
+				})
+				if err == nil {
+					stats = sortAndPageStats(allStats, sortBy, page)
+				}
+			} else {
+				stats, err = r.Repo.JobsStatsGrouped(ctx, filter, page, sortBy, groupBy, reqFields)
+			}
+			cclog.Infof("Timer JobsStatsGrouped call: %s", time.Since(startGrouped))
 		}
 	} else {
 		stats = make([]*model.JobsStatistics, 0, 1)
 		stats = append(stats, &model.JobsStatistics{})
 	}
 
-	if groupBy != nil {
-		if requireField(ctx, "shortJobs") {
-			stats, err = r.Repo.AddJobCountGrouped(ctx, filter, groupBy, stats, "short")
-		}
-		if requireField(ctx, "runningJobs") {
-			stats, err = r.Repo.AddJobCountGrouped(ctx, filter, groupBy, stats, "running")
-		}
-	} else {
-		if requireField(ctx, "shortJobs") {
-			stats, err = r.Repo.AddJobCount(ctx, filter, stats, "short")
-		}
-		if requireField(ctx, "runningJobs") {
-			stats, err = r.Repo.AddJobCount(ctx, filter, stats, "running")
+	// runningJobs and shortJobs are already inlined in JobsStats/JobsStatsGrouped.
+	// Only run separate count queries if main stats were not fetched.
+	if !fetchedMainStats {
+		if groupBy != nil {
+			if requireField(ctx, "shortJobs") {
+				stats, err = r.Repo.AddJobCountGrouped(ctx, filter, groupBy, stats, "short")
+			}
+			if requireField(ctx, "runningJobs") {
+				stats, err = r.Repo.AddJobCountGrouped(ctx, filter, groupBy, stats, "running")
+			}
+		} else {
+			if requireField(ctx, "shortJobs") {
+				stats, err = r.Repo.AddJobCount(ctx, filter, stats, "short")
+			}
+			if requireField(ctx, "runningJobs") {
+				stats, err = r.Repo.AddJobCount(ctx, filter, stats, "running")
+			}
 		}
 	}
 
@@ -456,6 +753,62 @@ func (r *queryResolver) JobsStatistics(ctx context.Context, filter []*model.JobF
 	return stats, nil
 }
 
+// JobsMetricStats is the resolver for the jobsMetricStats field.
+func (r *queryResolver) JobsMetricStats(ctx context.Context, filter []*model.JobFilter, metrics []string) ([]*model.JobStats, error) {
+	// No Paging, Fixed Order by StartTime ASC
+	order := &model.OrderByInput{
+		Field: "startTime",
+		Type:  "col",
+		Order: "ASC",
+	}
+
+	jobs, err := r.Repo.QueryJobs(ctx, filter, nil, order)
+	if err != nil {
+		cclog.Warn("Error while querying jobs for comparison")
+		return nil, err
+	}
+
+	res := []*model.JobStats{}
+	for _, job := range jobs {
+		data, err := metricdispatch.LoadJobStats(job, metrics, ctx)
+		if err != nil {
+			cclog.Warnf("Error while loading comparison jobStats data for job id %d", job.JobID)
+			continue
+			// return nil, err
+		}
+
+		sres := []*model.NamedStats{}
+		for name, md := range data {
+			sres = append(sres, &model.NamedStats{
+				Name: name,
+				Data: &md,
+			})
+		}
+
+		numThreadsInt := int(job.NumHWThreads)
+		numAccsInt := int(job.NumAcc)
+		res = append(res, &model.JobStats{
+			ID:              int(*job.ID),
+			JobID:           strconv.Itoa(int(job.JobID)),
+			StartTime:       int(job.StartTime),
+			Duration:        int(job.Duration),
+			Cluster:         job.Cluster,
+			SubCluster:      job.SubCluster,
+			NumNodes:        int(job.NumNodes),
+			NumHWThreads:    &numThreadsInt,
+			NumAccelerators: &numAccsInt,
+			Stats:           sres,
+		})
+	}
+	return res, err
+}
+
+// JobsFootprints is the resolver for the jobsFootprints field.
+func (r *queryResolver) JobsFootprints(ctx context.Context, filter []*model.JobFilter, metrics []string) (*model.Footprints, error) {
+	// NOTE: Legacy Naming! This resolver is for normalized histograms in analysis view only - *Not* related to DB "footprint" column!
+	return r.jobsFootprints(ctx, filter, metrics)
+}
+
 // RooflineHeatmap is the resolver for the rooflineHeatmap field.
 func (r *queryResolver) RooflineHeatmap(ctx context.Context, filter []*model.JobFilter, rows int, cols int, minX float64, minY float64, maxX float64, maxY float64) ([][]float64, error) {
 	return r.rooflineHeatmap(ctx, filter, rows, cols, minX, minY, maxX, maxY)
@@ -468,27 +821,37 @@ func (r *queryResolver) NodeMetrics(ctx context.Context, cluster string, nodes [
 		return nil, errors.New("you need to be administrator or support staff for this query")
 	}
 
+	defaultMetrics := make([]string, 0)
+	for _, mc := range archive.GetCluster(cluster).MetricConfig {
+		defaultMetrics = append(defaultMetrics, mc.Name)
+	}
 	if metrics == nil {
-		for _, mc := range archive.GetCluster(cluster).MetricConfig {
-			metrics = append(metrics, mc.Name)
-		}
+		metrics = defaultMetrics
+	} else {
+		metrics = slices.DeleteFunc(metrics, func(metric string) bool {
+			return !slices.Contains(defaultMetrics, metric) // Remove undefined metrics.
+		})
 	}
 
-	data, err := metricDataDispatcher.LoadNodeData(cluster, metrics, nodes, scopes, from, to, ctx)
+	data, err := metricdispatch.LoadNodeData(cluster, metrics, nodes, scopes, from, to, ctx)
 	if err != nil {
-		log.Warn("error while loading node data")
+		cclog.Warn("error while loading node data")
 		return nil, err
 	}
 
+	nodeRepo := repository.GetNodeRepository()
+	stateMap, _ := nodeRepo.MapNodes(cluster)
+
 	nodeMetrics := make([]*model.NodeMetrics, 0, len(data))
 	for hostname, metrics := range data {
 		host := &model.NodeMetrics{
 			Host:    hostname,
+			State:   stateMap[hostname],
 			Metrics: make([]*model.JobMetricWithName, 0, len(metrics)*len(scopes)),
 		}
 		host.SubCluster, err = archive.GetSubClusterByNode(cluster, hostname)
 		if err != nil {
-			log.Warnf("error in nodeMetrics resolver: %s", err)
+			cclog.Warnf("error in nodeMetrics resolver: %s", err)
 		}
 
 		for metric, scopedMetrics := range metrics {
@@ -508,7 +871,7 @@ func (r *queryResolver) NodeMetrics(ctx context.Context, cluster string, nodes [
 }
 
 // NodeMetricsList is the resolver for the nodeMetricsList field.
-func (r *queryResolver) NodeMetricsList(ctx context.Context, cluster string, subCluster string, nodeFilter string, scopes []schema.MetricScope, metrics []string, from time.Time, to time.Time, page *model.PageRequest, resolution *int) (*model.NodesResultList, error) {
+func (r *queryResolver) NodeMetricsList(ctx context.Context, cluster string, subCluster string, stateFilter string, nodeFilter string, scopes []schema.MetricScope, metrics []string, from time.Time, to time.Time, page *model.PageRequest, resolution *int) (*model.NodesResultList, error) {
 	if resolution == nil { // Load from Config
 		if config.Keys.EnableResampling != nil {
 			defaultRes := slices.Max(config.Keys.EnableResampling.Resolutions)
@@ -524,30 +887,39 @@ func (r *queryResolver) NodeMetricsList(ctx context.Context, cluster string, sub
 		return nil, errors.New("you need to be administrator or support staff for this query")
 	}
 
+	nodeRepo := repository.GetNodeRepository()
+	// nodes -> array hostname
+	nodes, stateMap, countNodes, hasNextPage, nerr := nodeRepo.GetNodesForList(ctx, cluster, subCluster, stateFilter, nodeFilter, page)
+	if nerr != nil {
+		return nil, errors.New("could not retrieve node list required for resolving NodeMetricsList")
+	}
+
 	if metrics == nil {
 		for _, mc := range archive.GetCluster(cluster).MetricConfig {
 			metrics = append(metrics, mc.Name)
 		}
 	}
 
-	data, totalNodes, hasNextPage, err := metricDataDispatcher.LoadNodeListData(cluster, subCluster, nodeFilter, metrics, scopes, *resolution, from, to, page, ctx)
+	// data -> map hostname:jobdata
+	data, err := metricdispatch.LoadNodeListData(cluster, subCluster, nodes, metrics, scopes, *resolution, from, to, ctx)
 	if err != nil {
-		log.Warn("error while loading node data")
+		cclog.Warn("error while loading node data (Resolver.NodeMetricsList")
 		return nil, err
 	}
 
 	nodeMetricsList := make([]*model.NodeMetrics, 0, len(data))
-	for hostname, metrics := range data {
+	for _, hostname := range nodes {
 		host := &model.NodeMetrics{
 			Host:    hostname,
-			Metrics: make([]*model.JobMetricWithName, 0, len(metrics)*len(scopes)),
+			State:   stateMap[hostname],
+			Metrics: make([]*model.JobMetricWithName, 0),
 		}
 		host.SubCluster, err = archive.GetSubClusterByNode(cluster, hostname)
 		if err != nil {
-			log.Warnf("error in nodeMetrics resolver: %s", err)
+			cclog.Warnf("error in nodeMetrics resolver: %s", err)
 		}
 
-		for metric, scopedMetrics := range metrics {
+		for metric, scopedMetrics := range data[hostname] {
 			for scope, scopedMetric := range scopedMetrics {
 				host.Metrics = append(host.Metrics, &model.JobMetricWithName{
 					Name:   metric,
@@ -561,14 +933,108 @@ func (r *queryResolver) NodeMetricsList(ctx context.Context, cluster string, sub
 	}
 
 	nodeMetricsListResult := &model.NodesResultList{
-		Items:       nodeMetricsList,
-		TotalNodes:  &totalNodes,
+		Items: nodeMetricsList,
+		// TotalNodes depends on sum of nodes grouped on latest timestamp, see repo/node.go:357
+		TotalNodes:  &countNodes,
 		HasNextPage: &hasNextPage,
 	}
 
 	return nodeMetricsListResult, nil
 }
 
+// ClusterMetrics is the resolver for the clusterMetrics field.
+func (r *queryResolver) ClusterMetrics(ctx context.Context, cluster string, metrics []string, from time.Time, to time.Time) (*model.ClusterMetrics, error) {
+	user := repository.GetUserFromContext(ctx)
+	if user != nil && !user.HasAnyRole([]schema.Role{schema.RoleAdmin, schema.RoleSupport}) {
+		return nil, errors.New("you need to be administrator or support staff for this query")
+	}
+
+	if metrics == nil {
+		for _, mc := range archive.GetCluster(cluster).MetricConfig {
+			metrics = append(metrics, mc.Name)
+		}
+	}
+
+	// 'nodes' == nil -> Defaults to all nodes of cluster for existing query workflow
+	scopes := []schema.MetricScope{"node"}
+	data, err := metricdispatch.LoadNodeData(cluster, metrics, nil, scopes, from, to, ctx)
+	if err != nil {
+		cclog.Warn("error while loading node data")
+		return nil, err
+	}
+
+	clusterMetricData := make([]*model.ClusterMetricWithName, 0)
+	clusterMetrics := model.ClusterMetrics{NodeCount: 0, Metrics: clusterMetricData}
+
+	collectorTimestep := make(map[string]int)
+	collectorUnit := make(map[string]schema.Unit)
+	collectorData := make(map[string][]schema.Float)
+
+	for _, metrics := range data {
+		clusterMetrics.NodeCount += 1
+		for metric, scopedMetrics := range metrics {
+			for _, scopedMetric := range scopedMetrics {
+				// Collect Info Once
+				_, okTimestep := collectorTimestep[metric]
+				if !okTimestep {
+					collectorTimestep[metric] = scopedMetric.Timestep
+				}
+				_, okUnit := collectorUnit[metric]
+				if !okUnit {
+					collectorUnit[metric] = scopedMetric.Unit
+				}
+				// Collect Data
+				for _, ser := range scopedMetric.Series {
+					_, okData := collectorData[metric]
+					// Init With Datasize > 0
+					if !okData && len(ser.Data) != 0 {
+						collectorData[metric] = make([]schema.Float, len(ser.Data))
+					} else if !okData {
+						cclog.Debugf("[SCHEMARESOLVER] clusterMetrics skip init: no data -> %s at %s; size %d", metric, ser.Hostname, len(ser.Data))
+					}
+					// Sum if init'd and matching size
+					if okData && len(ser.Data) == len(collectorData[metric]) {
+						for i, val := range ser.Data {
+							if val.IsNaN() {
+								continue
+							} else {
+								collectorData[metric][i] += val
+							}
+						}
+					} else if okData {
+						cclog.Debugf("[SCHEMARESOLVER] clusterMetrics skip sum: data diff -> %s at %s; want size %d, have size %d", metric, ser.Hostname, len(collectorData[metric]), len(ser.Data))
+					}
+				}
+			}
+		}
+	}
+
+	for metricName, data := range collectorData {
+		// use ccUnits for backend normalization to "Tera"
+		p_old := ccunit.NewPrefix(collectorUnit[metricName].Prefix)
+		p_new := ccunit.NewPrefix("T")
+		convFunc := ccunit.GetPrefixPrefixFactor(p_old, p_new)
+		u_new := schema.Unit{Prefix: p_new.Prefix(), Base: collectorUnit[metricName].Base}
+
+		roundedData := make([]schema.Float, 0)
+		for _, v_old := range data {
+			v_new := math.Round(convFunc(float64(v_old)).(float64)*100.0) / 100.0
+			roundedData = append(roundedData, schema.Float(v_new))
+		}
+
+		cm := model.ClusterMetricWithName{
+			Name:     metricName,
+			Unit:     &u_new,
+			Timestep: collectorTimestep[metricName],
+			Data:     roundedData,
+		}
+
+		clusterMetrics.Metrics = append(clusterMetrics.Metrics, &cm)
+	}
+
+	return &clusterMetrics, nil
+}
+
 // NumberOfNodes is the resolver for the numberOfNodes field.
 func (r *subClusterResolver) NumberOfNodes(ctx context.Context, obj *schema.SubCluster) (int, error) {
 	nodeList, err := archive.ParseNodeList(obj.Nodes)
@@ -590,6 +1056,9 @@ func (r *Resolver) MetricValue() generated.MetricValueResolver { return &metricV
 // Mutation returns generated.MutationResolver implementation.
 func (r *Resolver) Mutation() generated.MutationResolver { return &mutationResolver{r} }
 
+// Node returns generated.NodeResolver implementation.
+func (r *Resolver) Node() generated.NodeResolver { return &nodeResolver{r} }
+
 // Query returns generated.QueryResolver implementation.
 func (r *Resolver) Query() generated.QueryResolver { return &queryResolver{r} }
 
@@ -600,5 +1069,6 @@ type clusterResolver struct{ *Resolver }
 type jobResolver struct{ *Resolver }
 type metricValueResolver struct{ *Resolver }
 type mutationResolver struct{ *Resolver }
+type nodeResolver struct{ *Resolver }
 type queryResolver struct{ *Resolver }
 type subClusterResolver struct{ *Resolver }

internal/graph/stats_cache.go

@@ -0,0 +1,135 @@
+// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
+// All rights reserved. This file is part of cc-backend.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package graph
+
+import (
+	"context"
+	"fmt"
+	"slices"
+	"sync"
+
+	"github.com/ClusterCockpit/cc-backend/internal/graph/model"
+)
+
+// statsGroupCache is a per-request cache for grouped JobsStatistics results.
+// It deduplicates identical (filter+groupBy) SQL queries that arise when the
+// frontend requests multiple sort/page slices of the same underlying data
+// (e.g. topUserJobs, topUserNodes, topUserAccs all group by USER).
+type statsGroupCache struct {
+	mu      sync.Mutex
+	entries map[string]*cacheEntry
+}
+
+type cacheEntry struct {
+	once   sync.Once
+	result []*model.JobsStatistics
+	err    error
+}
+
+type ctxKey int
+
+const statsGroupCacheKey ctxKey = iota
+
+// newStatsGroupCache creates a new empty cache.
+func newStatsGroupCache() *statsGroupCache {
+	return &statsGroupCache{
+		entries: make(map[string]*cacheEntry),
+	}
+}
+
+// WithStatsGroupCache injects a new cache into the context.
+func WithStatsGroupCache(ctx context.Context) context.Context {
+	return context.WithValue(ctx, statsGroupCacheKey, newStatsGroupCache())
+}
+
+// getStatsGroupCache retrieves the cache from context, or nil if absent.
+func getStatsGroupCache(ctx context.Context) *statsGroupCache {
+	if c, ok := ctx.Value(statsGroupCacheKey).(*statsGroupCache); ok {
+		return c
+	}
+	return nil
+}
+
+// cacheKey builds a deterministic string key from filter + groupBy.
+func statsCacheKey(filter []*model.JobFilter, groupBy *model.Aggregate) string {
+	return fmt.Sprintf("%v|%v", filter, *groupBy)
+}
+
+// getOrCompute returns cached results for the given key, computing them on
+// first access via the provided function.
+func (c *statsGroupCache) getOrCompute(
+	key string,
+	compute func() ([]*model.JobsStatistics, error),
+) ([]*model.JobsStatistics, error) {
+	c.mu.Lock()
+	entry, ok := c.entries[key]
+	if !ok {
+		entry = &cacheEntry{}
+		c.entries[key] = entry
+	}
+	c.mu.Unlock()
+
+	entry.once.Do(func() {
+		entry.result, entry.err = compute()
+	})
+	return entry.result, entry.err
+}
+
+// sortAndPageStats sorts a copy of allStats by the given sortBy field (descending)
+// and returns the requested page slice.
+func sortAndPageStats(allStats []*model.JobsStatistics, sortBy *model.SortByAggregate, page *model.PageRequest) []*model.JobsStatistics {
+	// Work on a shallow copy so the cached slice order is not mutated.
+	sorted := make([]*model.JobsStatistics, len(allStats))
+	copy(sorted, allStats)
+
+	if sortBy != nil {
+		getter := statsFieldGetter(*sortBy)
+		slices.SortFunc(sorted, func(a, b *model.JobsStatistics) int {
+			return getter(b) - getter(a) // descending
+		})
+	}
+
+	if page != nil && page.ItemsPerPage != -1 {
+		start := (page.Page - 1) * page.ItemsPerPage
+		if start >= len(sorted) {
+			return nil
+		}
+		end := start + page.ItemsPerPage
+		if end > len(sorted) {
+			end = len(sorted)
+		}
+		sorted = sorted[start:end]
+	}
+
+	return sorted
+}
+
+// statsFieldGetter returns a function that extracts the sortable int field
+// from a JobsStatistics struct for the given sort key.
+func statsFieldGetter(sortBy model.SortByAggregate) func(*model.JobsStatistics) int {
+	switch sortBy {
+	case model.SortByAggregateTotaljobs:
+		return func(s *model.JobsStatistics) int { return s.TotalJobs }
+	case model.SortByAggregateTotalusers:
+		return func(s *model.JobsStatistics) int { return s.TotalUsers }
+	case model.SortByAggregateTotalwalltime:
+		return func(s *model.JobsStatistics) int { return s.TotalWalltime }
+	case model.SortByAggregateTotalnodes:
+		return func(s *model.JobsStatistics) int { return s.TotalNodes }
+	case model.SortByAggregateTotalnodehours:
+		return func(s *model.JobsStatistics) int { return s.TotalNodeHours }
+	case model.SortByAggregateTotalcores:
+		return func(s *model.JobsStatistics) int { return s.TotalCores }
+	case model.SortByAggregateTotalcorehours:
+		return func(s *model.JobsStatistics) int { return s.TotalCoreHours }
+	case model.SortByAggregateTotalaccs:
+		return func(s *model.JobsStatistics) int { return s.TotalAccs }
+	case model.SortByAggregateTotalacchours:
+		return func(s *model.JobsStatistics) int { return s.TotalAccHours }
+	default:
+		return func(s *model.JobsStatistics) int { return s.TotalJobs }
+	}
+}

internal/graph/util.go

@@ -1,20 +1,21 @@
 // Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved.
+// All rights reserved. This file is part of cc-backend.
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
+
 package graph
 
 import (
 	"context"
 	"fmt"
 	"math"
+	"slices"
 
 	"github.com/99designs/gqlgen/graphql"
 	"github.com/ClusterCockpit/cc-backend/internal/graph/model"
-	"github.com/ClusterCockpit/cc-backend/internal/metricDataDispatcher"
-	"github.com/ClusterCockpit/cc-backend/pkg/log"
-	"github.com/ClusterCockpit/cc-backend/pkg/schema"
-	// "github.com/ClusterCockpit/cc-backend/pkg/archive"
+	"github.com/ClusterCockpit/cc-backend/internal/metricdispatch"
+	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
+	"github.com/ClusterCockpit/cc-lib/v2/schema"
 )
 
 const MAX_JOBS_FOR_ANALYSIS = 500
@@ -28,7 +29,7 @@ func (r *queryResolver) rooflineHeatmap(
 ) ([][]float64, error) {
 	jobs, err := r.Repo.QueryJobs(ctx, filter, &model.PageRequest{Page: 1, ItemsPerPage: MAX_JOBS_FOR_ANALYSIS + 1}, nil)
 	if err != nil {
-		log.Error("Error while querying jobs for roofline")
+		cclog.Error("Error while querying jobs for roofline")
 		return nil, err
 	}
 	if len(jobs) > MAX_JOBS_FOR_ANALYSIS {
@@ -54,15 +55,15 @@ func (r *queryResolver) rooflineHeatmap(
 		// 	resolution = max(resolution, mc.Timestep)
 		// }
 
-		jobdata, err := metricDataDispatcher.LoadData(job, []string{"flops_any", "mem_bw"}, []schema.MetricScope{schema.MetricScopeNode}, ctx, 0)
+		jobdata, err := metricdispatch.LoadData(job, []string{"flops_any", "mem_bw"}, []schema.MetricScope{schema.MetricScopeNode}, ctx, 0)
 		if err != nil {
-			log.Errorf("Error while loading roofline metrics for job %d", job.ID)
+			cclog.Warnf("Error while loading roofline metrics for job %d", *job.ID)
 			return nil, err
 		}
 
 		flops_, membw_ := jobdata["flops_any"], jobdata["mem_bw"]
 		if flops_ == nil && membw_ == nil {
-			log.Infof("rooflineHeatmap(): 'flops_any' or 'mem_bw' missing for job %d", job.ID)
+			cclog.Warnf("rooflineHeatmap(): 'flops_any' or 'mem_bw' missing for job %d", *job.ID)
 			continue
 			// return nil, fmt.Errorf("GRAPH/UTIL > 'flops_any' or 'mem_bw' missing for job %d", job.ID)
 		}
@@ -70,7 +71,7 @@ func (r *queryResolver) rooflineHeatmap(
 		flops, ok1 := flops_["node"]
 		membw, ok2 := membw_["node"]
 		if !ok1 || !ok2 {
-			log.Info("rooflineHeatmap() query not implemented for where flops_any or mem_bw not available at 'node' level")
+			cclog.Info("rooflineHeatmap() query not implemented for where flops_any or mem_bw not available at 'node' level")
 			continue
 			// TODO/FIXME:
 			// return nil, errors.New("GRAPH/UTIL > todo: rooflineHeatmap() query not implemented for where flops_any or mem_bw not available at 'node' level")
@@ -105,7 +106,7 @@ func (r *queryResolver) rooflineHeatmap(
 func (r *queryResolver) jobsFootprints(ctx context.Context, filter []*model.JobFilter, metrics []string) (*model.Footprints, error) {
 	jobs, err := r.Repo.QueryJobs(ctx, filter, &model.PageRequest{Page: 1, ItemsPerPage: MAX_JOBS_FOR_ANALYSIS + 1}, nil)
 	if err != nil {
-		log.Error("Error while querying jobs for footprint")
+		cclog.Error("Error while querying jobs for footprint")
 		return nil, err
 	}
 	if len(jobs) > MAX_JOBS_FOR_ANALYSIS {
@@ -127,8 +128,8 @@ func (r *queryResolver) jobsFootprints(ctx context.Context, filter []*model.JobF
 			continue
 		}
 
-		if err := metricDataDispatcher.LoadAverages(job, metrics, avgs, ctx); err != nil {
-			log.Error("Error while loading averages for footprint")
+		if err := metricdispatch.LoadAverages(job, metrics, avgs, ctx); err != nil {
+			cclog.Error("Error while loading averages for footprint")
 			return nil, err
 		}
 
@@ -186,11 +187,5 @@ func (r *queryResolver) jobsFootprints(ctx context.Context, filter []*model.JobF
 func requireField(ctx context.Context, name string) bool {
 	fields := graphql.CollectAllFields(ctx)
 
-	for _, f := range fields {
-		if f == name {
-			return true
-		}
-	}
-
-	return false
+	return slices.Contains(fields, name)
 }

internal/importer/README.md

@@ -0,0 +1,132 @@
+# Importer Package
+
+The `importer` package provides functionality for importing job data into the ClusterCockpit database from archived job files.
+
+## Overview
+
+This package supports two primary import workflows:
+
+1. **Bulk Database Initialization** - Reinitialize the entire job database from archived jobs
+2. **Individual Job Import** - Import specific jobs from metadata/data file pairs
+
+Both workflows enrich job metadata by calculating performance footprints and energy consumption metrics before persisting to the database.
+
+## Main Entry Points
+
+### InitDB()
+
+Reinitializes the job database from all archived jobs.
+
+```go
+if err := importer.InitDB(); err != nil {
+    log.Fatal(err)
+}
+```
+
+This function:
+- Flushes existing job, tag, and jobtag tables
+- Iterates through all jobs in the configured archive
+- Enriches each job with calculated metrics
+- Inserts jobs into the database in batched transactions (100 jobs per batch)
+- Continues on individual job failures, logging errors
+
+**Use Case**: Initial database setup or complete database rebuild from archive.
+
+### HandleImportFlag(flag string)
+
+Imports jobs from specified file pairs.
+
+```go
+// Format: "<meta.json>:<data.json>[,<meta2.json>:<data2.json>,...]"
+flag := "/path/to/meta.json:/path/to/data.json"
+if err := importer.HandleImportFlag(flag); err != nil {
+    log.Fatal(err)
+}
+```
+
+This function:
+- Parses the comma-separated file pairs
+- Validates metadata and job data against schemas (if validation enabled)
+- Enriches each job with footprints and energy metrics
+- Imports jobs into both the archive and database
+- Fails fast on the first error
+
+**Use Case**: Importing specific jobs from external sources or manual job additions.
+
+## Job Enrichment
+
+Both import workflows use `enrichJobMetadata()` to calculate:
+
+### Performance Footprints
+
+Performance footprints are calculated from metric averages based on the subcluster configuration:
+
+```go
+job.Footprint["mem_used_avg"] = 45.2  // GB
+job.Footprint["cpu_load_avg"] = 0.87   // percentage
+```
+
+### Energy Metrics
+
+Energy consumption is calculated from power metrics using the formula:
+
+```
+Energy (kWh) = (Power (W) × Duration (s) / 3600) / 1000
+```
+
+For each energy metric:
+```go
+job.EnergyFootprint["acc_power"] = 12.5  // kWh
+job.Energy = 150.2  // Total energy in kWh
+```
+
+**Note**: Energy calculations for metrics with unit "energy" (Joules) are not yet implemented.
+
+## Data Validation
+
+### SanityChecks(job *schema.Job)
+
+Validates job metadata before database insertion:
+
+- Cluster exists in configuration
+- Subcluster is valid (assigns if needed)
+- Job state is valid
+- Resources and user fields are populated
+- Node counts and hardware thread counts are positive
+- Resource count matches declared node count
+
+## Normalization Utilities
+
+The package includes utilities for normalizing metric values to appropriate SI prefixes:
+
+### Normalize(avg float64, prefix string)
+
+Adjusts values and SI prefixes for readability:
+
+```go
+factor, newPrefix := importer.Normalize(2048.0, "M")  
+// Converts 2048 MB → ~2.0 GB
+// Returns: factor for conversion, "G"
+```
+
+This is useful for automatically scaling metrics (e.g., memory, storage) to human-readable units.
+
+## Dependencies
+
+- `github.com/ClusterCockpit/cc-backend/internal/repository` - Database operations
+- `github.com/ClusterCockpit/cc-backend/pkg/archive` - Job archive access
+- `github.com/ClusterCockpit/cc-lib/schema` - Job schema definitions
+- `github.com/ClusterCockpit/cc-lib/ccLogger` - Logging
+- `github.com/ClusterCockpit/cc-lib/ccUnits` - SI unit handling
+
+## Error Handling
+
+- **InitDB**: Continues processing on individual job failures, logs errors, returns summary
+- **HandleImportFlag**: Fails fast on first error, returns immediately
+- Both functions log detailed error context for debugging
+
+## Performance
+
+- **Transaction Batching**: InitDB processes jobs in batches of 100 for optimal database performance
+- **Tag Caching**: Tag IDs are cached during import to minimize database queries
+- **Progress Reporting**: InitDB prints progress updates during bulk operations

internal/importer/handleImport.go

@@ -1,29 +1,44 @@
 // Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved.
+// All rights reserved. This file is part of cc-backend.
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
+
 package importer
 
 import (
 	"bytes"
 	"encoding/json"
 	"fmt"
-	"math"
 	"os"
 	"strings"
 
 	"github.com/ClusterCockpit/cc-backend/internal/config"
 	"github.com/ClusterCockpit/cc-backend/internal/repository"
 	"github.com/ClusterCockpit/cc-backend/pkg/archive"
-	"github.com/ClusterCockpit/cc-backend/pkg/log"
-	"github.com/ClusterCockpit/cc-backend/pkg/schema"
+	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
+	"github.com/ClusterCockpit/cc-lib/v2/schema"
 )
 
-// Import all jobs specified as `<path-to-meta.json>:<path-to-data.json>,...`
+// HandleImportFlag imports jobs from file pairs specified in a comma-separated flag string.
+//
+// The flag format is: "<path-to-meta.json>:<path-to-data.json>[,<path-to-meta2.json>:<path-to-data2.json>,...]"
+//
+// For each job pair, this function:
+//  1. Reads and validates the metadata JSON file (schema.Job)
+//  2. Reads and validates the job data JSON file (schema.JobData)
+//  3. Enriches the job with calculated footprints and energy metrics
+//  4. Validates the job using SanityChecks()
+//  5. Imports the job into the archive
+//  6. Inserts the job into the database with associated tags
+//
+// Schema validation is performed if config.Keys.Validate is true.
+//
+// Returns an error if file reading, validation, enrichment, or database operations fail.
+// The function stops processing on the first error encountered.
 func HandleImportFlag(flag string) error {
 	r := repository.GetJobRepository()
 
-	for _, pair := range strings.Split(flag, ",") {
+	for pair := range strings.SplitSeq(flag, ",") {
 		files := strings.Split(pair, ":")
 		if len(files) != 2 {
 			return fmt.Errorf("REPOSITORY/INIT > invalid import flag format")
@@ -31,7 +46,7 @@ func HandleImportFlag(flag string) error {
 
 		raw, err := os.ReadFile(files[0])
 		if err != nil {
-			log.Warn("Error while reading metadata file for import")
+			cclog.Warn("Error while reading metadata file for import")
 			return err
 		}
 
@@ -42,15 +57,18 @@ func HandleImportFlag(flag string) error {
 		}
 		dec := json.NewDecoder(bytes.NewReader(raw))
 		dec.DisallowUnknownFields()
-		job := schema.JobMeta{BaseJob: schema.JobDefaults}
+		job := schema.Job{
+			Shared:           "none",
+			MonitoringStatus: schema.MonitoringStatusRunningOrArchiving,
+		}
 		if err = dec.Decode(&job); err != nil {
-			log.Warn("Error while decoding raw json metadata for import")
+			cclog.Warn("Error while decoding raw json metadata for import")
 			return err
 		}
 
 		raw, err = os.ReadFile(files[1])
 		if err != nil {
-			log.Warn("Error while reading jobdata file for import")
+			cclog.Warn("Error while reading jobdata file for import")
 			return err
 		}
 
@@ -63,108 +81,41 @@ func HandleImportFlag(flag string) error {
 		dec.DisallowUnknownFields()
 		jobData := schema.JobData{}
 		if err = dec.Decode(&jobData); err != nil {
-			log.Warn("Error while decoding raw json jobdata for import")
+			cclog.Warn("Error while decoding raw json jobdata for import")
 			return err
 		}
 
 		job.MonitoringStatus = schema.MonitoringStatusArchivingSuccessful
 
-		sc, err := archive.GetSubCluster(job.Cluster, job.SubCluster)
-		if err != nil {
-			log.Errorf("cannot get subcluster: %s", err.Error())
+		if err = enrichJobMetadata(&job); err != nil {
+			cclog.Errorf("Error enriching job metadata: %v", err)
 			return err
 		}
 
-		job.Footprint = make(map[string]float64)
-
-		for _, fp := range sc.Footprint {
-			statType := "avg"
-
-			if i, err := archive.MetricIndex(sc.MetricConfig, fp); err != nil {
-				statType = sc.MetricConfig[i].Footprint
-			}
-
-			name := fmt.Sprintf("%s_%s", fp, statType)
-
-			job.Footprint[name] = repository.LoadJobStat(&job, fp, statType)
-		}
-
-		job.RawFootprint, err = json.Marshal(job.Footprint)
-		if err != nil {
-			log.Warn("Error while marshaling job footprint")
-			return err
-		}
-
-		job.EnergyFootprint = make(map[string]float64)
-
-		// Total Job Energy Outside Loop
-		totalEnergy := 0.0
-		for _, fp := range sc.EnergyFootprint {
-			// Always Init Metric Energy Inside Loop
-			metricEnergy := 0.0
-			if i, err := archive.MetricIndex(sc.MetricConfig, fp); err == nil {
-				// Note: For DB data, calculate and save as kWh
-				if sc.MetricConfig[i].Energy == "energy" { // this metric has energy as unit (Joules)
-					log.Warnf("Update EnergyFootprint for Job %d and Metric %s on cluster %s: Set to 'energy' in cluster.json: Not implemented, will return 0.0", job.JobID, job.Cluster, fp)
-					// FIXME: Needs sum as stats type
-				} else if sc.MetricConfig[i].Energy == "power" { // this metric has power as unit (Watt)
-					// Energy: Power (in Watts) * Time (in Seconds)
-					// Unit: (W * (s / 3600)) / 1000 = kWh
-					// Round 2 Digits: round(Energy * 100) / 100
-					// Here: (All-Node Metric Average * Number of Nodes) * (Job Duration in Seconds / 3600) / 1000
-					// Note: Shared Jobs handled correctly since "Node Average" is based on partial resources, while "numNodes" factor is 1
-					rawEnergy := ((repository.LoadJobStat(&job, fp, "avg") * float64(job.NumNodes)) * (float64(job.Duration) / 3600.0)) / 1000.0
-					metricEnergy = math.Round(rawEnergy*100.0) / 100.0
-				}
-			} else {
-				log.Warnf("Error while collecting energy metric %s for job, DB ID '%v', return '0.0'", fp, job.ID)
-			}
-
-			job.EnergyFootprint[fp] = metricEnergy
-			totalEnergy += metricEnergy
-		}
-
-		job.Energy = (math.Round(totalEnergy*100.0) / 100.0)
-		if job.RawEnergyFootprint, err = json.Marshal(job.EnergyFootprint); err != nil {
-			log.Warnf("Error while marshaling energy footprint for job INTO BYTES, DB ID '%v'", job.ID)
-			return err
-		}
-
-		job.RawResources, err = json.Marshal(job.Resources)
-		if err != nil {
-			log.Warn("Error while marshaling job resources")
-			return err
-		}
-		job.RawMetaData, err = json.Marshal(job.MetaData)
-		if err != nil {
-			log.Warn("Error while marshaling job metadata")
-			return err
-		}
-
-		if err = SanityChecks(&job.BaseJob); err != nil {
-			log.Warn("BaseJob SanityChecks failed")
+		if err = SanityChecks(&job); err != nil {
+			cclog.Warn("BaseJob SanityChecks failed")
 			return err
 		}
 
 		if err = archive.GetHandle().ImportJob(&job, &jobData); err != nil {
-			log.Error("Error while importing job")
+			cclog.Error("Error while importing job")
 			return err
 		}
 
-		id, err := r.InsertJob(&job)
+		id, err := r.InsertJobDirect(&job)
 		if err != nil {
-			log.Warn("Error while job db insert")
+			cclog.Warn("Error while job db insert")
 			return err
 		}
 
 		for _, tag := range job.Tags {
 			if err := r.ImportTag(id, tag.Type, tag.Name, tag.Scope); err != nil {
-				log.Error("Error while adding or creating tag on import")
+				cclog.Error("Error while adding or creating tag on import")
 				return err
 			}
 		}
 
-		log.Infof("successfully imported a new job (jobId: %d, cluster: %s, dbid: %d)", job.JobID, job.Cluster, id)
+		cclog.Infof("successfully imported a new job (jobId: %d, cluster: %s, dbid: %d)", job.JobID, job.Cluster, id)
 	}
 	return nil
 }