Merge branch 'main' into feature/526-average-resample

.golangci.yml

@@ -0,0 +1,38 @@
+version: "2"
+
+run:
+  timeout: 5m
+
+formatters:
+  enable:
+    - gofumpt # gofumpt formatter
+
+  settings:
+    gofumpt:
+      module-path: github.com/ClusterCockpit/cc-backend
+
+linters:
+  enable:
+    - staticcheck # staticcheck = true
+    - unparam # unusedparams = true
+    - nilnil # nilness = true (catches nil returns of nil interface values)
+    - govet # base vet; nilness + unusedwrite via govet analyzers
+
+  settings:
+    staticcheck:
+      checks: ["ST1003"]
+
+    govet:
+      enable:
+        - nilness
+        - unusedwrite
+
+exclusions:
+  paths:
+    - .git
+    - .vscode
+    - .idea
+    - node_modules
+    - internal/graph/generated # gqlgen generated code
+    - internal/api/docs\.go # swaggo generated code
+    - _test\.go # test files excluded from all linters

.goreleaser.yaml

@@ -5,6 +5,7 @@ before:
 builds:
   - env:
       - CGO_ENABLED=1
+      - CC=x86_64-linux-musl-gcc
     goos:
       - linux
     goarch:

CLAUDE.md

@@ -314,6 +314,24 @@ job,function=stop_job event="{\"jobId\":123,\"cluster\":\"test\",\"startTime\":1
 - Messages are logged; no responses are sent back to publishers
 - If NATS client is unavailable, API subscriptions are skipped (logged as warning)
 
+### Security Considerations
+
+**The NATS API has no application-layer authentication or authorization.** Unlike
+the REST endpoints (which require a JWT with `RoleAPI`), the subscribers process
+any message delivered on the configured subjects. Anyone with publish rights to
+those subjects on the broker can:
+
+- Insert arbitrary jobs (potentially attributed to other users)
+- Mark running jobs as stopped, triggering archive/finalization
+- Overwrite node state and health metadata for any cluster
+
+Operators MUST restrict publish ACLs at the NATS broker (per-account or
+per-subject permissions) so that only trusted producers — e.g. the scheduler
+integration on a known host or service account — can publish to the configured
+`subject-job-event` and `subject-node-state` subjects. A shared, unrestricted
+NATS broker is not a safe deployment topology for this API. A startup warning
+is logged when these subscriptions are enabled.
+
 ## Development Guidelines
 
 ### Performance
@@ -323,6 +341,21 @@ records, archives) at scale. All code changes must prioritize maximum throughput
 and minimal latency. Avoid unnecessary allocations, prefer streaming over
 buffering, and be mindful of lock contention. When in doubt, benchmark.
 
+### Commit Message Convention
+
+Commits must use conventional commit prefixes so goreleaser can generate the
+changelog automatically. Only commits with these prefixes appear in releases:
+
+| Prefix  | Changelog group        |
+|---------|------------------------|
+| `feat:` | New Features           |
+| `fix:`  | Bug fixes              |
+| `sec:`  | Security updates       |
+| `docs:` | Documentation updates  |
+
+Scoped variants are also recognised, e.g. `feat(api):`, `fix(deps):`.
+Commits without one of these prefixes are excluded from the changelog.
+
 ### Change Impact Analysis
 
 For any significant change, you MUST:

Makefile

@@ -1,6 +1,6 @@
 TARGET = ./cc-backend
 FRONTEND = ./web/frontend
-VERSION = 1.5.2
+VERSION = 1.5.4
 GIT_HASH := $(shell git rev-parse --short HEAD || echo 'development')
 CURRENT_TIME = $(shell date +"%Y-%m-%d:T%H:%M:%S")
 LD_FLAGS = '-s -X main.date=${CURRENT_TIME} -X main.version=${VERSION} -X main.commit=${GIT_HASH}'
@@ -36,7 +36,7 @@ SVELTE_SRC = $(wildcard $(FRONTEND)/src/*.svelte)                 \
 			 $(wildcard $(FRONTEND)/src/header/*.svelte)          \
 			 $(wildcard $(FRONTEND)/src/job/*.svelte)
 
-.PHONY: clean distclean test tags frontend swagger graphql $(TARGET)
+.PHONY: clean distclean fmt lint test tags frontend swagger graphql $(TARGET)
 
 .NOTPARALLEL:
 
@@ -75,6 +75,14 @@ test:
 	@go vet ./...
 	@go test ./...
 
+fmt:
+	$(info ===>  FORMAT)
+	@gofumpt -l -w .
+
+lint:
+	$(info ===>  LINT)
+	@golangci-lint run ./...
+
 tags:
 	$(info ===>  TAGS)
 	@ctags -R

README.md

@@ -1,10 +1,9 @@
 # NOTE
 
-While we do our best to keep the master branch in a usable state, there is no guarantee the master branch works.
-Please do not use it for production!
+While we do our best to keep the main branch in a usable state, there is no
+guarantee the main branch works. Please do not use it for production!
 
-Please have a look at the [Release
-Notes](https://github.com/ClusterCockpit/cc-backend/blob/master/ReleaseNotes.md)
+Please have a look at the [Release Notes](https://github.com/ClusterCockpit/cc-backend/blob/main/ReleaseNotes.md)
 for breaking changes!
 
 # ClusterCockpit REST and GraphQL API backend
@@ -41,7 +40,7 @@ For real-time integration with HPC systems, the backend can subscribe to
 state updates, providing an alternative to REST API polling.
 
 Completed batch jobs are stored in a file-based job archive according to
-[this specification](https://github.com/ClusterCockpit/cc-specifications/tree/master/job-archive).
+[this specification](https://github.com/ClusterCockpit/cc-specifications/tree/main/job-archive).
 The backend supports authentication via local accounts, an external LDAP
 directory, and JWT tokens. Authorization for APIs is implemented with
 [JWT](https://jwt.io/) tokens created with public/private key encryption.
@@ -100,6 +99,26 @@ the following targets:
   frontend source files will result in a complete rebuild.
 - `make clean`: Clean go build cache and remove binary.
 - `make test`: Run the tests that are also run in the GitHub workflow setup.
+- `make fmt`: Format all Go source files using
+  [gofumpt](https://github.com/mvdan/gofumpt), a stricter superset of `gofmt`.
+  Requires `gofumpt` to be installed (see below).
+- `make lint`: Run [golangci-lint](https://golangci-lint.run/) with the project
+  configuration in `.golangci.yml`. Requires `golangci-lint` to be installed
+  (see below).
+
+### Installing development tools
+
+`gofumpt` and `golangci-lint` are not part of the Go module and must be
+installed separately:
+
+```sh
+# Formatter (gofumpt)
+go install mvdan.cc/gofumpt@latest
+
+# Linter (golangci-lint) — use the official install script to get a
+# pre-built binary; installing via `go install` is not supported upstream.
+curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/HEAD/install.sh | sh -s -- -b $(go env GOPATH)/bin
+```
 
 A common workflow for setting up cc-backend from scratch is:
 
@@ -223,73 +242,73 @@ The effective configuration is logged at startup for verification.
 
 ## Project file structure
 
-- [`.github/`](https://github.com/ClusterCockpit/cc-backend/tree/master/.github)
+- [`.github/`](https://github.com/ClusterCockpit/cc-backend/tree/main/.github)
   GitHub Actions workflows and dependabot configuration for CI/CD.
-- [`api/`](https://github.com/ClusterCockpit/cc-backend/tree/master/api)
+- [`api/`](https://github.com/ClusterCockpit/cc-backend/tree/main/api)
   contains the API schema files for the REST and GraphQL APIs. The REST API is
   documented in the OpenAPI 3.0 format in
   [./api/swagger.yaml](./api/swagger.yaml). The GraphQL schema is in
   [./api/schema.graphqls](./api/schema.graphqls).
-- [`cmd/cc-backend`](https://github.com/ClusterCockpit/cc-backend/tree/master/cmd/cc-backend)
+- [`cmd/cc-backend`](https://github.com/ClusterCockpit/cc-backend/tree/main/cmd/cc-backend)
   contains the main application entry point and CLI implementation.
-- [`configs/`](https://github.com/ClusterCockpit/cc-backend/tree/master/configs)
+- [`configs/`](https://github.com/ClusterCockpit/cc-backend/tree/main/configs)
   contains documentation about configuration and command line options and required
   environment variables. Sample configuration files are provided.
-- [`init/`](https://github.com/ClusterCockpit/cc-backend/tree/master/init)
+- [`init/`](https://github.com/ClusterCockpit/cc-backend/tree/main/init)
   contains an example of setting up systemd for production use.
-- [`internal/`](https://github.com/ClusterCockpit/cc-backend/tree/master/internal)
+- [`internal/`](https://github.com/ClusterCockpit/cc-backend/tree/main/internal)
   contains library source code that is not intended for use by others.
-  - [`api`](https://github.com/ClusterCockpit/cc-backend/tree/master/internal/api)
+  - [`api`](https://github.com/ClusterCockpit/cc-backend/tree/main/internal/api)
     REST API handlers and NATS integration
-  - [`archiver`](https://github.com/ClusterCockpit/cc-backend/tree/master/internal/archiver)
+  - [`archiver`](https://github.com/ClusterCockpit/cc-backend/tree/main/internal/archiver)
     Job archiving functionality
-  - [`auth`](https://github.com/ClusterCockpit/cc-backend/tree/master/internal/auth)
+  - [`auth`](https://github.com/ClusterCockpit/cc-backend/tree/main/internal/auth)
     Authentication (local, LDAP, OIDC) and JWT token handling
-  - [`config`](https://github.com/ClusterCockpit/cc-backend/tree/master/internal/config)
+  - [`config`](https://github.com/ClusterCockpit/cc-backend/tree/main/internal/config)
     Configuration management and validation
-  - [`graph`](https://github.com/ClusterCockpit/cc-backend/tree/master/internal/graph)
+  - [`graph`](https://github.com/ClusterCockpit/cc-backend/tree/main/internal/graph)
     GraphQL schema and resolvers
-  - [`importer`](https://github.com/ClusterCockpit/cc-backend/tree/master/internal/importer)
+  - [`importer`](https://github.com/ClusterCockpit/cc-backend/tree/main/internal/importer)
     Job data import and database initialization
-  - [`metricdispatch`](https://github.com/ClusterCockpit/cc-backend/tree/master/internal/metricdispatch)
+  - [`metricdispatch`](https://github.com/ClusterCockpit/cc-backend/tree/main/internal/metricdispatch)
     Dispatches metric data loading to appropriate backends
-  - [`repository`](https://github.com/ClusterCockpit/cc-backend/tree/master/internal/repository)
+  - [`repository`](https://github.com/ClusterCockpit/cc-backend/tree/main/internal/repository)
     Database repository layer for jobs and metadata
-  - [`routerConfig`](https://github.com/ClusterCockpit/cc-backend/tree/master/internal/routerConfig)
+  - [`routerConfig`](https://github.com/ClusterCockpit/cc-backend/tree/main/internal/routerConfig)
     HTTP router configuration and middleware
-  - [`tagger`](https://github.com/ClusterCockpit/cc-backend/tree/master/internal/tagger)
+  - [`tagger`](https://github.com/ClusterCockpit/cc-backend/tree/main/internal/tagger)
     Job classification and application detection
-  - [`taskmanager`](https://github.com/ClusterCockpit/cc-backend/tree/master/internal/taskmanager)
+  - [`taskmanager`](https://github.com/ClusterCockpit/cc-backend/tree/main/internal/taskmanager)
     Background task management and scheduled jobs
-  - [`metricstoreclient`](https://github.com/ClusterCockpit/cc-backend/tree/master/internal/metricstoreclient)
+  - [`metricstoreclient`](https://github.com/ClusterCockpit/cc-backend/tree/main/internal/metricstoreclient)
     Client for cc-metric-store queries
-- [`pkg/`](https://github.com/ClusterCockpit/cc-backend/tree/master/pkg)
+- [`pkg/`](https://github.com/ClusterCockpit/cc-backend/tree/main/pkg)
   contains Go packages that can be used by other projects.
-  - [`archive`](https://github.com/ClusterCockpit/cc-backend/tree/master/pkg/archive)
+  - [`archive`](https://github.com/ClusterCockpit/cc-backend/tree/main/pkg/archive)
     Job archive backend implementations (filesystem, S3, SQLite)
-  - [`metricstore`](https://github.com/ClusterCockpit/cc-backend/tree/master/pkg/metricstore)
+  - [`metricstore`](https://github.com/ClusterCockpit/cc-backend/tree/main/pkg/metricstore)
     In-memory metric data store with checkpointing and metric loading
-- [`tools/`](https://github.com/ClusterCockpit/cc-backend/tree/master/tools)
+- [`tools/`](https://github.com/ClusterCockpit/cc-backend/tree/main/tools)
   Additional command line helper tools.
-  - [`archive-manager`](https://github.com/ClusterCockpit/cc-backend/tree/master/tools/archive-manager)
+  - [`archive-manager`](https://github.com/ClusterCockpit/cc-backend/tree/main/tools/archive-manager)
     Commands for getting infos about an existing job archive, importing jobs
     between archive backends, and converting archives between JSON and Parquet formats.
-  - [`archive-migration`](https://github.com/ClusterCockpit/cc-backend/tree/master/tools/archive-migration)
+  - [`archive-migration`](https://github.com/ClusterCockpit/cc-backend/tree/main/tools/archive-migration)
     Tool for migrating job archives between formats.
-  - [`convert-pem-pubkey`](https://github.com/ClusterCockpit/cc-backend/tree/master/tools/convert-pem-pubkey)
+  - [`convert-pem-pubkey`](https://github.com/ClusterCockpit/cc-backend/tree/main/tools/convert-pem-pubkey)
     Tool to convert external pubkey for use in `cc-backend`.
-  - [`gen-keypair`](https://github.com/ClusterCockpit/cc-backend/tree/master/tools/gen-keypair)
+  - [`gen-keypair`](https://github.com/ClusterCockpit/cc-backend/tree/main/tools/gen-keypair)
     contains a small application to generate a compatible JWT keypair. You find
     documentation on how to use it
-    [here](https://github.com/ClusterCockpit/cc-backend/blob/master/docs/JWT-Handling.md).
-- [`web/`](https://github.com/ClusterCockpit/cc-backend/tree/master/web)
-  Server-side templates and frontend-related files:
-  - [`frontend`](https://github.com/ClusterCockpit/cc-backend/tree/master/web/frontend)
+    [here](https://github.com/ClusterCockpit/cc-backend/blob/main/docs/JWT-Handling.md).
+  - [`web/`](https://github.com/ClusterCockpit/cc-backend/tree/main/web)
+    Server-side templates and frontend-related files:
+  - [`frontend`](https://github.com/ClusterCockpit/cc-backend/tree/main/web/frontend)
     Svelte components and static assets for the frontend UI
-  - [`templates`](https://github.com/ClusterCockpit/cc-backend/tree/master/web/templates)
+  - [`templates`](https://github.com/ClusterCockpit/cc-backend/tree/main/web/templates)
     Server-side Go templates, including monitoring views
-- [`gqlgen.yml`](https://github.com/ClusterCockpit/cc-backend/blob/master/gqlgen.yml)
+- [`gqlgen.yml`](https://github.com/ClusterCockpit/cc-backend/blob/main/gqlgen.yml)
   Configures the behaviour and generation of
   [gqlgen](https://github.com/99designs/gqlgen).
-- [`startDemo.sh`](https://github.com/ClusterCockpit/cc-backend/blob/master/startDemo.sh)
+- [`startDemo.sh`](https://github.com/ClusterCockpit/cc-backend/blob/main/startDemo.sh)
   is a shell script that sets up demo data, and builds and starts `cc-backend`.

ReleaseNotes.md

@@ -1,16 +1,173 @@
-# `cc-backend` version 1.5.2
+# `cc-backend` version 1.5.4
 
 Supports job archive version 3 and database version 11.
 
-This is a bugfix release of `cc-backend`, the API backend and frontend
-implementation of ClusterCockpit.
+This is a security and bugfix release of `cc-backend`, the API backend and
+frontend implementation of ClusterCockpit.
 For release specific notes visit the [ClusterCockpit Documentation](https://clusterockpit.org/docs/release/).
 If you are upgrading from v1.5.1 no database migration is required.
 If you are upgrading from v1.5.0 you need to do another DB migration. This
 should not take long. For optimal database performance after the migration it is
 recommended to apply the new `optimize-db` flag, which runs the sqlite `ANALYZE`
 and `VACUUM` commands. Depending on your database size (more then 40GB) the
-`VACUUM` may take up to 2h.
+`VACUUM` may take up to 2h. You can also run the `ANALYZE` command manually.
+While we are confident that the memory issue with the metricstore cleanup move
+policy is fixed, it is still recommended to use delete policy for cleanup.
+This is also the default.
+
+## Changes in 1.5.4
+
+### Security fixes
+
+- **JWT HMAC empty-key bypass (critical)**: `jwtSession.go` now refuses to
+  register when `CROSS_LOGIN_JWT_HS512_KEY` is unset. Previously, an empty HMAC
+  key allowed unauthenticated admin token forgery because `golang-jwt` verifies
+  any HS256/HS512 signature against an empty key.
+- **SQL injection via metric names (critical)**: Metric names supplied through
+  GraphQL (`[String!]`) were interpolated raw into `json_extract` SQL expressions.
+  Names are now validated against `^[a-zA-Z0-9_]+$` in
+  `jobsMetricStatisticsHistogram` and `buildFloatJSONCondition`.
+- **Path traversal via line-protocol tags (critical)**: `cluster` and `host`
+  tags from the metric line protocol flowed unvalidated into `path.Join` for
+  checkpoint/WAL file paths, enabling arbitrary file writes outside the
+  checkpoint root via NATS (unauthenticated) or `POST /api/write`. Path-traversal
+  sequences are now rejected in `DecodeLine` before the tags become path
+  components.
+- **CORS `AllowCredentials` disabled**: CORS middleware no longer sets
+  `AllowCredentials: true`, which was incompatible with `AllowedOrigins: ["*"]`
+  and could enable cross-origin credential theft.
+- **HSTS header added**: `Strict-Transport-Security` is now set for all
+  HTTPS connections.
+- **Security response headers**: Added `X-Content-Type-Options: nosniff`,
+  `X-Frame-Options: DENY`, `Referrer-Policy: same-origin`, and a conservative
+  `Content-Security-Policy` (blocks `frame-ancestors`, `object-src`, `base-uri`)
+  to harden against clickjacking and base-tag injection.
+- **Stored XSS in job message**: `job.metaData.message` is now rendered as
+  escaped text (CSS `white-space: pre-wrap`) instead of raw `{@html ...}` in
+  `Job.root` and `JobFootprint`.
+- **SQL injection in tag queries**: The tag-scope `IN` list and manager project
+  subquery in `CountTags` are now parameterized instead of interpolating
+  `user.Username` / `user.Projects` values sourced from OIDC/LDAP.
+- **GraphQL DoS hardening**: Query cost is bounded with `FixedComplexityLimit`
+  (5000). Non-positive `items-per-page` and `page` values are rejected with HTTP
+  400 to prevent integer underflow into unbounded `LIMIT`/`OFFSET` queries.
+- **CSRF defense-in-depth**: State-changing requests with a cross-site
+  `Sec-Fetch-Site` header are now rejected (fails open for non-browser clients),
+  complementing the existing `SameSite=Lax` session cookie.
+- **NATS API security warning**: A startup warning is now logged when NATS
+  subscriptions are enabled, reminding operators that the NATS API has no
+  application-layer authentication and that publish ACLs must be restricted at
+  the broker.
+
+### Bug fixes
+
+- **Roofline legend placement**: Roofline plot legends now use fixed
+  coordinates instead of dynamic placement, preventing the legend from
+  overlapping data points or being rendered off-canvas (#546).
+- **Subcluster usage tab labels**: Subcluster names in the status dashboard
+  usage tabs are no longer force-capitalized; the original cluster-defined
+  casing is preserved.
+- **NodeListRow host filter**: The running-jobs query in the node list row now
+  filters by exact node hostname (`eq`) instead of substring match
+  (`contains`), avoiding incorrect matches when one hostname is a prefix of
+  another.
+- **WAL files not reset on shutdown**: On graceful shutdown the metricstore wrote
+  a final binary snapshot but never rotated the per-host `current.wal` files (the
+  `RotateWALFilesAfterShutdown` helper was defined but never called). The stale
+  WAL files were replayed and then appended to again on the next start, so they
+  grew without bound across restarts and were only ever reset at the next periodic
+  checkpoint. `Shutdown` now rotates the WAL files for all successfully
+  snapshotted hosts.
+
+### Dependencies
+
+- **Go module upgrades**: Refreshed Go module dependencies to their latest
+  compatible versions.
+
+## Changes in 1.5.3
+
+### Bug fixes
+
+- **OIDC role extraction**: Fixed role extraction from OIDC tokens where roles
+  were not correctly parsed from the token claims. Roles are now always
+  requested from the token regardless of other configuration.
+- **OIDC user sync role changes**: `SyncUser` and `UpdateUser` callbacks now
+  allow all role changes, removing a restriction that prevented role updates
+  during OIDC-driven user synchronization.
+- **OIDC projects array**: Projects array from the OIDC token is now submitted
+  and applied when syncing user attributes.
+- **WAL message drops during checkpoint**: WAL writes are now paused during
+  binary checkpoint creation. Previously, disk I/O contention between
+  checkpoint writes and WAL staging caused over 1.4 million dropped messages
+  per checkpoint cycle.
+- **WAL rotation skipped for all nodes**: `RotateWALFiles` used a non-blocking
+  send on a small channel. With thousands of nodes, the channel filled instantly
+  and nearly all hosts were skipped, leaving WAL files unrotated. Replaced with
+  a blocking send using a shared 2-minute deadline.
+- **Log viewer auto-refresh**: Fixed the log viewer component not auto-refreshing
+  correctly.
+- **SameSite cookie setting**: Relaxed the SameSite cookie attribute to improve
+  compatibility with OIDC redirect flows.
+- **WAL not rotated on partial checkpoint failure**: When binary checkpointing
+  failed for some hosts, WAL files for successfully checkpointed hosts were not
+  rotated and the checkpoint timestamp was not advanced. Partial successes now
+  correctly advance the checkpoint and rotate WAL files for completed hosts.
+- **Unbounded WAL file growth**: If binary checkpointing consistently failed for
+  a host, its `current.wal` file grew without limit until disk exhaustion. A new
+  `max-wal-size` configuration option (in the `checkpoints` block) allows setting
+  a per-host WAL size cap in bytes. When exceeded, the WAL is force-rotated.
+  Defaults to 0 (unlimited) for backward compatibility.
+
+- **Doubleranged filter fixes**: Range filters now correctly handle zero as a
+  boundary value. Improved validation and UI text for "more than equal" and
+  "less than equal" range selections.
+- **Lineprotocol body parsing interrupted**: Switched from `ReadTimeout` to
+  `ReadHeaderTimeout` so that long-running metric submissions are no longer
+  cut off mid-stream.
+- **Checkpoint archiving continues on error**: A single cluster's archiving
+  failure no longer aborts the entire cleanup operation. Errors are collected
+  and reported per cluster.
+- **Parquet row group overflow**: Added periodic flush during checkpoint
+  archiving to prevent exceeding the parquet-go 32k column-write limit.
+- **Removed metrics excluded from subcluster config**: Metrics removed from a
+  subcluster are no longer returned by `GetMetricConfigSubCluster`.
+
+### MetricStore performance
+
+- **WAL writer throughput**: Decoupled WAL file flushing from message processing
+  using a periodic 5-second batch flush (up to 4096 messages per cycle),
+  significantly increasing metric ingestion throughput.
+- **Improved shutdown time**: HTTP shutdown timeout reduced; metricstore and
+  archiver now shut down concurrently. Overall shutdown deadline raised to
+  60 seconds.
+
+### New features
+
+- **Manual checkpoint cleanup flag**: New `-cleanup-checkpoints` CLI flag
+  triggers checkpoint cleanup without starting the server, useful for
+  maintenance windows or automated cleanup scripts.
+- **Explicit node state queries in node view**: Node health and scheduler state
+  are now fetched independently from metric data for fresher status information.
+
+### Development tooling
+
+- **Make targets for formatting and linting**: New `make fmt` and `make lint`
+  targets using `gofumpt` and `golangci-lint`. Configuration added in
+  `.golangci.yml` and `gopls.json`.
+
+### New tools
+
+- **binaryCheckpointReader**: New utility tool (`tools/binaryCheckpointReader`)
+  that reads `.wal` or `.bin` checkpoint files produced by the metricstore
+  WAL/snapshot system and dumps their contents to a human-readable `.txt` file.
+  Useful for debugging and inspecting checkpoint data. Usage:
+  `go run ./tools/binaryCheckpointReader <file.wal|file.bin>`
+
+### Logging improvements
+
+- **Reduced tagger log noise**: Missing metrics and expression evaluation errors
+  in the job classification tagger are now logged at debug level instead of
+  error level.
 
 ## Changes in 1.5.2
 
@@ -19,6 +176,14 @@ and `VACUUM` commands. Depending on your database size (more then 40GB) the
 - **Memory spike in parquet writer**: Fixed memory spikes when using the
   metricstore move (archive) policy with the parquet writer. The writer now
   processes data in a streaming fashion to avoid accumulating large allocations.
+- **Top list query fixes**: Fixed top list queries in analysis and dashboard
+  views.
+- **Exclude down nodes from HealthCheck**: Down nodes are now excluded from
+  health checks in both the REST and NATS handlers.
+- **Node state priority order**: Node state determination now enforces a
+  priority order. Exception: idle+down results in idle.
+- **Blocking ReceiveNats call**: Fixed a blocking NATS receive call in the
+  metricstore.
 
 ### Database performance
 
@@ -33,6 +198,16 @@ and `VACUUM` commands. Depending on your database size (more then 40GB) the
   write load.
 - **Increased default SQLite timeout**: The default SQLite connection timeout
   has been raised to reduce spurious timeout errors under load.
+- **Optimized stats queries**: Improved sortby handling in stats queries, fixed
+  cache key passing, and simplified a stats query condition that caused an
+  expensive unnecessary subquery.
+
+### MetricStore performance
+
+- **Sharded WAL consumer**: The WAL consumer is now sharded for significantly
+  higher write throughput.
+- **NATS contention fix**: Fixed contention in the metricstore NATS ingestion
+  path.
 
 ### NATS API
 
@@ -52,6 +227,24 @@ and `VACUUM` commands. Depending on your database size (more then 40GB) the
   operation.
 - **Checkpoint archiving log**: Added an informational log message when the
   metricstore checkpoint archiving process runs.
+- **Auth failure context**: Auth failure log messages now include more context
+  information.
+
+### Behavior changes
+
+- **DB-based metricHealth**: Replaced heuristic-based metric health with
+  DB-based metric health for the node view, providing more accurate health
+  status information.
+- **Removed minRunningFor filter remnants**: Cleaned up remaining `minRunningFor`
+  references from the GraphQL schema and query builder.
+
+### Frontend
+
+- **Streamlined statsSeries**: Unified stats series calculation and rendering
+  across plot components.
+- **Clarified plot titles**: Improved titles in dashboard and health views.
+- **Bumped frontend dependencies**: Updated frontend dependencies to latest
+  versions.
 
 ### Dependencies
 
@@ -67,7 +260,7 @@ and `VACUUM` commands. Depending on your database size (more then 40GB) the
   running has to be allowed to execute the journalctl command.
 - The user configuration keys for the ui have changed. Therefore old user
   configuration persisted in the database is not used anymore. It is recommended
-  to configure the metrics shown in the ui-config sestion and remove all records
+  to configure the metrics shown in the ui-config section and remove all records
   in the table after the update.
 - Currently energy footprint metrics of type energy are ignored for calculating
   total energy.

api/swagger.json

@@ -391,11 +391,6 @@
         },
         "/api/jobs/edit_meta/": {
             "patch": {
-                "security": [
-                    {
-                        "ApiKeyAuth": []
-                    }
-                ],
                 "description": "Edit key value pairs in metadata json of job specified by jobID, StartTime and Cluster\nIf a key already exists its content will be overwritten",
                 "consumes": [
                     "application/json"
@@ -406,7 +401,7 @@
                 "tags": [
                     "Job add and modify"
                 ],
-                "summary": "Edit meta-data json by request",
+                "summary": "Edit meta-data json of job identified by request",
                 "parameters": [
                     {
                         "description": "Specifies job and payload to add or update",
@@ -449,12 +444,17 @@
                             "$ref": "#/definitions/api.ErrorResponse"
                         }
                     }
-                }
+                },
+                "security": [
+                    {
+                        "ApiKeyAuth": []
+                    }
+                ]
             }
         },
         "/api/jobs/edit_meta/{id}": {
             "patch": {
-                "description": "Edit key value pairs in job metadata json\nIf a key already exists its content will be overwritten",
+                "description": "Edit key value pairs in job metadata json of job specified by database id\nIf a key already exists its content will be overwritten",
                 "consumes": [
                     "application/json"
                 ],
@@ -464,7 +464,7 @@
                 "tags": [
                     "Job add and modify"
                 ],
-                "summary": "Edit meta-data json",
+                "summary": "Edit meta-data json of job identified by database id",
                 "parameters": [
                     {
                         "type": "integer",
@@ -474,7 +474,7 @@
                         "required": true
                     },
                     {
-                        "description": "Kay value pair to add",
+                        "description": "Metadata Key value pair to add or update",
                         "name": "request",
                         "in": "body",
                         "required": true,
@@ -736,6 +736,64 @@
                 ]
             }
         },
+        "/api/jobs/used_nodes": {
+            "get": {
+                "description": "Get a map of cluster names to lists of unique hostnames that are currently in use by running jobs that started before the specified timestamp.",
+                "produces": [
+                    "application/json"
+                ],
+                "tags": [
+                    "Job query"
+                ],
+                "summary": "Lists used nodes by cluster",
+                "parameters": [
+                    {
+                        "type": "integer",
+                        "description": "Unix timestamp to filter jobs (jobs with start_time \u003c ts)",
+                        "name": "ts",
+                        "in": "query",
+                        "required": true
+                    }
+                ],
+                "responses": {
+                    "200": {
+                        "description": "Map of cluster names to hostname lists",
+                        "schema": {
+                            "$ref": "#/definitions/api.GetUsedNodesAPIResponse"
+                        }
+                    },
+                    "400": {
+                        "description": "Bad Request",
+                        "schema": {
+                            "$ref": "#/definitions/api.ErrorResponse"
+                        }
+                    },
+                    "401": {
+                        "description": "Unauthorized",
+                        "schema": {
+                            "$ref": "#/definitions/api.ErrorResponse"
+                        }
+                    },
+                    "403": {
+                        "description": "Forbidden",
+                        "schema": {
+                            "$ref": "#/definitions/api.ErrorResponse"
+                        }
+                    },
+                    "500": {
+                        "description": "Internal Server Error",
+                        "schema": {
+                            "$ref": "#/definitions/api.ErrorResponse"
+                        }
+                    }
+                },
+                "security": [
+                    {
+                        "ApiKeyAuth": []
+                    }
+                ]
+            }
+        },
         "/api/jobs/{id}": {
             "get": {
                 "description": "Job to get is specified by database ID\nReturns full job resource information according to 'Job' scheme and all metrics according to 'JobData'.",
@@ -958,8 +1016,11 @@
         "/api/user/{id}": {
             "post": {
                 "description": "Allows admins to add/remove roles and projects for a user",
+                "consumes": [
+                    "application/json"
+                ],
                 "produces": [
-                    "text/plain"
+                    "application/json"
                 ],
                 "tags": [
                     "User"
@@ -974,35 +1035,26 @@
                         "required": true
                     },
                     {
-                        "type": "string",
-                        "description": "Role to add",
-                        "name": "add-role",
-                        "in": "formData"
-                    },
-                    {
-                        "type": "string",
-                        "description": "Role to remove",
-                        "name": "remove-role",
-                        "in": "formData"
-                    },
-                    {
-                        "type": "string",
-                        "description": "Project to add",
-                        "name": "add-project",
-                        "in": "formData"
-                    },
-                    {
-                        "type": "string",
-                        "description": "Project to remove",
-                        "name": "remove-project",
-                        "in": "formData"
+                        "description": "Single Field Changes",
+                        "name": "request",
+                        "in": "body",
+                        "required": true,
+                        "schema": {
+                            "$ref": "#/definitions/api.UpdateUserAPIRequest"
+                        }
                     }
                 ],
                 "responses": {
                     "200": {
-                        "description": "Success message",
+                        "description": "OK",
                         "schema": {
-                            "type": "string"
+                            "$ref": "#/definitions/api.DefaultAPIResponse"
+                        }
+                    },
+                    "400": {
+                        "description": "Bad Request",
+                        "schema": {
+                            "$ref": "#/definitions/api.ErrorResponse"
                         }
                     },
                     "403": {
@@ -1374,63 +1426,6 @@
                 ]
             }
         },
-        "/healthcheck/": {
-            "get": {
-                "description": "This endpoint allows the users to check if a node is healthy",
-                "produces": [
-                    "application/json"
-                ],
-                "tags": [
-                    "healthcheck"
-                ],
-                "summary": "HealthCheck endpoint",
-                "parameters": [
-                    {
-                        "type": "string",
-                        "description": "Selector",
-                        "name": "selector",
-                        "in": "query"
-                    }
-                ],
-                "responses": {
-                    "200": {
-                        "description": "Debug dump",
-                        "schema": {
-                            "type": "string"
-                        }
-                    },
-                    "400": {
-                        "description": "Bad Request",
-                        "schema": {
-                            "$ref": "#/definitions/api.ErrorResponse"
-                        }
-                    },
-                    "401": {
-                        "description": "Unauthorized",
-                        "schema": {
-                            "$ref": "#/definitions/api.ErrorResponse"
-                        }
-                    },
-                    "403": {
-                        "description": "Forbidden",
-                        "schema": {
-                            "$ref": "#/definitions/api.ErrorResponse"
-                        }
-                    },
-                    "500": {
-                        "description": "Internal Server Error",
-                        "schema": {
-                            "$ref": "#/definitions/api.ErrorResponse"
-                        }
-                    }
-                },
-                "security": [
-                    {
-                        "ApiKeyAuth": []
-                    }
-                ]
-            }
-        },
         "/jobs/tag_job/{id}": {
             "delete": {
                 "description": "Removes tag(s) from a job specified by DB ID. Name and Type of Tag(s) must match.\nTag Scope is required for matching, options: \"global\", \"admin\". Private tags can not be deleted via API.\nIf tagged job is already finished: Tag will be removed from respective archive files.",
@@ -1933,6 +1928,31 @@
                 }
             }
         },
+        "api.UpdateUserAPIRequest": {
+            "type": "object",
+            "properties": {
+                "add-role": {
+                    "description": "Role to add to user $ID",
+                    "type": "string",
+                    "example": "user"
+                },
+                "remove-role": {
+                    "description": "Role to remove from user $ID",
+                    "type": "string",
+                    "example": "user"
+                },
+                "add-project": {
+                    "description": "Project to add to user $ID managed array",
+                    "type": "string",
+                    "example": "abcd100"
+                },
+                "remove-project": {
+                    "description": "Project to remove from user $ID managed array",
+                    "type": "string",
+                    "example": "abcd100"
+                }
+            }
+        },
         "api.DefaultAPIResponse": {
             "type": "object",
             "properties": {
@@ -2036,6 +2056,52 @@
                 }
             }
         },
+        "api.GetUsedNodesAPIResponse": {
+            "type": "object",
+            "properties": {
+                "usedNodes": {
+                    "description": "Map of cluster names to lists of used node hostnames",
+                    "type": "object",
+                    "additionalProperties": {
+                        "type": "array",
+                        "items": {
+                            "type": "string"
+                        }
+                    }
+                }
+            }
+        },
+        "api.JobMetaRequest": {
+            "type": "object",
+            "required": [
+                "jobId"
+            ],
+            "properties": {
+                "cluster": {
+                    "description": "Cluster of job",
+                    "type": "string",
+                    "example": "fritz"
+                },
+                "jobId": {
+                    "description": "Cluster Job ID of job",
+                    "type": "integer",
+                    "example": 123000
+                },
+                "payload": {
+                    "description": "Content to Add to Job Meta_Data",
+                    "allOf": [
+                        {
+                            "$ref": "#/definitions/api.EditMetaRequest"
+                        }
+                    ]
+                },
+                "startTime": {
+                    "description": "Start Time of job as epoch",
+                    "type": "integer",
+                    "example": 1649723812
+                }
+            }
+        },
         "api.JobMetricWithName": {
             "type": "object",
             "properties": {
@@ -2168,13 +2234,6 @@
                         "format": "float64"
                     }
                 },
-                "exclusive": {
-                    "description": "for backwards compatibility",
-                    "type": "integer",
-                    "maximum": 2,
-                    "minimum": 0,
-                    "example": 1
-                },
                 "footprint": {
                     "type": "object",
                     "additionalProperties": {

api/swagger.yaml

@@ -31,6 +31,25 @@ definitions:
         example: Debug
         type: string
     type: object
+  api.UpdateUserAPIRequest:
+    properties:
+      add-project:
+        description: Project to add to user $ID managed array
+        example: abcd100
+        type: string
+      add-role:
+        description: Role to add to user $ID
+        example: user
+        type: string
+      remove-project:
+        description: Project to remove from user $ID managed array
+        example: abcd100
+        type: string
+      remove-role:
+        description: Role to remove from user $ID
+        example: user
+        type: string
+    type: object
   api.DefaultAPIResponse:
     properties:
       msg:
@@ -102,6 +121,16 @@ definitions:
         description: Page id returned
         type: integer
     type: object
+  api.GetUsedNodesAPIResponse:
+    properties:
+      usedNodes:
+        additionalProperties:
+          items:
+            type: string
+          type: array
+        description: Map of cluster names to lists of used node hostnames
+        type: object
+    type: object
   api.JobMetaRequest:
     properties:
       cluster:
@@ -214,12 +243,6 @@ definitions:
           format: float64
           type: number
         type: object
-      exclusive:
-        description: for backwards compatibility
-        example: 1
-        maximum: 2
-        minimum: 0
-        type: integer
       footprint:
         additionalProperties:
           format: float64
@@ -1151,7 +1174,7 @@ paths:
             $ref: '#/definitions/api.ErrorResponse'
       security:
       - ApiKeyAuth: []
-      summary: Edit meta-data json by request
+      summary: Edit meta-data json of job identified by request
       tags:
       - Job add and modify
   /api/jobs/edit_meta/{id}:
@@ -1159,7 +1182,7 @@ paths:
       consumes:
       - application/json
       description: |-
-        Edit key value pairs in job metadata json
+        Edit key value pairs in job metadata json of job specified by database id
         If a key already exists its content will be overwritten
       parameters:
       - description: Job Database ID
@@ -1167,7 +1190,7 @@ paths:
         name: id
         required: true
         type: integer
-      - description: Kay value pair to add
+      - description: Metadata Key value pair to add or update
         in: body
         name: request
         required: true
@@ -1198,7 +1221,7 @@ paths:
             $ref: '#/definitions/api.ErrorResponse'
       security:
       - ApiKeyAuth: []
-      summary: Edit meta-data json
+      summary: Edit meta-data json of job identified by database id
       tags:
       - Job add and modify
   /api/jobs/start_job/:
@@ -1346,6 +1369,44 @@ paths:
       summary: Adds one or more tags to a job
       tags:
       - Job add and modify
+  /api/jobs/used_nodes:
+    get:
+      description: Get a map of cluster names to lists of unique hostnames that are
+        currently in use by running jobs that started before the specified timestamp.
+      parameters:
+      - description: Unix timestamp to filter jobs (jobs with start_time < ts)
+        in: query
+        name: ts
+        required: true
+        type: integer
+      produces:
+      - application/json
+      responses:
+        "200":
+          description: Map of cluster names to hostname lists
+          schema:
+            $ref: '#/definitions/api.GetUsedNodesAPIResponse'
+        "400":
+          description: Bad Request
+          schema:
+            $ref: '#/definitions/api.ErrorResponse'
+        "401":
+          description: Unauthorized
+          schema:
+            $ref: '#/definitions/api.ErrorResponse'
+        "403":
+          description: Forbidden
+          schema:
+            $ref: '#/definitions/api.ErrorResponse'
+        "500":
+          description: Internal Server Error
+          schema:
+            $ref: '#/definitions/api.ErrorResponse'
+      security:
+      - ApiKeyAuth: []
+      summary: Lists used nodes by cluster
+      tags:
+      - Job query
   /api/nodestats/:
     post:
       description: |-
@@ -1388,6 +1449,8 @@ paths:
       - Nodestates
   /api/user/{id}:
     post:
+      consumes:
+      - application/json
       description: Allows admins to add/remove roles and projects for a user
       parameters:
       - description: Username
@@ -1395,29 +1458,23 @@ paths:
         name: id
         required: true
         type: string
-      - description: Role to add
-        in: formData
-        name: add-role
-        type: string
-      - description: Role to remove
-        in: formData
-        name: remove-role
-        type: string
-      - description: Project to add
-        in: formData
-        name: add-project
-        type: string
-      - description: Project to remove
-        in: formData
-        name: remove-project
-        type: string
+      - description: Single Field Changes
+        in: body
+        name: request
+        required: true
+        schema:
+          $ref: '#/definitions/api.UpdateUserAPIRequest'
       produces:
-      - text/plain
+      - application/json
       responses:
         "200":
-          description: Success message
+          description: OK
           schema:
-            type: string
+            $ref: '#/definitions/api.DefaultAPIResponse'
+        "400":
+          description: Bad Request
+          schema:
+            $ref: '#/definitions/api.ErrorResponse'
         "403":
           description: Forbidden
           schema:
@@ -1657,42 +1714,6 @@ paths:
       - ApiKeyAuth: []
       tags:
       - free
-  /healthcheck/:
-    get:
-      description: This endpoint allows the users to check if a node is healthy
-      parameters:
-      - description: Selector
-        in: query
-        name: selector
-        type: string
-      produces:
-      - application/json
-      responses:
-        "200":
-          description: Debug dump
-          schema:
-            type: string
-        "400":
-          description: Bad Request
-          schema:
-            $ref: '#/definitions/api.ErrorResponse'
-        "401":
-          description: Unauthorized
-          schema:
-            $ref: '#/definitions/api.ErrorResponse'
-        "403":
-          description: Forbidden
-          schema:
-            $ref: '#/definitions/api.ErrorResponse'
-        "500":
-          description: Internal Server Error
-          schema:
-            $ref: '#/definitions/api.ErrorResponse'
-      security:
-      - ApiKeyAuth: []
-      summary: HealthCheck endpoint
-      tags:
-      - healthcheck
   /jobs/tag_job/{id}:
     delete:
       consumes:

cmd/cc-backend/cli.go

@@ -11,7 +11,8 @@ import "flag"
 
 var (
 	flagReinitDB, flagInit, flagServer, flagSyncLDAP, flagGops, flagMigrateDB, flagRevertDB,
-	flagForceDB, flagDev, flagVersion, flagLogDateTime, flagApplyTags, flagOptimizeDB bool
+	flagForceDB, flagDev, flagVersion, flagLogDateTime, flagApplyTags, flagOptimizeDB,
+	flagCleanupCheckpoints bool
 	flagNewUser, flagDelUser, flagGenJWT, flagConfigFile, flagImportJob, flagLogLevel string
 )
 
@@ -28,6 +29,7 @@ func cliInit() {
 	flag.BoolVar(&flagApplyTags, "apply-tags", false, "Run taggers on all completed jobs and exit")
 	flag.BoolVar(&flagForceDB, "force-db", false, "Force database version, clear dirty flag and exit")
 	flag.BoolVar(&flagOptimizeDB, "optimize-db", false, "Optimize database: run VACUUM to reclaim space, then ANALYZE to update query planner statistics")
+	flag.BoolVar(&flagCleanupCheckpoints, "cleanup-checkpoints", false, "Clean up old checkpoint files (delete or archive) based on retention settings, then exit")
 	flag.BoolVar(&flagLogDateTime, "logdate", false, "Set this flag to add date and time to log messages")
 	flag.StringVar(&flagConfigFile, "config", "./config.json", "Specify alternative path to `config.json`")
 	flag.StringVar(&flagNewUser, "add-user", "", "Add a new user. Argument format: <username>:[admin,support,manager,api,user]:<password>")

cmd/cc-backend/main.go

@@ -14,6 +14,7 @@ import (
 	"fmt"
 	"os"
 	"os/signal"
+	goruntime "runtime"
 	"runtime/debug"
 	"strings"
 	"sync"
@@ -171,14 +172,20 @@ func handleUserCommands() error {
 		return fmt.Errorf("--add-user and --del-user can only be used if authentication is enabled")
 	}
 
-	if !config.Keys.DisableAuthentication {
-		if cfg := ccconf.GetPackageConfig("auth"); cfg != nil {
-			auth.Init(&cfg)
-		} else {
-			cclog.Warn("Authentication disabled due to missing configuration")
-			auth.Init(nil)
+	// Always initialize the auth subsystem so the HTTP server and REST API have a
+	// valid (non-nil) auth instance, even when authentication is disabled. With
+	// authentication disabled, Init only sets up an ephemeral session store and
+	// registers no authenticators (see auth.Init).
+	if cfg := ccconf.GetPackageConfig("auth"); cfg != nil {
+		auth.Init(&cfg)
+	} else {
+		if !config.Keys.DisableAuthentication {
+			cclog.Warn("Authentication enabled but no auth configuration found")
 		}
+		auth.Init(nil)
+	}
 
+	if !config.Keys.DisableAuthentication {
 		// Check for default security keys
 		checkDefaultSecurityKeys()
 
@@ -336,6 +343,12 @@ func initSubsystems() error {
 }
 
 func runServer(ctx context.Context) error {
+	// Derive a cancelable context so the startup-error path below can trigger the
+	// same graceful-shutdown sequence as a signal (via the signal handler that
+	// waits on ctx.Done()).
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+
 	var wg sync.WaitGroup
 
 	// Initialize metric store if configuration is provided
@@ -437,26 +450,32 @@ func runServer(ctx context.Context) error {
 	// Wait for either:
 	// 1. An error from server startup
 	// 2. Completion of all goroutines (normal shutdown or crash)
+	var runErr error
 	select {
-	case err := <-errChan:
+	case runErr = <-errChan:
 		// errChan will be closed when waitDone is closed, which happens
 		// when all goroutines complete (either from normal shutdown or error)
-		if err != nil {
-			return err
-		}
 	case <-time.After(100 * time.Millisecond):
 		// Give the server 100ms to start and report any immediate startup errors
 		// After that, just wait for normal shutdown completion
 		select {
-		case err := <-errChan:
-			if err != nil {
-				return err
-			}
+		case runErr = <-errChan:
 		case <-waitDone:
 			// Normal shutdown completed
 		}
 	}
 
+	if runErr != nil {
+		// A subsystem failed (e.g. the HTTP server could not bind). Trigger the
+		// graceful-shutdown path for the subsystems that were already started
+		// (metricstore checkpoint, archiver flush, taskmanager) by cancelling the
+		// context the signal handler waits on, then wait for it to finish so we
+		// don't exit before the final checkpoint is written.
+		cancel()
+		<-waitDone
+		return runErr
+	}
+
 	cclog.Print("Graceful shutdown completed!")
 	return nil
 }
@@ -536,6 +555,43 @@ func run() error {
 		return err
 	}
 
+	// Handle checkpoint cleanup
+	if flagCleanupCheckpoints {
+		mscfg := ccconf.GetPackageConfig("metric-store")
+		if mscfg == nil {
+			return fmt.Errorf("metric-store configuration required for checkpoint cleanup")
+		}
+		if err := json.Unmarshal(mscfg, &metricstore.Keys); err != nil {
+			return fmt.Errorf("decoding metric-store config: %w", err)
+		}
+		if metricstore.Keys.NumWorkers <= 0 {
+			metricstore.Keys.NumWorkers = min(goruntime.NumCPU()/2+1, metricstore.DefaultMaxWorkers)
+		}
+
+		d, err := time.ParseDuration(metricstore.Keys.RetentionInMemory)
+		if err != nil {
+			return fmt.Errorf("parsing retention-in-memory: %w", err)
+		}
+		from := time.Now().Add(-d)
+		deleteMode := metricstore.Keys.Cleanup == nil || metricstore.Keys.Cleanup.Mode != "archive"
+		cleanupDir := ""
+		if !deleteMode {
+			cleanupDir = metricstore.Keys.Cleanup.RootDir
+		}
+
+		cclog.Infof("Cleaning up checkpoints older than %s...", from.Format(time.RFC3339))
+		n, err := metricstore.CleanupCheckpoints(
+			metricstore.Keys.Checkpoints.RootDir, cleanupDir, from.Unix(), deleteMode)
+		if err != nil {
+			return fmt.Errorf("checkpoint cleanup: %w", err)
+		}
+		if deleteMode {
+			cclog.Exitf("Cleanup done: %d checkpoint files deleted.", n)
+		} else {
+			cclog.Exitf("Cleanup done: %d checkpoint files archived to parquet.", n)
+		}
+	}
+
 	// Exit if start server is not requested
 	if !flagServer {
 		cclog.Exit("No errors, server flag not set. Exiting cc-backend.")

cmd/cc-backend/server.go

@@ -18,10 +18,12 @@ import (
 	"net/http"
 	"os"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/99designs/gqlgen/graphql"
 	"github.com/99designs/gqlgen/graphql/handler"
+	"github.com/99designs/gqlgen/graphql/handler/extension"
 	"github.com/99designs/gqlgen/graphql/handler/transport"
 	"github.com/99designs/gqlgen/graphql/playground"
 	"github.com/ClusterCockpit/cc-backend/internal/api"
@@ -49,6 +51,12 @@ const (
 	envDebug = "DEBUG"
 )
 
+// maxQueryComplexity bounds the cost of a single GraphQL query to mitigate
+// denial-of-service via deeply nested or heavily aliased queries. The default
+// per-field cost is 1, so this leaves ample headroom for legitimate dashboard
+// queries while rejecting pathological ones.
+const maxQueryComplexity = 5000
+
 // Server encapsulates the HTTP server state and dependencies
 type Server struct {
 	router        chi.Router
@@ -89,6 +97,7 @@ func (s *Server) init() error {
 		generated.NewExecutableSchema(generated.Config{Resolvers: resolver}))
 
 	graphQLServer.AddTransport(transport.POST{})
+	graphQLServer.Use(extension.FixedComplexityLimit(maxQueryComplexity))
 
 	// Inject a per-request stats cache so that grouped statistics queries
 	// sharing the same (filter, groupBy) pair are executed only once.
@@ -128,11 +137,49 @@ func (s *Server) init() error {
 	s.router.Use(middleware.Compress(5))
 	s.router.Use(middleware.Recoverer)
 	s.router.Use(cors.Handler(cors.Options{
-		AllowCredentials: true,
+		AllowCredentials: false,
 		AllowedHeaders:   []string{"X-Requested-With", "Content-Type", "Authorization", "Origin"},
 		AllowedMethods:   []string{"GET", "POST", "HEAD", "OPTIONS"},
 		AllowedOrigins:   []string{"*"},
 	}))
+	s.router.Use(func(next http.Handler) http.Handler {
+		return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
+			h := rw.Header()
+			h.Set("X-Content-Type-Options", "nosniff")
+			h.Set("X-Frame-Options", "DENY")
+			h.Set("Referrer-Policy", "same-origin")
+			// Conservative CSP: blocks clickjacking (frame-ancestors), plugin
+			// content (object-src) and <base> injection (base-uri) without
+			// restricting scripts/styles, so it cannot break the self-hosted
+			// SPA which relies on inline scripts. A full script-src policy needs
+			// per-template nonces and should be added separately.
+			h.Set("Content-Security-Policy", "frame-ancestors 'none'; object-src 'none'; base-uri 'self'")
+			if r.TLS != nil || r.Header.Get("X-Forwarded-Proto") == "https" {
+				h.Set("Strict-Transport-Security", "max-age=31536000; includeSubDomains")
+			}
+			next.ServeHTTP(rw, r)
+		})
+	})
+
+	// CSRF defense-in-depth on top of the SameSite=Lax session cookie: reject
+	// cross-site state-changing requests. Modern browsers set Sec-Fetch-Site on
+	// every request, so this stops a malicious site from driving cookie-
+	// authenticated POST/PUT/DELETE/PATCH calls. It fails open when the header is
+	// absent or not "cross-site", so non-browser API clients and the same-origin
+	// SPA are unaffected.
+	s.router.Use(func(next http.Handler) http.Handler {
+		return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
+			switch r.Method {
+			case http.MethodGet, http.MethodHead, http.MethodOptions, http.MethodTrace:
+			default:
+				if r.Header.Get("Sec-Fetch-Site") == "cross-site" {
+					http.Error(rw, "cross-site request blocked", http.StatusForbidden)
+					return
+				}
+			}
+			next.ServeHTTP(rw, r)
+		})
+	})
 
 	s.restAPIHandle = api.New()
 
@@ -344,20 +391,20 @@ func (s *Server) init() error {
 
 // Server timeout defaults (in seconds)
 const (
-	defaultReadTimeout  = 20
-	defaultWriteTimeout = 20
+	defaultReadHeaderTimeout = 20
+	defaultWriteTimeout      = 20
 )
 
 func (s *Server) Start(ctx context.Context) error {
 	// Use configurable timeouts with defaults
-	readTimeout := time.Duration(defaultReadTimeout) * time.Second
+	readHeaderTimeout := time.Duration(defaultReadHeaderTimeout) * time.Second
 	writeTimeout := time.Duration(defaultWriteTimeout) * time.Second
 
 	s.server = &http.Server{
-		ReadTimeout:  readTimeout,
-		WriteTimeout: writeTimeout,
-		Handler:      s.router,
-		Addr:         config.Keys.Addr,
+		ReadHeaderTimeout: readHeaderTimeout,
+		WriteTimeout:      writeTimeout,
+		Handler:           s.router,
+		Addr:              config.Keys.Addr,
 	}
 
 	// Start http or https server
@@ -368,7 +415,9 @@ func (s *Server) Start(ctx context.Context) error {
 
 	if !strings.HasSuffix(config.Keys.Addr, ":80") && config.Keys.RedirectHTTPTo != "" {
 		go func() {
-			http.ListenAndServe(":80", http.RedirectHandler(config.Keys.RedirectHTTPTo, http.StatusMovedPermanently))
+			if err := http.ListenAndServe(":80", http.RedirectHandler(config.Keys.RedirectHTTPTo, http.StatusMovedPermanently)); err != nil {
+				cclog.Errorf("HTTP-to-HTTPS redirect listener on :80 failed: %v", err)
+			}
 		}()
 	}
 
@@ -399,16 +448,6 @@ func (s *Server) Start(ctx context.Context) error {
 		return fmt.Errorf("dropping privileges: %w", err)
 	}
 
-	// Handle context cancellation for graceful shutdown
-	go func() {
-		<-ctx.Done()
-		shutdownCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
-		defer cancel()
-		if err := s.server.Shutdown(shutdownCtx); err != nil {
-			cclog.Errorf("Server shutdown error: %v", err)
-		}
-	}()
-
 	if err = s.server.Serve(listener); err != nil && err != http.ErrServerClosed {
 		return fmt.Errorf("server failed: %w", err)
 	}
@@ -416,29 +455,58 @@ func (s *Server) Start(ctx context.Context) error {
 }
 
 func (s *Server) Shutdown(ctx context.Context) {
-	// Create a shutdown context with timeout
-	shutdownCtx, cancel := context.WithTimeout(ctx, 30*time.Second)
-	defer cancel()
+	shutdownStart := time.Now()
 
+	natsStart := time.Now()
 	nc := nats.GetClient()
 	if nc != nil {
 		nc.Close()
 	}
+	// Stop the NATS API worker goroutines after the client is closed (no more
+	// subscription callbacks can enqueue once the connection is down).
+	if s.natsAPIHandle != nil {
+		s.natsAPIHandle.Shutdown()
+	}
+	cclog.Infof("Shutdown: NATS closed (%v)", time.Since(natsStart))
 
-	// First shut down the server gracefully (waiting for all ongoing requests)
+	httpStart := time.Now()
+	shutdownCtx, cancel := context.WithTimeout(ctx, 60*time.Second)
+	defer cancel()
 	if err := s.server.Shutdown(shutdownCtx); err != nil {
 		cclog.Errorf("Server shutdown error: %v", err)
 	}
+	cclog.Infof("Shutdown: HTTP server stopped (%v)", time.Since(httpStart))
 
-	// Archive all the metric store data
-	ms := metricstore.GetMemoryStore()
+	// Run metricstore and archiver shutdown concurrently.
+	// They are independent: metricstore writes .bin snapshots,
+	// archiver flushes pending job archives.
+	storeStart := time.Now()
+	done := make(chan struct{})
+	go func() {
+		defer close(done)
+		var wg sync.WaitGroup
 
-	if ms != nil {
-		metricstore.Shutdown()
+		if ms := metricstore.GetMemoryStore(); ms != nil {
+			wg.Go(func() {
+				metricstore.Shutdown()
+			})
+		}
+
+		wg.Go(func() {
+			if err := archiver.Shutdown(60 * time.Second); err != nil {
+				cclog.Warnf("Archiver shutdown: %v", err)
+			}
+		})
+
+		wg.Wait()
+	}()
+
+	select {
+	case <-done:
+		cclog.Infof("Shutdown: metricstore + archiver completed (%v)", time.Since(storeStart))
+	case <-time.After(60 * time.Second):
+		cclog.Warnf("Shutdown deadline exceeded after %v, forcing exit", time.Since(shutdownStart))
 	}
 
-	// Shutdown archiver with 10 second timeout for fast shutdown
-	if err := archiver.Shutdown(10 * time.Second); err != nil {
-		cclog.Warnf("Archiver shutdown: %v", err)
-	}
+	cclog.Infof("Shutdown: total time %v", time.Since(shutdownStart))
 }

go.mod

@@ -8,19 +8,19 @@ tool (
 )
 
 require (
-	github.com/99designs/gqlgen v0.17.88
-	github.com/ClusterCockpit/cc-lib/v2 v2.10.0
+	github.com/99designs/gqlgen v0.17.90
+	github.com/ClusterCockpit/cc-lib/v2 v2.12.0
 	github.com/ClusterCockpit/cc-line-protocol/v2 v2.4.0
 	github.com/Masterminds/squirrel v1.5.4
-	github.com/aws/aws-sdk-go-v2 v1.41.4
-	github.com/aws/aws-sdk-go-v2/config v1.32.12
-	github.com/aws/aws-sdk-go-v2/credentials v1.19.12
-	github.com/aws/aws-sdk-go-v2/service/s3 v1.97.1
-	github.com/coreos/go-oidc/v3 v3.17.0
+	github.com/aws/aws-sdk-go-v2 v1.41.7
+	github.com/aws/aws-sdk-go-v2/config v1.32.18
+	github.com/aws/aws-sdk-go-v2/credentials v1.19.17
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.102.0
+	github.com/coreos/go-oidc/v3 v3.18.0
 	github.com/expr-lang/expr v1.17.8
-	github.com/go-chi/chi/v5 v5.2.5
+	github.com/go-chi/chi/v5 v5.3.0
 	github.com/go-chi/cors v1.2.2
-	github.com/go-co-op/gocron/v2 v2.19.1
+	github.com/go-co-op/gocron/v2 v2.21.2
 	github.com/go-ldap/ldap/v3 v3.4.13
 	github.com/golang-jwt/jwt/v5 v5.3.1
 	github.com/golang-migrate/migrate/v4 v4.19.1
@@ -28,56 +28,55 @@ require (
 	github.com/gorilla/sessions v1.4.0
 	github.com/jmoiron/sqlx v1.4.0
 	github.com/joho/godotenv v1.5.1
-	github.com/mattn/go-sqlite3 v1.14.37
-	github.com/parquet-go/parquet-go v0.29.0
+	github.com/mattn/go-sqlite3 v1.14.44
+	github.com/parquet-go/parquet-go v0.30.1
 	github.com/qustavo/sqlhooks/v2 v2.1.0
 	github.com/santhosh-tekuri/jsonschema/v5 v5.3.1
 	github.com/stretchr/testify v1.11.1
 	github.com/swaggo/http-swagger v1.3.4
 	github.com/swaggo/swag v1.16.6
-	github.com/vektah/gqlparser/v2 v2.5.32
-	golang.org/x/crypto v0.49.0
+	github.com/vektah/gqlparser/v2 v2.5.33
+	golang.org/x/crypto v0.52.0
 	golang.org/x/oauth2 v0.36.0
 	golang.org/x/sync v0.20.0
 	golang.org/x/time v0.15.0
 )
 
 require (
-	github.com/Azure/go-ntlmssp v0.1.0 // indirect
+	github.com/Azure/go-ntlmssp v0.1.1 // indirect
 	github.com/KyleBanks/depth v1.2.1 // indirect
 	github.com/agnivade/levenshtein v1.2.1 // indirect
-	github.com/andybalholm/brotli v1.2.0 // indirect
+	github.com/andybalholm/brotli v1.2.1 // indirect
 	github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect
-	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.7 // indirect
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.20 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.20 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.20 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.21 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.12 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.20 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.20 // indirect
-	github.com/aws/aws-sdk-go-v2/service/signin v1.0.8 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.30.13 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.17 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.41.9 // indirect
-	github.com/aws/smithy-go v1.24.2 // indirect
+	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.23 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.16 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23 // indirect
+	github.com/aws/aws-sdk-go-v2/service/signin v1.0.11 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.30.17 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.36.0 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.42.1 // indirect
+	github.com/aws/smithy-go v1.26.0 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.7 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
-	github.com/fsnotify/fsnotify v1.9.0 // indirect
+	github.com/fsnotify/fsnotify v1.10.1 // indirect
 	github.com/go-asn1-ber/asn1-ber v1.5.8-0.20250403174932-29230038a667 // indirect
-	github.com/go-jose/go-jose/v4 v4.1.3 // indirect
-	github.com/go-openapi/jsonpointer v0.22.5 // indirect
+	github.com/go-jose/go-jose/v4 v4.1.4 // indirect
+	github.com/go-openapi/jsonpointer v0.23.1 // indirect
 	github.com/go-openapi/jsonreference v0.21.5 // indirect
 	github.com/go-openapi/spec v0.22.4 // indirect
-	github.com/go-openapi/swag/conv v0.25.5 // indirect
-	github.com/go-openapi/swag/jsonname v0.25.5 // indirect
-	github.com/go-openapi/swag/jsonutils v0.25.5 // indirect
-	github.com/go-openapi/swag/loading v0.25.5 // indirect
-	github.com/go-openapi/swag/stringutils v0.25.5 // indirect
-	github.com/go-openapi/swag/typeutils v0.25.5 // indirect
-	github.com/go-openapi/swag/yamlutils v0.25.5 // indirect
+	github.com/go-openapi/swag/conv v0.26.0 // indirect
+	github.com/go-openapi/swag/jsonname v0.26.0 // indirect
+	github.com/go-openapi/swag/jsonutils v0.26.0 // indirect
+	github.com/go-openapi/swag/loading v0.26.0 // indirect
+	github.com/go-openapi/swag/stringutils v0.26.0 // indirect
+	github.com/go-openapi/swag/typeutils v0.26.0 // indirect
+	github.com/go-openapi/swag/yamlutils v0.26.0 // indirect
 	github.com/go-viper/mapstructure/v2 v2.5.0 // indirect
 	github.com/goccy/go-yaml v1.19.2 // indirect
 	github.com/google/uuid v1.6.0 // indirect
@@ -87,36 +86,36 @@ require (
 	github.com/influxdata/influxdb-client-go/v2 v2.14.0 // indirect
 	github.com/influxdata/line-protocol v0.0.0-20210922203350-b1ad95c89adf // indirect
 	github.com/jonboulle/clockwork v0.5.0 // indirect
-	github.com/klauspost/compress v1.18.4 // indirect
+	github.com/klauspost/compress v1.18.6 // indirect
 	github.com/kr/pretty v0.3.1 // indirect
 	github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect
 	github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect
-	github.com/nats-io/nats.go v1.49.0 // indirect
+	github.com/nats-io/nats.go v1.52.0 // indirect
 	github.com/nats-io/nkeys v0.4.15 // indirect
 	github.com/nats-io/nuid v1.0.1 // indirect
-	github.com/oapi-codegen/runtime v1.3.0 // indirect
+	github.com/oapi-codegen/runtime v1.4.1 // indirect
 	github.com/parquet-go/bitpack v1.0.0 // indirect
-	github.com/parquet-go/jsonlite v1.5.0 // indirect
+	github.com/parquet-go/jsonlite v1.5.2 // indirect
 	github.com/pierrec/lz4/v4 v4.1.26 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/robfig/cron/v3 v3.0.1 // indirect
 	github.com/rogpeppe/go-internal v1.10.0 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/sosodev/duration v1.4.0 // indirect
-	github.com/stmcginnis/gofish v0.21.4 // indirect
+	github.com/stmcginnis/gofish v0.22.0 // indirect
 	github.com/stretchr/objx v0.5.2 // indirect
 	github.com/swaggo/files v1.0.1 // indirect
 	github.com/twpayne/go-geom v1.6.1 // indirect
 	github.com/urfave/cli/v2 v2.27.7 // indirect
-	github.com/urfave/cli/v3 v3.7.0 // indirect
+	github.com/urfave/cli/v3 v3.8.0 // indirect
 	github.com/xrash/smetrics v0.0.0-20250705151800-55b8f293f342 // indirect
 	go.yaml.in/yaml/v2 v2.4.4 // indirect
 	go.yaml.in/yaml/v3 v3.0.4 // indirect
-	golang.org/x/mod v0.34.0 // indirect
-	golang.org/x/net v0.52.0 // indirect
-	golang.org/x/sys v0.42.0 // indirect
-	golang.org/x/text v0.35.0 // indirect
-	golang.org/x/tools v0.43.0 // indirect
+	golang.org/x/mod v0.36.0 // indirect
+	golang.org/x/net v0.55.0 // indirect
+	golang.org/x/sys v0.45.0 // indirect
+	golang.org/x/text v0.37.0 // indirect
+	golang.org/x/tools v0.45.0 // indirect
 	google.golang.org/protobuf v1.36.11 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	sigs.k8s.io/yaml v1.6.0 // indirect

go.sum

@@ -1,11 +1,11 @@
 filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA=
 filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
-github.com/99designs/gqlgen v0.17.88 h1:neMQDgehMwT1vYIOx/w5ZYPUU/iMNAJzRO44I5Intoc=
-github.com/99designs/gqlgen v0.17.88/go.mod h1:qeqYFEgOeSKqWedOjogPizimp2iu4E23bdPvl4jTYic=
-github.com/Azure/go-ntlmssp v0.1.0 h1:DjFo6YtWzNqNvQdrwEyr/e4nhU3vRiwenz5QX7sFz+A=
-github.com/Azure/go-ntlmssp v0.1.0/go.mod h1:NYqdhxd/8aAct/s4qSYZEerdPuH1liG2/X9DiVTbhpk=
-github.com/ClusterCockpit/cc-lib/v2 v2.10.0 h1:McCcEwc1j942hV54JAzsB/pcArP6A5FoQtjzW2H7K9U=
-github.com/ClusterCockpit/cc-lib/v2 v2.10.0/go.mod h1:Oj+N2lpFqiBOBzjfrLIGJ2YSWT400TX4M0ii4lNl81A=
+github.com/99designs/gqlgen v0.17.90 h1:wSv6blm/PoplU6QoNw83EcQpNtC0HX3/+44vITJOzpk=
+github.com/99designs/gqlgen v0.17.90/go.mod h1:GqYrEwYsqCG8VaOsq2kJUCUKwAE1T+u2i+Nj7NtXiVI=
+github.com/Azure/go-ntlmssp v0.1.1 h1:l+FM/EEMb0U9QZE7mKNEDw5Mu3mFiaa2GKOoTSsNDPw=
+github.com/Azure/go-ntlmssp v0.1.1/go.mod h1:NYqdhxd/8aAct/s4qSYZEerdPuH1liG2/X9DiVTbhpk=
+github.com/ClusterCockpit/cc-lib/v2 v2.12.0 h1:ZbGD68nDniuvzFjJCdyYawpCBrabdSyWOg5FFSyFbjQ=
+github.com/ClusterCockpit/cc-lib/v2 v2.12.0/go.mod h1:ml8xtcYa5WhPM7JDQ+M9/R9ZBxITCR/5xqGJ//GxXJI=
 github.com/ClusterCockpit/cc-line-protocol/v2 v2.4.0 h1:hIzxgTBWcmCIHtoDKDkSCsKCOCOwUC34sFsbD2wcW0Q=
 github.com/ClusterCockpit/cc-line-protocol/v2 v2.4.0/go.mod h1:y42qUu+YFmu5fdNuUAS4VbbIKxVjxCvbVqFdpdh8ahY=
 github.com/DATA-DOG/go-sqlmock v1.5.2 h1:OcvFkGmslmlZibjAjaHm3L//6LiuBgolP7OputlJIzU=
@@ -16,8 +16,8 @@ github.com/Masterminds/squirrel v1.5.4 h1:uUcX/aBc8O7Fg9kaISIUsHXdKuqehiXAMQTYX8
 github.com/Masterminds/squirrel v1.5.4/go.mod h1:NNaOrjSoIDfDA40n7sr2tPNZRfjzjA400rg+riTZj10=
 github.com/NVIDIA/go-nvml v0.13.0-1 h1:OLX8Jq3dONuPOQPC7rndB6+iDmDakw0XTYgzMxObkEw=
 github.com/NVIDIA/go-nvml v0.13.0-1/go.mod h1:+KNA7c7gIBH7SKSJ1ntlwkfN80zdx8ovl4hrK3LmPt4=
-github.com/PuerkitoBio/goquery v1.11.0 h1:jZ7pwMQXIITcUXNH83LLk+txlaEy6NVOfTuP43xxfqw=
-github.com/PuerkitoBio/goquery v1.11.0/go.mod h1:wQHgxUOU3JGuj3oD/QFfxUdlzW6xPHfqyHre6VMY4DQ=
+github.com/PuerkitoBio/goquery v1.12.0 h1:pAcL4g3WRXekcB9AU/y1mbKez2dbY2AajVhtkO8RIBo=
+github.com/PuerkitoBio/goquery v1.12.0/go.mod h1:802ej+gV2y7bbIhOIoPY5sT183ZW0YFofScC4q/hIpQ=
 github.com/RaveNoX/go-jsoncommentstrip v1.0.0/go.mod h1:78ihd09MekBnJnxpICcwzCMzGrKSKYe4AqU6PDYYpjk=
 github.com/agnivade/levenshtein v1.2.1 h1:EHBY3UOn1gwdy/VbFwgo4cxecRznFk7fKWN1KOX7eoM=
 github.com/agnivade/levenshtein v1.2.1/go.mod h1:QVVI16kDrtSuwcpd0p1+xMC6Z/VfhtCyDIjcwga4/DU=
@@ -29,61 +29,61 @@ github.com/alexbrainman/sspi v0.0.0-20250919150558-7d374ff0d59e h1:4dAU9FXIyQktp
 github.com/alexbrainman/sspi v0.0.0-20250919150558-7d374ff0d59e/go.mod h1:cEWa1LVoE5KvSD9ONXsZrj0z6KqySlCCNKHlLzbqAt4=
 github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883 h1:bvNMNQO63//z+xNgfBlViaCIJKLlCJ6/fmUseuG0wVQ=
 github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8=
-github.com/andybalholm/brotli v1.2.0 h1:ukwgCxwYrmACq68yiUqwIWnGY0cTPox/M94sVwToPjQ=
-github.com/andybalholm/brotli v1.2.0/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY=
+github.com/andybalholm/brotli v1.2.1 h1:R+f5xP285VArJDRgowrfb9DqL18yVK0gKAW/F+eTWro=
+github.com/andybalholm/brotli v1.2.1/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY=
 github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kktS1LM=
 github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA=
-github.com/antithesishq/antithesis-sdk-go v0.5.0-default-no-op h1:Ucf+QxEKMbPogRO5guBNe5cgd9uZgfoJLOYs8WWhtjM=
-github.com/antithesishq/antithesis-sdk-go v0.5.0-default-no-op/go.mod h1:IUpT2DPAKh6i/YhSbt6Gl3v2yvUZjmKncl7U91fup7E=
+github.com/antithesishq/antithesis-sdk-go v0.6.0-default-no-op h1:kpBdlEPbRvff0mDD1gk7o9BhI16b9p5yYAXRlidpqJE=
+github.com/antithesishq/antithesis-sdk-go v0.6.0-default-no-op/go.mod h1:IUpT2DPAKh6i/YhSbt6Gl3v2yvUZjmKncl7U91fup7E=
 github.com/apapsch/go-jsonmerge/v2 v2.0.0 h1:axGnT1gRIfimI7gJifB699GoE/oq+F2MU7Dml6nw9rQ=
 github.com/apapsch/go-jsonmerge/v2 v2.0.0/go.mod h1:lvDnEdqiQrp0O42VQGgmlKpxL1AP2+08jFMw88y4klk=
 github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0 h1:jfIu9sQUG6Ig+0+Ap1h4unLjW6YQJpKZVmUzxsD4E/Q=
 github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0/go.mod h1:t2tdKJDJF9BV14lnkjHmOQgcvEKgtqs5a1N3LNdJhGE=
-github.com/aws/aws-sdk-go-v2 v1.41.4 h1:10f50G7WyU02T56ox1wWXq+zTX9I1zxG46HYuG1hH/k=
-github.com/aws/aws-sdk-go-v2 v1.41.4/go.mod h1:mwsPRE8ceUUpiTgF7QmQIJ7lgsKUPQOUl3o72QBrE1o=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.7 h1:3kGOqnh1pPeddVa/E37XNTaWJ8W6vrbYV9lJEkCnhuY=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.7/go.mod h1:lyw7GFp3qENLh7kwzf7iMzAxDn+NzjXEAGjKS2UOKqI=
-github.com/aws/aws-sdk-go-v2/config v1.32.12 h1:O3csC7HUGn2895eNrLytOJQdoL2xyJy0iYXhoZ1OmP0=
-github.com/aws/aws-sdk-go-v2/config v1.32.12/go.mod h1:96zTvoOFR4FURjI+/5wY1vc1ABceROO4lWgWJuxgy0g=
-github.com/aws/aws-sdk-go-v2/credentials v1.19.12 h1:oqtA6v+y5fZg//tcTWahyN9PEn5eDU/Wpvc2+kJ4aY8=
-github.com/aws/aws-sdk-go-v2/credentials v1.19.12/go.mod h1:U3R1RtSHx6NB0DvEQFGyf/0sbrpJrluENHdPy1j/3TE=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.20 h1:zOgq3uezl5nznfoK3ODuqbhVg1JzAGDUhXOsU0IDCAo=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.20/go.mod h1:z/MVwUARehy6GAg/yQ1GO2IMl0k++cu1ohP9zo887wE=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.20 h1:CNXO7mvgThFGqOFgbNAP2nol2qAWBOGfqR/7tQlvLmc=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.20/go.mod h1:oydPDJKcfMhgfcgBUZaG+toBbwy8yPWubJXBVERtI4o=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.20 h1:tN6W/hg+pkM+tf9XDkWUbDEjGLb+raoBMFsTodcoYKw=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.20/go.mod h1:YJ898MhD067hSHA6xYCx5ts/jEd8BSOLtQDL3iZsvbc=
-github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 h1:qYQ4pzQ2Oz6WpQ8T3HvGHnZydA72MnLuFK9tJwmrbHw=
-github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6/go.mod h1:O3h0IK87yXci+kg6flUKzJnWeziQUKciKrLjcatSNcY=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.21 h1:SwGMTMLIlvDNyhMteQ6r8IJSBPlRdXX5d4idhIGbkXA=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.21/go.mod h1:UUxgWxofmOdAMuqEsSppbDtGKLfR04HGsD0HXzvhI1k=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 h1:5EniKhLZe4xzL7a+fU3C2tfUN4nWIqlLesfrjkuPFTY=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7/go.mod h1:x0nZssQ3qZSnIcePWLvcoFisRXJzcTVvYpAAdYX8+GI=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.12 h1:qtJZ70afD3ISKWnoX3xB0J2otEqu3LqicRcDBqsj0hQ=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.12/go.mod h1:v2pNpJbRNl4vEUWEh5ytQok0zACAKfdmKS51Hotc3pQ=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.20 h1:2HvVAIq+YqgGotK6EkMf+KIEqTISmTYh5zLpYyeTo1Y=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.20/go.mod h1:V4X406Y666khGa8ghKmphma/7C0DAtEQYhkq9z4vpbk=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.20 h1:siU1A6xjUZ2N8zjTHSXFhB9L/2OY8Dqs0xXiLjF30jA=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.20/go.mod h1:4TLZCmVJDM3FOu5P5TJP0zOlu9zWgDWU7aUxWbr+rcw=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.97.1 h1:csi9NLpFZXb9fxY7rS1xVzgPRGMt7MSNWeQ6eo247kE=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.97.1/go.mod h1:qXVal5H0ChqXP63t6jze5LmFalc7+ZE7wOdLtZ0LCP0=
-github.com/aws/aws-sdk-go-v2/service/signin v1.0.8 h1:0GFOLzEbOyZABS3PhYfBIx2rNBACYcKty+XGkTgw1ow=
-github.com/aws/aws-sdk-go-v2/service/signin v1.0.8/go.mod h1:LXypKvk85AROkKhOG6/YEcHFPoX+prKTowKnVdcaIxE=
-github.com/aws/aws-sdk-go-v2/service/sso v1.30.13 h1:kiIDLZ005EcKomYYITtfsjn7dtOwHDOFy7IbPXKek2o=
-github.com/aws/aws-sdk-go-v2/service/sso v1.30.13/go.mod h1:2h/xGEowcW/g38g06g3KpRWDlT+OTfxxI0o1KqayAB8=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.17 h1:jzKAXIlhZhJbnYwHbvUQZEB8KfgAEuG0dc08Bkda7NU=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.17/go.mod h1:Al9fFsXjv4KfbzQHGe6V4NZSZQXecFcvaIF4e70FoRA=
-github.com/aws/aws-sdk-go-v2/service/sts v1.41.9 h1:Cng+OOwCHmFljXIxpEVXAGMnBia8MSU6Ch5i9PgBkcU=
-github.com/aws/aws-sdk-go-v2/service/sts v1.41.9/go.mod h1:LrlIndBDdjA/EeXeyNBle+gyCwTlizzW5ycgWnvIxkk=
-github.com/aws/smithy-go v1.24.2 h1:FzA3bu/nt/vDvmnkg+R8Xl46gmzEDam6mZ1hzmwXFng=
-github.com/aws/smithy-go v1.24.2/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
+github.com/aws/aws-sdk-go-v2 v1.41.7 h1:DWpAJt66FmnnaRIOT/8ASTucrvuDPZASqhhLey6tLY8=
+github.com/aws/aws-sdk-go-v2 v1.41.7/go.mod h1:4LAfZOPHNVNQEckOACQx60Y8pSRjIkNZQz1w92xpMJc=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 h1:gx1AwW1Iyk9Z9dD9F4akX5gnN3QZwUB20GGKH/I+Rho=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10/go.mod h1:qqY157uZoqm5OXq/amuaBJyC9hgBCBQnsaWnPe905GY=
+github.com/aws/aws-sdk-go-v2/config v1.32.18 h1:Hcia46bxhGgF3BaSnG8nSNCWmqTK6bj9xN9/FJ3WK6Q=
+github.com/aws/aws-sdk-go-v2/config v1.32.18/go.mod h1:zEjCAYmxqDadH1WX8CdBvmLKhUEUVFgKRQG38zjDmrY=
+github.com/aws/aws-sdk-go-v2/credentials v1.19.17 h1:gP2nkGsS+KMvF/jfFz2Vv2qiiOqWKyPACSzPsqHgoW8=
+github.com/aws/aws-sdk-go-v2/credentials v1.19.17/go.mod h1:Bsew3S/moG5iT77giPj1q8wb/s0RE5/QfH+ASjYtuQc=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.23 h1:UuSfcORqNSz/ey3VPRS8TcVH2Ikf0/sC+Hdj400QI6U=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.23/go.mod h1:+G/OSGiOFnSOkYloKj/9M35s74LgVAdJBSD5lsFfqKg=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 h1:GpT/TrnBYuE5gan2cZbTtvP+JlHsutdmlV2YfEyNde0=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23/go.mod h1:xYWD6BS9ywC5bS3sz9Xh04whO/hzK2plt2Zkyrp4JuA=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 h1:bpd8vxhlQi2r1hiueOw02f/duEPTMK59Q4QMAoTTtTo=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23/go.mod h1:15DfR2nw+CRHIk0tqNyifu3G1YdAOy68RftkhMDDwYk=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24 h1:OQqn11BtaYv1WLUowvcA30MpzIu8Ti4pcLPIIyoKZrA=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24/go.mod h1:X5ZJyfwVrWA96GzPmUCWFQaEARPR7gCrpq2E92PJwAE=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9 h1:FLudkZLt5ci0ozzgkVo8BJGwvqNaZbTWb3UcucAateA=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9/go.mod h1:w7wZ/s9qK7c8g4al+UyoF1Sp/Z45UwMGcqIzLWVQHWk=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.16 h1:tX68nPDCoX0s2ksM7CipWP0QFw2hGDWwUdxI6+eT9ZU=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.16/go.mod h1:e3IzZvQ3kAWNykvE0Tr0RDZCMFInMvhku3qNpcIQXhM=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23 h1:pbrxO/kuIwgEsOPLkaHu0O+m4fNgLU8B3vxQ+72jTPw=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23/go.mod h1:/CMNUqoj46HpS3MNRDEDIwcgEnrtZlKRaHNaHxIFpNA=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23 h1:03xatSQO4+AM1lTAbnRg5OK528EUg744nW7F73U8DKw=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23/go.mod h1:M8l3mwgx5ToK7wot2sBBce/ojzgnPzZXUV445gTSyE8=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.102.0 h1:gfPQ6do5PZTCc5n/vZUHz/G8McrNrfERGSO+iHvVbCA=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.102.0/go.mod h1:wO6U9egJtCtsZEHG2AAcFf1kUWDRrH0Iif6K3bVmmdE=
+github.com/aws/aws-sdk-go-v2/service/signin v1.0.11 h1:TdJ+HdzOBhU8+iVAOGUTU63VXopcumCOF1paFulHWZc=
+github.com/aws/aws-sdk-go-v2/service/signin v1.0.11/go.mod h1:R82ZRExE/nheo0N+T8zHPcLRTcH8MGsnR3BiVGX0TwI=
+github.com/aws/aws-sdk-go-v2/service/sso v1.30.17 h1:7byT8HUWrgoRp6sXjxtZwgOKfhss5fW6SkLBtqzgRoE=
+github.com/aws/aws-sdk-go-v2/service/sso v1.30.17/go.mod h1:xNWknVi4Ezm1vg1QsB/5EWpAJURq22uqd38U8qKvOJc=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.36.0 h1:nDARhv/oF55bcxF7rCI/4PDxOKnVXVWwDuDwCs2I2SQ=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.36.0/go.mod h1:4vIRDq+CJB2xFAXZ+YgGUTiEft7oAQlhIs71xcSeuVg=
+github.com/aws/aws-sdk-go-v2/service/sts v1.42.1 h1:F/M5Y9I3nwr2IEpshZgh1GeHpOItExNM9L1euNuh/fk=
+github.com/aws/aws-sdk-go-v2/service/sts v1.42.1/go.mod h1:mTNxImtovCOEEuD65mKW7DCsL+2gjEH+RPEAexAzAio=
+github.com/aws/smithy-go v1.26.0 h1:9ouqbi+NyKP7fV3Te7UElCwdAb6Y8uk7LGwPE5tVe/s=
+github.com/aws/smithy-go v1.26.0/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/bmatcuk/doublestar v1.1.1/go.mod h1:UD6OnuiIn0yFxxA2le/rnRU1G4RaI4UvFv1sNto9p6w=
 github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
 github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
-github.com/coreos/go-oidc/v3 v3.17.0 h1:hWBGaQfbi0iVviX4ibC7bk8OKT5qNr4klBaCHVNvehc=
-github.com/coreos/go-oidc/v3 v3.17.0/go.mod h1:wqPbKFrVnE90vty060SB40FCJ8fTHTxSwyXJqZH+sI8=
+github.com/coder/websocket v1.8.14 h1:9L0p0iKiNOibykf283eHkKUHHrpG7f65OE3BhhO7v9g=
+github.com/coder/websocket v1.8.14/go.mod h1:NX3SzP+inril6yawo5CQXx8+fk145lPDC6pumgx0mVg=
+github.com/coreos/go-oidc/v3 v3.18.0 h1:V9orjXynvu5wiC9SemFTWnG4F45v403aIcjWo0d41+A=
+github.com/coreos/go-oidc/v3 v3.18.0/go.mod h1:DYCf24+ncYi+XkIH97GY1+dqoRlbaSI26KVTCI9SrY4=
 github.com/cpuguy83/go-md2man/v2 v2.0.7 h1:zbFlGlXEAKlwXpmvle3d8Oe3YnkKIK4xSRTd3sHPnBo=
 github.com/cpuguy83/go-md2man/v2 v2.0.7/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
@@ -97,47 +97,47 @@ github.com/expr-lang/expr v1.17.8 h1:W1loDTT+0PQf5YteHSTpju2qfUfNoBt4yw9+wOEU9VM
 github.com/expr-lang/expr v1.17.8/go.mod h1:8/vRC7+7HBzESEqt5kKpYXxrxkr31SaO8r40VO/1IT4=
 github.com/frankban/quicktest v1.13.0 h1:yNZif1OkDfNoDfb9zZa9aXIpejNR4F23Wely0c+Qdqk=
 github.com/frankban/quicktest v1.13.0/go.mod h1:qLE0fzW0VuyUAJgPU19zByoIr0HtCHN/r/VLSOOIySU=
-github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k=
-github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
+github.com/fsnotify/fsnotify v1.10.1 h1:b0/UzAf9yR5rhf3RPm9gf3ehBPpf0oZKIjtpKrx59Ho=
+github.com/fsnotify/fsnotify v1.10.1/go.mod h1:TLheqan6HD6GBK6PrDWyDPBaEV8LspOxvPSjC+bVfgo=
 github.com/go-asn1-ber/asn1-ber v1.5.8-0.20250403174932-29230038a667 h1:BP4M0CvQ4S3TGls2FvczZtj5Re/2ZzkV9VwqPHH/3Bo=
 github.com/go-asn1-ber/asn1-ber v1.5.8-0.20250403174932-29230038a667/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
-github.com/go-chi/chi/v5 v5.2.5 h1:Eg4myHZBjyvJmAFjFvWgrqDTXFyOzjj7YIm3L3mu6Ug=
-github.com/go-chi/chi/v5 v5.2.5/go.mod h1:X7Gx4mteadT3eDOMTsXzmI4/rwUpOwBHLpAfupzFJP0=
+github.com/go-chi/chi/v5 v5.3.0 h1:halUjDxhshgXHMrao5bB8eNBXo/rnzwr8m5m36glehM=
+github.com/go-chi/chi/v5 v5.3.0/go.mod h1:R+tYY2hNuVUUjxoPtqUdgBqevM9s9njzkTLutVsOCto=
 github.com/go-chi/cors v1.2.2 h1:Jmey33TE+b+rB7fT8MUy1u0I4L+NARQlK6LhzKPSyQE=
 github.com/go-chi/cors v1.2.2/go.mod h1:sSbTewc+6wYHBBCW7ytsFSn836hqM7JxpglAy2Vzc58=
-github.com/go-co-op/gocron/v2 v2.19.1 h1:B4iLeA0NB/2iO3EKQ7NfKn5KsQgZfjb2fkvoZJU3yBI=
-github.com/go-co-op/gocron/v2 v2.19.1/go.mod h1:5lEiCKk1oVJV39Zg7/YG10OnaVrDAV5GGR6O0663k6U=
-github.com/go-jose/go-jose/v4 v4.1.3 h1:CVLmWDhDVRa6Mi/IgCgaopNosCaHz7zrMeF9MlZRkrs=
-github.com/go-jose/go-jose/v4 v4.1.3/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08=
+github.com/go-co-op/gocron/v2 v2.21.2 h1:bD8/YwkojYHgXFr3iEulL148KBdTbKVxUZzFKpXcdbY=
+github.com/go-co-op/gocron/v2 v2.21.2/go.mod h1:5lEiCKk1oVJV39Zg7/YG10OnaVrDAV5GGR6O0663k6U=
+github.com/go-jose/go-jose/v4 v4.1.4 h1:moDMcTHmvE6Groj34emNPLs/qtYXRVcd6S7NHbHz3kA=
+github.com/go-jose/go-jose/v4 v4.1.4/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08=
 github.com/go-ldap/ldap/v3 v3.4.13 h1:+x1nG9h+MZN7h/lUi5Q3UZ0fJ1GyDQYbPvbuH38baDQ=
 github.com/go-ldap/ldap/v3 v3.4.13/go.mod h1:LxsGZV6vbaK0sIvYfsv47rfh4ca0JXokCoKjZxsszv0=
-github.com/go-openapi/jsonpointer v0.22.5 h1:8on/0Yp4uTb9f4XvTrM2+1CPrV05QPZXu+rvu2o9jcA=
-github.com/go-openapi/jsonpointer v0.22.5/go.mod h1:gyUR3sCvGSWchA2sUBJGluYMbe1zazrYWIkWPjjMUY0=
+github.com/go-openapi/jsonpointer v0.23.1 h1:1HBACs7XIwR2RcmItfdSFlALhGbe6S92p0ry4d1GWg4=
+github.com/go-openapi/jsonpointer v0.23.1/go.mod h1:iWRmZTrGn7XwYhtPt/fvdSFj1OfNBngqRT2UG3BxSqY=
 github.com/go-openapi/jsonreference v0.21.5 h1:6uCGVXU/aNF13AQNggxfysJ+5ZcU4nEAe+pJyVWRdiE=
 github.com/go-openapi/jsonreference v0.21.5/go.mod h1:u25Bw85sX4E2jzFodh1FOKMTZLcfifd1Q+iKKOUxExw=
 github.com/go-openapi/spec v0.22.4 h1:4pxGjipMKu0FzFiu/DPwN3CTBRlVM2yLf/YTWorYfDQ=
 github.com/go-openapi/spec v0.22.4/go.mod h1:WQ6Ai0VPWMZgMT4XySjlRIE6GP1bGQOtEThn3gcWLtQ=
 github.com/go-openapi/swag v0.19.15 h1:D2NRCBzS9/pEY3gP9Nl8aDqGUcPFrwG2p+CNFrLyrCM=
-github.com/go-openapi/swag/conv v0.25.5 h1:wAXBYEXJjoKwE5+vc9YHhpQOFj2JYBMF2DUi+tGu97g=
-github.com/go-openapi/swag/conv v0.25.5/go.mod h1:CuJ1eWvh1c4ORKx7unQnFGyvBbNlRKbnRyAvDvzWA4k=
-github.com/go-openapi/swag/jsonname v0.25.5 h1:8p150i44rv/Drip4vWI3kGi9+4W9TdI3US3uUYSFhSo=
-github.com/go-openapi/swag/jsonname v0.25.5/go.mod h1:jNqqikyiAK56uS7n8sLkdaNY/uq6+D2m2LANat09pKU=
-github.com/go-openapi/swag/jsonutils v0.25.5 h1:XUZF8awQr75MXeC+/iaw5usY/iM7nXPDwdG3Jbl9vYo=
-github.com/go-openapi/swag/jsonutils v0.25.5/go.mod h1:48FXUaz8YsDAA9s5AnaUvAmry1UcLcNVWUjY42XkrN4=
-github.com/go-openapi/swag/jsonutils/fixtures_test v0.25.5 h1:SX6sE4FrGb4sEnnxbFL/25yZBb5Hcg1inLeErd86Y1U=
-github.com/go-openapi/swag/jsonutils/fixtures_test v0.25.5/go.mod h1:/2KvOTrKWjVA5Xli3DZWdMCZDzz3uV/T7bXwrKWPquo=
-github.com/go-openapi/swag/loading v0.25.5 h1:odQ/umlIZ1ZVRteI6ckSrvP6e2w9UTF5qgNdemJHjuU=
-github.com/go-openapi/swag/loading v0.25.5/go.mod h1:I8A8RaaQ4DApxhPSWLNYWh9NvmX2YKMoB9nwvv6oW6g=
-github.com/go-openapi/swag/stringutils v0.25.5 h1:NVkoDOA8YBgtAR/zvCx5rhJKtZF3IzXcDdwOsYzrB6M=
-github.com/go-openapi/swag/stringutils v0.25.5/go.mod h1:PKK8EZdu4QJq8iezt17HM8RXnLAzY7gW0O1KKarrZII=
-github.com/go-openapi/swag/typeutils v0.25.5 h1:EFJ+PCga2HfHGdo8s8VJXEVbeXRCYwzzr9u4rJk7L7E=
-github.com/go-openapi/swag/typeutils v0.25.5/go.mod h1:itmFmScAYE1bSD8C4rS0W+0InZUBrB2xSPbWt6DLGuc=
-github.com/go-openapi/swag/yamlutils v0.25.5 h1:kASCIS+oIeoc55j28T4o8KwlV2S4ZLPT6G0iq2SSbVQ=
-github.com/go-openapi/swag/yamlutils v0.25.5/go.mod h1:Gek1/SjjfbYvM+Iq4QGwa/2lEXde9n2j4a3wI3pNuOQ=
-github.com/go-openapi/testify/enable/yaml/v2 v2.4.0 h1:7SgOMTvJkM8yWrQlU8Jm18VeDPuAvB/xWrdxFJkoFag=
-github.com/go-openapi/testify/enable/yaml/v2 v2.4.0/go.mod h1:14iV8jyyQlinc9StD7w1xVPW3CO3q1Gj04Jy//Kw4VM=
-github.com/go-openapi/testify/v2 v2.4.0 h1:8nsPrHVCWkQ4p8h1EsRVymA2XABB4OT40gcvAu+voFM=
-github.com/go-openapi/testify/v2 v2.4.0/go.mod h1:HCPmvFFnheKK2BuwSA0TbbdxJ3I16pjwMkYkP4Ywn54=
+github.com/go-openapi/swag/conv v0.26.0 h1:5yGGsPYI1ZCva93U0AoKi/iZrNhaJEjr324YVsiD89I=
+github.com/go-openapi/swag/conv v0.26.0/go.mod h1:tpAmIL7X58VPnHHiSO4uE3jBeRamGsFsfdDeDtb5ECE=
+github.com/go-openapi/swag/jsonname v0.26.0 h1:gV1NFX9M8avo0YSpmWogqfQISigCmpaiNci8cGECU5w=
+github.com/go-openapi/swag/jsonname v0.26.0/go.mod h1:urBBR8bZNoDYGr653ynhIx+gTeIz0ARZxHkAPktJK2M=
+github.com/go-openapi/swag/jsonutils v0.26.0 h1:FawFML2iAXsPqmERscuMPIHmFsoP1tOqWkxBaKNMsnA=
+github.com/go-openapi/swag/jsonutils v0.26.0/go.mod h1:2VmA0CJlyFqgawOaPI9psnjFDqzyivIqLYN34t9p91E=
+github.com/go-openapi/swag/jsonutils/fixtures_test v0.26.0 h1:apqeINu/ICHouqiRZbyFvuDge5jCmmLTqGQ9V95EaOM=
+github.com/go-openapi/swag/jsonutils/fixtures_test v0.26.0/go.mod h1:AyM6QT8uz5IdKxk5akv0y6u4QvcL9GWERt0Jx/F/R8Y=
+github.com/go-openapi/swag/loading v0.26.0 h1:Apg6zaKhCJurpJer0DCxq99qwmhFddBhaMX7kilDcko=
+github.com/go-openapi/swag/loading v0.26.0/go.mod h1:dBxQ/6V2uBaAQdevN18VELE6xSpJWZxLX4txe12JwDg=
+github.com/go-openapi/swag/stringutils v0.26.0 h1:qZQngLxs5s7SLijc3N2ZO+fUq2o8LjuWAASSrJuh+xg=
+github.com/go-openapi/swag/stringutils v0.26.0/go.mod h1:sWn5uY+QIIspwPhvgnqJsH8xqFT2ZbYcvbcFanRyhFE=
+github.com/go-openapi/swag/typeutils v0.26.0 h1:2kdEwdiNWy+JJdOvu5MA2IIg2SylWAFuuyQIKYybfq4=
+github.com/go-openapi/swag/typeutils v0.26.0/go.mod h1:oovDuIUvTrEHVMqWilQzKzV4YlSKgyZmFh7AlfABNVE=
+github.com/go-openapi/swag/yamlutils v0.26.0 h1:H7O8l/8NJJQ/oiReEN+oMpnGMyt8G0hl460nRZxhLMQ=
+github.com/go-openapi/swag/yamlutils v0.26.0/go.mod h1:1evKEGAtP37Pkwcc7EWMF0hedX0/x3Rkvei2wtG/TbU=
+github.com/go-openapi/testify/enable/yaml/v2 v2.4.2 h1:5zRca5jw7lzVREKCZVNBpysDNBjj74rBh0N2BGQbSR0=
+github.com/go-openapi/testify/enable/yaml/v2 v2.4.2/go.mod h1:XVevPw5hUXuV+5AkI1u1PeAm27EQVrhXTTCPAF85LmE=
+github.com/go-openapi/testify/v2 v2.4.2 h1:tiByHpvE9uHrrKjOszax7ZvKB7QOgizBWGBLuq0ePx4=
+github.com/go-openapi/testify/v2 v2.4.2/go.mod h1:SgsVHtfooshd0tublTtJ50FPKhujf47YRqauXXOUxfw=
 github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
 github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y=
 github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg=
@@ -151,8 +151,8 @@ github.com/golang-migrate/migrate/v4 v4.19.1 h1:OCyb44lFuQfYXYLx1SCxPZQGU7mcaZ7g
 github.com/golang-migrate/migrate/v4 v4.19.1/go.mod h1:CTcgfjxhaUtsLipnLoQRWCrjYXycRz/g5+RWDuYgPrE=
 github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
 github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
-github.com/google/go-tpm v0.9.7 h1:u89J4tUUeDTlH8xxC3CTW7OHZjbjKoHdQ9W7gCUhtxA=
-github.com/google/go-tpm v0.9.7/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY=
+github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
+github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY=
 github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=
 github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/gops v0.3.29 h1:n98J2qSOK1NJvRjdLDcjgDryjpIBGhbaqph1mXKL0rY=
@@ -198,8 +198,8 @@ github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwA
 github.com/jonboulle/clockwork v0.5.0 h1:Hyh9A8u51kptdkR+cqRpT1EebBwTn1oK9YfGYbdFz6I=
 github.com/jonboulle/clockwork v0.5.0/go.mod h1:3mZlmanh0g2NDKO5TWZVJAfofYk64M7XN3SzBPjZF60=
 github.com/juju/gnuflag v0.0.0-20171113085948-2ce1bb71843d/go.mod h1:2PavIy+JPciBPrBUjwbNvtwB6RQlve+hkpll6QSNmOE=
-github.com/klauspost/compress v1.18.4 h1:RPhnKRAQ4Fh8zU2FY/6ZFDwTVTxgJ/EMydqSTzE9a2c=
-github.com/klauspost/compress v1.18.4/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4=
+github.com/klauspost/compress v1.18.6 h1:2jupLlAwFm95+YDR+NwD2MEfFO9d4z4Prjl1XXDjuao=
+github.com/klauspost/compress v1.18.6/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
 github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
@@ -213,31 +213,33 @@ github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
 github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
 github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
 github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
-github.com/mattn/go-sqlite3 v1.14.37 h1:3DOZp4cXis1cUIpCfXLtmlGolNLp2VEqhiB/PARNBIg=
-github.com/mattn/go-sqlite3 v1.14.37/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
+github.com/mattn/go-sqlite3 v1.14.44 h1:3VSe+xafpbzsLbdr2AWlAZk9yRHiBhTBakioXaCKTF8=
+github.com/mattn/go-sqlite3 v1.14.44/go.mod h1:pjEuOr8IwzLJP2MfGeTb0A35jauH+C2kbHKBr7yXKVQ=
 github.com/minio/highwayhash v1.0.4-0.20251030100505-070ab1a87a76 h1:KGuD/pM2JpL9FAYvBrnBBeENKZNh6eNtjqytV6TYjnk=
 github.com/minio/highwayhash v1.0.4-0.20251030100505-070ab1a87a76/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
-github.com/nats-io/jwt/v2 v2.8.0 h1:K7uzyz50+yGZDO5o772eRE7atlcSEENpL7P+b74JV1g=
-github.com/nats-io/jwt/v2 v2.8.0/go.mod h1:me11pOkwObtcBNR8AiMrUbtVOUGkqYjMQZ6jnSdVUIA=
-github.com/nats-io/nats-server/v2 v2.12.3 h1:KRv+1n7lddMVgkJPQer+pt36TcO0ENxjilBmeWdjcHs=
-github.com/nats-io/nats-server/v2 v2.12.3/go.mod h1:MQXjG9WjyXKz9koWzUc3jYUMKD8x3CLmTNy91IQQz3Y=
-github.com/nats-io/nats.go v1.49.0 h1:yh/WvY59gXqYpgl33ZI+XoVPKyut/IcEaqtsiuTJpoE=
-github.com/nats-io/nats.go v1.49.0/go.mod h1:fDCn3mN5cY8HooHwE2ukiLb4p4G4ImmzvXyJt+tGwdw=
+github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU=
+github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg=
+github.com/nats-io/nats-server/v2 v2.12.7 h1:prQ9cPiWHcnwfT81Wi5lU9LL8TLY+7pxDru6fQYLCQQ=
+github.com/nats-io/nats-server/v2 v2.12.7/go.mod h1:dOnmkprKMluTmTF7/QHZioxlau3sKHUM/LBPy9AiBPw=
+github.com/nats-io/nats.go v1.52.0 h1:n3avV4VBsCgsdwh71TppsTwtv+QdPs7ntSKM8qJLGsc=
+github.com/nats-io/nats.go v1.52.0/go.mod h1:26HypzazeOkyO3/mqd1zZd53STJN0EjCYF9Uy2ZOBno=
 github.com/nats-io/nkeys v0.4.15 h1:JACV5jRVO9V856KOapQ7x+EY8Jo3qw1vJt/9Jpwzkk4=
 github.com/nats-io/nkeys v0.4.15/go.mod h1:CpMchTXC9fxA5zrMo4KpySxNjiDVvr8ANOSZdiNfUrs=
 github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
 github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
-github.com/oapi-codegen/runtime v1.3.0 h1:vyK1zc0gDWWXgk2xoQa4+X4RNNc5SL2RbTpJS/4vMYA=
-github.com/oapi-codegen/runtime v1.3.0/go.mod h1:kOdeacKy7t40Rclb1je37ZLFboFxh+YLy0zaPCMibPY=
+github.com/oapi-codegen/nullable v1.1.0 h1:eAh8JVc5430VtYVnq00Hrbpag9PFRGWLjxR1/3KntMs=
+github.com/oapi-codegen/nullable v1.1.0/go.mod h1:KUZ3vUzkmEKY90ksAmit2+5juDIhIZhfDl+0PwOQlFY=
+github.com/oapi-codegen/runtime v1.4.1 h1:9nwLoI+KrWxzbBcp0jO/R8uXqbik/HUyCvPeU68Y/qo=
+github.com/oapi-codegen/runtime v1.4.1/go.mod h1:GwV7hC2hviaMzj+ITfHVRESK5J2W/GefVwIND/bMGvU=
 github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
 github.com/parquet-go/bitpack v1.0.0 h1:AUqzlKzPPXf2bCdjfj4sTeacrUwsT7NlcYDMUQxPcQA=
 github.com/parquet-go/bitpack v1.0.0/go.mod h1:XnVk9TH+O40eOOmvpAVZ7K2ocQFrQwysLMnc6M/8lgs=
-github.com/parquet-go/jsonlite v1.5.0 h1:ulS7lNWdPwiqDMLzTiXHYmIUhu99mavZh2iAVdXet3g=
-github.com/parquet-go/jsonlite v1.5.0/go.mod h1:nDjpkpL4EOtqs6NQugUsi0Rleq9sW/OtC1NnZEnxzF0=
-github.com/parquet-go/parquet-go v0.29.0 h1:xXlPtFVR51jpSVzf+cgHnNIcb7Xet+iuvkbe0HIm90Y=
-github.com/parquet-go/parquet-go v0.29.0/go.mod h1:navtkAYr2LGoJVp141oXPlO/sxLvaOe3la2JEoD8+rg=
+github.com/parquet-go/jsonlite v1.5.2 h1:8TZzYknFOHUpgjTLf80qbzc+8GdeT/3a3fdXSzhMylE=
+github.com/parquet-go/jsonlite v1.5.2/go.mod h1:nDjpkpL4EOtqs6NQugUsi0Rleq9sW/OtC1NnZEnxzF0=
+github.com/parquet-go/parquet-go v0.30.1 h1:Oy6ganNrAdFiVwy7wNmWagfPTWA2X9Z3tVHBc7JtuX8=
+github.com/parquet-go/parquet-go v0.30.1/go.mod h1:navtkAYr2LGoJVp141oXPlO/sxLvaOe3la2JEoD8+rg=
 github.com/pierrec/lz4/v4 v4.1.26 h1:GrpZw1gZttORinvzBdXPUXATeqlJjqUG/D87TKMnhjY=
 github.com/pierrec/lz4/v4 v4.1.26/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4=
 github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
@@ -252,6 +254,8 @@ github.com/prometheus/common v0.67.5 h1:pIgK94WWlQt1WLwAC5j2ynLaBRDiinoAb86HZHTU
 github.com/prometheus/common v0.67.5/go.mod h1:SjE/0MzDEEAyrdr5Gqc6G+sXI67maCxzaT3A2+HqjUw=
 github.com/prometheus/procfs v0.20.1 h1:XwbrGOIplXW/AU3YhIhLODXMJYyC1isLFfYCsTEycfc=
 github.com/prometheus/procfs v0.20.1/go.mod h1:o9EMBZGRyvDrSPH1RqdxhojkuXstoe4UlK79eF5TGGo=
+github.com/questdb/go-questdb-client/v4 v4.2.0 h1:+d0HJwCjUWMj7zmY6qmhoqTJzTyoYKl+LSTYGN0T8T8=
+github.com/questdb/go-questdb-client/v4 v4.2.0/go.mod h1:/2x93LK1wjM4JX/b5c6q7Yqk22htjWY1lE6p1X8iLbE=
 github.com/qustavo/sqlhooks/v2 v2.1.0 h1:54yBemHnGHp/7xgT+pxwmIlMSDNYKx5JW5dfRAiCZi0=
 github.com/qustavo/sqlhooks/v2 v2.1.0/go.mod h1:aMREyKo7fOKTwiLuWPsaHRXEmtqG4yREztO0idF83AU=
 github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
@@ -268,8 +272,8 @@ github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NF
 github.com/sosodev/duration v1.4.0 h1:35ed0KiVFriGHHzZZJaZLgmTEEICIyt8Sx0RQfj9IjE=
 github.com/sosodev/duration v1.4.0/go.mod h1:RQIBBX0+fMLc/D9+Jb/fwvVmo0eZvDDEERAikUR6SDg=
 github.com/spkg/bom v0.0.0-20160624110644-59b7046e48ad/go.mod h1:qLr4V1qq6nMqFKkMo8ZTx3f+BZEkzsRUY10Xsm2mwU0=
-github.com/stmcginnis/gofish v0.21.4 h1:daexK8sh31CgeSMkPUNs21HWHHA9ecCPJPyLCTxukCg=
-github.com/stmcginnis/gofish v0.21.4/go.mod h1:PzF5i8ecRG9A2ol8XT64npKUunyraJ+7t0kYMpQAtqU=
+github.com/stmcginnis/gofish v0.22.0 h1:OahXohfrIzAXOsWuKDQ7lm/QvdZBg1P2OzFYmbKAd/0=
+github.com/stmcginnis/gofish v0.22.0/go.mod h1:PzF5i8ecRG9A2ol8XT64npKUunyraJ+7t0kYMpQAtqU=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
 github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
@@ -288,10 +292,10 @@ github.com/twpayne/go-geom v1.6.1 h1:iLE+Opv0Ihm/ABIcvQFGIiFBXd76oBIar9drAwHFhR4
 github.com/twpayne/go-geom v1.6.1/go.mod h1:Kr+Nly6BswFsKM5sd31YaoWS5PeDDH2NftJTK7Gd028=
 github.com/urfave/cli/v2 v2.27.7 h1:bH59vdhbjLv3LAvIu6gd0usJHgoTTPhCFib8qqOwXYU=
 github.com/urfave/cli/v2 v2.27.7/go.mod h1:CyNAG/xg+iAOg0N4MPGZqVmv2rCoP267496AOXUZjA4=
-github.com/urfave/cli/v3 v3.7.0 h1:AGSnbUyjtLiM+WJUb4dzXKldl/gL+F8OwmRDtVr6g2U=
-github.com/urfave/cli/v3 v3.7.0/go.mod h1:ysVLtOEmg2tOy6PknnYVhDoouyC/6N42TMeoMzskhso=
-github.com/vektah/gqlparser/v2 v2.5.32 h1:k9QPJd4sEDTL+qB4ncPLflqTJ3MmjB9SrVzJrawpFSc=
-github.com/vektah/gqlparser/v2 v2.5.32/go.mod h1:c1I28gSOVNzlfc4WuDlqU7voQnsqI6OG2amkBAFmgts=
+github.com/urfave/cli/v3 v3.8.0 h1:XqKPrm0q4P0q5JpoclYoCAv0/MIvH/jZ2umzuf8pNTI=
+github.com/urfave/cli/v3 v3.8.0/go.mod h1:ysVLtOEmg2tOy6PknnYVhDoouyC/6N42TMeoMzskhso=
+github.com/vektah/gqlparser/v2 v2.5.33 h1:lRp8aIeNUNbimf/axZd7ETg24q06hBtPaas+TcvI/7E=
+github.com/vektah/gqlparser/v2 v2.5.33/go.mod h1:c1I28gSOVNzlfc4WuDlqU7voQnsqI6OG2amkBAFmgts=
 github.com/xrash/smetrics v0.0.0-20250705151800-55b8f293f342 h1:FnBeRrxr7OU4VvAzt5X7s6266i6cSVkkFPS0TuXWbIg=
 github.com/xrash/smetrics v0.0.0-20250705151800-55b8f293f342/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM=
 github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU=
@@ -305,17 +309,17 @@ go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
 go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
-golang.org/x/crypto v0.49.0 h1:+Ng2ULVvLHnJ/ZFEq4KdcDd/cfjrrjjNSXNzxg0Y4U4=
-golang.org/x/crypto v0.49.0/go.mod h1:ErX4dUh2UM+CFYiXZRTcMpEcN8b/1gxEuv3nODoYtCA=
+golang.org/x/crypto v0.52.0 h1:RMs7fP2rXdep0CftQlK8Uf+kibLm7qkCcradZWYz988=
+golang.org/x/crypto v0.52.0/go.mod h1:1QgfPxDqh0T2M/elOJtp9RvuR95kVjir0e6/BvEmGbc=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
-golang.org/x/mod v0.34.0 h1:xIHgNUUnW6sYkcM5Jleh05DvLOtwc6RitGHbDk4akRI=
-golang.org/x/mod v0.34.0/go.mod h1:ykgH52iCZe79kzLLMhyCUzhMci+nQj+0XkbXpNYtVjY=
+golang.org/x/mod v0.36.0 h1:JJjpVx6myfUsUdAzZuOSTTmRE0PfZeNWzzvKrP7amb4=
+golang.org/x/mod v0.36.0/go.mod h1:moc6ELqsWcOw5Ef3xVprK5ul/MvtVvkIXLziUOICjUQ=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
-golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0=
-golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw=
+golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8=
+golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww=
 golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs=
 golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -328,8 +332,8 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo=
-golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
+golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY=
+golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
@@ -337,15 +341,15 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
-golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8=
-golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA=
+golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc=
+golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38=
 golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
 golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
-golang.org/x/tools v0.43.0 h1:12BdW9CeB3Z+J/I/wj34VMl8X+fEXBxVR90JeMX5E7s=
-golang.org/x/tools v0.43.0/go.mod h1:uHkMso649BX2cZK6+RpuIPXS3ho2hZo4FVwfoy1vIk0=
+golang.org/x/tools v0.45.0 h1:18qN3FAooORvApf5XjCXgsuayZOEtXf6JK18I3+ONa8=
+golang.org/x/tools v0.45.0/go.mod h1:LuUGqqaXcXMEFEruIVJVm5mgDD8vww/z/SR1gQ4uE/0=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=
 google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=

gopls.json

@@ -0,0 +1,49 @@
+{
+    "gofumpt": true,
+    "codelenses": {
+        "gc_details": false,
+        "generate": true,
+        "regenerate_cgo": true,
+        "run_govulncheck": true,
+        "test": true,
+        "tidy": true,
+        "upgrade_dependency": true,
+        "vendor": true
+    },
+    "hints": {
+        "assignVariableTypes": true,
+        "compositeLiteralFields": true,
+        "compositeLiteralTypes": true,
+        "constantValues": true,
+        "functionTypeParameters": true,
+        "parameterNames": true,
+        "rangeVariableTypes": true
+    },
+    "analyses": {
+        "nilness": true,
+        "unusedparams": true,
+        "unusedwrite": true,
+        "useany": true
+    },
+    "usePlaceholders": true,
+    "completeUnimported": true,
+    "staticcheck": true,
+    "directoryFilters": [
+        "-.git",
+        "-.vscode",
+        "-.idea",
+        "-.vscode-test",
+        "-node_modules",
+        "-web/frontend",
+        "-web/templates",
+        "-var",
+        "-api",
+        "-configs",
+        "-init",
+        "-internal/repository/migrations",
+        "-internal/repository/testdata",
+        "-internal/importer/testdata",
+        "-pkg/archive/testdata"
+    ],
+    "semanticTokens": true
+}

internal/api/api_test.go

@@ -170,6 +170,7 @@ func setup(t *testing.T) *api.RestAPI {
 
 	archiver.Start(repository.GetJobRepository(), context.Background())
 
+	t.Setenv("SESSION_KEY", "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=")
 	if cfg := ccconf.GetPackageConfig("auth"); cfg != nil {
 		auth.Init(&cfg)
 	} else {

internal/api/docs.go

@@ -398,11 +398,6 @@ const docTemplate = `{
         },
         "/api/jobs/edit_meta/": {
             "patch": {
-                "security": [
-                    {
-                        "ApiKeyAuth": []
-                    }
-                ],
                 "description": "Edit key value pairs in metadata json of job specified by jobID, StartTime and Cluster\nIf a key already exists its content will be overwritten",
                 "consumes": [
                     "application/json"
@@ -413,7 +408,7 @@ const docTemplate = `{
                 "tags": [
                     "Job add and modify"
                 ],
-                "summary": "Edit meta-data json by request",
+                "summary": "Edit meta-data json of job identified by request",
                 "parameters": [
                     {
                         "description": "Specifies job and payload to add or update",
@@ -456,12 +451,17 @@ const docTemplate = `{
                             "$ref": "#/definitions/api.ErrorResponse"
                         }
                     }
-                }
+                },
+                "security": [
+                    {
+                        "ApiKeyAuth": []
+                    }
+                ]
             }
         },
         "/api/jobs/edit_meta/{id}": {
             "patch": {
-                "description": "Edit key value pairs in job metadata json\nIf a key already exists its content will be overwritten",
+                "description": "Edit key value pairs in job metadata json of job specified by database id\nIf a key already exists its content will be overwritten",
                 "consumes": [
                     "application/json"
                 ],
@@ -471,7 +471,7 @@ const docTemplate = `{
                 "tags": [
                     "Job add and modify"
                 ],
-                "summary": "Edit meta-data json",
+                "summary": "Edit meta-data json of job identified by database id",
                 "parameters": [
                     {
                         "type": "integer",
@@ -481,7 +481,7 @@ const docTemplate = `{
                         "required": true
                     },
                     {
-                        "description": "Kay value pair to add",
+                        "description": "Metadata Key value pair to add or update",
                         "name": "request",
                         "in": "body",
                         "required": true,
@@ -743,6 +743,64 @@ const docTemplate = `{
                 ]
             }
         },
+        "/api/jobs/used_nodes": {
+            "get": {
+                "description": "Get a map of cluster names to lists of unique hostnames that are currently in use by running jobs that started before the specified timestamp.",
+                "produces": [
+                    "application/json"
+                ],
+                "tags": [
+                    "Job query"
+                ],
+                "summary": "Lists used nodes by cluster",
+                "parameters": [
+                    {
+                        "type": "integer",
+                        "description": "Unix timestamp to filter jobs (jobs with start_time \u003c ts)",
+                        "name": "ts",
+                        "in": "query",
+                        "required": true
+                    }
+                ],
+                "responses": {
+                    "200": {
+                        "description": "Map of cluster names to hostname lists",
+                        "schema": {
+                            "$ref": "#/definitions/api.GetUsedNodesAPIResponse"
+                        }
+                    },
+                    "400": {
+                        "description": "Bad Request",
+                        "schema": {
+                            "$ref": "#/definitions/api.ErrorResponse"
+                        }
+                    },
+                    "401": {
+                        "description": "Unauthorized",
+                        "schema": {
+                            "$ref": "#/definitions/api.ErrorResponse"
+                        }
+                    },
+                    "403": {
+                        "description": "Forbidden",
+                        "schema": {
+                            "$ref": "#/definitions/api.ErrorResponse"
+                        }
+                    },
+                    "500": {
+                        "description": "Internal Server Error",
+                        "schema": {
+                            "$ref": "#/definitions/api.ErrorResponse"
+                        }
+                    }
+                },
+                "security": [
+                    {
+                        "ApiKeyAuth": []
+                    }
+                ]
+            }
+        },
         "/api/jobs/{id}": {
             "get": {
                 "description": "Job to get is specified by database ID\nReturns full job resource information according to 'Job' scheme and all metrics according to 'JobData'.",
@@ -965,8 +1023,11 @@ const docTemplate = `{
         "/api/user/{id}": {
             "post": {
                 "description": "Allows admins to add/remove roles and projects for a user",
+                "consumes": [
+                    "application/json"
+                ],
                 "produces": [
-                    "text/plain"
+                    "application/json"
                 ],
                 "tags": [
                     "User"
@@ -981,35 +1042,26 @@ const docTemplate = `{
                         "required": true
                     },
                     {
-                        "type": "string",
-                        "description": "Role to add",
-                        "name": "add-role",
-                        "in": "formData"
-                    },
-                    {
-                        "type": "string",
-                        "description": "Role to remove",
-                        "name": "remove-role",
-                        "in": "formData"
-                    },
-                    {
-                        "type": "string",
-                        "description": "Project to add",
-                        "name": "add-project",
-                        "in": "formData"
-                    },
-                    {
-                        "type": "string",
-                        "description": "Project to remove",
-                        "name": "remove-project",
-                        "in": "formData"
+                        "description": "Single Field Changes",
+                        "name": "request",
+                        "in": "body",
+                        "required": true,
+                        "schema": {
+                            "$ref": "#/definitions/api.UpdateUserAPIRequest"
+                        }
                     }
                 ],
                 "responses": {
                     "200": {
-                        "description": "Success message",
+                        "description": "OK",
                         "schema": {
-                            "type": "string"
+                            "$ref": "#/definitions/api.DefaultAPIResponse"
+                        }
+                    },
+                    "400": {
+                        "description": "Bad Request",
+                        "schema": {
+                            "$ref": "#/definitions/api.ErrorResponse"
                         }
                     },
                     "403": {
@@ -1381,63 +1433,6 @@ const docTemplate = `{
                 ]
             }
         },
-        "/healthcheck/": {
-            "get": {
-                "description": "This endpoint allows the users to check if a node is healthy",
-                "produces": [
-                    "application/json"
-                ],
-                "tags": [
-                    "healthcheck"
-                ],
-                "summary": "HealthCheck endpoint",
-                "parameters": [
-                    {
-                        "type": "string",
-                        "description": "Selector",
-                        "name": "selector",
-                        "in": "query"
-                    }
-                ],
-                "responses": {
-                    "200": {
-                        "description": "Debug dump",
-                        "schema": {
-                            "type": "string"
-                        }
-                    },
-                    "400": {
-                        "description": "Bad Request",
-                        "schema": {
-                            "$ref": "#/definitions/api.ErrorResponse"
-                        }
-                    },
-                    "401": {
-                        "description": "Unauthorized",
-                        "schema": {
-                            "$ref": "#/definitions/api.ErrorResponse"
-                        }
-                    },
-                    "403": {
-                        "description": "Forbidden",
-                        "schema": {
-                            "$ref": "#/definitions/api.ErrorResponse"
-                        }
-                    },
-                    "500": {
-                        "description": "Internal Server Error",
-                        "schema": {
-                            "$ref": "#/definitions/api.ErrorResponse"
-                        }
-                    }
-                },
-                "security": [
-                    {
-                        "ApiKeyAuth": []
-                    }
-                ]
-            }
-        },
         "/jobs/tag_job/{id}": {
             "delete": {
                 "description": "Removes tag(s) from a job specified by DB ID. Name and Type of Tag(s) must match.\nTag Scope is required for matching, options: \"global\", \"admin\". Private tags can not be deleted via API.\nIf tagged job is already finished: Tag will be removed from respective archive files.",
@@ -1940,6 +1935,31 @@ const docTemplate = `{
                 }
             }
         },
+        "api.UpdateUserAPIRequest": {
+            "type": "object",
+            "properties": {
+                "add-role": {
+                    "description": "Role to add to user $ID",
+                    "type": "string",
+                    "example": "user"
+                },
+                "remove-role": {
+                    "description": "Role to remove from user $ID",
+                    "type": "string",
+                    "example": "user"
+                },
+                "add-project": {
+                    "description": "Project to add to user $ID managed array",
+                    "type": "string",
+                    "example": "abcd100"
+                },
+                "remove-project": {
+                    "description": "Project to remove from user $ID managed array",
+                    "type": "string",
+                    "example": "abcd100"
+                }
+            }
+        },
         "api.DefaultAPIResponse": {
             "type": "object",
             "properties": {
@@ -2043,6 +2063,52 @@ const docTemplate = `{
                 }
             }
         },
+        "api.GetUsedNodesAPIResponse": {
+            "type": "object",
+            "properties": {
+                "usedNodes": {
+                    "description": "Map of cluster names to lists of used node hostnames",
+                    "type": "object",
+                    "additionalProperties": {
+                        "type": "array",
+                        "items": {
+                            "type": "string"
+                        }
+                    }
+                }
+            }
+        },
+        "api.JobMetaRequest": {
+            "type": "object",
+            "required": [
+                "jobId"
+            ],
+            "properties": {
+                "cluster": {
+                    "description": "Cluster of job",
+                    "type": "string",
+                    "example": "fritz"
+                },
+                "jobId": {
+                    "description": "Cluster Job ID of job",
+                    "type": "integer",
+                    "example": 123000
+                },
+                "payload": {
+                    "description": "Content to Add to Job Meta_Data",
+                    "allOf": [
+                        {
+                            "$ref": "#/definitions/api.EditMetaRequest"
+                        }
+                    ]
+                },
+                "startTime": {
+                    "description": "Start Time of job as epoch",
+                    "type": "integer",
+                    "example": 1649723812
+                }
+            }
+        },
         "api.JobMetricWithName": {
             "type": "object",
             "properties": {
@@ -2175,13 +2241,6 @@ const docTemplate = `{
                         "format": "float64"
                     }
                 },
-                "exclusive": {
-                    "description": "for backwards compatibility",
-                    "type": "integer",
-                    "maximum": 2,
-                    "minimum": 0,
-                    "example": 1
-                },
                 "footprint": {
                     "type": "object",
                     "additionalProperties": {

internal/api/job.go

@@ -166,6 +166,10 @@ func (api *RestAPI) getJobs(rw http.ResponseWriter, r *http.Request) {
 				handleError(err, http.StatusBadRequest, rw)
 				return
 			}
+			if x < 1 {
+				handleError(fmt.Errorf("page must be >= 1"), http.StatusBadRequest, rw)
+				return
+			}
 			page.Page = x
 		case "items-per-page":
 			x, err := strconv.Atoi(vals[0])
@@ -173,6 +177,10 @@ func (api *RestAPI) getJobs(rw http.ResponseWriter, r *http.Request) {
 				handleError(err, http.StatusBadRequest, rw)
 				return
 			}
+			if x < 1 {
+				handleError(fmt.Errorf("items-per-page must be >= 1"), http.StatusBadRequest, rw)
+				return
+			}
 			page.ItemsPerPage = x
 		case "with-metadata":
 			withMetadata = true

internal/api/nats.go

@@ -78,6 +78,12 @@ type NatsAPI struct {
 	jobCh chan natsMessage
 	// nodeCh receives node state messages for processing by worker goroutines.
 	nodeCh chan natsMessage
+	// stop signals worker goroutines and subscription callbacks to stop.
+	// Closing it (via Shutdown) makes workers exit and callbacks drop further
+	// messages instead of blocking; the channels are never closed so in-flight
+	// callbacks can never send on a closed channel.
+	stop     chan struct{}
+	stopOnce sync.Once
 }
 
 // NewNatsAPI creates a new NatsAPI instance with channel-based worker pools.
@@ -99,6 +105,7 @@ func NewNatsAPI() *NatsAPI {
 		JobRepository: repository.GetJobRepository(),
 		jobCh:         make(chan natsMessage, jobConc),
 		nodeCh:        make(chan natsMessage, nodeConc),
+		stop:          make(chan struct{}),
 	}
 
 	// Start worker goroutines
@@ -112,17 +119,36 @@ func NewNatsAPI() *NatsAPI {
 	return api
 }
 
+// Shutdown stops the worker goroutines and tells subscription callbacks to stop
+// enqueueing. It is safe to call multiple times. Callers must ensure the NATS
+// client is closed first so no new callbacks are invoked.
+func (api *NatsAPI) Shutdown() {
+	api.stopOnce.Do(func() {
+		close(api.stop)
+	})
+}
+
 // jobWorker processes job event messages from the job channel.
 func (api *NatsAPI) jobWorker() {
-	for msg := range api.jobCh {
-		api.handleJobEvent(msg.subject, msg.data)
+	for {
+		select {
+		case <-api.stop:
+			return
+		case msg := <-api.jobCh:
+			api.handleJobEvent(msg.subject, msg.data)
+		}
 	}
 }
 
 // nodeWorker processes node state messages from the node channel.
 func (api *NatsAPI) nodeWorker() {
-	for msg := range api.nodeCh {
-		api.handleNodeState(msg.subject, msg.data)
+	for {
+		select {
+		case <-api.stop:
+			return
+		case msg := <-api.nodeCh:
+			api.handleNodeState(msg.subject, msg.data)
+		}
 	}
 }
 
@@ -140,18 +166,27 @@ func (api *NatsAPI) StartSubscriptions() error {
 		s := config.Keys.APISubjects
 
 		if err := client.Subscribe(s.SubjectJobEvent, func(subject string, data []byte) {
-			api.jobCh <- natsMessage{subject: subject, data: data}
+			select {
+			case api.jobCh <- natsMessage{subject: subject, data: data}:
+			case <-api.stop:
+			}
 		}); err != nil {
 			return err
 		}
 
 		if err := client.Subscribe(s.SubjectNodeState, func(subject string, data []byte) {
-			api.nodeCh <- natsMessage{subject: subject, data: data}
+			select {
+			case api.nodeCh <- natsMessage{subject: subject, data: data}:
+			case <-api.stop:
+			}
 		}); err != nil {
 			return err
 		}
 
-		cclog.Info("NATS API subscriptions started")
+		cclog.Warnf("NATS API subscriptions started on subjects %q and %q — these are UNAUTHENTICATED: "+
+			"anyone with publish rights on the broker can start/stop jobs and update node state. "+
+			"Restrict publish ACLs on the NATS broker to trusted producers only.",
+			s.SubjectJobEvent, s.SubjectNodeState)
 	}
 	return nil
 }

internal/api/nats_test.go

@@ -156,6 +156,7 @@ func setupNatsTest(t *testing.T) *NatsAPI {
 
 	archiver.Start(repository.GetJobRepository(), context.Background())
 
+	t.Setenv("SESSION_KEY", "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=")
 	if cfg := ccconf.GetPackageConfig("auth"); cfg != nil {
 		auth.Init(&cfg)
 	} else {

internal/api/rest.go

@@ -79,6 +79,8 @@ func (api *RestAPI) MountAPIRoutes(r chi.Router) {
 	// REST API Uses TokenAuth
 	// User List
 	r.Get("/users/", api.getUsers)
+	// User Edit
+	r.Post("/user/{id}", api.updateUserByRequest)
 	// Cluster List
 	r.Get("/clusters/", api.getClusters)
 	// Slurm node state
@@ -152,7 +154,7 @@ func (api *RestAPI) MountConfigAPIRoutes(r chi.Router) {
 		r.Put("/config/users/", api.createUser)
 		r.Get("/config/users/", api.getUsers)
 		r.Delete("/config/users/", api.deleteUser)
-		r.Post("/config/user/{id}", api.updateUser)
+		r.Post("/config/user/{id}", api.updateUserByForm)
 		r.Post("/config/notice/", api.editNotice)
 		r.Get("/config/taggers/", api.getTaggers)
 		r.Post("/config/taggers/run/", api.runTagger)

internal/api/user.go

@@ -24,6 +24,14 @@ type APIReturnedUser struct {
 	Projects []string `json:"projects"`
 }
 
+// UpdateUserAPIRequest model
+type UpdateUserAPIRequest struct {
+	NewRole string `json:"add-role" example:"user"`          // Role to add to user $ID
+	DelRole string `json:"remove-role" example:"user"`       // Role to remove from user $ID
+	NewProj string `json:"add-project" example:"abcd100"`    // Project to add to user $ID managed array
+	DelProj string `json:"remove-project" example:"abcd100"` // Project to remove from user $ID managed array
+}
+
 // getUsers godoc
 // @summary     Returns a list of users
 // @tags User
@@ -58,22 +66,74 @@ func (api *RestAPI) getUsers(rw http.ResponseWriter, r *http.Request) {
 	}
 }
 
-// updateUser godoc
+// updateUserByRequest godoc
 // @summary     Update user roles and projects
 // @tags User
 // @description Allows admins to add/remove roles and projects for a user
-// @produce     plain
-// @param       id          path   string  true  "Username"
-// @param       add-role    formData string false "Role to add"
-// @param       remove-role formData string false "Role to remove"
-// @param       add-project formData string false "Project to add"
-// @param       remove-project formData string false "Project to remove"
+// @accept      json
+// @produce     json
+// @param       id      path string                    true "Username"
+// @param       request body api.UpdateUserAPIRequest  true "Single Field Changes"
 // @success     200     {string} string "Success message"
 // @failure     403     {object} api.ErrorResponse "Forbidden"
 // @failure     422     {object} api.ErrorResponse "Unprocessable Entity"
 // @security    ApiKeyAuth
 // @router      /api/user/{id} [post]
-func (api *RestAPI) updateUser(rw http.ResponseWriter, r *http.Request) {
+func (api *RestAPI) updateUserByRequest(rw http.ResponseWriter, r *http.Request) {
+
+	if user := repository.GetUserFromContext(r.Context()); !user.HasRole(schema.RoleAdmin) {
+		handleError(fmt.Errorf("only admins are allowed to update a user"), http.StatusForbidden, rw)
+		return
+	}
+
+	// Get Values
+	var req UpdateUserAPIRequest
+	if err := decode(r.Body, &req); err != nil {
+		handleError(fmt.Errorf("decoding request failed: %w", err), http.StatusBadRequest, rw)
+		return
+	}
+
+	rw.Header().Set("Content-Type", "application/json")
+
+	// Handle role updates
+	if req.NewRole != "" {
+		if err := repository.GetUserRepository().AddRole(r.Context(), chi.URLParam(r, "id"), req.NewRole); err != nil {
+			handleError(fmt.Errorf("adding role failed: %w", err), http.StatusUnprocessableEntity, rw)
+			return
+		}
+		if err := json.NewEncoder(rw).Encode(DefaultAPIResponse{Message: fmt.Sprintf("Add Role Success for user %s", chi.URLParam(r, "id"))}); err != nil {
+			cclog.Errorf("Failed to encode response: %v", err)
+		}
+	} else if req.DelRole != "" {
+		if err := repository.GetUserRepository().RemoveRole(r.Context(), chi.URLParam(r, "id"), req.DelRole); err != nil {
+			handleError(fmt.Errorf("removing role failed: %w", err), http.StatusUnprocessableEntity, rw)
+			return
+		}
+		if err := json.NewEncoder(rw).Encode(DefaultAPIResponse{Message: fmt.Sprintf("Remove Role Success for user %s", chi.URLParam(r, "id"))}); err != nil {
+			cclog.Errorf("Failed to encode response: %v", err)
+		}
+	} else if req.NewProj != "" {
+		if err := repository.GetUserRepository().AddProject(r.Context(), chi.URLParam(r, "id"), req.NewProj); err != nil {
+			handleError(fmt.Errorf("adding project failed: %w", err), http.StatusUnprocessableEntity, rw)
+			return
+		}
+		if err := json.NewEncoder(rw).Encode(DefaultAPIResponse{Message: fmt.Sprintf("Add Project Success for user %s", chi.URLParam(r, "id"))}); err != nil {
+			cclog.Errorf("Failed to encode response: %v", err)
+		}
+	} else if req.DelProj != "" {
+		if err := repository.GetUserRepository().RemoveProject(r.Context(), chi.URLParam(r, "id"), req.DelProj); err != nil {
+			handleError(fmt.Errorf("removing project failed: %w", err), http.StatusUnprocessableEntity, rw)
+			return
+		}
+		if err := json.NewEncoder(rw).Encode(DefaultAPIResponse{Message: fmt.Sprintf("Remove Project Success for user %s", chi.URLParam(r, "id"))}); err != nil {
+			cclog.Errorf("Failed to encode response: %v", err)
+		}
+	} else {
+		handleError(fmt.Errorf("no operation specified: must provide add-role, remove-role, add-project, or remove-project"), http.StatusBadRequest, rw)
+	}
+}
+
+func (api *RestAPI) updateUserByForm(rw http.ResponseWriter, r *http.Request) {
 	// SecuredCheck() only worked with TokenAuth: Removed
 
 	if user := repository.GetUserFromContext(r.Context()); !user.HasRole(schema.RoleAdmin) {

internal/archiver/archiveWorker.go

@@ -222,6 +222,13 @@ func TriggerArchiving(job *schema.Job) {
 func Shutdown(timeout time.Duration) error {
 	cclog.Info("Initiating archiver shutdown...")
 
+	// Guard against Shutdown being called when Start was never run: closing a nil
+	// channel and receiving from a nil workerDone would panic/block forever.
+	if archiveChannel == nil {
+		cclog.Warn("Archiver shutdown called but archiver was never started")
+		return nil
+	}
+
 	// Close channel to signal no more jobs will be accepted
 	close(archiveChannel)
 

internal/auth/auth.go

@@ -17,6 +17,7 @@ import (
 	"fmt"
 	"net"
 	"net/http"
+	"net/url"
 	"os"
 	"sync"
 	"time"
@@ -164,12 +165,17 @@ func (auth *Authentication) AuthViaSession(
 		return nil, errors.New("invalid session data")
 	}
 
+	authSourceInt, ok := session.Values["authSource"].(int)
+	if !ok {
+		authSourceInt = int(schema.AuthViaLocalPassword)
+	}
+
 	return &schema.User{
 		Username:   username,
 		Projects:   projects,
 		Roles:      roles,
 		AuthType:   schema.AuthSession,
-		AuthSource: -1,
+		AuthSource: schema.AuthSource(authSourceInt),
 	}, nil
 }
 
@@ -182,24 +188,37 @@ func Init(authCfg *json.RawMessage) {
 
 		sessKey := os.Getenv("SESSION_KEY")
 		if sessKey == "" {
-			cclog.Warn("environment variable 'SESSION_KEY' not set (will use non-persistent random key)")
-			bytes := make([]byte, 32)
-			if _, err := rand.Read(bytes); err != nil {
-				cclog.Fatal("Error while initializing authentication -> failed to generate random bytes for session key")
+			if !config.Keys.DisableAuthentication {
+				cclog.Fatal("environment variable 'SESSION_KEY' not set: refusing to start with an ephemeral session key. " +
+					"Set SESSION_KEY in .env (base64-encoded 32 random bytes); a random key would invalidate all sessions on every restart " +
+					"and prevent sessions from validating across replicas.")
 			}
-			authInstance.sessionStore = sessions.NewCookieStore(bytes)
+			// Authentication is disabled: no user sessions are issued, so an
+			// ephemeral random key is sufficient and SESSION_KEY is not required.
+			ephemeralKey := make([]byte, 32)
+			if _, err := rand.Read(ephemeralKey); err != nil {
+				cclog.Fatalf("Error while initializing authentication -> generating ephemeral session key failed: %v", err)
+			}
+			authInstance.sessionStore = sessions.NewCookieStore(ephemeralKey)
 		} else {
-			bytes, err := base64.StdEncoding.DecodeString(sessKey)
+			keyBytes, err := base64.StdEncoding.DecodeString(sessKey)
 			if err != nil {
 				cclog.Fatal("Error while initializing authentication -> decoding session key failed")
 			}
-			authInstance.sessionStore = sessions.NewCookieStore(bytes)
+			authInstance.sessionStore = sessions.NewCookieStore(keyBytes)
 		}
 
 		if d, err := time.ParseDuration(config.Keys.SessionMaxAge); err == nil {
 			authInstance.SessionMaxAge = d
 		}
 
+		// When authentication is disabled no authenticators are required; the
+		// session store created above is enough for the server to run with a
+		// valid (non-nil) auth instance.
+		if config.Keys.DisableAuthentication {
+			return
+		}
+
 		if authCfg == nil {
 			return
 		}
@@ -319,10 +338,12 @@ func (auth *Authentication) SaveSession(rw http.ResponseWriter, r *http.Request,
 		}
 		session.Options.Secure = false
 	}
-	session.Options.SameSite = http.SameSiteStrictMode
+	session.Options.SameSite = http.SameSiteLaxMode
+	session.Options.HttpOnly = true
 	session.Values["username"] = user.Username
 	session.Values["projects"] = user.Projects
 	session.Values["roles"] = user.Roles
+	session.Values["authSource"] = int(user.AuthSource)
 	if err := auth.sessionStore.Save(r, rw, session); err != nil {
 		cclog.Warnf("session save failed: %s", err.Error())
 		http.Error(rw, err.Error(), http.StatusInternalServerError)
@@ -382,9 +403,12 @@ func (auth *Authentication) Login(
 			cclog.Infof("login successfull: user: %#v (roles: %v, projects: %v)", user.Username, user.Roles, user.Projects)
 			ctx := context.WithValue(r.Context(), repository.ContextUserKey, user)
 
-			if r.FormValue("redirect") != "" {
-				http.RedirectHandler(r.FormValue("redirect"), http.StatusFound).ServeHTTP(rw, r.WithContext(ctx))
-				return
+			if redirect := r.FormValue("redirect"); redirect != "" {
+				if u, perr := url.Parse(redirect); perr == nil && u.Scheme == "" && u.Host == "" {
+					http.RedirectHandler(redirect, http.StatusFound).ServeHTTP(rw, r.WithContext(ctx))
+					return
+				}
+				cclog.Warnf("login redirect rejected (not a relative path): %q", redirect)
 			}
 
 			http.RedirectHandler("/", http.StatusFound).ServeHTTP(rw, r.WithContext(ctx))
@@ -626,7 +650,7 @@ func securedCheck(user *schema.User, r *http.Request) error {
 	}
 	// If SplitHostPort fails, IPAddress is already just a host (no port)
 
-	// If nothing declared in config: Continue
+	// If nothing declared in config: Continue // FIXME: Allow All If Not Declared?
 	if len(config.Keys.APIAllowedIPs) == 0 {
 		return nil
 	}

internal/auth/jwtCookieSession.go

@@ -9,6 +9,7 @@ import (
 	"crypto/ed25519"
 	"encoding/base64"
 	"errors"
+	"fmt"
 	"net/http"
 	"os"
 
@@ -119,22 +120,26 @@ func (ja *JWTCookieSessionAuthenticator) Login(
 		rawtoken = jwtCookie.Value
 	}
 
-	token, err := jwt.Parse(rawtoken, func(t *jwt.Token) (any, error) {
-		if t.Method != jwt.SigningMethodEdDSA {
-			return nil, errors.New("only Ed25519/EdDSA supported")
-		}
+	parser := jwt.NewParser(jwt.WithValidMethods([]string{jwt.SigningMethodEdDSA.Alg()}))
 
-		unvalidatedIssuer, success := t.Claims.(jwt.MapClaims)["iss"].(string)
-		if success && unvalidatedIssuer == jc.TrustedIssuer {
-			// The (unvalidated) issuer seems to be the expected one,
-			// use public cross login key from config
-			return ja.publicKeyCrossLogin, nil
-		}
+	unverified, _, perr := parser.ParseUnverified(rawtoken, jwt.MapClaims{})
+	if perr != nil {
+		cclog.Warn("JWT cookie session: error while parsing token")
+		return nil, perr
+	}
+	issuer, _ := unverified.Claims.(jwt.MapClaims)["iss"].(string)
 
-		// No cross login key configured or issuer not expected
-		// Try own key
-		return ja.publicKey, nil
-	})
+	var key any
+	switch issuer {
+	case jc.TrustedIssuer:
+		key = ja.publicKeyCrossLogin
+	case "":
+		key = ja.publicKey
+	default:
+		return nil, fmt.Errorf("untrusted JWT issuer: %q", issuer)
+	}
+
+	token, err := parser.Parse(rawtoken, func(*jwt.Token) (any, error) { return key, nil })
 	if err != nil {
 		cclog.Warn("JWT cookie session: error while parsing token")
 		return nil, err

internal/auth/jwtSession.go

@@ -25,15 +25,21 @@ type JWTSessionAuthenticator struct {
 var _ Authenticator = (*JWTSessionAuthenticator)(nil)
 
 func (ja *JWTSessionAuthenticator) Init() error {
-	if pubKey := os.Getenv("CROSS_LOGIN_JWT_HS512_KEY"); pubKey != "" {
-		bytes, err := base64.StdEncoding.DecodeString(pubKey)
-		if err != nil {
-			cclog.Warn("Could not decode cross login JWT HS512 key")
-			return err
-		}
-		ja.loginTokenKey = bytes
+	pubKey := os.Getenv("CROSS_LOGIN_JWT_HS512_KEY")
+	if pubKey == "" {
+		// Without a configured key the HMAC verification below would run against
+		// an empty key, which lets anyone forge a valid token. Refuse to register
+		// the authenticator in that case so JWT session login is simply disabled.
+		return errors.New("CROSS_LOGIN_JWT_HS512_KEY not set: JWT session login disabled")
 	}
 
+	bytes, err := base64.StdEncoding.DecodeString(pubKey)
+	if err != nil {
+		cclog.Warn("Could not decode cross login JWT HS512 key")
+		return err
+	}
+	ja.loginTokenKey = bytes
+
 	cclog.Info("JWT Session authenticator successfully registered")
 	return nil
 }
@@ -60,6 +66,12 @@ func (ja *JWTSessionAuthenticator) Login(
 
 	token, err := jwt.Parse(rawtoken, func(t *jwt.Token) (any, error) {
 		if t.Method == jwt.SigningMethodHS256 || t.Method == jwt.SigningMethodHS512 {
+			// Defense in depth: an empty key would verify any HMAC signature.
+			// Init() already refuses to register without a key, so this should
+			// never trigger, but guard explicitly rather than trust the chain.
+			if len(ja.loginTokenKey) == 0 {
+				return nil, errors.New("HS login key not configured")
+			}
 			return ja.loginTokenKey, nil
 		}
 		return nil, fmt.Errorf("unkown signing method for login token: %s (known: HS256, HS512, EdDSA)", t.Method.Alg())

internal/auth/oidc.go

@@ -79,7 +79,7 @@ func NewOIDC(a *Authentication) *OIDC {
 		ClientID:     clientID,
 		ClientSecret: clientSecret,
 		Endpoint:     provider.Endpoint(),
-		Scopes:       []string{oidc.ScopeOpenID, "profile"},
+		Scopes:       []string{oidc.ScopeOpenID, "profile", "roles"},
 	}
 
 	oa := &OIDC{provider: provider, client: client, clientID: clientID, authentication: a}
@@ -162,36 +162,76 @@ func (oa *OIDC) OAuth2Callback(rw http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	projects := make([]string, 0)
+	// projects is populated below from ID token claims
+	var projects []string
 
-	// Extract custom claims from userinfo
-	var claims struct {
+	// Extract profile claims from userinfo (username, name)
+	var userInfoClaims struct {
+		Username string `json:"preferred_username"`
+		Name     string `json:"name"`
+	}
+	if err := userInfo.Claims(&userInfoClaims); err != nil {
+		cclog.Errorf("failed to extract userinfo claims: %s", err.Error())
+		http.Error(rw, "Failed to extract user claims", http.StatusInternalServerError)
+		return
+	}
+
+	// Extract role claims from the ID token.
+	// Keycloak includes realm_access and resource_access in the ID token (JWT),
+	// but NOT in the UserInfo endpoint response by default.
+	var idTokenClaims struct {
 		Username string `json:"preferred_username"`
 		Name     string `json:"name"`
 		// Keycloak realm-level roles
 		RealmAccess struct {
 			Roles []string `json:"roles"`
 		} `json:"realm_access"`
-		// Keycloak client-level roles
-		ResourceAccess struct {
-			Client struct {
-				Roles []string `json:"roles"`
-			} `json:"clustercockpit"`
+		// Keycloak client-level roles: map from client-id to role list
+		ResourceAccess map[string]struct {
+			Roles []string `json:"roles"`
 		} `json:"resource_access"`
+		// Custom multi-valued user attribute mapped via a Keycloak User Attribute mapper
+		Projects []string `json:"projects"`
 	}
-	if err := userInfo.Claims(&claims); err != nil {
-		cclog.Errorf("failed to extract claims: %s", err.Error())
-		http.Error(rw, "Failed to extract user claims", http.StatusInternalServerError)
+	if err := idToken.Claims(&idTokenClaims); err != nil {
+		cclog.Errorf("failed to extract ID token claims: %s", err.Error())
+		http.Error(rw, "Failed to extract ID token claims", http.StatusInternalServerError)
 		return
 	}
 
-	if claims.Username == "" {
+	cclog.Debugf("OIDC userinfo claims: username=%q name=%q", userInfoClaims.Username, userInfoClaims.Name)
+	cclog.Debugf("OIDC ID token realm_access roles: %v", idTokenClaims.RealmAccess.Roles)
+	cclog.Debugf("OIDC ID token resource_access: %v", idTokenClaims.ResourceAccess)
+	cclog.Debugf("OIDC ID token projects: %v", idTokenClaims.Projects)
+
+	projects = idTokenClaims.Projects
+	if projects == nil {
+		projects = []string{}
+	}
+
+	// Prefer username from userInfo; fall back to ID token claim
+	username := userInfoClaims.Username
+	if username == "" {
+		username = idTokenClaims.Username
+	}
+	name := userInfoClaims.Name
+	if name == "" {
+		name = idTokenClaims.Name
+	}
+
+	if username == "" {
 		http.Error(rw, "Username claim missing from OIDC provider", http.StatusBadRequest)
 		return
 	}
 
-	// Merge roles from both client-level and realm-level access
-	oidcRoles := append(claims.ResourceAccess.Client.Roles, claims.RealmAccess.Roles...)
+	// Collect roles from realm_access (realm roles) in the ID token
+	oidcRoles := append([]string{}, idTokenClaims.RealmAccess.Roles...)
+
+	// Also collect roles from resource_access (client roles) for all clients
+	for clientID, access := range idTokenClaims.ResourceAccess {
+		cclog.Debugf("OIDC ID token resource_access[%q] roles: %v", clientID, access.Roles)
+		oidcRoles = append(oidcRoles, access.Roles...)
+	}
 
 	roleSet := make(map[string]bool)
 	for _, r := range oidcRoles {
@@ -217,8 +257,8 @@ func (oa *OIDC) OAuth2Callback(rw http.ResponseWriter, r *http.Request) {
 	}
 
 	user := &schema.User{
-		Username:   claims.Username,
-		Name:       claims.Name,
+		Username:   username,
+		Name:       name,
 		Roles:      roles,
 		Projects:   projects,
 		AuthSource: schema.AuthViaOIDC,

internal/graph/schema.resolvers.go

@@ -3,7 +3,7 @@ package graph
 // This file will be automatically regenerated based on the schema, any resolver
 // implementations
 // will be copied through when generating and any unknown code will be moved to the end.
-// Code generated by github.com/99designs/gqlgen version v0.17.88
+// Code generated by github.com/99designs/gqlgen version v0.17.90
 
 import (
 	"context"
@@ -682,6 +682,11 @@ func (r *queryResolver) JobsStatistics(ctx context.Context, filter []*model.JobF
 			// Use request-scoped cache: multiple aliases with same (filter, groupBy)
 			// but different sortBy/page hit the DB only once.
 			if cache := getStatsGroupCache(ctx); cache != nil {
+				// Ensure the sort field is computed even if not in the GraphQL selection,
+				// because sortAndPageStats will sort by it in memory.
+				if sortBy != nil {
+					reqFields[sortByFieldName(*sortBy)] = true
+				}
 				key := statsCacheKey(filter, groupBy, reqFields)
 				var allStats []*model.JobsStatistics
 				allStats, err = cache.getOrCompute(key, func() ([]*model.JobsStatistics, error) {

internal/graph/stats_cache.go

@@ -107,6 +107,33 @@ func sortAndPageStats(allStats []*model.JobsStatistics, sortBy *model.SortByAggr
 	return sorted
 }
 
+// sortByFieldName maps a SortByAggregate enum to the corresponding reqFields key.
+// This ensures the DB computes the column that sortAndPageStats will sort by.
+func sortByFieldName(sortBy model.SortByAggregate) string {
+	switch sortBy {
+	case model.SortByAggregateTotaljobs:
+		return "totalJobs"
+	case model.SortByAggregateTotalusers:
+		return "totalUsers"
+	case model.SortByAggregateTotalwalltime:
+		return "totalWalltime"
+	case model.SortByAggregateTotalnodes:
+		return "totalNodes"
+	case model.SortByAggregateTotalnodehours:
+		return "totalNodeHours"
+	case model.SortByAggregateTotalcores:
+		return "totalCores"
+	case model.SortByAggregateTotalcorehours:
+		return "totalCoreHours"
+	case model.SortByAggregateTotalaccs:
+		return "totalAccs"
+	case model.SortByAggregateTotalacchours:
+		return "totalAccHours"
+	default:
+		return "totalJobs"
+	}
+}
+
 // statsFieldGetter returns a function that extracts the sortable int field
 // from a JobsStatistics struct for the given sort key.
 func statsFieldGetter(sortBy model.SortByAggregate) func(*model.JobsStatistics) int {

internal/metricstoreclient/cc-metric-store-queries.go

@@ -236,4 +236,3 @@ func (ccms *CCMetricStore) buildNodeQueries(
 
 	return queries, assignedScope, nil
 }
-

internal/repository/config.go

@@ -63,10 +63,10 @@ func DefaultConfig() *RepositoryConfig {
 		MaxIdleConnections:    4,
 		ConnectionMaxLifetime: time.Hour,
 		ConnectionMaxIdleTime: 10 * time.Minute,
-		MinRunningJobDuration: 600, // 10 minutes
-		DbCacheSizeMB:        2048,  // 2GB per connection
-		DbSoftHeapLimitMB:    16384, // 16GB process-wide
-		BusyTimeoutMs:        60000, // 60 seconds
+		MinRunningJobDuration: 600,   // 10 minutes
+		DbCacheSizeMB:         2048,  // 2GB per connection
+		DbSoftHeapLimitMB:     16384, // 16GB process-wide
+		BusyTimeoutMs:         60000, // 60 seconds
 	}
 }
 

internal/repository/jobQuery.go

@@ -76,8 +76,15 @@ func (r *JobRepository) QueryJobs(
 	}
 
 	if page != nil && page.ItemsPerPage != -1 {
+		// -1 is the only valid non-positive value ("load all"); reject other
+		// non-positive values so that uint64(page.ItemsPerPage) cannot underflow
+		// into a huge limit. Clamp Page to >= 1 to avoid the same on the offset.
+		if page.ItemsPerPage < 1 {
+			return nil, fmt.Errorf("invalid items-per-page value: %d", page.ItemsPerPage)
+		}
+		p := max(page.Page, 1)
 		limit := uint64(page.ItemsPerPage)
-		query = query.Offset((uint64(page.Page) - 1) * limit).Limit(limit)
+		query = query.Offset((uint64(p) - 1) * limit).Limit(limit)
 	}
 
 	for _, f := range filters {
@@ -280,11 +287,11 @@ func BuildWhereClause(filter *model.JobFilter, query sq.SelectBuilder) sq.Select
 
 // buildIntCondition creates clauses for integer range filters, using BETWEEN only if required.
 func buildIntCondition(field string, cond *config.IntRange, query sq.SelectBuilder) sq.SelectBuilder {
-	if cond.From != 1 && cond.To != 0 {
+	if cond.From > 0 && cond.To > 0 {
 		return query.Where(field+" BETWEEN ? AND ?", cond.From, cond.To)
-	} else if cond.From != 1 && cond.To == 0 {
+	} else if cond.From > 0 && cond.To == 0 {
 		return query.Where(field+" >= ?", cond.From)
-	} else if cond.From == 1 && cond.To != 0 {
+	} else if cond.From == 0 && cond.To > 0 {
 		return query.Where(field+" <= ?", cond.To)
 	} else {
 		return query
@@ -293,11 +300,11 @@ func buildIntCondition(field string, cond *config.IntRange, query sq.SelectBuild
 
 // buildFloatCondition creates a clauses for float range filters, using BETWEEN only if required.
 func buildFloatCondition(field string, cond *model.FloatRange, query sq.SelectBuilder) sq.SelectBuilder {
-	if cond.From != 1.0 && cond.To != 0.0 {
+	if cond.From > 0.0 && cond.To > 0.0 {
 		return query.Where(field+" BETWEEN ? AND ?", cond.From, cond.To)
-	} else if cond.From != 1.0 && cond.To == 0.0 {
+	} else if cond.From > 0.0 && cond.To == 0.0 {
 		return query.Where(field+" >= ?", cond.From)
-	} else if cond.From == 1.0 && cond.To != 0.0 {
+	} else if cond.From == 0.0 && cond.To > 0.0 {
 		return query.Where(field+" <= ?", cond.To)
 	} else {
 		return query
@@ -336,14 +343,24 @@ func buildTimeCondition(field string, cond *config.TimeRange, query sq.SelectBui
 	}
 }
 
+// validMetricName guards metric/footprint names that are interpolated into the
+// json_extract() path of footprint queries. SQLite treats double-quoted strings
+// as string literals, so an unescaped name (e.g. containing a `"`) would allow
+// SQL injection. Legitimate metric names only use these characters.
+var validMetricName = regexp.MustCompile(`^[a-zA-Z0-9_]+$`)
+
 // buildFloatJSONCondition creates a filter on a numeric field within the footprint JSON column, using BETWEEN only if required.
 func buildFloatJSONCondition(jsonField string, cond *model.FloatRange, query sq.SelectBuilder) sq.SelectBuilder {
+	if !validMetricName.MatchString(jsonField) {
+		cclog.Warnf("buildFloatJSONCondition: rejecting invalid metric name %q", jsonField)
+		return query.Where("0 = 1")
+	}
 	query = query.Where("JSON_VALID(footprint)")
-	if cond.From != 1.0 && cond.To != 0.0 {
+	if cond.From > 0.0 && cond.To > 0.0 {
 		return query.Where("JSON_EXTRACT(footprint, \"$."+jsonField+"\") BETWEEN ? AND ?", cond.From, cond.To)
-	} else if cond.From != 1.0 && cond.To == 0.0 {
+	} else if cond.From > 0.0 && cond.To == 0.0 {
 		return query.Where("JSON_EXTRACT(footprint, \"$."+jsonField+"\") >= ?", cond.From)
-	} else if cond.From == 1.0 && cond.To != 0.0 {
+	} else if cond.From == 0.0 && cond.To > 0.0 {
 		return query.Where("JSON_EXTRACT(footprint, \"$."+jsonField+"\") <= ?", cond.To)
 	} else {
 		return query

internal/repository/stats.go

@@ -909,6 +909,13 @@ func (r *JobRepository) jobsMetricStatisticsHistogram(
 	filters []*model.JobFilter,
 	bins *int,
 ) (*model.MetricHistoPoints, error) {
+	// The metric name is interpolated into the json_extract() path of the SQL
+	// below. SQLite parses double-quoted strings as literals, so reject anything
+	// that is not a plain metric identifier to prevent SQL injection.
+	if !validMetricName.MatchString(metric) {
+		return nil, fmt.Errorf("invalid metric name: %q", metric)
+	}
+
 	// Peak value defines the upper bound for binning: values are distributed across
 	// bins from 0 to peak. First try to get peak from filtered cluster, otherwise
 	// scan all clusters to find the maximum peak value.

internal/repository/tags.go

@@ -311,26 +311,33 @@ func (r *JobRepository) CountTags(user *schema.User) (tags []schema.Tag, counts
 		LeftJoin("jobtag jt ON t.id = jt.tag_id").
 		GroupBy("t.tag_type, t.tag_name")
 
-	// Build scope list for filtering
-	var scopeBuilder strings.Builder
-	scopeBuilder.WriteString(`"global"`)
+	// Build scope list for filtering. Values are parameterized rather than
+	// interpolated because user.Username originates from external identity
+	// providers (OIDC/LDAP) and must not be trusted as SQL.
+	scopes := []string{"global"}
 	if user != nil {
-		scopeBuilder.WriteString(`,"`)
-		scopeBuilder.WriteString(user.Username)
-		scopeBuilder.WriteString(`"`)
+		scopes = append(scopes, user.Username)
 		if user.HasAnyRole([]schema.Role{schema.RoleAdmin, schema.RoleSupport}) {
-			scopeBuilder.WriteString(`,"admin"`)
+			scopes = append(scopes, "admin")
 		}
 	}
-	q = q.Where("t.tag_scope IN (" + scopeBuilder.String() + ")")
+	q = q.Where(sq.Eq{"t.tag_scope": scopes})
 
 	// Handle Job Ownership
 	if user != nil && user.HasAnyRole([]schema.Role{schema.RoleAdmin, schema.RoleSupport}) { // ADMIN || SUPPORT: Count all jobs
 		// cclog.Debug("CountTags: User Admin or Support -> Count all Jobs for Tags")
 		// Unchanged: Needs to be own case still, due to UserRole/NoRole compatibility handling in else case
-	} else if user != nil && user.HasRole(schema.RoleManager) { // MANAGER: Count own jobs plus project's jobs
-		// Build ("project1", "project2", ...) list of variable length directly in SQL string
-		q = q.Where("jt.job_id IN (SELECT id FROM job WHERE job.hpc_user = ? OR job.project IN (\""+strings.Join(user.Projects, "\",\"")+"\"))", user.Username)
+	} else if user != nil && user.HasRole(schema.RoleManager) && len(user.Projects) > 0 { // MANAGER: Count own jobs plus project's jobs
+		// Build a parameterized ("?", "?", ...) placeholder list for the
+		// variable-length project set instead of interpolating values into SQL.
+		args := make([]any, 0, len(user.Projects)+1)
+		args = append(args, user.Username)
+		placeholders := make([]string, len(user.Projects))
+		for i, p := range user.Projects {
+			placeholders[i] = "?"
+			args = append(args, p)
+		}
+		q = q.Where("jt.job_id IN (SELECT id FROM job WHERE job.hpc_user = ? OR job.project IN ("+strings.Join(placeholders, ",")+"))", args...)
 	} else if user != nil { // USER OR NO ROLE (Compatibility): Only count own jobs
 		q = q.Where("jt.job_id IN (SELECT id FROM job WHERE job.hpc_user = ?)", user.Username)
 	}

internal/repository/user.go

@@ -14,6 +14,7 @@ import (
 	"path/filepath"
 	"reflect"
 	"runtime"
+	"sort"
 	"strings"
 	"sync"
 
@@ -210,6 +211,12 @@ func (r *UserRepository) AddUserIfNotExists(user *schema.User) error {
 	return err
 }
 
+func sortedRoles(roles []string) []string {
+	cp := append([]string{}, roles...)
+	sort.Strings(cp)
+	return cp
+}
+
 func (r *UserRepository) UpdateUser(dbUser *schema.User, user *schema.User) error {
 	// user contains updated info -> Apply to dbUser
 	// --- Simple Name Update ---
@@ -279,6 +286,15 @@ func (r *UserRepository) UpdateUser(dbUser *schema.User, user *schema.User) erro
 		}
 	}
 
+	// --- Fallback: sync any remaining role differences not covered above ---
+	// This handles admin role assignment/removal and any other combinations that
+	// the specific branches above do not cover (e.g. user→admin, admin→user).
+	if !reflect.DeepEqual(sortedRoles(dbUser.Roles), sortedRoles(user.Roles)) {
+		if err := updateRoles(user.Roles); err != nil {
+			return err
+		}
+	}
+
 	return nil
 }
 

internal/routerConfig/routes.go

@@ -239,6 +239,9 @@ func buildFilterPresets(query url.Values) map[string]any {
 	if query.Get("cluster") != "" {
 		filterPresets["cluster"] = query.Get("cluster")
 	}
+	if query.Get("subCluster") != "" {
+		filterPresets["subCluster"] = query.Get("subCluster")
+	}
 	if query.Get("partition") != "" {
 		filterPresets["partition"] = query.Get("partition")
 	}
@@ -309,7 +312,7 @@ func buildFilterPresets(query url.Values) map[string]any {
 			if parts[0] == "lessthan" {
 				lt, lte := strconv.Atoi(parts[1])
 				if lte == nil {
-					filterPresets["numNodes"] = map[string]int{"from": 1, "to": lt}
+					filterPresets["numNodes"] = map[string]int{"from": 0, "to": lt}
 				}
 			} else if parts[0] == "morethan" {
 				mt, mte := strconv.Atoi(parts[1])
@@ -331,7 +334,7 @@ func buildFilterPresets(query url.Values) map[string]any {
 			if parts[0] == "lessthan" {
 				lt, lte := strconv.Atoi(parts[1])
 				if lte == nil {
-					filterPresets["numHWThreads"] = map[string]int{"from": 1, "to": lt}
+					filterPresets["numHWThreads"] = map[string]int{"from": 0, "to": lt}
 				}
 			} else if parts[0] == "morethan" {
 				mt, mte := strconv.Atoi(parts[1])
@@ -353,7 +356,7 @@ func buildFilterPresets(query url.Values) map[string]any {
 			if parts[0] == "lessthan" {
 				lt, lte := strconv.Atoi(parts[1])
 				if lte == nil {
-					filterPresets["numAccelerators"] = map[string]int{"from": 1, "to": lt}
+					filterPresets["numAccelerators"] = map[string]int{"from": 0, "to": lt}
 				}
 			} else if parts[0] == "morethan" {
 				mt, mte := strconv.Atoi(parts[1])
@@ -409,7 +412,7 @@ func buildFilterPresets(query url.Values) map[string]any {
 			if parts[0] == "lessthan" {
 				lt, lte := strconv.Atoi(parts[1])
 				if lte == nil {
-					filterPresets["energy"] = map[string]int{"from": 1, "to": lt}
+					filterPresets["energy"] = map[string]int{"from": 0, "to": lt}
 				}
 			} else if parts[0] == "morethan" {
 				mt, mte := strconv.Atoi(parts[1])
@@ -435,7 +438,7 @@ func buildFilterPresets(query url.Values) map[string]any {
 					if lte == nil {
 						statEntry := map[string]any{
 							"field": parts[0],
-							"from":  1,
+							"from":  0,
 							"to":    lt,
 						}
 						statList = append(statList, statEntry)

internal/tagger/classifyJob.go

@@ -363,7 +363,7 @@ func (t *JobClassTagger) Match(job *schema.Job) {
 		for _, m := range ri.metrics {
 			stats, ok := jobStats[m]
 			if !ok {
-				cclog.Errorf("job classification: missing metric '%s' for rule %s on job %d", m, tag, job.JobID)
+				cclog.Debugf("job classification: missing metric '%s' for rule %s on job %d", m, tag, job.JobID)
 				skipRule = true
 				break
 			}
@@ -388,7 +388,7 @@ func (t *JobClassTagger) Match(job *schema.Job) {
 		for _, r := range ri.requirements {
 			ok, err := expr.Run(r, env)
 			if err != nil {
-				cclog.Errorf("error running requirement for rule %s: %#v", tag, err)
+				cclog.Debugf("error running requirement for rule %s: %#v", tag, err)
 				requirementsMet = false
 				break
 			}
@@ -407,7 +407,7 @@ func (t *JobClassTagger) Match(job *schema.Job) {
 		for _, v := range ri.variables {
 			value, err := expr.Run(v.expr, env)
 			if err != nil {
-				cclog.Errorf("error evaluating variable %s for rule %s: %#v", v.name, tag, err)
+				cclog.Debugf("error evaluating variable %s for rule %s: %#v", v.name, tag, err)
 				varError = true
 				break
 			}

pkg/archive/clusterConfig.go

@@ -198,36 +198,19 @@ func GetSubCluster(cluster, subcluster string) (*schema.SubCluster, error) {
 func GetMetricConfigSubCluster(cluster, subcluster string) map[string]*schema.Metric {
 	metrics := make(map[string]*schema.Metric)
 
-	for _, c := range Clusters {
-		if c.Name == cluster {
-			for _, m := range c.MetricConfig {
-				for _, s := range m.SubClusters {
-					if s.Name == subcluster {
-						metrics[m.Name] = &schema.Metric{
-							Name:    m.Name,
-							Unit:    s.Unit,
-							Peak:    s.Peak,
-							Normal:  s.Normal,
-							Caution: s.Caution,
-							Alert:   s.Alert,
-						}
-						break
-					}
-				}
+	sc, err := GetSubCluster(cluster, subcluster)
+	if err != nil {
+		return metrics
+	}
 
-				_, ok := metrics[m.Name]
-				if !ok {
-					metrics[m.Name] = &schema.Metric{
-						Name:    m.Name,
-						Unit:    m.Unit,
-						Peak:    m.Peak,
-						Normal:  m.Normal,
-						Caution: m.Caution,
-						Alert:   m.Alert,
-					}
-				}
-			}
-			break
+	for _, m := range sc.MetricConfig {
+		metrics[m.Name] = &schema.Metric{
+			Name:    m.Name,
+			Unit:    m.Unit,
+			Peak:    m.Peak,
+			Normal:  m.Normal,
+			Caution: m.Caution,
+			Alert:   m.Alert,
 		}
 	}
 

pkg/archive/clusterConfig_test.go

@@ -37,3 +37,27 @@ func TestClusterConfig(t *testing.T) {
 	// spew.Dump(archive.GlobalMetricList)
 	// t.Fail()
 }
+
+func TestGetMetricConfigSubClusterRespectsRemovedMetrics(t *testing.T) {
+	if err := archive.Init(json.RawMessage(`{"kind": "file","path": "testdata/archive"}`)); err != nil {
+		t.Fatal(err)
+	}
+
+	sc, err := archive.GetSubCluster("fritz", "spr2tb")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	metrics := archive.GetMetricConfigSubCluster("fritz", "spr2tb")
+	if len(metrics) != len(sc.MetricConfig) {
+		t.Fatalf("GetMetricConfigSubCluster() returned %d metrics, want %d", len(metrics), len(sc.MetricConfig))
+	}
+
+	if _, ok := metrics["flops_any"]; ok {
+		t.Fatalf("GetMetricConfigSubCluster() returned removed metric flops_any for subcluster spr2tb")
+	}
+
+	if _, ok := metrics["cpu_power"]; !ok {
+		t.Fatalf("GetMetricConfigSubCluster() missing active metric cpu_power for subcluster spr2tb")
+	}
+}

pkg/archive/parquet/writer.go

@@ -133,10 +133,10 @@ func (pt *prefixedTarget) WriteFile(name string, data []byte) error {
 // ClusterAwareParquetWriter organizes Parquet output by cluster.
 // Each cluster gets its own subdirectory with a cluster.json config file.
 type ClusterAwareParquetWriter struct {
-	target       ParquetTarget
-	maxSizeMB    int
-	writers      map[string]*ParquetWriter
-	clusterCfgs  map[string]*schema.Cluster
+	target      ParquetTarget
+	maxSizeMB   int
+	writers     map[string]*ParquetWriter
+	clusterCfgs map[string]*schema.Cluster
 }
 
 // NewClusterAwareParquetWriter creates a writer that routes jobs to per-cluster ParquetWriters.

pkg/metricstore/api.go

@@ -79,7 +79,7 @@ type APIQueryResponse struct {
 //   - Type + TypeIds: First level of hierarchy (e.g., "cpu" + ["0", "1", "2"])
 //   - SubType + SubTypeIds: Second level of hierarchy (e.g., "core" + ["0", "1"])
 //
-// If Aggregate is true, data from multiple type/subtype IDs will be aggregated according
+// If Aggregate is true, data from multiple type/stype IDs will be aggregated according
 // to the metric's aggregation strategy. Otherwise, separate results are returned for each combination.
 type APIQuery struct {
 	Type        *string      `json:"type,omitempty"`
@@ -175,13 +175,13 @@ func (data *APIMetricData) PadDataWithNull(ms *MemoryStore, from, to int64, metr
 // This is the primary API for retrieving metric data from the memory store. It supports:
 //   - Individual queries via req.Queries
 //   - Batch queries for all nodes via req.ForAllNodes
-//   - Hierarchical selector construction (cluster → host → type → subtype)
+//   - Hierarchical selector construction (cluster → host → type → stype)
 //   - Optional statistics computation (avg, min, max)
 //   - Optional data scaling
 //   - Optional data padding with NaN values
 //
 // The function constructs selectors based on the query parameters and calls MemoryStore.Read()
-// for each selector. If a query specifies Aggregate=false with multiple type/subtype IDs,
+// for each selector. If a query specifies Aggregate=false with multiple type/stype IDs,
 // separate results are returned for each combination.
 //
 // Parameters:

pkg/metricstore/archive.go

@@ -3,6 +3,12 @@
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 
+// This file implements the cleanup (archiving or deletion) of old checkpoint files.
+//
+// The CleanUp worker runs on a timer equal to RetentionInMemory. In "archive" mode
+// it converts checkpoint files older than the retention window into per-cluster
+// Parquet files and then deletes the originals. In "delete" mode it simply removes
+// old checkpoint files.
 package metricstore
 
 import (
@@ -11,6 +17,7 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"strings"
 	"sync"
 	"sync/atomic"
 	"time"
@@ -18,10 +25,15 @@ import (
 	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
 )
 
-// Worker for either Archiving or Deleting files
-
+// CleanUp starts a background worker that periodically removes or archives
+// checkpoint files older than the configured retention window.
+//
+// In "archive" mode, old checkpoint files are converted to Parquet and stored
+// under Keys.Cleanup.RootDir. In "delete" mode they are simply removed.
+// The cleanup interval equals Keys.RetentionInMemory.
 func CleanUp(wg *sync.WaitGroup, ctx context.Context) {
 	if Keys.Cleanup.Mode == "archive" {
+		cclog.Info("[METRICSTORE]> enable archive cleanup to parquet")
 		// Run as Archiver
 		cleanUpWorker(wg, ctx,
 			Keys.RetentionInMemory,
@@ -43,7 +55,6 @@ func CleanUp(wg *sync.WaitGroup, ctx context.Context) {
 // cleanUpWorker takes simple values to configure what it does
 func cleanUpWorker(wg *sync.WaitGroup, ctx context.Context, interval string, mode string, cleanupDir string, delete bool) {
 	wg.Go(func() {
-
 		d, err := time.ParseDuration(interval)
 		if err != nil {
 			cclog.Fatalf("[METRICSTORE]> error parsing %s interval duration: %v\n", mode, err)
@@ -99,8 +110,8 @@ func deleteCheckpoints(checkpointsDir string, from int64) (int, error) {
 	}
 
 	type workItem struct {
-		dir            string
-		cluster, host  string
+		dir           string
+		cluster, host string
 	}
 
 	var wg sync.WaitGroup
@@ -181,6 +192,7 @@ func archiveCheckpoints(checkpointsDir, cleanupDir string, from int64) (int, err
 	}
 
 	totalFiles := 0
+	var clusterErrors []string
 
 	for _, clusterEntry := range clusterEntries {
 		if !clusterEntry.IsDir() {
@@ -190,7 +202,9 @@ func archiveCheckpoints(checkpointsDir, cleanupDir string, from int64) (int, err
 		cluster := clusterEntry.Name()
 		hostEntries, err := os.ReadDir(filepath.Join(checkpointsDir, cluster))
 		if err != nil {
-			return totalFiles, err
+			cclog.Errorf("[METRICSTORE]> error reading host entries for cluster %s: %s", cluster, err.Error())
+			clusterErrors = append(clusterErrors, cluster)
+			continue
 		}
 
 		// Workers load checkpoint files from disk; main thread writes to parquet.
@@ -255,7 +269,9 @@ func archiveCheckpoints(checkpointsDir, cleanupDir string, from int64) (int, err
 			// Drain results channel to unblock workers
 			for range results {
 			}
-			return totalFiles, fmt.Errorf("creating parquet writer for cluster %s: %w", cluster, err)
+			cclog.Errorf("[METRICSTORE]> error creating parquet writer for cluster %s: %s", cluster, err.Error())
+			clusterErrors = append(clusterErrors, cluster)
+			continue
 		}
 
 		type deleteItem struct {
@@ -275,6 +291,12 @@ func archiveCheckpoints(checkpointsDir, cleanupDir string, from int64) (int, err
 						break
 					}
 				}
+				// Flush once per host to keep row group count within parquet limits.
+				if writeErr == nil {
+					if err := writer.FlushRowGroup(); err != nil {
+						writeErr = err
+					}
+				}
 			}
 			// Always track files for deletion (even if write failed, we still drain)
 			toDelete = append(toDelete, deleteItem{dir: r.dir, files: r.files})
@@ -285,7 +307,10 @@ func archiveCheckpoints(checkpointsDir, cleanupDir string, from int64) (int, err
 		}
 
 		if errs > 0 {
-			return totalFiles, fmt.Errorf("%d errors reading checkpoints for cluster %s", errs, cluster)
+			cclog.Errorf("[METRICSTORE]> %d errors reading checkpoints for cluster %s", errs, cluster)
+			clusterErrors = append(clusterErrors, cluster)
+			os.Remove(parquetFile)
+			continue
 		}
 
 		if writer.count == 0 {
@@ -296,7 +321,9 @@ func archiveCheckpoints(checkpointsDir, cleanupDir string, from int64) (int, err
 
 		if writeErr != nil {
 			os.Remove(parquetFile)
-			return totalFiles, fmt.Errorf("writing parquet archive for cluster %s: %w", cluster, writeErr)
+			cclog.Errorf("[METRICSTORE]> error writing parquet archive for cluster %s: %s", cluster, writeErr.Error())
+			clusterErrors = append(clusterErrors, cluster)
+			continue
 		}
 
 		// Delete archived checkpoint files
@@ -316,5 +343,10 @@ func archiveCheckpoints(checkpointsDir, cleanupDir string, from int64) (int, err
 	}
 
 	cclog.Infof("[METRICSTORE]> archiving checkpoints completed in %s (%d files)", time.Since(startTime).Round(time.Millisecond), totalFiles)
+
+	if len(clusterErrors) > 0 {
+		return totalFiles, fmt.Errorf("archiving failed for clusters: %s", strings.Join(clusterErrors, ", "))
+	}
+
 	return totalFiles, nil
 }

pkg/metricstore/buffer.go

@@ -146,7 +146,9 @@ var (
 
 	// ErrDataDoesNotAlign indicates that aggregated data from child scopes
 	// does not align with the parent scope's expected timestamps/intervals.
-	ErrDataDoesNotAlign error = errors.New("[METRICSTORE]> data from lower granularities does not align")
+	ErrDataDoesNotAlignMissingFront    error = errors.New("[METRICSTORE]> data from lower granularities does not align (missing data prior to start of the buffers)")
+	ErrDataDoesNotAlignMissingBack     error = errors.New("[METRICSTORE]> data from lower granularities does not align (missing data after the end of the buffers)")
+	ErrDataDoesNotAlignDataLenMismatch error = errors.New("[METRICSTORE]> data from lower granularities does not align (collected data length is different than expected data length)")
 )
 
 // buffer stores time-series data for a single metric at a specific hierarchical level.

pkg/metricstore/checkpoint.go

@@ -86,11 +86,12 @@ var (
 
 // Checkpointing starts a background worker that periodically saves metric data to disk.
 //
-// Checkpoints are written every 12 hours (hardcoded).
+// The checkpoint interval is read from Keys.CheckpointInterval (default: 12 hours).
 //
 // Format behaviour:
-//   - "json": Periodic checkpointing every checkpointInterval
-//   - "wal":  Periodic binary snapshots + WAL rotation every checkpointInterval
+//   - "json": Writes a JSON snapshot file per host every interval
+//   - "wal":  Writes a binary snapshot file per host every interval, then rotates
+//     the current.wal files for all successfully checkpointed hosts
 func Checkpointing(wg *sync.WaitGroup, ctx context.Context) {
 	lastCheckpointMu.Lock()
 	lastCheckpoint = time.Now()
@@ -99,7 +100,6 @@ func Checkpointing(wg *sync.WaitGroup, ctx context.Context) {
 	ms := GetMemoryStore()
 
 	wg.Go(func() {
-
 		d := 12 * time.Hour // default checkpoint interval
 		if Keys.CheckpointInterval != "" {
 			parsed, err := time.ParseDuration(Keys.CheckpointInterval)
@@ -126,10 +126,15 @@ func Checkpointing(wg *sync.WaitGroup, ctx context.Context) {
 				cclog.Infof("[METRICSTORE]> start checkpointing (starting at %s)...", from.Format(time.RFC3339))
 
 				if Keys.Checkpoints.FileFormat == "wal" {
+					// Pause WAL writes: the binary snapshot captures all in-memory
+					// data, so WAL records written during checkpoint are redundant
+					// and would be deleted during rotation anyway.
+					walCheckpointActive.Store(true)
 					n, hostDirs, err := ms.ToCheckpointWAL(Keys.Checkpoints.RootDir, from.Unix(), now.Unix())
 					if err != nil {
 						cclog.Errorf("[METRICSTORE]> binary checkpointing failed: %s", err.Error())
-					} else {
+					}
+					if n > 0 {
 						cclog.Infof("[METRICSTORE]> done: %d binary snapshot files created", n)
 						lastCheckpointMu.Lock()
 						lastCheckpoint = now
@@ -137,6 +142,8 @@ func Checkpointing(wg *sync.WaitGroup, ctx context.Context) {
 						// Rotate WAL files for successfully checkpointed hosts.
 						RotateWALFiles(hostDirs)
 					}
+					walCheckpointActive.Store(false)
+					walDropped.Store(0)
 				} else {
 					n, err := ms.ToCheckpoint(Keys.Checkpoints.RootDir, from.Unix(), now.Unix())
 					if err != nil {

pkg/metricstore/config.go

@@ -59,11 +59,14 @@ const (
 // Checkpoints configures periodic persistence of in-memory metric data.
 //
 // Fields:
-//   - FileFormat: "json" (human-readable, periodic) or "wal" (binary snapshot + WAL, crash-safe); default is "wal"
-//   - RootDir:    Filesystem path for checkpoint files (created if missing)
+//   - FileFormat:  "json" (human-readable, periodic) or "wal" (binary snapshot + WAL, crash-safe); default is "wal"
+//   - RootDir:     Filesystem path for checkpoint files (created if missing)
+//   - MaxWALSize:  Maximum size in bytes for a single host's WAL file; 0 = unlimited (default).
+//     When exceeded the WAL is force-rotated to prevent unbounded disk growth.
 type Checkpoints struct {
 	FileFormat string `json:"file-format"`
 	RootDir    string `json:"directory"`
+	MaxWALSize int64  `json:"max-wal-size,omitempty"`
 }
 
 // Debug provides development and profiling options.

pkg/metricstore/configSchema.go

@@ -24,6 +24,11 @@ const configSchema = `{
         "directory": {
           "description": "Path in which the checkpointed files should be placed.",
           "type": "string"
+        },
+        "max-wal-size": {
+          "description": "Maximum size in bytes for a single host's WAL file. When exceeded the WAL is force-rotated to prevent unbounded disk growth. Only applies when file-format is 'wal'. 0 means unlimited (default).",
+          "type": "integer",
+          "minimum": 0
         }
       }
     },

pkg/metricstore/lineprotocol.go

@@ -6,11 +6,11 @@
 // This file implements ingestion of InfluxDB line-protocol metric data received
 // over NATS. Each line encodes one metric sample with the following structure:
 //
-//	<measurement>[,cluster=<c>][,hostname=<h>][,type=<t>][,type-id=<id>][,subtype=<s>][,stype-id=<id>] value=<v> [<timestamp>]
+//	<measurement>[,cluster=<c>][,hostname=<h>][,type=<t>][,type-id=<id>][,stype=<s>][,stype-id=<id>] value=<v> [<timestamp>]
 //
 // The measurement name identifies the metric (e.g. "cpu_load"). Tags provide
 // routing information (cluster, host) and optional sub-device selectors (type,
-// subtype). Only one field is expected per line: "value".
+// stype). Only one field is expected per line: "value".
 //
 // After decoding, each sample is:
 //  1. Written to the in-memory store via ms.WriteToLevel.
@@ -22,6 +22,7 @@ import (
 	"bytes"
 	"context"
 	"fmt"
+	"strings"
 	"sync"
 	"sync/atomic"
 	"time"
@@ -103,7 +104,7 @@ func ReceiveNats(ms *MemoryStore,
 // reorder prepends prefix to buf in-place when buf has enough spare capacity,
 // avoiding an allocation. Falls back to a regular append otherwise.
 //
-// It is used to assemble the "type<type-id>" and "subtype<stype-id>" selector
+// It is used to assemble the "type<type-id>" and "stype<stype-id>" selector
 // strings when the type tag arrives before the type-id tag in the line, so the
 // two byte slices need to be concatenated in tag-declaration order regardless
 // of wire order.
@@ -145,7 +146,7 @@ type decodeState struct {
 	// current line. Reset at the start of each line's tag-decode loop.
 	typeBuf []byte
 
-	// subTypeBuf accumulates the concatenated "subtype"+"stype-id" tag value.
+	// subTypeBuf accumulates the concatenated "stype"+"stype-id" tag value.
 	// Reset at the start of each line's tag-decode loop.
 	subTypeBuf []byte
 
@@ -186,7 +187,7 @@ var decodeStatePool = sync.Pool{
 //   - The Level pointer (host-level node in the metric tree) is cached across
 //     consecutive lines that share the same cluster+host pair to avoid
 //     repeated lock acquisitions on the root and cluster levels.
-//   - []byte→string conversions for type/subtype selectors are cached via
+//   - []byte→string conversions for type/stype selectors are cached via
 //     prevType*/prevSubType* fields because batches typically repeat the same
 //     sub-device identifiers.
 //   - Timestamp parsing tries Second precision first; if that fails it retries
@@ -196,6 +197,16 @@ var decodeStatePool = sync.Pool{
 // When the checkpoint format is "wal" each successfully decoded sample is also
 // sent to WALMessages so the WAL staging goroutine can persist it durably
 // before the next binary snapshot.
+// isValidPathComponent reports whether s is safe to use as a single filesystem
+// path component (cluster or host name) when building checkpoint/WAL paths. It
+// rejects path-traversal sequences and embedded separators. An empty string is
+// allowed because a missing host tag is legitimate and does not escape the root.
+func isValidPathComponent(s string) bool {
+	return !strings.Contains(s, "..") &&
+		!strings.Contains(s, "/") &&
+		!strings.Contains(s, "\\")
+}
+
 func DecodeLine(dec *lineprotocol.Decoder,
 	ms *MemoryStore,
 	clusterDefault string,
@@ -269,8 +280,8 @@ func DecodeLine(dec *lineprotocol.Decoder,
 				}
 			case "type-id":
 				st.typeBuf = append(st.typeBuf, val...)
-			case "subtype":
-				// We cannot be sure that the "subtype" tag comes before the "stype-id" tag:
+			case "stype":
+				// We cannot be sure that the "stype" tag comes before the "stype-id" tag:
 				if len(st.subTypeBuf) == 0 {
 					st.subTypeBuf = append(st.subTypeBuf, val...)
 				} else {
@@ -282,6 +293,17 @@ func DecodeLine(dec *lineprotocol.Decoder,
 			}
 		}
 
+		// cluster and host are taken verbatim from attacker-controllable line
+		// protocol tags and are later used as filesystem path components for the
+		// checkpoint/WAL files (path.Join(RootDir, cluster, host)). Reject
+		// path-traversal sequences so a malicious tag cannot escape the
+		// checkpoint root. Skip the offending sample; the rest of the batch is
+		// still processed.
+		if !isValidPathComponent(cluster) || !isValidPathComponent(host) {
+			cclog.Warnf("[METRICSTORE]> rejecting metric with invalid cluster/host tag (cluster=%q host=%q)", cluster, host)
+			continue
+		}
+
 		// If the cluster or host changed, the lvl was set to nil
 		if lvl == nil {
 			st.selector = st.selector[:2]
@@ -291,7 +313,7 @@ func DecodeLine(dec *lineprotocol.Decoder,
 		}
 
 		// subtypes: cache []byte→string conversions; messages in a batch typically
-		// share the same type/subtype so the hit rate is very high.
+		// share the same type/stype so the hit rate is very high.
 		st.selector = st.selector[:0]
 		if len(st.typeBuf) > 0 {
 			if !bytes.Equal(st.typeBuf, st.prevTypeBytes) {

pkg/metricstore/metricstore.go

@@ -8,7 +8,7 @@
 //
 // The package organizes metrics in a tree structure (cluster → host → component) and
 // provides concurrent read/write access to metric data with configurable aggregation strategies.
-// Background goroutines handle periodic checkpointing (JSON or Avro format), archiving old data,
+// Background goroutines handle periodic checkpointing (JSON or WAL/binary format), archiving old data,
 // and enforcing retention policies.
 //
 // Key features:
@@ -16,7 +16,7 @@
 //   - Hierarchical data organization (selectors)
 //   - Concurrent checkpoint/archive workers
 //   - Support for sum and average aggregation
-//   - NATS integration for metric ingestion
+//   - NATS integration for metric ingestion via InfluxDB line protocol
 package metricstore
 
 import (
@@ -113,7 +113,8 @@ type MemoryStore struct {
 //  6. Optionally subscribes to NATS for real-time metric ingestion
 //
 // Parameters:
-//   - rawConfig: JSON configuration for the metric store (see MetricStoreConfig)
+//   - rawConfig: JSON configuration for the metric store (see MetricStoreConfig); may be nil to use defaults
+//   - metrics: Map of metric names to their configurations (frequency and aggregation strategy)
 //   - wg: WaitGroup that will be incremented for each background goroutine started
 //
 // The function will call cclog.Fatal on critical errors during initialization.
@@ -271,19 +272,37 @@ func (ms *MemoryStore) SetNodeProvider(provider NodeProvider) {
 //
 // Note: This function blocks until the final checkpoint is written.
 func Shutdown() {
+	totalStart := time.Now()
+
 	shutdownFuncMu.Lock()
-	defer shutdownFuncMu.Unlock()
-	if shutdownFunc != nil {
-		shutdownFunc()
+	if shutdownFunc == nil {
+		// Already shut down (or never initialized): nothing to do. This keeps
+		// Shutdown idempotent so it is safe to call from more than one path.
+		shutdownFuncMu.Unlock()
+		return
 	}
+	shutdownFunc()
+	shutdownFunc = nil
+	shutdownFuncMu.Unlock()
+	cclog.Infof("[METRICSTORE]> Background workers cancelled (%v)", time.Since(totalStart))
 
 	if Keys.Checkpoints.FileFormat == "wal" {
+		// Signal producers to stop sending before closing channels,
+		// preventing send-on-closed-channel panics from in-flight NATS workers.
+		walShuttingDown.Store(true)
+		// Brief grace period for in-flight DecodeLine calls to complete.
+		time.Sleep(100 * time.Millisecond)
+
 		for _, ch := range walShardChs {
 			close(ch)
 		}
+		drainStart := time.Now()
+		WaitForWALStagingDrain()
+		cclog.Infof("[METRICSTORE]> WAL staging goroutines exited (%v)", time.Since(drainStart))
 	}
 
-	cclog.Infof("[METRICSTORE]> Writing to '%s'...\n", Keys.Checkpoints.RootDir)
+	cclog.Infof("[METRICSTORE]> Writing checkpoint to '%s'...", Keys.Checkpoints.RootDir)
+	checkpointStart := time.Now()
 	var files int
 	var err error
 
@@ -294,19 +313,22 @@ func Shutdown() {
 	lastCheckpointMu.Unlock()
 
 	if Keys.Checkpoints.FileFormat == "wal" {
-		var hostDirs []string
-		files, hostDirs, err = ms.ToCheckpointWAL(Keys.Checkpoints.RootDir, from.Unix(), time.Now().Unix())
-		if err == nil {
-			RotateWALFilesAfterShutdown(hostDirs)
-		}
+		var successDirs []string
+		files, successDirs, err = ms.ToCheckpointWAL(Keys.Checkpoints.RootDir, from.Unix(), time.Now().Unix())
+		// The final binary snapshot now captures all in-memory data for these
+		// hosts, making their current.wal redundant. The staging goroutines have
+		// already exited, so remove the WAL files directly (the channel-based
+		// RotateWALFiles is no longer safe to call). Without this, current.wal
+		// files survive shutdown and keep growing across restarts.
+		RotateWALFilesAfterShutdown(successDirs)
 	} else {
 		files, err = ms.ToCheckpoint(Keys.Checkpoints.RootDir, from.Unix(), time.Now().Unix())
 	}
 
 	if err != nil {
-		cclog.Errorf("[METRICSTORE]> Writing checkpoint failed: %s\n", err.Error())
+		cclog.Errorf("[METRICSTORE]> Writing checkpoint failed: %s", err.Error())
 	}
-	cclog.Infof("[METRICSTORE]> Done! (%d files written)\n", files)
+	cclog.Infof("[METRICSTORE]> Done! (%d files written in %v, total shutdown: %v)", files, time.Since(checkpointStart), time.Since(totalStart))
 }
 
 // Retention starts a background goroutine that periodically frees old metric data.
@@ -702,16 +724,16 @@ func (m *MemoryStore) Read(selector util.Selector, metric string, from, to, reso
 		} else if from != cfrom || to != cto || len(data) != len(cdata) {
 			missingfront, missingback := int((from-cfrom)/minfo.Frequency), int((to-cto)/minfo.Frequency)
 			if missingfront != 0 {
-				return ErrDataDoesNotAlign
+				return ErrDataDoesNotAlignMissingFront
 			}
 
 			newlen := len(cdata) - missingback
 			if newlen < 1 {
-				return ErrDataDoesNotAlign
+				return ErrDataDoesNotAlignMissingBack
 			}
 			cdata = cdata[0:newlen]
 			if len(cdata) != len(data) {
-				return ErrDataDoesNotAlign
+				return ErrDataDoesNotAlignDataLenMismatch
 			}
 
 			from, to = cfrom, cto

pkg/metricstore/parquetArchive.go

@@ -99,7 +99,7 @@ func newParquetArchiveWriter(filename string) (*parquetArchiveWriter, error) {
 
 // WriteCheckpointFile streams a CheckpointFile tree directly to Parquet rows,
 // writing metrics in sorted order without materializing all rows in memory.
-// Produces one row group per call (typically one host's data).
+// Call FlushRowGroup() after writing all checkpoint files for a host.
 func (w *parquetArchiveWriter) WriteCheckpointFile(cf *CheckpointFile, cluster, hostname, scope, scopeID string) error {
 	w.writeLevel(cf, cluster, hostname, scope, scopeID)
 
@@ -112,10 +112,15 @@ func (w *parquetArchiveWriter) WriteCheckpointFile(cf *CheckpointFile, cluster,
 		w.batch = w.batch[:0]
 	}
 
+	return nil
+}
+
+// FlushRowGroup flushes the current row group to the Parquet file.
+// Should be called once per host after all checkpoint files for that host are written.
+func (w *parquetArchiveWriter) FlushRowGroup() error {
 	if err := w.writer.Flush(); err != nil {
 		return fmt.Errorf("flushing parquet row group: %w", err)
 	}
-
 	return nil
 }
 

pkg/metricstore/stats.go

@@ -91,8 +91,10 @@ func (m *MemoryStore) Stats(selector util.Selector, metric string, from, to int6
 
 		if n == 0 {
 			from, to = cfrom, cto
-		} else if from != cfrom || to != cto {
-			return ErrDataDoesNotAlign
+		} else if from != cfrom {
+			return ErrDataDoesNotAlignMissingFront
+		} else if to != cto {
+			return ErrDataDoesNotAlignMissingBack
 		}
 
 		samples += stats.Samples

pkg/metricstore/walCheckpoint.go

@@ -69,6 +69,7 @@ import (
 	"strings"
 	"sync"
 	"sync/atomic"
+	"time"
 
 	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
 	"github.com/ClusterCockpit/cc-lib/v2/schema"
@@ -91,6 +92,18 @@ var walShardRotateChs []chan walRotateReq
 // walNumShards stores the number of shards (set during WALStaging init).
 var walNumShards int
 
+// walStagingWg tracks WALStaging goroutine exits for shutdown synchronization.
+var walStagingWg sync.WaitGroup
+
+// walShuttingDown is set before closing shard channels to prevent
+// SendWALMessage from sending on a closed channel (which panics in Go).
+var walShuttingDown atomic.Bool
+
+// walCheckpointActive is set during binary checkpoint writes.
+// While active, SendWALMessage skips sending (returns true) because the
+// snapshot captures all in-memory data, making WAL writes redundant.
+var walCheckpointActive atomic.Bool
+
 // WALMessage represents a single metric write to be appended to the WAL.
 // Cluster and Node are NOT stored in the WAL record (inferred from file path).
 type WALMessage struct {
@@ -111,10 +124,17 @@ type walRotateReq struct {
 
 // walFileState holds an open WAL file handle and buffered writer for one host directory.
 type walFileState struct {
-	f *os.File
-	w *bufio.Writer
+	f     *os.File
+	w     *bufio.Writer
+	dirty bool
+	size  int64 // approximate bytes written (tracked from open + writes)
 }
 
+// walFlushInterval controls how often dirty WAL files are flushed to disk.
+// Decoupling flushes from message processing lets the consumer run at memory
+// speed, amortizing syscall overhead across many writes.
+const walFlushInterval = 1 * time.Second
+
 // walShardIndex computes which shard a message belongs to based on cluster+node.
 // Uses FNV-1a hash for fast, well-distributed mapping.
 func walShardIndex(cluster, node string) int {
@@ -126,11 +146,14 @@ func walShardIndex(cluster, node string) int {
 }
 
 // SendWALMessage routes a WAL message to the appropriate shard channel.
-// Returns false if the channel is full (message dropped).
+// Returns false if the channel is full or shutdown is in progress.
 func SendWALMessage(msg *WALMessage) bool {
-	if walShardChs == nil {
+	if walShardChs == nil || walShuttingDown.Load() {
 		return false
 	}
+	if walCheckpointActive.Load() {
+		return true // Data safe in memory; snapshot will capture it
+	}
 	shard := walShardIndex(msg.Cluster, msg.Node)
 	select {
 	case walShardChs[shard] <- msg:
@@ -164,7 +187,9 @@ func WALStaging(wg *sync.WaitGroup, ctx context.Context) {
 		msgCh := walShardChs[i]
 		rotateCh := walShardRotateChs[i]
 
+		walStagingWg.Add(1)
 		wg.Go(func() {
+			defer walStagingWg.Done()
 			hostFiles := make(map[string]*walFileState)
 
 			defer func() {
@@ -198,7 +223,11 @@ func WALStaging(wg *sync.WaitGroup, ctx context.Context) {
 
 				// Write file header magic if file is new (empty).
 				info, err := f.Stat()
-				if err == nil && info.Size() == 0 {
+				var fileSize int64
+				if err == nil {
+					fileSize = info.Size()
+				}
+				if err == nil && fileSize == 0 {
 					var hdr [4]byte
 					binary.LittleEndian.PutUint32(hdr[:], walFileMagic)
 					if _, err := w.Write(hdr[:]); err != nil {
@@ -206,9 +235,10 @@ func WALStaging(wg *sync.WaitGroup, ctx context.Context) {
 						f.Close()
 						return nil
 					}
+					fileSize = 4
 				}
 
-				ws = &walFileState{f: f, w: w}
+				ws = &walFileState{f: f, w: w, size: fileSize}
 				hostFiles[hostDir] = ws
 				return ws
 			}
@@ -219,9 +249,31 @@ func WALStaging(wg *sync.WaitGroup, ctx context.Context) {
 				if ws == nil {
 					return
 				}
-				if err := writeWALRecordDirect(ws.w, msg); err != nil {
+
+				// Enforce max WAL size: force-rotate before writing if limit is exceeded.
+				// The in-memory store still holds the data; only crash-recovery coverage is lost.
+				if maxSize := Keys.Checkpoints.MaxWALSize; maxSize > 0 && ws.size >= maxSize {
+					cclog.Warnf("[METRICSTORE]> WAL: force-rotating %s (size %d >= limit %d)",
+						hostDir, ws.size, maxSize)
+					ws.w.Flush()
+					ws.f.Close()
+					walPath := path.Join(hostDir, "current.wal")
+					if err := os.Remove(walPath); err != nil && !os.IsNotExist(err) {
+						cclog.Errorf("[METRICSTORE]> WAL: remove %s: %v", walPath, err)
+					}
+					delete(hostFiles, hostDir)
+					ws = getOrOpenWAL(hostDir)
+					if ws == nil {
+						return
+					}
+				}
+
+				n, err := writeWALRecordDirect(ws.w, msg)
+				if err != nil {
 					cclog.Errorf("[METRICSTORE]> WAL: write record: %v", err)
 				}
+				ws.size += int64(n)
+				ws.dirty = true
 			}
 
 			processRotate := func(req walRotateReq) {
@@ -238,58 +290,57 @@ func WALStaging(wg *sync.WaitGroup, ctx context.Context) {
 				close(req.done)
 			}
 
-			flushAll := func() {
+			flushDirty := func() {
 				for _, ws := range hostFiles {
-					if ws.f != nil {
+					if ws.dirty {
 						ws.w.Flush()
+						ws.dirty = false
 					}
 				}
 			}
 
-			drain := func() {
-				for {
+			ticker := time.NewTicker(walFlushInterval)
+			defer ticker.Stop()
+
+			// drainBatch processes up to 4096 pending messages without blocking.
+			// Returns false if the channel was closed.
+			drainBatch := func() bool {
+				for range 4096 {
 					select {
 					case msg, ok := <-msgCh:
 						if !ok {
-							return
+							flushDirty()
+							return false
 						}
 						processMsg(msg)
 					case req := <-rotateCh:
 						processRotate(req)
 					default:
-						flushAll()
-						return
+						return true
 					}
 				}
+				return true
 			}
 
 			for {
 				select {
 				case <-ctx.Done():
-					drain()
+					// On shutdown, skip draining buffered messages — a full binary
+					// checkpoint will be written from in-memory state, making
+					// buffered WAL records redundant.
+					flushDirty()
 					return
 				case msg, ok := <-msgCh:
 					if !ok {
 						return
 					}
 					processMsg(msg)
-
-					// Drain up to 256 more messages without blocking to batch writes.
-					for range 256 {
-						select {
-						case msg, ok := <-msgCh:
-							if !ok {
-								return
-							}
-							processMsg(msg)
-						case req := <-rotateCh:
-							processRotate(req)
-						default:
-							goto flushed
-						}
+					if !drainBatch() {
+						return
 					}
-				flushed:
-					flushAll()
+					// No flush here — timer handles periodic flushing.
+				case <-ticker.C:
+					flushDirty()
 				case req := <-rotateCh:
 					processRotate(req)
 				}
@@ -298,23 +349,45 @@ func WALStaging(wg *sync.WaitGroup, ctx context.Context) {
 	}
 }
 
+// WaitForWALStagingDrain blocks until all WALStaging goroutines have exited.
+// Must be called after closing walShardChs to ensure all file handles are
+// flushed and closed before checkpoint writes begin.
+func WaitForWALStagingDrain() {
+	walStagingWg.Wait()
+}
+
 // RotateWALFiles sends rotation requests for the given host directories
 // and blocks until all rotations complete. Each request is routed to the
 // shard that owns the host directory.
+//
+// If shutdown is in progress (WAL staging goroutines may have exited),
+// rotation is skipped to avoid deadlocking on abandoned channels.
 func RotateWALFiles(hostDirs []string) {
-	if walShardRotateChs == nil {
+	if walShardRotateChs == nil || walShuttingDown.Load() {
 		return
 	}
-	dones := make([]chan struct{}, len(hostDirs))
-	for i, dir := range hostDirs {
-		dones[i] = make(chan struct{})
-		// Extract cluster/node from hostDir to find the right shard.
-		// hostDir = rootDir/cluster/node
+	deadline := time.After(2 * time.Minute)
+	dones := make([]chan struct{}, 0, len(hostDirs))
+	for _, dir := range hostDirs {
+		done := make(chan struct{})
 		shard := walShardIndexFromDir(dir)
-		walShardRotateChs[shard] <- walRotateReq{hostDir: dir, done: dones[i]}
+		select {
+		case walShardRotateChs[shard] <- walRotateReq{hostDir: dir, done: done}:
+			dones = append(dones, done)
+		case <-deadline:
+			cclog.Warnf("[METRICSTORE]> WAL rotation send timed out, %d of %d hosts remaining",
+				len(hostDirs)-len(dones), len(hostDirs))
+			goto waitDones
+		}
 	}
+waitDones:
 	for _, done := range dones {
-		<-done
+		select {
+		case <-done:
+		case <-time.After(30 * time.Second):
+			cclog.Warn("[METRICSTORE]> WAL rotation completion timed out, continuing")
+			return
+		}
 	}
 }
 
@@ -327,8 +400,9 @@ func walShardIndexFromDir(hostDir string) int {
 	return walShardIndex(cluster, node)
 }
 
-// RotateWALFiles sends rotation requests for the given host directories
-// and blocks until all rotations complete.
+// RotateWALFilesAfterShutdown directly removes current.wal files for the given
+// host directories. Used after shutdown, when WALStaging goroutines have already
+// exited and the channel-based RotateWALFiles is no longer safe to call.
 func RotateWALFilesAfterShutdown(hostDirs []string) {
 	for _, dir := range hostDirs {
 		walPath := path.Join(dir, "current.wal")
@@ -338,142 +412,66 @@ func RotateWALFilesAfterShutdown(hostDirs []string) {
 	}
 }
 
-// writeWALRecordDirect encodes a WAL record directly into the bufio.Writer,
-// avoiding heap allocations by using a stack-allocated scratch buffer for
-// the fixed-size header/trailer and computing CRC inline.
-func writeWALRecordDirect(w *bufio.Writer, msg *WALMessage) error {
-	// Compute payload size.
+// writeWALRecordDirect encodes a WAL record into a contiguous buffer first,
+// then writes it to the bufio.Writer in a single call. This prevents partial
+// records in the write buffer if a write error occurs mid-record (e.g. disk full).
+// Returns the number of bytes written and any error.
+func writeWALRecordDirect(w *bufio.Writer, msg *WALMessage) (int, error) {
+	// Compute payload and total record size.
 	payloadSize := 8 + 2 + len(msg.MetricName) + 1 + 4
 	for _, s := range msg.Selector {
 		payloadSize += 1 + len(s)
 	}
+	// Total: 8 (header) + payload + 4 (CRC).
+	totalSize := 8 + payloadSize + 4
 
-	// Write magic + payload length (8 bytes header).
-	var hdr [8]byte
-	binary.LittleEndian.PutUint32(hdr[0:4], walRecordMagic)
-	binary.LittleEndian.PutUint32(hdr[4:8], uint32(payloadSize))
-	if _, err := w.Write(hdr[:]); err != nil {
-		return err
+	// Use stack buffer for typical small records, heap-allocate only for large ones.
+	var stackBuf [256]byte
+	var buf []byte
+	if totalSize <= len(stackBuf) {
+		buf = stackBuf[:totalSize]
+	} else {
+		buf = make([]byte, totalSize)
 	}
 
-	// We need to compute CRC over the payload as we write it.
-	crc := crc32.NewIEEE()
+	// Header: magic + payload length.
+	binary.LittleEndian.PutUint32(buf[0:4], walRecordMagic)
+	binary.LittleEndian.PutUint32(buf[4:8], uint32(payloadSize))
+
+	// Payload starts at offset 8.
+	p := 8
 
 	// Timestamp (8 bytes).
-	var scratch [8]byte
-	binary.LittleEndian.PutUint64(scratch[:8], uint64(msg.Timestamp))
-	crc.Write(scratch[:8])
-	if _, err := w.Write(scratch[:8]); err != nil {
-		return err
-	}
+	binary.LittleEndian.PutUint64(buf[p:p+8], uint64(msg.Timestamp))
+	p += 8
 
 	// Metric name length (2 bytes) + metric name.
-	binary.LittleEndian.PutUint16(scratch[:2], uint16(len(msg.MetricName)))
-	crc.Write(scratch[:2])
-	if _, err := w.Write(scratch[:2]); err != nil {
-		return err
-	}
-	nameBytes := []byte(msg.MetricName)
-	crc.Write(nameBytes)
-	if _, err := w.Write(nameBytes); err != nil {
-		return err
-	}
+	binary.LittleEndian.PutUint16(buf[p:p+2], uint16(len(msg.MetricName)))
+	p += 2
+	p += copy(buf[p:], msg.MetricName)
 
 	// Selector count (1 byte).
-	scratch[0] = byte(len(msg.Selector))
-	crc.Write(scratch[:1])
-	if _, err := w.Write(scratch[:1]); err != nil {
-		return err
-	}
+	buf[p] = byte(len(msg.Selector))
+	p++
 
 	// Selectors (1-byte length + bytes each).
 	for _, sel := range msg.Selector {
-		scratch[0] = byte(len(sel))
-		crc.Write(scratch[:1])
-		if _, err := w.Write(scratch[:1]); err != nil {
-			return err
-		}
-		selBytes := []byte(sel)
-		crc.Write(selBytes)
-		if _, err := w.Write(selBytes); err != nil {
-			return err
-		}
+		buf[p] = byte(len(sel))
+		p++
+		p += copy(buf[p:], sel)
 	}
 
 	// Value (4 bytes, float32 bits).
-	binary.LittleEndian.PutUint32(scratch[:4], math.Float32bits(float32(msg.Value)))
-	crc.Write(scratch[:4])
-	if _, err := w.Write(scratch[:4]); err != nil {
-		return err
-	}
+	binary.LittleEndian.PutUint32(buf[p:p+4], math.Float32bits(float32(msg.Value)))
+	p += 4
 
-	// CRC32 (4 bytes).
-	binary.LittleEndian.PutUint32(scratch[:4], crc.Sum32())
-	_, err := w.Write(scratch[:4])
-	return err
-}
+	// CRC32 over payload (bytes 8..8+payloadSize).
+	crc := crc32.ChecksumIEEE(buf[8 : 8+payloadSize])
+	binary.LittleEndian.PutUint32(buf[p:p+4], crc)
 
-// buildWALPayload encodes a WALMessage into a binary payload (without magic/length/CRC).
-func buildWALPayload(msg *WALMessage) []byte {
-	size := 8 + 2 + len(msg.MetricName) + 1 + 4
-	for _, s := range msg.Selector {
-		size += 1 + len(s)
-	}
-
-	buf := make([]byte, 0, size)
-
-	// Timestamp (8 bytes, little-endian int64)
-	var ts [8]byte
-	binary.LittleEndian.PutUint64(ts[:], uint64(msg.Timestamp))
-	buf = append(buf, ts[:]...)
-
-	// Metric name (2-byte length prefix + bytes)
-	var mLen [2]byte
-	binary.LittleEndian.PutUint16(mLen[:], uint16(len(msg.MetricName)))
-	buf = append(buf, mLen[:]...)
-	buf = append(buf, msg.MetricName...)
-
-	// Selector count (1 byte)
-	buf = append(buf, byte(len(msg.Selector)))
-
-	// Selectors (1-byte length prefix + bytes each)
-	for _, sel := range msg.Selector {
-		buf = append(buf, byte(len(sel)))
-		buf = append(buf, sel...)
-	}
-
-	// Value (4 bytes, float32 bit representation)
-	var val [4]byte
-	binary.LittleEndian.PutUint32(val[:], math.Float32bits(float32(msg.Value)))
-	buf = append(buf, val[:]...)
-
-	return buf
-}
-
-// writeWALRecord appends a binary WAL record to the writer.
-// Format: [4B magic][4B payload_len][payload][4B CRC32]
-func writeWALRecord(w io.Writer, msg *WALMessage) error {
-	payload := buildWALPayload(msg)
-	crc := crc32.ChecksumIEEE(payload)
-
-	record := make([]byte, 0, 4+4+len(payload)+4)
-
-	var magic [4]byte
-	binary.LittleEndian.PutUint32(magic[:], walRecordMagic)
-	record = append(record, magic[:]...)
-
-	var pLen [4]byte
-	binary.LittleEndian.PutUint32(pLen[:], uint32(len(payload)))
-	record = append(record, pLen[:]...)
-
-	record = append(record, payload...)
-
-	var crcBytes [4]byte
-	binary.LittleEndian.PutUint32(crcBytes[:], crc)
-	record = append(record, crcBytes[:]...)
-
-	_, err := w.Write(record)
-	return err
+	// Single atomic write to the buffered writer.
+	n, err := w.Write(buf)
+	return n, err
 }
 
 // readWALRecord reads one WAL record from the reader.
@@ -697,7 +695,10 @@ func (m *MemoryStore) ToCheckpointWAL(dir string, from, to int64) (int, []string
 		selector []string
 	}
 
-	n, errs := int32(0), int32(0)
+	totalWork := len(levels)
+	cclog.Infof("[METRICSTORE]> Starting binary checkpoint for %d hosts with %d workers", totalWork, Keys.NumWorkers)
+
+	n, errs, completed := int32(0), int32(0), int32(0)
 	var successDirs []string
 	var successMu sync.Mutex
 
@@ -705,6 +706,22 @@ func (m *MemoryStore) ToCheckpointWAL(dir string, from, to int64) (int, []string
 	wg.Add(Keys.NumWorkers)
 	work := make(chan workItem, Keys.NumWorkers*2)
 
+	// Progress logging goroutine.
+	stopProgress := make(chan struct{})
+	go func() {
+		ticker := time.NewTicker(10 * time.Second)
+		defer ticker.Stop()
+		for {
+			select {
+			case <-ticker.C:
+				cclog.Infof("[METRICSTORE]> Checkpoint progress: %d/%d hosts (%d written, %d errors)",
+					atomic.LoadInt32(&completed), totalWork, atomic.LoadInt32(&n), atomic.LoadInt32(&errs))
+			case <-stopProgress:
+				return
+			}
+		}
+	}()
+
 	for range Keys.NumWorkers {
 		go func() {
 			defer wg.Done()
@@ -712,6 +729,7 @@ func (m *MemoryStore) ToCheckpointWAL(dir string, from, to int64) (int, []string
 				err := wi.level.toCheckpointBinary(wi.hostDir, from, to, m)
 				if err != nil {
 					if err == ErrNoNewArchiveData {
+						atomic.AddInt32(&completed, 1)
 						continue
 					}
 					cclog.Errorf("[METRICSTORE]> binary checkpoint error for %s: %v", wi.hostDir, err)
@@ -722,6 +740,7 @@ func (m *MemoryStore) ToCheckpointWAL(dir string, from, to int64) (int, []string
 					successDirs = append(successDirs, wi.hostDir)
 					successMu.Unlock()
 				}
+				atomic.AddInt32(&completed, 1)
 			}
 		}()
 	}
@@ -736,6 +755,7 @@ func (m *MemoryStore) ToCheckpointWAL(dir string, from, to int64) (int, []string
 	}
 	close(work)
 	wg.Wait()
+	close(stopProgress)
 
 	if errs > 0 {
 		return int(n), successDirs, fmt.Errorf("[METRICSTORE]> %d errors during binary checkpoint (%d successes)", errs, n)

tools/binaryCheckpointReader/binaryCheckpointReader.go

@@ -0,0 +1,381 @@
+// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
+// All rights reserved. This file is part of cc-backend.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+// binaryCheckpointReader reads .wal or .bin checkpoint files produced by the
+// metricstore WAL/snapshot system and dumps their contents to a human-readable
+// .txt file (same name as input, with .txt extension).
+//
+// Usage:
+//
+//	go run ./tools/binaryCheckpointReader <file.wal|file.bin>
+package main
+
+import (
+	"bufio"
+	"encoding/binary"
+	"fmt"
+	"hash/crc32"
+	"io"
+	"math"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+)
+
+// Magic numbers matching metricstore/walCheckpoint.go.
+const (
+	walFileMagic   = uint32(0xCC1DA701)
+	walRecordMagic = uint32(0xCC1DA7A1)
+	snapFileMagic  = uint32(0xCC5B0001)
+)
+
+func main() {
+	if len(os.Args) != 2 {
+		fmt.Fprintf(os.Stderr, "Usage: %s <file.wal|file.bin>\n", os.Args[0])
+		os.Exit(1)
+	}
+
+	inputPath := os.Args[1]
+	ext := strings.ToLower(filepath.Ext(inputPath))
+
+	if ext != ".wal" && ext != ".bin" {
+		fmt.Fprintf(os.Stderr, "Error: file must have .wal or .bin extension, got %q\n", ext)
+		os.Exit(1)
+	}
+
+	f, err := os.Open(inputPath)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error opening %s: %v\n", inputPath, err)
+		os.Exit(1)
+	}
+	defer f.Close()
+
+	// Output file: replace extension with .txt
+	outputPath := strings.TrimSuffix(inputPath, filepath.Ext(inputPath)) + ".txt"
+	out, err := os.Create(outputPath)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error creating output %s: %v\n", outputPath, err)
+		os.Exit(1)
+	}
+	defer out.Close()
+
+	w := bufio.NewWriter(out)
+	defer w.Flush()
+
+	switch ext {
+	case ".wal":
+		err = dumpWAL(f, w)
+	case ".bin":
+		err = dumpBinarySnapshot(f, w)
+	}
+
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error reading %s: %v\n", inputPath, err)
+		os.Exit(1)
+	}
+
+	w.Flush()
+	fmt.Printf("Output written to %s\n", outputPath)
+}
+
+// ---------- WAL reader ----------
+
+func dumpWAL(f *os.File, w *bufio.Writer) error {
+	br := bufio.NewReader(f)
+
+	// Read and verify file header magic.
+	var fileMagic uint32
+	if err := binary.Read(br, binary.LittleEndian, &fileMagic); err != nil {
+		if err == io.EOF {
+			fmt.Fprintln(w, "WAL file is empty (0 bytes).")
+			return nil
+		}
+		return fmt.Errorf("read file header: %w", err)
+	}
+
+	if fileMagic != walFileMagic {
+		return fmt.Errorf("invalid WAL file magic 0x%08X (expected 0x%08X)", fileMagic, walFileMagic)
+	}
+
+	fmt.Fprintf(w, "=== WAL File Dump ===\n")
+	fmt.Fprintf(w, "File:        %s\n", f.Name())
+	fmt.Fprintf(w, "File Magic:  0x%08X (valid)\n\n", fileMagic)
+
+	recordNum := 0
+	for {
+		msg, err := readWALRecord(br)
+		if err != nil {
+			fmt.Fprintf(w, "--- Record #%d: ERROR ---\n", recordNum+1)
+			fmt.Fprintf(w, "  Error: %v\n", err)
+			fmt.Fprintf(w, "  (stopping replay — likely truncated trailing record)\n\n")
+			break
+		}
+		if msg == nil {
+			break // Clean EOF
+		}
+
+		recordNum++
+		ts := time.Unix(msg.Timestamp, 0).UTC()
+
+		fmt.Fprintf(w, "--- Record #%d ---\n", recordNum)
+		fmt.Fprintf(w, "  Timestamp:   %d (%s)\n", msg.Timestamp, ts.Format(time.RFC3339))
+		fmt.Fprintf(w, "  Metric:      %s\n", msg.MetricName)
+		if len(msg.Selector) > 0 {
+			fmt.Fprintf(w, "  Selectors:   [%s]\n", strings.Join(msg.Selector, ", "))
+		} else {
+			fmt.Fprintf(w, "  Selectors:   (none)\n")
+		}
+		fmt.Fprintf(w, "  Value:       %g\n\n", msg.Value)
+	}
+
+	fmt.Fprintf(w, "=== Total valid records: %d ===\n", recordNum)
+	return nil
+}
+
+type walMessage struct {
+	MetricName string
+	Selector   []string
+	Value      float32
+	Timestamp  int64
+}
+
+func readWALRecord(r io.Reader) (*walMessage, error) {
+	var magic uint32
+	if err := binary.Read(r, binary.LittleEndian, &magic); err != nil {
+		if err == io.EOF {
+			return nil, nil
+		}
+		return nil, fmt.Errorf("read record magic: %w", err)
+	}
+
+	if magic != walRecordMagic {
+		return nil, fmt.Errorf("invalid record magic 0x%08X (expected 0x%08X)", magic, walRecordMagic)
+	}
+
+	var payloadLen uint32
+	if err := binary.Read(r, binary.LittleEndian, &payloadLen); err != nil {
+		return nil, fmt.Errorf("read payload length: %w", err)
+	}
+
+	if payloadLen > 1<<20 {
+		return nil, fmt.Errorf("record payload too large: %d bytes", payloadLen)
+	}
+
+	payload := make([]byte, payloadLen)
+	if _, err := io.ReadFull(r, payload); err != nil {
+		return nil, fmt.Errorf("read payload: %w", err)
+	}
+
+	var storedCRC uint32
+	if err := binary.Read(r, binary.LittleEndian, &storedCRC); err != nil {
+		return nil, fmt.Errorf("read CRC: %w", err)
+	}
+
+	if crc32.ChecksumIEEE(payload) != storedCRC {
+		return nil, fmt.Errorf("CRC mismatch (truncated write or corruption)")
+	}
+
+	return parseWALPayload(payload)
+}
+
+func parseWALPayload(payload []byte) (*walMessage, error) {
+	if len(payload) < 8+2+1+4 {
+		return nil, fmt.Errorf("payload too short: %d bytes", len(payload))
+	}
+
+	offset := 0
+
+	// Timestamp (8 bytes).
+	ts := int64(binary.LittleEndian.Uint64(payload[offset : offset+8]))
+	offset += 8
+
+	// Metric name (2-byte length + bytes).
+	if offset+2 > len(payload) {
+		return nil, fmt.Errorf("metric name length overflows payload")
+	}
+	mLen := int(binary.LittleEndian.Uint16(payload[offset : offset+2]))
+	offset += 2
+
+	if offset+mLen > len(payload) {
+		return nil, fmt.Errorf("metric name overflows payload")
+	}
+	metricName := string(payload[offset : offset+mLen])
+	offset += mLen
+
+	// Selector count (1 byte).
+	if offset >= len(payload) {
+		return nil, fmt.Errorf("selector count overflows payload")
+	}
+	selCount := int(payload[offset])
+	offset++
+
+	selectors := make([]string, selCount)
+	for i := range selCount {
+		if offset >= len(payload) {
+			return nil, fmt.Errorf("selector[%d] length overflows payload", i)
+		}
+		sLen := int(payload[offset])
+		offset++
+
+		if offset+sLen > len(payload) {
+			return nil, fmt.Errorf("selector[%d] data overflows payload", i)
+		}
+		selectors[i] = string(payload[offset : offset+sLen])
+		offset += sLen
+	}
+
+	// Value (4 bytes, float32 bits).
+	if offset+4 > len(payload) {
+		return nil, fmt.Errorf("value overflows payload")
+	}
+	bits := binary.LittleEndian.Uint32(payload[offset : offset+4])
+	value := math.Float32frombits(bits)
+
+	return &walMessage{
+		MetricName: metricName,
+		Timestamp:  ts,
+		Selector:   selectors,
+		Value:      value,
+	}, nil
+}
+
+// ---------- Binary snapshot reader ----------
+
+func dumpBinarySnapshot(f *os.File, w *bufio.Writer) error {
+	br := bufio.NewReader(f)
+
+	var magic uint32
+	if err := binary.Read(br, binary.LittleEndian, &magic); err != nil {
+		return fmt.Errorf("read magic: %w", err)
+	}
+	if magic != snapFileMagic {
+		return fmt.Errorf("invalid snapshot magic 0x%08X (expected 0x%08X)", magic, snapFileMagic)
+	}
+
+	var from, to int64
+	if err := binary.Read(br, binary.LittleEndian, &from); err != nil {
+		return fmt.Errorf("read from: %w", err)
+	}
+	if err := binary.Read(br, binary.LittleEndian, &to); err != nil {
+		return fmt.Errorf("read to: %w", err)
+	}
+
+	fromTime := time.Unix(from, 0).UTC()
+	toTime := time.Unix(to, 0).UTC()
+
+	fmt.Fprintf(w, "=== Binary Snapshot Dump ===\n")
+	fmt.Fprintf(w, "File:    %s\n", f.Name())
+	fmt.Fprintf(w, "Magic:   0x%08X (valid)\n", magic)
+	fmt.Fprintf(w, "From:    %d (%s)\n", from, fromTime.Format(time.RFC3339))
+	fmt.Fprintf(w, "To:      %d (%s)\n\n", to, toTime.Format(time.RFC3339))
+
+	return dumpBinaryLevel(br, w, 0)
+}
+
+func dumpBinaryLevel(r io.Reader, w *bufio.Writer, depth int) error {
+	indent := strings.Repeat("  ", depth)
+
+	var numMetrics uint32
+	if err := binary.Read(r, binary.LittleEndian, &numMetrics); err != nil {
+		return fmt.Errorf("read num_metrics: %w", err)
+	}
+
+	if numMetrics > 0 {
+		fmt.Fprintf(w, "%sMetrics (%d):\n", indent, numMetrics)
+	}
+
+	for i := range numMetrics {
+		name, err := readString16(r)
+		if err != nil {
+			return fmt.Errorf("read metric name [%d]: %w", i, err)
+		}
+
+		var freq, start int64
+		if err := binary.Read(r, binary.LittleEndian, &freq); err != nil {
+			return fmt.Errorf("read frequency for %s: %w", name, err)
+		}
+		if err := binary.Read(r, binary.LittleEndian, &start); err != nil {
+			return fmt.Errorf("read start for %s: %w", name, err)
+		}
+
+		var numValues uint32
+		if err := binary.Read(r, binary.LittleEndian, &numValues); err != nil {
+			return fmt.Errorf("read num_values for %s: %w", name, err)
+		}
+
+		startTime := time.Unix(start, 0).UTC()
+
+		fmt.Fprintf(w, "%s  [%s]\n", indent, name)
+		fmt.Fprintf(w, "%s    Frequency:  %d s\n", indent, freq)
+		fmt.Fprintf(w, "%s    Start:      %d (%s)\n", indent, start, startTime.Format(time.RFC3339))
+		fmt.Fprintf(w, "%s    Values (%d):", indent, numValues)
+
+		if numValues == 0 {
+			fmt.Fprintln(w, " (none)")
+		} else {
+			fmt.Fprintln(w)
+			// Print values in rows of 10 for readability.
+			for j := range numValues {
+				var bits uint32
+				if err := binary.Read(r, binary.LittleEndian, &bits); err != nil {
+					return fmt.Errorf("read value[%d] for %s: %w", j, name, err)
+				}
+				val := math.Float32frombits(bits)
+
+				if j%10 == 0 {
+					if j > 0 {
+						fmt.Fprintln(w)
+					}
+					// Print the timestamp for this row's first value.
+					rowTS := start + int64(j)*freq
+					fmt.Fprintf(w, "%s      [%s] ", indent, time.Unix(rowTS, 0).UTC().Format("15:04:05"))
+				}
+
+				if math.IsNaN(float64(val)) {
+					fmt.Fprintf(w, "NaN ")
+				} else {
+					fmt.Fprintf(w, "%g ", val)
+				}
+			}
+			fmt.Fprintln(w)
+		}
+	}
+
+	var numChildren uint32
+	if err := binary.Read(r, binary.LittleEndian, &numChildren); err != nil {
+		return fmt.Errorf("read num_children: %w", err)
+	}
+
+	if numChildren > 0 {
+		fmt.Fprintf(w, "%sChildren (%d):\n", indent, numChildren)
+	}
+
+	for i := range numChildren {
+		childName, err := readString16(r)
+		if err != nil {
+			return fmt.Errorf("read child name [%d]: %w", i, err)
+		}
+
+		fmt.Fprintf(w, "%s  [%s]\n", indent, childName)
+		if err := dumpBinaryLevel(r, w, depth+2); err != nil {
+			return fmt.Errorf("read child %s: %w", childName, err)
+		}
+	}
+
+	return nil
+}
+
+func readString16(r io.Reader) (string, error) {
+	var sLen uint16
+	if err := binary.Read(r, binary.LittleEndian, &sLen); err != nil {
+		return "", err
+	}
+	buf := make([]byte, sLen)
+	if _, err := io.ReadFull(r, buf); err != nil {
+		return "", err
+	}
+	return string(buf), nil
+}

web/frontend/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "cc-frontend",
-  "version": "1.5.2",
+  "version": "1.5.3",
   "lockfileVersion": 4,
   "requires": true,
   "packages": {
     "": {
       "name": "cc-frontend",
-      "version": "1.5.2",
+      "version": "1.5.3",
       "license": "MIT",
       "dependencies": {
         "@rollup/plugin-replace": "^6.0.3",
@@ -608,9 +608,9 @@
       ]
     },
     "node_modules/@sveltejs/acorn-typescript": {
-      "version": "1.0.9",
-      "resolved": "https://registry.npmjs.org/@sveltejs/acorn-typescript/-/acorn-typescript-1.0.9.tgz",
-      "integrity": "sha512-lVJX6qEgs/4DOcRTpo56tmKzVPtoWAaVbL4hfO7t7NVwl9AAXzQR6cihesW1BmNMPl+bK6dreu2sOKBP2Q9CIA==",
+      "version": "1.0.10",
+      "resolved": "https://registry.npmjs.org/@sveltejs/acorn-typescript/-/acorn-typescript-1.0.10.tgz",
+      "integrity": "sha512-4WfKk68eTih+MiJD4fSbxN7E8kVBmTMPWHUPYjvl2N0rMs53YLTT8/YjKU5Dtnz5LqDjl7LEw4U7lXR2W3J5WA==",
       "license": "MIT",
       "peerDependencies": {
         "acorn": "^8.9.0"
@@ -654,19 +654,6 @@
       "integrity": "sha512-ScaPdn1dQczgbl0QFTeTOmVHFULt394XJgOQNoyVhZ6r2vLnMLJfBPd53SB52T/3G36VI1/g2MZaX0cwDuXsfw==",
       "license": "MIT"
     },
-    "node_modules/@typescript-eslint/types": {
-      "version": "8.57.1",
-      "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-8.57.1.tgz",
-      "integrity": "sha512-S29BOBPJSFUiblEl6RzPPjJt6w25A6XsBqRVDt53tA/tlL8q7ceQNZHTjPeONt/3S7KRI4quk+yP9jK2WjBiPQ==",
-      "license": "MIT",
-      "engines": {
-        "node": "^18.18.0 || ^20.9.0 || >=21.1.0"
-      },
-      "funding": {
-        "type": "opencollective",
-        "url": "https://opencollective.com/typescript-eslint"
-      }
-    },
     "node_modules/@urql/core": {
       "version": "5.2.0",
       "resolved": "https://registry.npmjs.org/@urql/core/-/core-5.2.0.tgz",
@@ -803,9 +790,9 @@
       }
     },
     "node_modules/devalue": {
-      "version": "5.6.4",
-      "resolved": "https://registry.npmjs.org/devalue/-/devalue-5.6.4.tgz",
-      "integrity": "sha512-Gp6rDldRsFh/7XuouDbxMH3Mx8GMCcgzIb1pDTvNyn8pZGQ22u+Wa+lGV9dQCltFQ7uVw0MhRyb8XDskNFOReA==",
+      "version": "5.8.1",
+      "resolved": "https://registry.npmjs.org/devalue/-/devalue-5.8.1.tgz",
+      "integrity": "sha512-4CXDYRBGqN+57wVJkuXBYmpAVUSg3L6JAQa/DFqm238G73E1wuyc/JhGQJzN7vUf/CMphYau2zXbfWzDR5aTEw==",
       "license": "MIT"
     },
     "node_modules/escape-latex": {
@@ -821,13 +808,20 @@
       "license": "MIT"
     },
     "node_modules/esrap": {
-      "version": "2.2.4",
-      "resolved": "https://registry.npmjs.org/esrap/-/esrap-2.2.4.tgz",
-      "integrity": "sha512-suICpxAmZ9A8bzJjEl/+rLJiDKC0X4gYWUxT6URAWBLvlXmtbZd5ySMu/N2ZGEtMCAmflUDPSehrP9BQcsGcSg==",
+      "version": "2.2.9",
+      "resolved": "https://registry.npmjs.org/esrap/-/esrap-2.2.9.tgz",
+      "integrity": "sha512-4KijP+NxCWthMCUC3qHbE6n4vCjqgJS1uAYKhuT/GWfFTf1Qyive2TgOjep+gzbSzRfnNyaN/UU9YmdOt8Eg0A==",
       "license": "MIT",
       "dependencies": {
-        "@jridgewell/sourcemap-codec": "^1.4.15",
+        "@jridgewell/sourcemap-codec": "^1.4.15"
+      },
+      "peerDependencies": {
         "@typescript-eslint/types": "^8.2.0"
+      },
+      "peerDependenciesMeta": {
+        "@typescript-eslint/types": {
+          "optional": true
+        }
       }
     },
     "node_modules/estree-walker": {
@@ -968,9 +962,9 @@
       }
     },
     "node_modules/mathjs": {
-      "version": "15.1.1",
-      "resolved": "https://registry.npmjs.org/mathjs/-/mathjs-15.1.1.tgz",
-      "integrity": "sha512-rM668DTtpSzMVoh/cKAllyQVEbBApM5g//IMGD8vD7YlrIz9ITRr3SrdhjaDxcBNTdyETWwPebj2unZyHD7ZdA==",
+      "version": "15.2.0",
+      "resolved": "https://registry.npmjs.org/mathjs/-/mathjs-15.2.0.tgz",
+      "integrity": "sha512-UAQzSVob9rNLdGpqcFMYmSu9dkuLYy7Lr2hBEQS5SHQdknA9VppJz3cy2KkpMzTODunad6V6cNv+5kOLsePLow==",
       "license": "Apache-2.0",
       "dependencies": {
         "@babel/runtime": "^7.26.10",
@@ -998,9 +992,9 @@
       "license": "MIT"
     },
     "node_modules/picomatch": {
-      "version": "4.0.3",
-      "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.3.tgz",
-      "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==",
+      "version": "4.0.4",
+      "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.4.tgz",
+      "integrity": "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==",
       "license": "MIT",
       "engines": {
         "node": ">=12"
@@ -1134,9 +1128,9 @@
       }
     },
     "node_modules/rollup-plugin-svelte/node_modules/picomatch": {
-      "version": "2.3.1",
-      "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz",
-      "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==",
+      "version": "2.3.2",
+      "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.2.tgz",
+      "integrity": "sha512-V7+vQEJ06Z+c5tSye8S+nHUfI51xoXIXjHQ99cQtKUkQqqO1kO/KCJUfZXuB47h/YBlDhah2H3hdUGXn8ie0oA==",
       "dev": true,
       "license": "MIT",
       "engines": {
@@ -1153,9 +1147,9 @@
       "license": "MIT"
     },
     "node_modules/serialize-javascript": {
-      "version": "7.0.4",
-      "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-7.0.4.tgz",
-      "integrity": "sha512-DuGdB+Po43Q5Jxwpzt1lhyFSYKryqoNjQSA9M92tyw0lyHIOur+XCalOUe0KTJpyqzT8+fQ5A0Jf7vCx/NKmIg==",
+      "version": "7.0.5",
+      "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-7.0.5.tgz",
+      "integrity": "sha512-F4LcB0UqUl1zErq+1nYEEzSHJnIwb3AF2XWB94b+afhrekOUijwooAYqFyRbjYkm2PAKBabx6oYv/xDxNi8IBw==",
       "dev": true,
       "license": "BSD-3-Clause",
       "engines": {
@@ -1207,23 +1201,23 @@
       }
     },
     "node_modules/svelte": {
-      "version": "5.54.0",
-      "resolved": "https://registry.npmjs.org/svelte/-/svelte-5.54.0.tgz",
-      "integrity": "sha512-TTDxwYnHkova6Wsyj1PGt9TByuWqvMoeY1bQiuAf2DM/JeDSMw7FjRKzk8K/5mJ99vGOKhbCqTDpyAKwjp4igg==",
+      "version": "5.55.10",
+      "resolved": "https://registry.npmjs.org/svelte/-/svelte-5.55.10.tgz",
+      "integrity": "sha512-v9mFVBY1USosyIWdXE7Cg4AN0ywyKCMcAhONvli8doMowEhFhMdNLKD1j7O/UnsrdVTHaUOk/jv8hD/HClVy+g==",
       "license": "MIT",
       "dependencies": {
         "@jridgewell/remapping": "^2.3.4",
         "@jridgewell/sourcemap-codec": "^1.5.0",
-        "@sveltejs/acorn-typescript": "^1.0.5",
+        "@sveltejs/acorn-typescript": "^1.0.10",
         "@types/estree": "^1.0.5",
         "@types/trusted-types": "^2.0.7",
         "acorn": "^8.12.1",
         "aria-query": "5.3.1",
         "axobject-query": "^4.1.0",
         "clsx": "^2.1.1",
-        "devalue": "^5.6.4",
+        "devalue": "^5.8.1",
         "esm-env": "^1.2.1",
-        "esrap": "^2.2.2",
+        "esrap": "^2.2.9",
         "is-reference": "^3.0.3",
         "locate-character": "^3.0.0",
         "magic-string": "^0.30.11",

web/frontend/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cc-frontend",
-  "version": "1.5.2",
+  "version": "1.5.3",
   "license": "MIT",
   "scripts": {
     "build": "rollup -c",

web/frontend/src/Job.root.svelte

@@ -237,7 +237,7 @@
                   The following note was added by administrators:
                 </Card>
                 <Card body>
-                  {@html thisJob.metaData.message}
+                  <span style="white-space: pre-wrap">{thisJob.metaData.message}</span>
                 </Card>
               </CardBody>
             </TabPane>

web/frontend/src/Logs.root.svelte

@@ -57,7 +57,7 @@
   let entries = $state([]);
   let loading = $state(false);
   let error = $state(null);
-  let timer = $state(null);
+  let timer = null;
 
   function levelColor(priority) {
     if (priority <= 2) return "danger";

web/frontend/src/Node.root.svelte

@@ -54,11 +54,16 @@
   const paging = { itemsPerPage: 50, page: 1 };
   const sorting = { field: "startTime", type: "col", order: "DESC" };
   const nodeMetricsQuery = gql`
-    query ($cluster: String!, $nodes: [String!], $from: Time!, $to: Time!) {
+    query (
+      $cluster: String!,
+      $nodes: [String!],
+      $from: Time!,
+      $to: Time!, 
+      $nodeFilter: [NodeFilter!]!,
+      $sorting: OrderByInput!
+    ) {
       nodeMetrics(cluster: $cluster, nodes: $nodes, from: $from, to: $to) {
         host
-        nodeState
-        metricHealth
         subCluster
         metrics {
           name
@@ -79,7 +84,14 @@
             }
           }
         }
-      }
+      },
+      nodeStatus: nodes(filter: $nodeFilter, order: $sorting) {
+          count
+          items {
+            schedulerState
+            healthState
+          }
+        }
     }
   `;
   const nodeJobsQuery = gql`
@@ -146,6 +158,8 @@
         nodes: [hostname],
         from: from?.toISOString(),
         to: to?.toISOString(),
+        nodeFilter: { hostname: { eq: hostname }},
+        sorting // $sorting unused in backend: Use placeholder
       },
     })
   );
@@ -157,8 +171,8 @@
     })
   );
 
-  const thisNodeState = $derived($nodeMetricsData?.data?.nodeMetrics[0]?.nodeState || 'notindb');
-  const thisMetricHealth = $derived($nodeMetricsData?.data?.nodeMetrics[0]?.metricHealth || 'unknown');
+  const thisNodeState = $derived($nodeMetricsData?.data?.nodeStatus?.items[0]?.schedulerState || 'notindb');
+  const thisMetricHealth = $derived($nodeMetricsData?.data?.nodeStatus?.items[0]?.healthState || 'unknown');
 </script>
 
 <Row cols={{ xs: 2, lg: 3}}>

web/frontend/src/generic/Filters.svelte

@@ -73,6 +73,7 @@
     userMatch: "contains",
     // Filter Modals
     cluster: null,
+    subCluster: null,
     partition: null,
     states: allJobStates,
     shared: "",
@@ -107,6 +108,7 @@
     user: filterPresets?.user || "",
     userMatch: filterPresets?.userMatch || "contains",
     cluster: filterPresets?.cluster || null,
+    subCluster: filterPresets?.subCluster || null,
     partition: filterPresets?.partition || null,
     states:
       filterPresets?.states || filterPresets?.state
@@ -158,6 +160,7 @@
     if (filters.dbId.length != 0)
       items.push({ dbId: filters.dbId });
     if (filters.cluster) items.push({ cluster: { eq: filters.cluster } });
+    if (filters.subCluster) items.push({ subCluster: { eq: filters.subCluster } });
     if (filters.partition) items.push({ partition: { eq: filters.partition } });
     if (filters.states.length != allJobStates?.length)
       items.push({ state: filters.states });
@@ -166,12 +169,12 @@
       items.push({ project: { [filters.projectMatch]: filters.project } });
     if (filters.user)
       items.push({ user: { [filters.userMatch]: filters.user } });
-    if (filters.numNodes.from != null || filters.numNodes.to != null) {
+    if (filters.numNodes.from != null && filters.numNodes.to != null) {
       items.push({
         numNodes: { from: filters.numNodes.from, to: filters.numNodes.to },
       });
     }
-    if (filters.numAccelerators.from != null || filters.numAccelerators.to != null) {
+    if (filters.numAccelerators.from != null && filters.numAccelerators.to != null) {
       items.push({
         numAccelerators: {
           from: filters.numAccelerators.from,
@@ -179,7 +182,7 @@
         },
       });
     }
-    if (filters.numHWThreads.from != null || filters.numHWThreads.to != null) {
+    if (filters.numHWThreads.from != null && filters.numHWThreads.to != null) {
       items.push({
         numHWThreads: {
           from: filters.numHWThreads.from,
@@ -206,14 +209,21 @@
       items.push({ duration: { to: filters.duration.lessThan, from: 0 } });
     if (filters.duration.moreThan)
       items.push({ duration: { to: 0, from: filters.duration.moreThan } });
-    if (filters.energy.from != null || filters.energy.to != null)
+    if (filters.energy.from != null && filters.energy.to != null)
       items.push({
         energy: { from: filters.energy.from, to: filters.energy.to },
       });
     if (filters.jobId)
       items.push({ jobId: { [filters.jobIdMatch]: filters.jobId } });
-    if (filters.stats.length != 0)
-      items.push({ metricStats: filters.stats.map((st) => { return { metricName: st.field, range: { from: st.from, to: st.to }} }) });
+    if (filters.stats.length != 0) {
+      const metricStats = [];
+      filters.stats.forEach((st) => {
+        if (st.from != null && st.to != null) 
+          metricStats.push({ metricName: st.field, range: { from: st.from, to: st.to }});
+      });
+      if (metricStats.length != 0)
+         items.push({metricStats})
+    };
     if (filters.node) items.push({ node: { [filters.nodeMatch]: filters.node } });
     if (filters.jobName) items.push({ jobName: { contains: filters.jobName } });
     if (filters.schedule) items.push({ schedule: filters.schedule });
@@ -260,6 +270,7 @@
       opts.push(`userMatch=${filters.userMatch}`);
     // Filter Modals
     if (filters.cluster) opts.push(`cluster=${filters.cluster}`);
+    if (filters.subCluster) opts.push(`subCluster=${filters.subCluster}`);
     if (filters.partition) opts.push(`partition=${filters.partition}`);
     if (filters.states.length != allJobStates?.length)
       for (let state of filters.states) opts.push(`state=${state}`);
@@ -280,40 +291,40 @@
       opts.push(`duration=morethan-${filters.duration.moreThan}`);
     if (filters.tags.length != 0)
       for (let tag of filters.tags) opts.push(`tag=${tag}`);
-    if (filters.numNodes.from > 1 && filters.numNodes.to > 0)
+    if (filters.numNodes.from > 0 && filters.numNodes.to > 0)
       opts.push(`numNodes=${filters.numNodes.from}-${filters.numNodes.to}`);
-    else if (filters.numNodes.from > 1 && filters.numNodes.to == 0)
+    else if (filters.numNodes.from > 0 && filters.numNodes.to == 0)
       opts.push(`numNodes=morethan-${filters.numNodes.from}`);
-    else if (filters.numNodes.from == 1 && filters.numNodes.to > 0)
+    else if (filters.numNodes.from == 0 && filters.numNodes.to > 0)
       opts.push(`numNodes=lessthan-${filters.numNodes.to}`);
-    if (filters.numHWThreads.from > 1 && filters.numHWThreads.to > 0)
+    if (filters.numHWThreads.from > 0 && filters.numHWThreads.to > 0)
       opts.push(`numHWThreads=${filters.numHWThreads.from}-${filters.numHWThreads.to}`);
-    else if (filters.numHWThreads.from > 1 && filters.numHWThreads.to == 0)
+    else if (filters.numHWThreads.from > 0 && filters.numHWThreads.to == 0)
       opts.push(`numHWThreads=morethan-${filters.numHWThreads.from}`);
-    else if (filters.numHWThreads.from == 1 && filters.numHWThreads.to > 0)
+    else if (filters.numHWThreads.from == 0 && filters.numHWThreads.to > 0)
       opts.push(`numHWThreads=lessthan-${filters.numHWThreads.to}`);
-    if (filters.numAccelerators.from && filters.numAccelerators.to)
+    if (filters.numAccelerators.from > 0 && filters.numAccelerators.to > 0)
       opts.push(`numAccelerators=${filters.numAccelerators.from}-${filters.numAccelerators.to}`);
-    else if (filters.numAccelerators.from > 1 && filters.numAccelerators.to == 0)
+    else if (filters.numAccelerators.from > 0 && filters.numAccelerators.to == 0)
       opts.push(`numAccelerators=morethan-${filters.numAccelerators.from}`);
-    else if (filters.numAccelerators.from == 1 && filters.numAccelerators.to > 0)
+    else if (filters.numAccelerators.from == 0 && filters.numAccelerators.to > 0)
       opts.push(`numAccelerators=lessthan-${filters.numAccelerators.to}`);
     if (filters.node) opts.push(`node=${filters.node}`);
     if (filters.node && filters.nodeMatch != "eq") // "eq" is default-case
       opts.push(`nodeMatch=${filters.nodeMatch}`);
-    if (filters.energy.from > 1 && filters.energy.to > 0)
+    if (filters.energy.from > 0 && filters.energy.to > 0)
       opts.push(`energy=${filters.energy.from}-${filters.energy.to}`);
-    else if (filters.energy.from > 1 && filters.energy.to == 0)
+    else if (filters.energy.from > 0 && filters.energy.to == 0)
       opts.push(`energy=morethan-${filters.energy.from}`);
-    else if (filters.energy.from == 1 && filters.energy.to > 0)
+    else if (filters.energy.from == 0 && filters.energy.to > 0)
       opts.push(`energy=lessthan-${filters.energy.to}`);
     if (filters.stats.length > 0)
       for (let stat of filters.stats) {
-        if (stat.from > 1 && stat.to > 0)
+        if (stat.from > 0 && stat.to > 0)
           opts.push(`stat=${stat.field}-${stat.from}-${stat.to}`);
-        else if (stat.from > 1 && stat.to == 0)
+        else if (stat.from > 0 && stat.to == 0)
           opts.push(`stat=${stat.field}-morethan-${stat.from}`);
-        else if (stat.from == 1 && stat.to > 0)
+        else if (stat.from == 0 && stat.to > 0)
           opts.push(`stat=${stat.field}-lessthan-${stat.to}`);
       }
     // Build && Return
@@ -339,7 +350,7 @@
         {/if}
         <DropdownItem header>Manage Filters</DropdownItem>
         <DropdownItem onclick={() => (isClusterOpen = true)}>
-          <Icon name="cpu" /> Cluster/Partition
+          <Icon name="cpu" /> Cluster/SubCluster/Partition
         </DropdownItem>
         <DropdownItem onclick={() => (isJobStatesOpen = true)}>
           <Icon name="gear-fill" /> Job States
@@ -433,6 +444,9 @@
   {#if filters.cluster}
     <Info icon="cpu" onclick={() => (isClusterOpen = true)}>
       {filters.cluster}
+      {#if filters.subCluster}
+        [{filters.subCluster}]
+      {/if}
       {#if filters.partition}
         ({filters.partition})
       {/if}
@@ -511,43 +525,43 @@
     </Info>
   {/if}
 
-  {#if filters.numNodes.from > 1 && filters.numNodes.to > 0}
+  {#if filters.numNodes.from > 0 && filters.numNodes.to > 0}
     <Info icon="hdd-stack" onclick={() => (isResourcesOpen = true)}>
       Nodes: {filters.numNodes.from} - {filters.numNodes.to}
     </Info>
-  {:else if filters.numNodes.from > 1 && filters.numNodes.to == 0}
+  {:else if filters.numNodes.from > 0 && filters.numNodes.to == 0}
     <Info icon="hdd-stack" onclick={() => (isResourcesOpen = true)}>
       &nbsp;&ge;&nbsp;{filters.numNodes.from} Node(s)
     </Info>
-  {:else if filters.numNodes.from == 1 && filters.numNodes.to > 0}
+  {:else if filters.numNodes.from == 0 && filters.numNodes.to > 0}
     <Info icon="hdd-stack" onclick={() => (isResourcesOpen = true)}>
       &nbsp;&le;&nbsp;{filters.numNodes.to} Node(s)
     </Info>
   {/if}
 
-  {#if filters.numHWThreads.from > 1 && filters.numHWThreads.to > 0}
+  {#if filters.numHWThreads.from > 0 && filters.numHWThreads.to > 0}
     <Info icon="cpu" onclick={() => (isResourcesOpen = true)}>
       HWThreads: {filters.numHWThreads.from} - {filters.numHWThreads.to}
     </Info>
-  {:else if filters.numHWThreads.from > 1 && filters.numHWThreads.to == 0}
+  {:else if filters.numHWThreads.from > 0 && filters.numHWThreads.to == 0}
     <Info icon="cpu" onclick={() => (isResourcesOpen = true)}>
       &nbsp;&ge;&nbsp;{filters.numHWThreads.from} HWThread(s)
     </Info>
-  {:else if filters.numHWThreads.from == 1 && filters.numHWThreads.to > 0}
+  {:else if filters.numHWThreads.from == 0 && filters.numHWThreads.to > 0}
     <Info icon="cpu" onclick={() => (isResourcesOpen = true)}>
       &nbsp;&le;&nbsp;{filters.numHWThreads.to} HWThread(s)
     </Info>
   {/if}
 
-  {#if filters.numAccelerators.from > 1 && filters.numAccelerators.to > 0}
+  {#if filters.numAccelerators.from > 0 && filters.numAccelerators.to > 0}
     <Info icon="gpu-card" onclick={() => (isResourcesOpen = true)}>
       Accelerators: {filters.numAccelerators.from} - {filters.numAccelerators.to}
     </Info>
-  {:else if filters.numAccelerators.from > 1 && filters.numAccelerators.to == 0}
+  {:else if filters.numAccelerators.from > 0 && filters.numAccelerators.to == 0}
     <Info icon="gpu-card" onclick={() => (isResourcesOpen = true)}>
       &nbsp;&ge;&nbsp;{filters.numAccelerators.from} Acc(s)
     </Info>
-  {:else if filters.numAccelerators.from == 1 && filters.numAccelerators.to > 0}
+  {:else if filters.numAccelerators.from == 0 && filters.numAccelerators.to > 0}
     <Info icon="gpu-card" onclick={() => (isResourcesOpen = true)}>
       &nbsp;&le;&nbsp;{filters.numAccelerators.to} Acc(s)
     </Info>
@@ -559,15 +573,15 @@
     </Info>
   {/if}
 
-  {#if filters.energy.from > 1 && filters.energy.to > 0}
+  {#if filters.energy.from > 0 && filters.energy.to > 0}
     <Info icon="lightning-charge-fill" onclick={() => (isEnergyOpen = true)}>
       Total Energy: {filters.energy.from} - {filters.energy.to} kWh
     </Info>
-  {:else if filters.energy.from > 1 && filters.energy.to == 0}
+  {:else if filters.energy.from > 0 && filters.energy.to == 0}
     <Info icon="lightning-charge-fill" onclick={() => (isEnergyOpen = true)}>
       Total Energy &ge;&nbsp;{filters.energy.from} kWh
     </Info>
-  {:else if filters.energy.from == 1 && filters.energy.to > 0}
+  {:else if filters.energy.from == 0 && filters.energy.to > 0}
     <Info icon="lightning-charge-fill" onclick={() => (isEnergyOpen = true)}>
       Total Energy &le;&nbsp;{filters.energy.to} kWh
     </Info>
@@ -575,15 +589,15 @@
 
   {#if filters.stats.length > 0}
     {#each filters.stats as stat}
-      {#if stat.from > 1 && stat.to > 0}
+      {#if stat.from > 0 && stat.to > 0}
         <Info icon="bar-chart" onclick={() => (isStatsOpen = true)}>
           {stat.field}: {stat.from} - {stat.to} {stat.unit}
         </Info>&thinsp;
-      {:else if stat.from > 1 && stat.to == 0}
+      {:else if stat.from > 0 && stat.to == 0}
         <Info icon="bar-chart" onclick={() => (isStatsOpen = true)}>
           {stat.field} &ge;&nbsp;{stat.from} {stat.unit}
         </Info>&thinsp;
-      {:else if stat.from == 1 && stat.to > 0}
+      {:else if stat.from == 0 && stat.to > 0}
         <Info icon="bar-chart" onclick={() => (isStatsOpen = true)}>
           {stat.field} &le;&nbsp;{stat.to} {stat.unit}
         </Info>&thinsp;
@@ -596,6 +610,7 @@
   bind:isOpen={isClusterOpen}
   presetCluster={filters.cluster}
   presetPartition={filters.partition}
+  presetSubCluster={filters.subCluster}
   {disableClusterSelection}
   setFilter={(filter) => updateFilters(filter)}
 />

web/frontend/src/generic/filters/Cluster.svelte

@@ -1,10 +1,11 @@
 <!--
-  @component Filter sub-component for selecting cluster and subCluster
+  @component Filter sub-component for selecting cluster, partition and subCluster
 
   Properties:
   - `isOpen Bool?`: Is this filter component opened [Bindable, Default: false]
   - `presetCluster String?`: The latest selected cluster [Default: ""]
   - `presetPartition String?`: The latest selected partition [Default: ""]
+  - `presetSubCluster String?`: The latest selected subCluster [Default: ""]
   - `disableClusterSelection Bool?`: Is the selection disabled [Default: false]
   - `setFilter Func`: The callback function to apply current filter selection
 -->
@@ -26,6 +27,7 @@
     isOpen = $bindable(false),
     presetCluster = "",
     presetPartition = "",
+    presetSubCluster = "",
     disableClusterSelection = false,
     setFilter
   } = $props();
@@ -36,10 +38,11 @@
   const clusterInfos = $derived($initialized ? getContext("clusters") : null);
   let pendingCluster = $derived(presetCluster);
   let pendingPartition = $derived(presetPartition);
+  let pendingSubCluster = $derived(presetSubCluster);
 </script>
 
 <Modal {isOpen} toggle={() => (isOpen = !isOpen)}>
-  <ModalHeader>Select Cluster & Slurm Partition</ModalHeader>
+  <ModalHeader>Select Cluster, SubCluster & Partition</ModalHeader>
   <ModalBody>
     {#if $initialized}
       <h4>Cluster</h4>
@@ -51,7 +54,7 @@
           <ListGroupItem
             disabled={disableClusterSelection}
             active={pendingCluster == null}
-            onclick={() => ((pendingCluster = null), (pendingPartition = null))}
+            onclick={() => ((pendingCluster = null), (pendingPartition = null), (pendingSubCluster = null))}
           >
             Any Cluster
           </ListGroupItem>
@@ -60,7 +63,7 @@
               disabled={disableClusterSelection}
               active={pendingCluster == cluster.name}
               onclick={() => (
-                (pendingCluster = cluster.name), (pendingPartition = null)
+                (pendingCluster = cluster.name), (pendingPartition = null), (pendingSubCluster = null)
               )}
             >
               {cluster.name}
@@ -71,7 +74,27 @@
     {/if}
     {#if $initialized && pendingCluster != null}
       <br />
-      <h4>Partiton</h4>
+      <h4>SubCluster</h4>
+      <ListGroup>
+        <ListGroupItem
+          active={pendingSubCluster == null}
+          onclick={() => (pendingSubCluster = null)}
+        >
+          Any SubCluster
+        </ListGroupItem>
+        {#each clusterInfos?.find((c) => c.name == pendingCluster)?.subClusters as subCluster}
+          <ListGroupItem
+            active={pendingSubCluster == subCluster.name}
+            onclick={() => (pendingSubCluster = subCluster.name)}
+          >
+            {subCluster.name}
+          </ListGroupItem>
+        {/each}
+      </ListGroup>
+    {/if}
+    {#if $initialized && pendingCluster != null}
+      <br />
+      <h4>Partition</h4>
       <ListGroup>
         <ListGroupItem
           active={pendingPartition == null}
@@ -95,7 +118,7 @@
       color="primary"
       onclick={() => {
         isOpen = false;
-        setFilter({ cluster: pendingCluster, partition: pendingPartition });
+        setFilter({ cluster: pendingCluster, subCluster: pendingSubCluster, partition: pendingPartition });
       }}>Close & Apply</Button
     >
     {#if !disableClusterSelection}
@@ -105,7 +128,8 @@
           isOpen = false;
           pendingCluster = null;
           pendingPartition = null;
-          setFilter({ cluster: pendingCluster, partition: pendingPartition})
+          pendingSubCluster = null;
+          setFilter({ cluster: pendingCluster, subCluster: pendingSubCluster, partition: pendingPartition })
         }}>Reset</Button
       >
     {/if}

web/frontend/src/generic/filters/Energy.svelte

@@ -28,31 +28,29 @@
   } = $props();
 
   /* Const */
-  const minEnergyPreset = 1;
+  const minEnergyPreset = 0;
   const maxEnergyPreset = 100;
 
   /* Derived */
   // Pending
   let pendingEnergyState = $derived({
-    from: presetEnergy?.from ? presetEnergy.from : minEnergyPreset,
-    to: !(presetEnergy.to == null || presetEnergy.to == 0) ? presetEnergy.to : maxEnergyPreset,
+    from: presetEnergy?.from || minEnergyPreset,
+    to: (presetEnergy.to == 0) ? null : presetEnergy.to,
   });
   // Changable
   let energyState = $derived({
-    from: presetEnergy?.from ? presetEnergy.from : minEnergyPreset,
-    to: !(presetEnergy.to == null || presetEnergy.to == 0) ? presetEnergy.to : maxEnergyPreset,
+    from: presetEnergy?.from || minEnergyPreset,
+    to: (presetEnergy.to == 0) ? null : presetEnergy.to,
   });
 
-  const energyActive = $derived(!(JSON.stringify(energyState) === JSON.stringify({ from: minEnergyPreset, to: maxEnergyPreset })));
-  // Block Apply if null
-  const disableApply = $derived(energyState.from === null || energyState.to === null);
+  const energyActive = $derived(!(JSON.stringify(energyState) === JSON.stringify({ from: minEnergyPreset, to: null })));
 
   /* Function */
   function setEnergy() {
     if (energyActive) {
       pendingEnergyState = {
-        from: energyState.from,
-        to: (energyState.to == maxEnergyPreset) ? 0 : energyState.to
+        from: (!energyState?.from) ? 0 : energyState.from,
+        to: (energyState.to === null) ? 0 : energyState.to
       };
     } else {
       pendingEnergyState = { from: null, to: null};
@@ -86,7 +84,6 @@
   <ModalFooter>
     <Button
       color="primary"
-      disabled={disableApply}
       onclick={() => {
         isOpen = false;
         setEnergy();

web/frontend/src/generic/filters/Resources.svelte

@@ -98,44 +98,38 @@
   // Pending
   let pendingNumNodes = $derived({
     from: presetNumNodes.from,
-    to: (presetNumNodes.to == 0) ? maxNumNodes : presetNumNodes.to
+    to: (presetNumNodes.to == 0) ? null : presetNumNodes.to
   });
   let pendingNumHWThreads = $derived({
     from: presetNumHWThreads.from,
-    to: (presetNumHWThreads.to == 0) ? maxNumHWThreads : presetNumHWThreads.to
+    to: (presetNumHWThreads.to == 0) ? null : presetNumHWThreads.to
   });
   let pendingNumAccelerators = $derived({
     from: presetNumAccelerators.from,
-    to: (presetNumAccelerators.to == 0) ? maxNumAccelerators : presetNumAccelerators.to
+    to: (presetNumAccelerators.to == 0) ? null : presetNumAccelerators.to
   });
   let pendingNamedNode = $derived(presetNamedNode);
   let pendingNodeMatch = $derived(presetNodeMatch);
   // Changable States
   let nodesState = $derived({
-    from: presetNumNodes.from,
-    to: (presetNumNodes.to == 0) ? maxNumNodes : presetNumNodes.to
+    from: presetNumNodes?.from || 0,
+    to: (presetNumNodes.to == 0) ? null : presetNumNodes.to
   });
   let threadState = $derived({
-    from: presetNumHWThreads.from,
-    to: (presetNumHWThreads.to == 0) ? maxNumHWThreads : presetNumHWThreads.to
+    from: presetNumHWThreads?.from || 0,
+    to: (presetNumHWThreads.to == 0) ? null : presetNumHWThreads.to
   });
   let accState = $derived({
-    from: presetNumAccelerators.from,
-    to: (presetNumAccelerators.to == 0) ? maxNumAccelerators : presetNumAccelerators.to
+    from: presetNumAccelerators?.from || 0,
+    to: (presetNumAccelerators.to == 0) ? null : presetNumAccelerators.to
   });
 
   const initialized = $derived(getContext("initialized") || false);
   const clusterInfos = $derived($initialized ? getContext("clusters") : null);
   // Is Selection Active
-  const nodesActive = $derived(!(JSON.stringify(nodesState) === JSON.stringify({ from: 1, to: maxNumNodes })));
-  const threadActive = $derived(!(JSON.stringify(threadState) === JSON.stringify({ from: 1, to: maxNumHWThreads })));
-  const accActive = $derived(!(JSON.stringify(accState) === JSON.stringify({ from: 1, to: maxNumAccelerators })));
-  // Block Apply if null
-  const disableApply = $derived(
-    nodesState.from === null || nodesState.to === null ||
-    threadState.from === null || threadState.to === null ||
-    accState.from === null || accState.to === null
-  );
+  const nodesActive = $derived(!(JSON.stringify(nodesState) === JSON.stringify({ from: 0, to: null })));
+  const threadActive = $derived(!(JSON.stringify(threadState) === JSON.stringify({ from: 0, to: null })));
+  const accActive = $derived(!(JSON.stringify(accState) === JSON.stringify({ from: 0, to: null })));
 
   /* Reactive Effects | Svelte 5 onMount */
   $effect(() => {
@@ -153,58 +147,28 @@
     }
   });
 
-  $effect(() => {
-    if (
-      $initialized &&
-      pendingNumNodes.from == null &&
-      pendingNumNodes.to == null
-    ) {
-      nodesState = { from: 1, to: maxNumNodes };
-    }
-  });
-
-  $effect(() => {
-    if (
-      $initialized &&
-      pendingNumHWThreads.from == null && 
-      pendingNumHWThreads.to == null 
-    ) {
-      threadState = { from: 1, to: maxNumHWThreads };
-    }
-  });
-
-  $effect(() => {
-    if (
-      $initialized &&
-      pendingNumAccelerators.from == null &&
-      pendingNumAccelerators.to == null
-    ) {
-      accState = { from: 1, to: maxNumAccelerators };
-    }
-  });
-
   /* Functions */
   function setResources() {
     if (nodesActive) {
       pendingNumNodes = {
-        from: nodesState.from,
-        to: (nodesState.to == maxNumNodes) ? 0 : nodesState.to
+        from: (!nodesState?.from) ? 0 : nodesState.from,
+        to: (nodesState.to === null) ? 0 : nodesState.to
       };
     } else {
       pendingNumNodes = { from: null, to: null};
     };
     if (threadActive) {
       pendingNumHWThreads = {
-        from: threadState.from,
-        to: (threadState.to == maxNumHWThreads) ? 0 : threadState.to
+        from: (!threadState?.from) ? 0 : threadState.from,
+        to: (threadState.to === null) ? 0 : threadState.to
       };
     } else {
       pendingNumHWThreads = { from: null, to: null};
     };
     if (accActive) {
       pendingNumAccelerators = {
-        from: accState.from,
-        to: (accState.to == maxNumAccelerators) ? 0 : accState.to
+        from: (!accState?.from) ? 0 : accState.from,
+        to: (accState.to === null) ? 0 : accState.to
       };
     } else {
       pendingNumAccelerators = { from: null, to: null};
@@ -249,7 +213,7 @@
           nodesState.from = detail[0];
           nodesState.to = detail[1];
         }}
-        sliderMin={1}
+        sliderMin={0}
         sliderMax={maxNumNodes}
         fromPreset={nodesState.from}
         toPreset={nodesState.to}
@@ -269,7 +233,7 @@
           threadState.from = detail[0];
           threadState.to = detail[1];
         }}
-        sliderMin={1}
+        sliderMin={0}
         sliderMax={maxNumHWThreads}
         fromPreset={threadState.from}
         toPreset={threadState.to}
@@ -289,7 +253,7 @@
             accState.from = detail[0];
             accState.to = detail[1];
           }}
-          sliderMin={1}
+          sliderMin={0}
           sliderMax={maxNumAccelerators}
           fromPreset={accState.from}
           toPreset={accState.to}
@@ -300,7 +264,6 @@
   <ModalFooter>
     <Button
       color="primary"
-      disabled={disableApply}
       onclick={() => {
         isOpen = false;
         setResources();

web/frontend/src/generic/filters/Stats.svelte

@@ -34,7 +34,8 @@
   function setRanges() {
     for (let as of availableStats) {
       if (as.enabled) {
-        as.to = (as.to == as.peak) ? 0 : as.to
+        as.from = (!as?.from) ? 0 : as.from,
+        as.to = (as.to == null) ? 0 : as.to
       }
     };
   }
@@ -42,8 +43,8 @@
   function resetRanges() {
     for (let as of availableStats) {
       as.enabled = false
-      as.from = 1
-      as.to = as.peak
+      as.from = null
+      as.to = null
     };
   }
 </script>
@@ -66,13 +67,13 @@
           changeRange={(detail) => {
             aStat.from = detail[0];
             aStat.to = detail[1];
-            if (aStat.from == 1 && aStat.to == aStat.peak) {
+            if (aStat.from == 0 && aStat.to === null) {
               aStat.enabled = false;
             } else {
               aStat.enabled = true;
             }
           }}
-          sliderMin={1}
+          sliderMin={0}
           sliderMax={aStat.peak}
           fromPreset={aStat.from}
           toPreset={aStat.to}

web/frontend/src/generic/helper/JobFootprint.svelte

@@ -228,7 +228,7 @@
     {/each}
     {#if job?.metaData?.message}
       <hr class="mt-1 mb-2" />
-      {@html job.metaData.message}
+      <span style="white-space: pre-wrap">{job.metaData.message}</span>
     {/if}
   </CardBody>
 </Card>

web/frontend/src/generic/plots/Roofline.svelte

@@ -287,12 +287,12 @@
               } else if (nodesData[i]?.schedulerState == "allocated") {
                 //u.ctx.strokeStyle = "rgb(0, 255, 0)";
                 u.ctx.fillStyle = "rgba(0, 255, 0, 0.5)";
-              } else if (nodesData[i]?.schedulerState == "notindb") {
+              } else if (nodesData[i]?.schedulerState == "mixed") {
                 //u.ctx.strokeStyle = "rgb(0, 0, 0)";
-                u.ctx.fillStyle = "rgba(0, 0, 0, 0.5)";
-              } else { // Fallback: All other DEFINED states
-                //u.ctx.strokeStyle = "rgb(255, 0, 0)";
                 u.ctx.fillStyle = "rgba(255, 0, 0, 0.5)";
+              } else { // Fallback: All other states: Reserved, Down, Notindb
+                //u.ctx.strokeStyle = "rgb(255, 0, 0)";
+                u.ctx.fillStyle = "rgba(0, 0, 0, 0.5)";
               }
             }
           } else {
@@ -450,10 +450,10 @@
             tooltip.style.borderColor = "rgb(0, 0, 255)";
           } else if (nodesData[i]?.schedulerState == "allocated") {
             tooltip.style.borderColor = "rgb(0, 255, 0)";
-          } else if (nodesData[i]?.schedulerState == "notindb") { // Missing from DB table
-            tooltip.style.borderColor = "rgb(0, 0, 0)";
-          } else { // Fallback: All other DEFINED states
+          } else if (nodesData[i]?.schedulerState == "mixed") {
             tooltip.style.borderColor = "rgb(255, 0, 0)";
+          } else { // Fallback: All other DEFINED states
+            tooltip.style.borderColor = "rgb(0, 0, 0)";
           }
         }
       } else {
@@ -904,7 +904,7 @@
                 if (jobsData) {
                   const posX = u.valToPos(0.1, "x", true)
                   const posXLimit = u.valToPos(100, "x", true)
-                  const posY = u.valToPos(17500.0, "y", true)
+                  const posY = 7 // u.valToPos(17500.0, "y", true)
                   u.ctx.fillStyle = 'black'
                   u.ctx.fillText('0 Hours', posX, posY)
                   const start = posX + 10
@@ -921,16 +921,16 @@
 
                 // Nodes: The Colors Of NodeStates
                 if (nodesData) {
-                  const posY = u.valToPos(17500.0, "y", true)
+                  const posY = 7 // u.valToPos(17500.0, "y", true)
 
                   const posAllocDot = u.valToPos(0.03, "x", true)
                   const posAllocText = posAllocDot + 60
-                  const posIdleDot = u.valToPos(0.3, "x", true)
-                  const posIdleText = posIdleDot + 30
-                  const posOtherDot = u.valToPos(3, "x", true)
+                  const posIdleDot = u.valToPos(1, "x", true)
+                  const posIdleText = posIdleDot + 28
+                  const posMixedDot = u.valToPos(7, "x", true)
+                  const posMixedText = posMixedDot + 40
+                  const posOtherDot = u.valToPos(100, "x", true)
                   const posOtherText = posOtherDot + 40
-                  const posMissingDot = u.valToPos(30, "x", true)
-                  const posMissingText = posMissingDot + 80
 
                   u.ctx.fillStyle = "rgb(0, 255, 0)"
                   u.ctx.beginPath()
@@ -948,16 +948,16 @@
 
                   u.ctx.fillStyle = "rgb(255, 0, 0)"
                   u.ctx.beginPath()
-                  u.ctx.arc(posOtherDot, posY, 3, 0, Math.PI * 2, false)
+                  u.ctx.arc(posMixedDot, posY, 3, 0, Math.PI * 2, false)
                   u.ctx.fill()
                   u.ctx.fillStyle = 'black'
-                  u.ctx.fillText('Other', posOtherText, posY)
+                  u.ctx.fillText('Mixed', posMixedText, posY)
 
                   u.ctx.fillStyle = 'black'
                   u.ctx.beginPath()
-                  u.ctx.arc(posMissingDot, posY, 3, 0, Math.PI * 2, false)
+                  u.ctx.arc(posOtherDot, posY, 3, 0, Math.PI * 2, false)
                   u.ctx.fill()
-                  u.ctx.fillText('Missing in DB', posMissingText, posY)
+                  u.ctx.fillText('Other', posOtherText, posY)
                 }
               }
             },

web/frontend/src/generic/plots/RooflineLegacy.svelte

@@ -338,7 +338,7 @@
                 // The Color Scale For Time Information
                 const posX = u.valToPos(0.1, "x", true)
                 const posXLimit = u.valToPos(100, "x", true)
-                const posY = u.valToPos(14000.0, "y", true)
+                const posY = 7 // u.valToPos(((subCluster?.flopRateSimd?.value || 10000) + 5000), "y", true)
                 u.ctx.fillStyle = 'black'
                 u.ctx.fillText('Start', posX, posY)
                 const start = posX + 10

web/frontend/src/generic/select/DoubleRangeSlider.svelte

@@ -21,7 +21,7 @@
   let {
     sliderMin,
     sliderMax,
-    fromPreset = 1,
+    fromPreset = 0,
     toPreset = 100,
     changeRange
   } = $props();
@@ -33,9 +33,9 @@
   /* Derived */
   let pendingValues = $derived([fromPreset, toPreset]);
   let sliderFrom = $derived(Math.max(((fromPreset  == null ? sliderMin : fromPreset)  - sliderMin) / (sliderMax - sliderMin), 0.));
-  let sliderTo = $derived(Math.min(((toPreset == null ? sliderMin : toPreset) - sliderMin) / (sliderMax - sliderMin), 1.));
-  let inputFieldFrom = $derived(fromPreset ? fromPreset.toString() : null);
-  let inputFieldTo = $derived(toPreset ? toPreset.toString() : null);
+  let sliderTo = $derived(Math.min(((toPreset == null ? sliderMax : toPreset) - sliderMin) / (sliderMax - sliderMin), 1.));
+  let inputFieldFrom = $derived(fromPreset != null ? fromPreset.toString() : null);
+  let inputFieldTo = $derived(toPreset != null ? toPreset.toString() : null);
 
   /* Var Init */
   let timeoutId = null;
@@ -79,17 +79,22 @@
     evt.preventDefault()
     evt.stopPropagation()
     const newV = Number.parseInt(evt.target.value);
-    const newP = clamp((newV - sliderMin) / (sliderMax - sliderMin), 0., 1.)
+    const newP = clamp((newV - sliderMin) / (sliderMax - sliderMin), 0., 1., target)
     updateStates(newV, newP, target);
   }
 
-  function clamp(x, testMin, testMax) {
-    return x < testMin
-      ? testMin
-      : (x > testMax 
-        ? testMax
-        : x
-      );
+  function clamp(x, testMin, testMax, target) {
+    if (isNaN(x)) {
+      if (target == 'from') return testMin
+      else if (target == 'to') return testMax
+    } else {
+      return x < testMin
+        ? testMin
+        : (x > testMax 
+          ? testMax
+          : x
+        );
+    }
   }
 
   function draggable(node) {
@@ -159,23 +164,23 @@
 
 <div class="double-range-container">
   <div class="header">
-    <input class="form-control" type="text" placeholder="from..." value={inputFieldFrom}
+    <input class="form-control" type="text" placeholder={`${sliderMin} ...`} value={inputFieldFrom}
       oninput={(e) => {
         inputChanged(e, 'from');
       }}
     />
 
-    {#if inputFieldFrom != sliderMin?.toString() && inputFieldTo != sliderMax?.toString() }
+    {#if (inputFieldFrom && inputFieldFrom != sliderMin?.toString()) && inputFieldTo != null }
       <span>Selected: Range <b> {inputFieldFrom} </b> - <b> {inputFieldTo} </b></span>
-    {:else if inputFieldFrom != sliderMin?.toString() && inputFieldTo == sliderMax?.toString() }
-      <span>Selected: More than <b> {inputFieldFrom} </b> </span>
-    {:else if inputFieldFrom == sliderMin?.toString() && inputFieldTo != sliderMax?.toString() }
-      <span>Selected: Less than <b> {inputFieldTo} </b></span>
+    {:else if (inputFieldFrom && inputFieldFrom != sliderMin?.toString()) && inputFieldTo == null }
+      <span>Selected: More Than Equal <b> {inputFieldFrom} </b> </span>
+    {:else if (!inputFieldFrom || inputFieldFrom == sliderMin?.toString()) && inputFieldTo != null }
+      <span>Selected: Less Than Equal <b> {inputFieldTo} </b></span>
     {:else}
       <span><i>No Selection</i></span>
     {/if}
 
-    <input class="form-control" type="text" placeholder="to..." value={inputFieldTo}
+    <input class="form-control" type="text" placeholder={`... ${sliderMax} ...`} value={inputFieldTo}
       oninput={(e) => {
         inputChanged(e, 'to');
       }}

web/frontend/src/generic/utils.js

@@ -347,8 +347,8 @@ export function getStatsItems(presetStats = []) {
                         field: presetEntry.field,
                         text: `${gm.name} (${gm.footprint})`,
                         metric: gm.name,
-                        from: presetEntry.from,
-                        to: (presetEntry.to == 0) ? mc.peak : presetEntry.to,
+                        from: presetEntry?.from || 0,
+                        to: (presetEntry.to == 0) ? null : presetEntry.to,
                         peak: mc.peak,
                         enabled: true,
                         unit: `${gm?.unit?.prefix ? gm.unit.prefix : ''}${gm.unit.base}`
@@ -358,8 +358,8 @@ export function getStatsItems(presetStats = []) {
                         field: `${gm.name}_${gm.footprint}`,
                         text: `${gm.name} (${gm.footprint})`,
                         metric: gm.name,
-                        from: 1,
-                        to: mc.peak,
+                        from: 0,
+                        to: null,
                         peak: mc.peak,
                         enabled: false,
                         unit: `${gm?.unit?.prefix ? gm.unit.prefix : ''}${gm.unit.base}`

web/frontend/src/status/DashDetails.svelte

@@ -87,7 +87,7 @@
 
       {#if subClusters?.length > 1}
         {#each subClusters.map(sc => sc.name) as scn}
-        <TabPane tabId="{scn}-usage-dash" tab="{scn.charAt(0).toUpperCase() + scn.slice(1)} Usage">
+        <TabPane tabId="{scn}-usage-dash" tab="{scn} Usage">
           <CardBody>
             <UsageDash {presetCluster} presetSubCluster={scn} {useCbColors} loadMe={(activeTab === `${scn}-usage-dash`)}></UsageDash>
           </CardBody>

web/frontend/src/systems/nodelist/NodeListRow.svelte

@@ -64,7 +64,7 @@
   const filter = $derived([
     { cluster: { eq: cluster } },
     { state: ["running"] },
-    { node: { contains: nodeData.host } },
+    { node: { eq: nodeData.host } },
   ]);
   const nodeJobsData = $derived(queryStore({
       client: client,