186		}
187
188		if !config.Keys.DisableAuthentication {
189			// Check for default security keys
190			checkDefaultSecurityKeys()
191
192			if flagNewUser != "" {
193				if err := addUser(flagNewUser); err != nil {
194					return err
195				}
196			}
197
198			if flagDelUser != "" {
199				if err := delUser(flagDelUser); err != nil {
200					return err
201				}
202			}
203
204			authHandle := auth.GetAuthInstance()
205
206			if flagSyncLDAP {
207				if err := syncLDAP(authHandle); err != nil {
208					return err
209				}
210			}
211
212			if flagGenJWT != "" {
213				if err := generateJWT(authHandle, flagGenJWT); err != nil {
214					return err
215				}
216			}
217		}
218
219		return nil
220	}
221
222	// checkDefaultSecurityKeys warns if default JWT keys are detected
223	func checkDefaultSecurityKeys() {

	}

	if !config.Keys.DisableAuthentication {
		// Check for default security keys
		checkDefaultSecurityKeys()

		if flagNewUser != "" {
			if err := addUser(flagNewUser); err != nil {
				return err
			}
		}

		if flagDelUser != "" {
			if err := delUser(flagDelUser); err != nil {
				return err
			}
		}

		authHandle := auth.GetAuthInstance()

		if flagSyncLDAP {
			if err := syncLDAP(authHandle); err != nil {
				return err
			}
		}

		if flagGenJWT != "" {
			if err := generateJWT(authHandle, flagGenJWT); err != nil {
				return err
			}
		}
	}

	return nil
}

// checkDefaultSecurityKeys warns if default JWT keys are detected
func checkDefaultSecurityKeys() {


405		errChan := make(chan error, 1)
406
407		// Start HTTP server
408		wg.Go(func() {
409			if err := srv.Start(ctx); err != nil {
410				errChan <- err
411			}
412		})
413
414		// Handle shutdown signals
415		wg.Add(1)
416		sigs := make(chan os.Signal, 1)
417		signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
418		go func() {
419			defer wg.Done()
420			select {
421			case <-sigs:
422				cclog.Info("Shutdown signal received")
423			case <-ctx.Done():
424			}
425
426			runtime.SystemdNotify(false, "Shutting down ...")
427			srv.Shutdown(ctx)
428			util.FsWatcherShutdown()
429			taskmanager.Shutdown()
430		}()
431
432		// Set GC percent if not configured
433		if os.Getenv(envGOGC) == "" {
434			// trigger GC when heap grows 15% above the previous live set
435			debug.SetGCPercent(15)
436		}
437		runtime.SystemdNotify(true, "running")
438
439		waitDone := make(chan struct{})
440		go func() {
441			wg.Wait()
442			close(waitDone)
443		}()
444
445		go func() {
446			<-waitDone
447			close(errChan)
448		}()
449
450		// Wait for either:
451		// 1. An error from server startup
452		// 2. Completion of all goroutines (normal shutdown or crash)
453		var runErr error
454		select {
455		case runErr = <-errChan:
456			// errChan will be closed when waitDone is closed, which happens
457			// when all goroutines complete (either from normal shutdown or error)
458		case <-time.After(100 * time.Millisecond):
459			// Give the server 100ms to start and report any immediate startup errors
460			// After that, just wait for normal shutdown completion
461			select {
462			case runErr = <-errChan:
463			case <-waitDone:
464				// Normal shutdown completed
465			}
466		}
467
468		if runErr != nil {
469			// A subsystem failed (e.g. the HTTP server could not bind). Trigger the
470			// graceful-shutdown path for the subsystems that were already started
471			// (metricstore checkpoint, archiver flush, taskmanager) by cancelling the
472			// context the signal handler waits on, then wait for it to finish so we
473			// don't exit before the final checkpoint is written.
474			cancel()
475			<-waitDone
476			return runErr
477		}
478
479		cclog.Print("Graceful shutdown completed!")

	errChan := make(chan error, 1)

	// Start HTTP server
	wg.Go(func() {
		if err := srv.Start(ctx); err != nil {
			errChan <- err
		}
	})

	// Handle shutdown signals
	wg.Add(1)
	sigs := make(chan os.Signal, 1)
	signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
	go func() {
		defer wg.Done()
		select {
		case <-sigs:
			cclog.Info("Shutdown signal received")
		case <-ctx.Done():
		}

		runtime.SystemdNotify(false, "Shutting down ...")
		srv.Shutdown(ctx)
		util.FsWatcherShutdown()
		taskmanager.Shutdown()
	}()

	// Set GC percent if not configured
	if os.Getenv(envGOGC) == "" {
		// trigger GC when heap grows 15% above the previous live set
		debug.SetGCPercent(15)
	}
	runtime.SystemdNotify(true, "running")

	waitDone := make(chan struct{})
	go func() {
		wg.Wait()
		close(waitDone)
	}()

	go func() {
		<-waitDone
		close(errChan)
	}()

	// Wait for either:
	// 1. An error from server startup
	// 2. Completion of all goroutines (normal shutdown or crash)
	var runErr error
	select {
	case runErr = <-errChan:
		// errChan will be closed when waitDone is closed, which happens
		// when all goroutines complete (either from normal shutdown or error)
	case <-time.After(100 * time.Millisecond):
		// Give the server 100ms to start and report any immediate startup errors
		// After that, just wait for normal shutdown completion
		select {
		case runErr = <-errChan:
		case <-waitDone:
			// Normal shutdown completed
		}
	}

	if runErr != nil {
		// A subsystem failed (e.g. the HTTP server could not bind). Trigger the
		// graceful-shutdown path for the subsystems that were already started
		// (metricstore checkpoint, archiver flush, taskmanager) by cancelling the
		// context the signal handler waits on, then wait for it to finish so we
		// don't exit before the final checkpoint is written.
		cancel()
		<-waitDone
		return runErr
	}

	cclog.Print("Graceful shutdown completed!")


All changes are structurally sound. Let me run `gofmt` on the changed files and a broader test pass to be safe:


Bash

cd /Users/jan/prg/ClusterCockpit/cc-backend; gofmt -l cmd/cc-backend/main.go cmd/cc-backend/server.go internal/auth/auth.go internal/api/nats.go internal/archiver/archiveWorker.go pkg/metricstore/metricstore.go; echo "=== gofmt check done (empty = clean) ==="; go build ./... && echo "BUILD OK"


Do a thorough code  review with a focus on critical and severe issues. Initially focus on the initialization, startup and shutdown.


=== gofmt check done (empty = clean) ===
BUILD OK


All changes are clean — gofmt reports nothing, the full build passes, and tests are green. Here's a summary of what I fixed, in the requested order:

## Fixes applied

**F1 — CRITICAL: `disable-authentication: true` no longer aborts the server**
- `internal/auth/auth.go`: when authentication is disabled and `SESSION_KEY` is unset, `Init` now creates an ephemeral session store (via `crypto/rand`) instead of `cclog.Fatal`, and returns early without registering authenticators. `SESSION_KEY` is still required when auth is enabled.
- `cmd/cc-backend/main.go`: `auth.Init` is now always called (moved out of the `!DisableAuthentication` guard), so the unconditional `auth.GetAuthInstance()` callers (`server.go:123`, `api.New()` in `rest.go:72`) always get a valid instance. User-management commands stay gated behind the enabled check.

**F2 — SEVERE: startup-error path now shuts down gracefully**
- `cmd/cc-backend/main.go`: `runServer` derives its own cancelable context. On a startup error (e.g. failed bind / bad TLS cert) it now `cancel()`s and waits on `waitDone`, driving the existing signal-handler shutdown so the metricstore final checkpoint / WAL rotation, archiver flush, and taskmanager actually run before exit.

**M1** — `server.go`: the `:80` HTTP→HTTPS redirect goroutine now logs its `ListenAndServe` error instead of silently swallowing it.

**M2** — `internal/archiver/archiveWorker.go`: `Shutdown` now guards against being called when `Start` never ran (avoids `close(nil)` panic / blocking on a nil `workerDone`).

**M3** — `internal/api/nats.go` + `server.go`: added a `stop` channel + `Shutdown()` (idempotent via `sync.Once`). Workers and subscription callbacks now select on `stop`, so the NATS API worker goroutines exit on shutdown without ever closing `jobCh`/`nodeCh` (no send-on-closed-channel risk). `Server.Shutdown` calls it right after closing the NATS client.

**M4** — `pkg/metricstore/metricstore.go`: `Shutdown()` is now idempotent (nils `shutdownFunc` and early-returns on a second call) and releases `shutdownFuncMu` before the checkpoint write instead of holding it for the whole shutdown.

Verification done: `go build ./...`, `go vet` on the touched packages, `gofmt -l` (clean), and `go test ./internal/auth/... ./internal/api/... ./internal/archiver/... ./pkg/metricstore/...` all pass.

One thing worth confirming with you: with F1, running with `disable-authentication: true` and no `SESSION_KEY` is now allowed (ephemeral key, no sessions). If your security posture would rather *require* `SESSION_KEY` always and instead make `disable-authentication` a hard, explicit opt-in, let me know and I'll adjust.


The current way with F1 is OK. Commit the changes


git status && echo "=== DIFFSTAT ===" && git diff --stat


On branch release/v1.5
Your branch is up to date with 'origin/release/v1.5'.

Changes not staged for commit:
  (use "git add <file>..." to update what will be committed)
  (use "git restore <file>..." to discard changes in working directory)
	modified:   cmd/cc-backend/main.go
	modified:   cmd/cc-backend/server.go
	modified:   internal/api/nats.go
	modified:   internal/archiver/archiveWorker.go
	modified:   internal/auth/auth.go
	modified:   pkg/metricstore/metricstore.go

no changes added to commit (use "git add" and/or "git commit -a")
=== DIFFSTAT ===
 cmd/cc-backend/main.go             | 46 ++++++++++++++++++++++++++------------
 cmd/cc-backend/server.go           |  9 +++++++-
 internal/api/nats.go               | 44 +++++++++++++++++++++++++++++++-----
 internal/archiver/archiveWorker.go |  7 ++++++
 internal/auth/auth.go              | 34 +++++++++++++++++++++-------
 pkg/metricstore/metricstore.go     | 11 ++++++---
 6 files changed, 119 insertions(+), 32 deletions(-)


The WAL fix and ReleaseNotes changes from earlier in this session aren't showing as modified. Let me verify what's actually in the working tree and the git history:


git log --oneline -5; echo "=== ReleaseNotes WAL entry present? ==="; grep -c "WAL files not reset on shutdown" ReleaseNotes.md; echo "=== metricstore.go diff ==="; git diff pkg/metricstore/metricstore.go


56ae1de0 Update ReleaseNotes
87334972 Fix bug that prevents WAL cleanup on shutdown
6cfa5113 Update release notes
3bef199c Regenerate GraphQL
16942f55 Fix medium-severity issues from follow-up security audit
=== ReleaseNotes WAL entry present? ===
1
=== metricstore.go diff ===
diff --git a/pkg/metricstore/metricstore.go b/pkg/metricstore/metricstore.go
index 703c470e..8a73cde9 100644
--- a/pkg/metricstore/metricstore.go
+++ b/pkg/metricstore/metricstore.go
@@ -275,10 +275,15 @@ func Shutdown() {
 	totalStart := time.Now()

 	shutdownFuncMu.Lock()
-	defer shutdownFuncMu.Unlock()
-	if shutdownFunc != nil {
-		shutdownFunc()
+	if shutdownFunc == nil {
+		// Already shut down (or never initialized): nothing to do. This keeps
+		// Shutdown idempotent so it is safe to call from more than one path.
+		shutdownFuncMu.Unlock()
+		return
 	}
+	shutdownFunc()
+	shutdownFunc = nil
+	shutdownFuncMu.Unlock()
 	cclog.Infof("[METRICSTORE]> Background workers cancelled (%v)", time.Since(totalStart))

 	if Keys.Checkpoints.FileFormat == "wal" {


The WAL fix and ReleaseNotes were already committed (87334972, 56ae1de0). The working tree now holds only the lifecycle review fixes (F1, F2, M1–M4) across 6 files — one logical change. I'm on `release/v1.5` (not the default branch), so I'll commit directly here.


git add cmd/cc-backend/main.go cmd/cc-backend/server.go internal/api/nats.go internal/archiver/archiveWorker.go internal/auth/auth.go pkg/metricstore/metricstore.go && git commit -m "$(cat <<'EOF'
Fix critical/severe issues in init, startup and shutdown

- auth: do not abort the server when authentication is disabled. auth.Init
  is now always called; with disable-authentication it sets up an ephemeral
  session store (SESSION_KEY not required) and registers no authenticators,
  so the unconditional auth.GetAuthInstance() callers (server init,
  api.New()) always get a valid instance.
- main: run the graceful-shutdown sequence on the startup-error path. runServer
  derives a cancelable context and, on a server-start failure, cancels it and
  waits so the metricstore final checkpoint / WAL rotation, archiver flush and
  taskmanager shutdown actually run before exit.
- server: log the :80 HTTP->HTTPS redirect listener error instead of dropping it.
- archiver: guard Shutdown against being called when Start never ran
  (avoids close(nil) panic / blocking on a nil workerDone).
- nats API: stop worker goroutines on shutdown via a stop channel + idempotent
  Shutdown(); workers and subscription callbacks select on stop and the
  channels are never closed, so no send-on-closed-channel can occur. Wired
  into Server.Shutdown after the NATS client is closed.
- metricstore: make Shutdown idempotent (nil shutdownFunc, early return) and
  release shutdownFuncMu before the checkpoint write.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
EOF
)"