Fix critical/severe issues in init, startup and shutdown

- auth: do not abort the server when authentication is disabled. auth.Init
  is now always called; with disable-authentication it sets up an ephemeral
  session store (SESSION_KEY not required) and registers no authenticators,
  so the unconditional auth.GetAuthInstance() callers (server init,
  api.New()) always get a valid instance.
- main: run the graceful-shutdown sequence on the startup-error path. runServer
  derives a cancelable context and, on a server-start failure, cancels it and
  waits so the metricstore final checkpoint / WAL rotation, archiver flush and
  taskmanager shutdown actually run before exit.
- server: log the :80 HTTP->HTTPS redirect listener error instead of dropping it.
- archiver: guard Shutdown against being called when Start never ran
  (avoids close(nil) panic / blocking on a nil workerDone).
- nats API: stop worker goroutines on shutdown via a stop channel + idempotent
  Shutdown(); workers and subscription callbacks select on stop and the
  channels are never closed, so no send-on-closed-channel can occur. Wired
  into Server.Shutdown after the NATS client is closed.
- metricstore: make Shutdown idempotent (nil shutdownFunc, early return) and
  release shutdownFuncMu before the checkpoint write.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Entire-Checkpoint: 3c179f9caa8f

internal/api/nats.go

@@ -78,6 +78,12 @@ type NatsAPI struct {
 	jobCh chan natsMessage
 	// nodeCh receives node state messages for processing by worker goroutines.
 	nodeCh chan natsMessage
+	// stop signals worker goroutines and subscription callbacks to stop.
+	// Closing it (via Shutdown) makes workers exit and callbacks drop further
+	// messages instead of blocking; the channels are never closed so in-flight
+	// callbacks can never send on a closed channel.
+	stop     chan struct{}
+	stopOnce sync.Once
 }
 
 // NewNatsAPI creates a new NatsAPI instance with channel-based worker pools.
@@ -99,6 +105,7 @@ func NewNatsAPI() *NatsAPI {
 		JobRepository: repository.GetJobRepository(),
 		jobCh:         make(chan natsMessage, jobConc),
 		nodeCh:        make(chan natsMessage, nodeConc),
+		stop:          make(chan struct{}),
 	}
 
 	// Start worker goroutines
@@ -112,17 +119,36 @@ func NewNatsAPI() *NatsAPI {
 	return api
 }
 
+// Shutdown stops the worker goroutines and tells subscription callbacks to stop
+// enqueueing. It is safe to call multiple times. Callers must ensure the NATS
+// client is closed first so no new callbacks are invoked.
+func (api *NatsAPI) Shutdown() {
+	api.stopOnce.Do(func() {
+		close(api.stop)
+	})
+}
+
 // jobWorker processes job event messages from the job channel.
 func (api *NatsAPI) jobWorker() {
-	for msg := range api.jobCh {
-		api.handleJobEvent(msg.subject, msg.data)
+	for {
+		select {
+		case <-api.stop:
+			return
+		case msg := <-api.jobCh:
+			api.handleJobEvent(msg.subject, msg.data)
+		}
 	}
 }
 
 // nodeWorker processes node state messages from the node channel.
 func (api *NatsAPI) nodeWorker() {
-	for msg := range api.nodeCh {
-		api.handleNodeState(msg.subject, msg.data)
+	for {
+		select {
+		case <-api.stop:
+			return
+		case msg := <-api.nodeCh:
+			api.handleNodeState(msg.subject, msg.data)
+		}
 	}
 }
 
@@ -140,13 +166,19 @@ func (api *NatsAPI) StartSubscriptions() error {
 		s := config.Keys.APISubjects
 
 		if err := client.Subscribe(s.SubjectJobEvent, func(subject string, data []byte) {
-			api.jobCh <- natsMessage{subject: subject, data: data}
+			select {
+			case api.jobCh <- natsMessage{subject: subject, data: data}:
+			case <-api.stop:
+			}
 		}); err != nil {
 			return err
 		}
 
 		if err := client.Subscribe(s.SubjectNodeState, func(subject string, data []byte) {
-			api.nodeCh <- natsMessage{subject: subject, data: data}
+			select {
+			case api.nodeCh <- natsMessage{subject: subject, data: data}:
+			case <-api.stop:
+			}
 		}); err != nil {
 			return err
 		}

internal/archiver/archiveWorker.go

@@ -222,6 +222,13 @@ func TriggerArchiving(job *schema.Job) {
 func Shutdown(timeout time.Duration) error {
 	cclog.Info("Initiating archiver shutdown...")
 
+	// Guard against Shutdown being called when Start was never run: closing a nil
+	// channel and receiving from a nil workerDone would panic/block forever.
+	if archiveChannel == nil {
+		cclog.Warn("Archiver shutdown called but archiver was never started")
+		return nil
+	}
+
 	// Close channel to signal no more jobs will be accepted
 	close(archiveChannel)
 

internal/auth/auth.go

@@ -9,6 +9,7 @@ package auth
 import (
 	"bytes"
 	"context"
+	"crypto/rand"
 	"database/sql"
 	"encoding/base64"
 	"encoding/json"
@@ -187,20 +188,37 @@ func Init(authCfg *json.RawMessage) {
 
 		sessKey := os.Getenv("SESSION_KEY")
 		if sessKey == "" {
-			cclog.Fatal("environment variable 'SESSION_KEY' not set: refusing to start with an ephemeral session key. " +
-				"Set SESSION_KEY in .env (base64-encoded 32 random bytes); a random key would invalidate all sessions on every restart " +
-				"and prevent sessions from validating across replicas.")
+			if !config.Keys.DisableAuthentication {
+				cclog.Fatal("environment variable 'SESSION_KEY' not set: refusing to start with an ephemeral session key. " +
+					"Set SESSION_KEY in .env (base64-encoded 32 random bytes); a random key would invalidate all sessions on every restart " +
+					"and prevent sessions from validating across replicas.")
+			}
+			// Authentication is disabled: no user sessions are issued, so an
+			// ephemeral random key is sufficient and SESSION_KEY is not required.
+			ephemeralKey := make([]byte, 32)
+			if _, err := rand.Read(ephemeralKey); err != nil {
+				cclog.Fatalf("Error while initializing authentication -> generating ephemeral session key failed: %v", err)
+			}
+			authInstance.sessionStore = sessions.NewCookieStore(ephemeralKey)
+		} else {
+			keyBytes, err := base64.StdEncoding.DecodeString(sessKey)
+			if err != nil {
+				cclog.Fatal("Error while initializing authentication -> decoding session key failed")
+			}
+			authInstance.sessionStore = sessions.NewCookieStore(keyBytes)
 		}
-		keyBytes, err := base64.StdEncoding.DecodeString(sessKey)
-		if err != nil {
-			cclog.Fatal("Error while initializing authentication -> decoding session key failed")
-		}
-		authInstance.sessionStore = sessions.NewCookieStore(keyBytes)
 
 		if d, err := time.ParseDuration(config.Keys.SessionMaxAge); err == nil {
 			authInstance.SessionMaxAge = d
 		}
 
+		// When authentication is disabled no authenticators are required; the
+		// session store created above is enough for the server to run with a
+		// valid (non-nil) auth instance.
+		if config.Keys.DisableAuthentication {
+			return
+		}
+
 		if authCfg == nil {
 			return
 		}