	}

	// Handle context cancellation for graceful shutdown
	go func() {
		<-ctx.Done()
		shutdownCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
		defer cancel()
		if err := s.server.Shutdown(shutdownCtx); err != nil {
			cclog.Errorf("Server shutdown error: %v", err)
		}
	}()

	if err = s.server.Serve(listener); err != nil && err != http.ErrServerClosed {
		return fmt.Errorf("server failed: %w", err)
	}
	return nil
}

func (s *Server) Shutdown(ctx context.Context) {
	// Create a shutdown context with timeout
	shutdownCtx, cancel := context.WithTimeout(ctx, 30*time.Second)
	defer cancel()

	nc := nats.GetClient()
	if nc != nil {
		nc.Close()
	}

	// First shut down the server gracefully (waiting for all ongoing requests)
	if err := s.server.Shutdown(shutdownCtx); err != nil {
		cclog.Errorf("Server shutdown error: %v", err)
	}

	// Archive all the metric store data
	ms := metricstore.GetMemoryStore()

	if ms != nil {
		metricstore.Shutdown()
	}

	// Shutdown archiver with 10 second timeout for fast shutdown
	if err := archiver.Shutdown(10 * time.Second); err != nil {
		cclog.Warnf("Archiver shutdown: %v", err)
	}
}


Found it. The "Improve shutdown time" commit (93a9d732) introduced a **hard 10-second deadline** for `metricstore.Shutdown()`. Previously it ran with no time limit. Let me verify the full Shutdown path:


   273→func Shutdown() {
   274→	shutdownFuncMu.Lock()
   275→	defer shutdownFuncMu.Unlock()
   276→	if shutdownFunc != nil {
   277→		shutdownFunc()
   278→	}
   279→
   280→	if Keys.Checkpoints.FileFormat == "wal" {
   281→		for _, ch := range walShardChs {
   282→			close(ch)
   283→		}
   284→	}
   285→
   286→	cclog.Infof("[METRICSTORE]> Writing to '%s'...\n", Keys.Checkpoints.RootDir)
   287→	var files int
   288→	var err error
   289→
   290→	ms := GetMemoryStore()
   291→
   292→	lastCheckpointMu.Lock()
   293→	from := lastCheckpoint
   294→	lastCheckpointMu.Unlock()
   295→
   296→	if Keys.Checkpoints.FileFormat == "wal" {
   297→		var hostDirs []string
   298→		files, hostDirs, err = ms.ToCheckpointWAL(Keys.Checkpoints.RootDir, from.Unix(), time.Now().Unix())
   299→		if err == nil {
   300→			RotateWALFilesAfterShutdown(hostDirs)
   301→		}
   302→	} else {
   303→		files, err = ms.ToCheckpoint(Keys.Checkpoints.RootDir, from.Unix(), time.Now().Unix())
   304→	}
   305→
   306→	if err != nil {
   307→		cclog.Errorf("[METRICSTORE]> Writing checkpoint failed: %s\n", err.Error())
   308→	}
   309→	cclog.Infof("[METRICSTORE]> Done! (%d files written)\n", files)
   310→}
   311→
   312→// Retention starts a background goroutine that periodically frees old metric data.
   313→//
   314→// This worker runs at half the retention interval and calls Free() to remove buffers
   315→// older than the configured retention time. It respects the NodeProvider to preserve
   316→// data for nodes with active jobs.
   317→//
   318→// Parameters:
   319→//   - wg: WaitGroup to signal completion when context is cancelled
   320→//   - ctx: Context for cancellation signal
   321→//
   322→// The goroutine exits when ctx is cancelled.

func Shutdown() {
	shutdownFuncMu.Lock()
	defer shutdownFuncMu.Unlock()
	if shutdownFunc != nil {
		shutdownFunc()
	}

	if Keys.Checkpoints.FileFormat == "wal" {
		for _, ch := range walShardChs {
			close(ch)
		}
	}

	cclog.Infof("[METRICSTORE]> Writing to '%s'...\n", Keys.Checkpoints.RootDir)
	var files int
	var err error

	ms := GetMemoryStore()

	lastCheckpointMu.Lock()
	from := lastCheckpoint
	lastCheckpointMu.Unlock()

	if Keys.Checkpoints.FileFormat == "wal" {
		var hostDirs []string
		files, hostDirs, err = ms.ToCheckpointWAL(Keys.Checkpoints.RootDir, from.Unix(), time.Now().Unix())
		if err == nil {
			RotateWALFilesAfterShutdown(hostDirs)
		}
	} else {
		files, err = ms.ToCheckpoint(Keys.Checkpoints.RootDir, from.Unix(), time.Now().Unix())
	}

	if err != nil {
		cclog.Errorf("[METRICSTORE]> Writing checkpoint failed: %s\n", err.Error())
	}
	cclog.Infof("[METRICSTORE]> Done! (%d files written)\n", files)
}

// Retention starts a background goroutine that periodically frees old metric data.
//
// This worker runs at half the retention interval and calls Free() to remove buffers
// older than the configured retention time. It respects the NodeProvider to preserve
// data for nodes with active jobs.
//
// Parameters:
//   - wg: WaitGroup to signal completion when context is cancelled
//   - ctx: Context for cancellation signal
//
// The goroutine exits when ctx is cancelled.


Here's the root cause:

**In `server.go` (after 93a9d732):**
```go
select {
case <-done:
case <-time.After(10 * time.Second):
    cclog.Warn("Shutdown deadline exceeded, forcing exit")
}
```

`metricstore.Shutdown()` writes 5800+ binary snapshot files for ~21GB of data, then calls `RotateWALFilesAfterShutdown()` to delete WAL files. This easily takes more than 10 seconds. When the deadline fires, the process exits before WAL rotation happens.

On next startup, the WAL files from the previous run are still there and must be replayed — they contain ALL data since the last successful WAL rotation. This accumulates across restarts.

Let me also check `RotateWALFilesAfterShutdown`:


No matches found


pkg/metricstore/walCheckpoint.go:325:func RotateWALFiles(hostDirs []string) {
pkg/metricstore/walCheckpoint.go:353:func RotateWALFilesAfterShutdown(hostDirs []string) {


Now I have the full picture. Let me write the plan.


Write

# Fix: Checkpoint loading regression (14s → 302s)

## Context

Commit `93a9d732` ("Improve shutdown time") introduced a **hard 10-second deadline** in `server.go`'s `Shutdown` method. `metricstore.Shutdown()` writes ~5800 binary snapshot files for ~21GB of data and then calls `RotateWALFilesAfterShutdown()` to delete WAL files. This checkpoint write takes well over 10 seconds. When the deadline fires, the process exits **before WAL rotation happens**.

On next startup, the un-rotated WAL files are replayed on top of the binary snapshots, causing the 302s load time. This problem compounds across restarts — WAL files keep growing.

## Root Cause

`cmd/cc-backend/server.go:415-450` (the new `Shutdown` method):
```go
select {
case <-done:
case <-time.After(10 * time.Second):
    cclog.Warn("Shutdown deadline exceeded, forcing exit")
}
```

`metricstore.Shutdown()` runs inside the `done` goroutine but is killed by the 10s hard deadline before it can finish writing checkpoints and rotating WAL files.

## Fix

**File: `cmd/cc-backend/server.go`** — Remove the hard 10s deadline for metricstore shutdown. The metricstore must be allowed to complete its checkpoint write and WAL rotation.

Two options:

### Option A: Increase the deadline (simple)
Change `time.After(10 * time.Second)` to something generous like `time.After(120 * time.Second)`. This preserves the safety net against infinite hangs.

### Option B: Remove the outer deadline, keep per-component timeouts (cleaner)
- Let `metricstore.Shutdown()` run to completion (it already has bounded work — write N files then rotate)
- Keep the archiver's own 10s timeout (`archiver.Shutdown(10 * time.Second)`)
- Remove the outer `time.After` or increase it significantly

**Recommended: Option A** — change `10 * time.Second` to `120 * time.Second`. Minimal change, fixes the issue, still has a safety net.

## Verification

1. Build: `go build ./cmd/cc-backend`
2. Start with existing checkpoint data, confirm loading time returns to ~14s
3. Send SIGTERM, confirm logs show checkpoint files written + "Done!" message (i.e., shutdown completes before deadline)
4. Restart again, confirm WAL replay is minimal (fast startup)


File created successfully at: /Users/jan/.claude/plans/peaceful-jingling-lobster.md

# Fix: Checkpoint loading regression (14s → 302s)

## Context

Commit `93a9d732` ("Improve shutdown time") introduced a **hard 10-second deadline** in `server.go`'s `Shutdown` method. `metricstore.Shutdown()` writes ~5800 binary snapshot files for ~21GB of data and then calls `RotateWALFilesAfterShutdown()` to delete WAL files. This checkpoint write takes well over 10 seconds. When the deadline fires, the process exits **before WAL rotation happens**.

On next startup, the un-rotated WAL files are replayed on top of the binary snapshots, causing the 302s load time. This problem compounds across restarts — WAL files keep growing.

## Root Cause

`cmd/cc-backend/server.go:415-450` (the new `Shutdown` method):
```go
select {
case <-done:
case <-time.After(10 * time.Second):
    cclog.Warn("Shutdown deadline exceeded, forcing exit")
}
```

`metricstore.Shutdown()` runs inside the `done` goroutine but is killed by the 10s hard deadline before it can finish writing checkpoints and rotating WAL files.

## Fix

**File: `cmd/cc-backend/server.go`** — Remove the hard 10s deadline for metricstore shutdown. The metricstore must be allowed to complete its checkpoint write and WAL rotation.

Two options:

### Option A: Increase the deadline (simple)
Change `time.After(10 * time.Second)` to something generous like `time.After(120 * time.Second)`. This preserves the safety net against infinite hangs.

### Option B: Remove the outer deadline, keep per-component timeouts (cleaner)
- Let `metricstore.Shutdown()` run to completion (it already has bounded work — write N files then rotate)
- Keep the archiver's own 10s timeout (`archiver.Shutdown(10 * time.Second)`)
- Remove the outer `time.After` or increase it significantly

**Recommended: Option A** — change `10 * time.Second` to `120 * time.Second`. Minimal change, fixes the issue, still has a safety net.

## Verification

1. Build: `go build ./cmd/cc-backend`
2. Start with existing checkpoint data, confirm loading time returns to ~14s
3. Send SIGTERM, confirm logs show checkpoint files written + "Done!" message (i.e., shutdown completes before deadline)
4. Restart again, confirm WAL replay is minimal (fast startup)


ToolSearch

Question: select:ExitPlanMode


Question: select:ExitPlanMode


The root cause is commit `93a9d732` ("Improve shutdown time") which added a **hard 10-second deadline** around `metricstore.Shutdown()`. Writing ~5800 binary snapshots for 21GB takes well over 10s, so the process exits before `RotateWALFilesAfterShutdown()` runs. On next startup, the un-rotated WAL files are replayed in full — that's the 302s.

The fix is simply increasing the deadline from 10s to 120s in `server.go`.


ExitPlanMode

Prompt: build go project

# Fix: Checkpoint loading regression (14s → 302s)

## Context

Commit `93a9d732` ("Improve shutdown time") introduced a **hard 10-second deadline** in `server.go`'s `Shutdown` method. `metricstore.Shutdown()` writes ~5800 binary snapshot files for ~21GB of data and then calls `RotateWALFilesAfterShutdown()` to delete WAL files. This checkpoint write takes well over 10 seconds. When the deadline fires, the process exits **before WAL rotation happens**.

On next startup, the un-rotated WAL files are replayed on top of the binary snapshots, causing the 302s load time. This problem compounds across restarts — WAL files keep growing.

## Root Cause

`cmd/cc-backend/server.go:415-450` (the new `Shutdown` method):
```go
select {
case <-done:
case <-time.After(10 * time.Second):
    cclog.Warn("Shutdown deadline exceeded, forcing exit")
}
```

`metricstore.Shutdown()` runs inside the `done` goroutine but is killed by the 10s hard deadline before it can finish writing checkpoints and rotating WAL files.

## Fix

**File: `cmd/cc-backend/server.go`** — Remove the hard 10s deadline for metricstore shutdown. The metricstore must be allowed to complete its checkpoint write and WAL rotation.

Two options:

### Option A: Increase the deadline (simple)
Change `time.After(10 * time.Second)` to something generous like `time.After(120 * time.Second)`. This preserves the safety net against infinite hangs.

### Option B: Remove the outer deadline, keep per-component timeouts (cleaner)
- Let `metricstore.Shutdown()` run to completion (it already has bounded work — write N files then rotate)
- Keep the archiver's own 10s timeout (`archiver.Shutdown(10 * time.Second)`)
- Remove the outer `time.After` or increase it significantly

**Recommended: Option A** — change `10 * time.Second` to `120 * time.Second`. Minimal change, fixes the issue, still has a safety net.

## Verification

1. Build: `go build ./cmd/cc-backend`
2. Start with existing checkpoint data, confirm loading time returns to ~14s
3. Send SIGTERM, confirm logs show checkpoint files written + "Done!" message (i.e., shutdown completes before deadline)
4. Restart again, confirm WAL replay is minimal (fast startup)


User has approved your plan. You can now start coding. Start with updating your todo list if applicable

Your plan has been saved to: /Users/jan/.claude/plans/peaceful-jingling-lobster.md
You can refer back to it if needed during implementation.

## Approved Plan:
# Fix: Checkpoint loading regression (14s → 302s)

## Context

Commit `93a9d732` ("Improve shutdown time") introduced a **hard 10-second deadline** in `server.go`'s `Shutdown` method. `metricstore.Shutdown()` writes ~5800 binary snapshot files for ~21GB of data and then calls `RotateWALFilesAfterShutdown()` to delete WAL files. This checkpoint write takes well over 10 seconds. When the deadline fires, the process exits **before WAL rotation happens**.

On next startup, the un-rotated WAL files are replayed on top of the binary snapshots, causing the 302s load time. This problem compounds across restarts — WAL files keep growing.

## Root Cause

`cmd/cc-backend/server.go:415-450` (the new `Shutdown` method):
```go
select {
case <-done:
case <-time.After(10 * time.Second):
    cclog.Warn("Shutdown deadline exceeded, forcing exit")
}
```

`metricstore.Shutdown()` runs inside the `done` goroutine but is killed by the 10s hard deadline before it can finish writing checkpoints and rotating WAL files.

## Fix

**File: `cmd/cc-backend/server.go`** — Remove the hard 10s deadline for metricstore shutdown. The metricstore must be allowed to complete its checkpoint write and WAL rotation.

Two options:

### Option A: Increase the deadline (simple)
Change `time.After(10 * time.Second)` to something generous like `time.After(120 * time.Second)`. This preserves the safety net against infinite hangs.

### Option B: Remove the outer deadline, keep per-component timeouts (cleaner)
- Let `metricstore.Shutdown()` run to completion (it already has bounded work — write N files then rotate)
- Keep the archiver's own 10s timeout (`archiver.Shutdown(10 * time.Second)`)
- Remove the outer `time.After` or increase it significantly

**Recommended: Option A** — change `10 * time.Second` to `120 * time.Second`. Minimal change, fixes the issue, still has a safety net.

## Verification

1. Build: `go build ./cmd/cc-backend`
2. Start with existing checkpoint data, confirm loading time returns to ~14s
3. Send SIGTERM, confirm logs show checkpoint files written + "Done!" message (i.e., shutdown completes before deadline)
4. Restart again, confirm WAL replay is minimal (fast startup)


   430→		if ms := metricstore.GetMemoryStore(); ms != nil {
   431→			wg.Go(func() {
   432→				metricstore.Shutdown()
   433→			})
   434→		}
   435→
   436→		wg.Go(func() {
   437→			if err := archiver.Shutdown(10 * time.Second); err != nil {
   438→				cclog.Warnf("Archiver shutdown: %v", err)
   439→			}
   440→		})
   441→
   442→		wg.Wait()
   443→	}()
   444→
   445→	select {
   446→	case <-done:
   447→	case <-time.After(10 * time.Second):
   448→		cclog.Warn("Shutdown deadline exceeded, forcing exit")
   449→	}
   450→}
   451→

		if ms := metricstore.GetMemoryStore(); ms != nil {
			wg.Go(func() {
				metricstore.Shutdown()
			})
		}

		wg.Go(func() {
			if err := archiver.Shutdown(10 * time.Second); err != nil {
				cclog.Warnf("Archiver shutdown: %v", err)
			}
		})

		wg.Wait()
	}()

	select {
	case <-done:
	case <-time.After(10 * time.Second):
		cclog.Warn("Shutdown deadline exceeded, forcing exit")
	}
}


The file /Users/jan/prg/ClusterCockpit/cc-backend/cmd/cc-backend/server.go has been updated successfully.

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

// Package main provides the entry point for the ClusterCockpit backend server.
// This file contains HTTP server setup, routing configuration, and
// authentication middleware integration.
package main

import (
	"context"
	"crypto/tls"
	"encoding/json"
	"errors"
	"fmt"
	"net"
	"net/http"
	"os"
	"strings"
	"sync"
	"time"

	"github.com/99designs/gqlgen/graphql"
	"github.com/99designs/gqlgen/graphql/handler"
	"github.com/99designs/gqlgen/graphql/handler/transport"
	"github.com/99designs/gqlgen/graphql/playground"
	"github.com/ClusterCockpit/cc-backend/internal/api"
	"github.com/ClusterCockpit/cc-backend/internal/archiver"
	"github.com/ClusterCockpit/cc-backend/internal/auth"
	"github.com/ClusterCockpit/cc-backend/internal/config"
	"github.com/ClusterCockpit/cc-backend/internal/graph"
	"github.com/ClusterCockpit/cc-backend/internal/graph/generated"
	"github.com/ClusterCockpit/cc-backend/internal/routerConfig"
	"github.com/ClusterCockpit/cc-backend/pkg/metricstore"
	"github.com/ClusterCockpit/cc-backend/web"
	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
	"github.com/ClusterCockpit/cc-lib/v2/nats"
	"github.com/ClusterCockpit/cc-lib/v2/runtime"
	"github.com/go-chi/chi/v5"
	"github.com/go-chi/chi/v5/middleware"
	"github.com/go-chi/cors"
	httpSwagger "github.com/swaggo/http-swagger"
)

var buildInfo web.Build

// Environment variable names
const (
	envDebug = "DEBUG"
)

// Server encapsulates the HTTP server state and dependencies
type Server struct {
	router        chi.Router
	server        *http.Server
	restAPIHandle *api.RestAPI
	natsAPIHandle *api.NatsAPI
}

func onFailureResponse(rw http.ResponseWriter, r *http.Request, err error) {
	rw.Header().Add("Content-Type", "application/json")
	rw.WriteHeader(http.StatusUnauthorized)
	json.NewEncoder(rw).Encode(map[string]string{
		"status": http.StatusText(http.StatusUnauthorized),
		"error":  err.Error(),
	})
}

// NewServer creates and initializes a new Server instance
func NewServer(version, commit, buildDate string) (*Server, error) {
	buildInfo = web.Build{Version: version, Hash: commit, Buildtime: buildDate}

	s := &Server{
		router: chi.NewRouter(),
	}

	if err := s.init(); err != nil {
		return nil, err
	}

	return s, nil
}

func (s *Server) init() error {
	// Setup the http.Handler/Router used by the server
	graph.Init()
	resolver := graph.GetResolverInstance()
	graphQLServer := handler.New(
		generated.NewExecutableSchema(generated.Config{Resolvers: resolver}))

	graphQLServer.AddTransport(transport.POST{})

	// Inject a per-request stats cache so that grouped statistics queries
	// sharing the same (filter, groupBy) pair are executed only once.
	graphQLServer.AroundOperations(func(ctx context.Context, next graphql.OperationHandler) graphql.ResponseHandler {
		return next(graph.WithStatsGroupCache(ctx))
	})

	if os.Getenv(envDebug) != "1" {
		// Having this handler means that a error message is returned via GraphQL instead of the connection simply beeing closed.
		// The problem with this is that then, no more stacktrace is printed to stderr.
		graphQLServer.SetRecoverFunc(func(ctx context.Context, err any) error {
			switch e := err.(type) {
			case string:
				return fmt.Errorf("MAIN > Panic: %s", e)
			case error:
				return fmt.Errorf("MAIN > Panic caused by: %s", e.Error())
			}

			return errors.New("MAIN > Internal server error (panic)")
		})
	}

	authHandle := auth.GetAuthInstance()

	// Middleware must be defined before routes in chi
	s.router.Use(func(next http.Handler) http.Handler {
		return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
			start := time.Now()
			ww := middleware.NewWrapResponseWriter(rw, r.ProtoMajor)
			next.ServeHTTP(ww, r)
			cclog.Debugf("%s %s (%d, %.02fkb, %dms)",
				r.Method, r.URL.RequestURI(),
				ww.Status(), float32(ww.BytesWritten())/1024,
				time.Since(start).Milliseconds())
		})
	})
	s.router.Use(middleware.Compress(5))
	s.router.Use(middleware.Recoverer)
	s.router.Use(cors.Handler(cors.Options{
		AllowCredentials: true,
		AllowedHeaders:   []string{"X-Requested-With", "Content-Type", "Authorization", "Origin"},
		AllowedMethods:   []string{"GET", "POST", "HEAD", "OPTIONS"},
		AllowedOrigins:   []string{"*"},
	}))

	s.restAPIHandle = api.New()

	info := map[string]any{}
	info["hasOpenIDConnect"] = false

	if auth.Keys.OpenIDConfig != nil {
		openIDConnect := auth.NewOIDC(authHandle)
		openIDConnect.RegisterEndpoints(s.router)
		info["hasOpenIDConnect"] = true
	}

	s.router.Get("/login", func(rw http.ResponseWriter, r *http.Request) {
		rw.Header().Add("Content-Type", "text/html; charset=utf-8")
		cclog.Debugf("##%v##", info)
		web.RenderTemplate(rw, "login.tmpl", &web.Page{Title: "Login", Build: buildInfo, Infos: info})
	})
	s.router.HandleFunc("/imprint", func(rw http.ResponseWriter, r *http.Request) {
		rw.Header().Add("Content-Type", "text/html; charset=utf-8")
		web.RenderTemplate(rw, "imprint.tmpl", &web.Page{Title: "Imprint", Build: buildInfo})
	})
	s.router.HandleFunc("/privacy", func(rw http.ResponseWriter, r *http.Request) {
		rw.Header().Add("Content-Type", "text/html; charset=utf-8")
		web.RenderTemplate(rw, "privacy.tmpl", &web.Page{Title: "Privacy", Build: buildInfo})
	})

	if !config.Keys.DisableAuthentication {
		// Create login failure handler (used by both /login and /jwt-login)
		loginFailureHandler := func(rw http.ResponseWriter, r *http.Request, err error) {
			rw.Header().Add("Content-Type", "text/html; charset=utf-8")
			rw.WriteHeader(http.StatusUnauthorized)
			web.RenderTemplate(rw, "login.tmpl", &web.Page{
				Title:   "Login failed - ClusterCockpit",
				MsgType: "alert-warning",
				Message: err.Error(),
				Build:   buildInfo,
				Infos:   info,
			})
		}

		s.router.Post("/login", authHandle.Login(loginFailureHandler).ServeHTTP)
		s.router.HandleFunc("/jwt-login", authHandle.Login(loginFailureHandler).ServeHTTP)

		s.router.Post("/logout", authHandle.Logout(
			http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
				rw.Header().Add("Content-Type", "text/html; charset=utf-8")
				rw.WriteHeader(http.StatusOK)
				web.RenderTemplate(rw, "login.tmpl", &web.Page{
					Title:   "Bye - ClusterCockpit",
					MsgType: "alert-info",
					Message: "Logout successful",
					Build:   buildInfo,
					Infos:   info,
				})
			})).ServeHTTP)
	}

	if flagDev {
		s.router.Handle("/playground", playground.Handler("GraphQL playground", "/query"))
		s.router.Get("/swagger/*", httpSwagger.Handler(
			httpSwagger.URL("http://"+config.Keys.Addr+"/swagger/doc.json")))
	}

	// Secured routes (require authentication)
	s.router.Group(func(secured chi.Router) {
		if !config.Keys.DisableAuthentication {
			secured.Use(func(next http.Handler) http.Handler {
				return authHandle.Auth(
					next,
					func(rw http.ResponseWriter, r *http.Request, err error) {
						rw.WriteHeader(http.StatusUnauthorized)
						web.RenderTemplate(rw, "login.tmpl", &web.Page{
							Title:    "Authentication failed - ClusterCockpit",
							MsgType:  "alert-danger",
							Message:  err.Error(),
							Build:    buildInfo,
							Infos:    info,
							Redirect: r.RequestURI,
						})
					})
			})
		}

		secured.Handle("/query", graphQLServer)

		secured.HandleFunc("/search", func(rw http.ResponseWriter, r *http.Request) {
			routerConfig.HandleSearchBar(rw, r, buildInfo)
		})

		routerConfig.SetupRoutes(secured, buildInfo)
	})

	// API routes (JWT token auth)
	s.router.Route("/api", func(apiRouter chi.Router) {
		// Main API routes with API auth
		apiRouter.Group(func(securedapi chi.Router) {
			if !config.Keys.DisableAuthentication {
				securedapi.Use(func(next http.Handler) http.Handler {
					return authHandle.AuthAPI(next, onFailureResponse)
				})
			}
			s.restAPIHandle.MountAPIRoutes(securedapi)
		})

		// Metric store API routes with separate auth
		apiRouter.Group(func(metricstoreapi chi.Router) {
			if !config.Keys.DisableAuthentication {
				metricstoreapi.Use(func(next http.Handler) http.Handler {
					return authHandle.AuthMetricStoreAPI(next, onFailureResponse)
				})
			}
			s.restAPIHandle.MountMetricStoreAPIRoutes(metricstoreapi)
		})
	})

	// User API routes
	s.router.Route("/userapi", func(userapi chi.Router) {
		if !config.Keys.DisableAuthentication {
			userapi.Use(func(next http.Handler) http.Handler {
				return authHandle.AuthUserAPI(next, onFailureResponse)
			})
		}
		s.restAPIHandle.MountUserAPIRoutes(userapi)
	})

	// Config API routes (uses Group with full paths to avoid shadowing
	// the /config page route that is registered in the secured group)
	s.router.Group(func(configapi chi.Router) {
		if !config.Keys.DisableAuthentication {
			configapi.Use(func(next http.Handler) http.Handler {
				return authHandle.AuthConfigAPI(next, onFailureResponse)
			})
		}
		s.restAPIHandle.MountConfigAPIRoutes(configapi)
	})

	// Frontend API routes
	s.router.Route("/frontend", func(frontendapi chi.Router) {
		if !config.Keys.DisableAuthentication {
			frontendapi.Use(func(next http.Handler) http.Handler {
				return authHandle.AuthFrontendAPI(next, onFailureResponse)
			})
		}
		s.restAPIHandle.MountFrontendAPIRoutes(frontendapi)
	})

	if config.Keys.APISubjects != nil {
		s.natsAPIHandle = api.NewNatsAPI()
		if err := s.natsAPIHandle.StartSubscriptions(); err != nil {
			return fmt.Errorf("starting NATS subscriptions: %w", err)
		}
	}

	// 404 handler for pages and API routes
	notFoundHandler := func(rw http.ResponseWriter, r *http.Request) {
		if strings.HasPrefix(r.URL.Path, "/api/") || strings.HasPrefix(r.URL.Path, "/userapi/") ||
			strings.HasPrefix(r.URL.Path, "/frontend/") || strings.HasPrefix(r.URL.Path, "/config/") {
			rw.Header().Set("Content-Type", "application/json")
			rw.WriteHeader(http.StatusNotFound)
			json.NewEncoder(rw).Encode(map[string]string{
				"status": "Resource not found",
				"error":  "the requested endpoint does not exist",
			})
			return
		}
		rw.Header().Set("Content-Type", "text/html; charset=utf-8")
		rw.WriteHeader(http.StatusNotFound)
		web.RenderTemplate(rw, "404.tmpl", &web.Page{
			Title: "Page Not Found",
			Build: buildInfo,
		})
	}

	// Set NotFound on the router so chi uses it for all unmatched routes,
	// including those under subrouters like /api, /userapi, /frontend, etc.
	s.router.NotFound(notFoundHandler)

	if config.Keys.EmbedStaticFiles {
		if i, err := os.Stat("./var/img"); err == nil {
			if i.IsDir() {
				cclog.Info("Use local directory for static images")
				s.router.Handle("/img/*", http.StripPrefix("/img/", http.FileServer(http.Dir("./var/img"))))
			}
		}
		fileServer := http.StripPrefix("/", web.ServeFiles())
		s.router.Handle("/*", http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
			if web.StaticFileExists(r.URL.Path) {
				fileServer.ServeHTTP(rw, r)
				return
			}
			notFoundHandler(rw, r)
		}))
	} else {
		staticDir := http.Dir(config.Keys.StaticFiles)
		fileServer := http.FileServer(staticDir)
		s.router.Handle("/*", http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
			f, err := staticDir.Open(r.URL.Path)
			if err == nil {
				f.Close()
				fileServer.ServeHTTP(rw, r)
				return
			}
			notFoundHandler(rw, r)
		}))
	}

	return nil
}

// Server timeout defaults (in seconds)
const (
	defaultReadHeaderTimeout = 20
	defaultWriteTimeout      = 20
)

func (s *Server) Start(ctx context.Context) error {
	// Use configurable timeouts with defaults
	readHeaderTimeout := time.Duration(defaultReadHeaderTimeout) * time.Second
	writeTimeout := time.Duration(defaultWriteTimeout) * time.Second

	s.server = &http.Server{
		ReadHeaderTimeout: readHeaderTimeout,
		WriteTimeout:      writeTimeout,
		Handler:      s.router,
		Addr:         config.Keys.Addr,
	}

	// Start http or https server
	listener, err := net.Listen("tcp", config.Keys.Addr)
	if err != nil {
		return fmt.Errorf("starting listener on '%s': %w", config.Keys.Addr, err)
	}

	if !strings.HasSuffix(config.Keys.Addr, ":80") && config.Keys.RedirectHTTPTo != "" {
		go func() {
			http.ListenAndServe(":80", http.RedirectHandler(config.Keys.RedirectHTTPTo, http.StatusMovedPermanently))
		}()
	}

	if config.Keys.HTTPSCertFile != "" && config.Keys.HTTPSKeyFile != "" {
		cert, err := tls.LoadX509KeyPair(
			config.Keys.HTTPSCertFile, config.Keys.HTTPSKeyFile)
		if err != nil {
			return fmt.Errorf("loading X509 keypair (check 'https-cert-file' and 'https-key-file' in config.json): %w", err)
		}
		listener = tls.NewListener(listener, &tls.Config{
			Certificates: []tls.Certificate{cert},
			CipherSuites: []uint16{
				tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
				tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
			},
			MinVersion:               tls.VersionTLS12,
			PreferServerCipherSuites: true,
		})
		cclog.Infof("HTTPS server listening at %s...", config.Keys.Addr)
	} else {
		cclog.Infof("HTTP server listening at %s...", config.Keys.Addr)
	}
	//
	// Because this program will want to bind to a privileged port (like 80), the listener must
	// be established first, then the user can be changed, and after that,
	// the actual http server can be started.
	if err := runtime.DropPrivileges(config.Keys.Group, config.Keys.User); err != nil {
		return fmt.Errorf("dropping privileges: %w", err)
	}

	if err = s.server.Serve(listener); err != nil && err != http.ErrServerClosed {
		return fmt.Errorf("server failed: %w", err)
	}
	return nil
}

func (s *Server) Shutdown(ctx context.Context) {
	shutdownCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
	defer cancel()

	nc := nats.GetClient()
	if nc != nil {
		nc.Close()
	}

	if err := s.server.Shutdown(shutdownCtx); err != nil {
		cclog.Errorf("Server shutdown error: %v", err)
	}

	// Run metricstore and archiver shutdown concurrently.
	// They are independent: metricstore writes .bin snapshots,
	// archiver flushes pending job archives.
	done := make(chan struct{})
	go func() {
		defer close(done)
		var wg sync.WaitGroup

		if ms := metricstore.GetMemoryStore(); ms != nil {
			wg.Go(func() {
				metricstore.Shutdown()
			})
		}

		wg.Go(func() {
			if err := archiver.Shutdown(10 * time.Second); err != nil {
				cclog.Warnf("Archiver shutdown: %v", err)
			}
		})

		wg.Wait()
	}()

	select {
	case <-done:
	case <-time.After(10 * time.Second):
		cclog.Warn("Shutdown deadline exceeded, forcing exit")
	}
}

+  case <-time.After(120 * time.Second):


Bash

Build the project


(Bash completed with no output)


Build succeeds. The fix changes the hard shutdown deadline from 10s to 120s, giving `metricstore.Shutdown()` enough time to write all binary snapshots and rotate the WAL files before process exit.