   140→	cclog.Debugf("[METRICSTORE]> Using %d workers for checkpoint/archive operations\n", Keys.NumWorkers)
   141→
   142→	// Pass the config.MetricStoreKeys
   143→	InitMetrics(metrics)
   144→
   145→	ms := GetMemoryStore()
   146→
   147→	d, err := time.ParseDuration(Keys.RetentionInMemory)
   148→	if err != nil {
   149→		cclog.Fatal(err)
   150→	}
   151→
   152→	restoreFrom := startupTime.Add(-d)
   153→	cclog.Infof("[METRICSTORE]> Loading checkpoints newer than %s\n", restoreFrom.Format(time.RFC3339))
   154→	files, err := ms.FromCheckpointFiles(Keys.Checkpoints.RootDir, restoreFrom.Unix())
   155→	loadedData := ms.SizeInBytes() / 1024 / 1024 // In MB
   156→	if err != nil {
   157→		cclog.Fatalf("[METRICSTORE]> Loading checkpoints failed: %s\n", err.Error())
   158→	} else {
   159→		cclog.Infof("[METRICSTORE]> Checkpoints loaded (%d files, %d MB, that took %fs)\n", files, loadedData, time.Since(startupTime).Seconds())
   160→	}
   161→
   162→	// Try to use less memory by forcing a GC run here and then
   163→	// lowering the target percentage. The default of 100 means
   164→	// that only once the ratio of new allocations execeds the
   165→	// previously active heap, a GC is triggered.
   166→	// Forcing a GC here will set the "previously active heap"
   167→	// to a minumum.
   168→	// runtime.GC()
   169→
   170→	ctx, shutdown := context.WithCancel(context.Background())
   171→
   172→	Retention(wg, ctx)
   173→	Checkpointing(wg, ctx)
   174→	CleanUp(wg, ctx)
   175→	WALStaging(wg, ctx)
   176→	MemoryUsageTracker(wg, ctx)
   177→
   178→	// Note: Signal handling has been removed from this function.
   179→	// The caller is responsible for handling shutdown signals and calling
   180→	// the shutdown() function when appropriate.
   181→	// Store the shutdown function for later use by Shutdown()
   182→	shutdownFuncMu.Lock()
   183→	shutdownFunc = shutdown
   184→	shutdownFuncMu.Unlock()
   185→
   186→	if Keys.Subscriptions != nil {
   187→		wg.Go(func() {
   188→			if err := ReceiveNats(ms, Keys.NumWorkers, ctx); err != nil {
   189→				cclog.Fatal(err)

	cclog.Debugf("[METRICSTORE]> Using %d workers for checkpoint/archive operations\n", Keys.NumWorkers)

	// Pass the config.MetricStoreKeys
	InitMetrics(metrics)

	ms := GetMemoryStore()

	d, err := time.ParseDuration(Keys.RetentionInMemory)
	if err != nil {
		cclog.Fatal(err)
	}

	restoreFrom := startupTime.Add(-d)
	cclog.Infof("[METRICSTORE]> Loading checkpoints newer than %s\n", restoreFrom.Format(time.RFC3339))
	files, err := ms.FromCheckpointFiles(Keys.Checkpoints.RootDir, restoreFrom.Unix())
	loadedData := ms.SizeInBytes() / 1024 / 1024 // In MB
	if err != nil {
		cclog.Fatalf("[METRICSTORE]> Loading checkpoints failed: %s\n", err.Error())
	} else {
		cclog.Infof("[METRICSTORE]> Checkpoints loaded (%d files, %d MB, that took %fs)\n", files, loadedData, time.Since(startupTime).Seconds())
	}

	// Try to use less memory by forcing a GC run here and then
	// lowering the target percentage. The default of 100 means
	// that only once the ratio of new allocations execeds the
	// previously active heap, a GC is triggered.
	// Forcing a GC here will set the "previously active heap"
	// to a minumum.
	// runtime.GC()

	ctx, shutdown := context.WithCancel(context.Background())

	Retention(wg, ctx)
	Checkpointing(wg, ctx)
	CleanUp(wg, ctx)
	WALStaging(wg, ctx)
	MemoryUsageTracker(wg, ctx)

	// Note: Signal handling has been removed from this function.
	// The caller is responsible for handling shutdown signals and calling
	// the shutdown() function when appropriate.
	// Store the shutdown function for later use by Shutdown()
	shutdownFuncMu.Lock()
	shutdownFunc = shutdown
	shutdownFuncMu.Unlock()

	if Keys.Subscriptions != nil {
		wg.Go(func() {
			if err := ReceiveNats(ms, Keys.NumWorkers, ctx); err != nil {
				cclog.Fatal(err)


Now I have a clear understanding. Let me design the plan.


Agent

Design shutdown optimization plan

Prompt:
Design an implementation plan to reduce the shutdown time of this Go backend from ~40s to ~10s.

## Current Shutdown Flow (SEQUENTIAL, in server.go:418-444)

```go
func (s *Server) Shutdown(ctx context.Context) {
    shutdownCtx, cancel := context.WithTimeout(ctx, 30*time.Second)  // 30s timeout
    defer cancel()
    nc := nats.GetClient()
    if nc != nil { nc.Close() }

    // Step 1: HTTP server graceful shutdown — up to 30s timeout
    s.server.Shutdown(shutdownCtx)

    // Step 2: metricstore.Shutdown() — NO TIMEOUT, writes binary snapshots for ALL hosts
    metricstore.Shutdown()

    // Step 3: archiver.Shutdown(10s) — 10s timeout
    archiver.Shutdown(10 * time.Second)
}
```

Then in main.go:417, `taskmanager.Shutdown()` runs after.

## Root Cause Analysis

The shutdown is sequential with generous/missing timeouts:
- HTTP server gets up to 30s (server.go:420)
- metricstore.Shutdown() has NO timeout — calls ToCheckpointWAL() which writes binary snapshot files for every host using a worker pool (walCheckpoint.go:627-702). This can take 20+ seconds for large installations.
- archiver gets 10s (server.go:441)
- Worst case sequential: 30 + unbounded + 10 = 40s+

The key issue is NOT the WAL drain (that's fast) — it's the final binary checkpoint write in ToCheckpointWAL().

## metricstore.Shutdown() detail (metricstore.go:273-310)
1. Cancels context (stops background goroutines)
2. Closes WAL shard channels (triggers WAL staging goroutine drain)
3. Calls ToCheckpointWAL() — THIS IS THE BOTTLENECK: writes .bin files for every host

## Constraints
- The final checkpoint MUST still be written for data integrity — we can't skip it
- The WAL format means we have both WAL files AND periodic binary snapshots
- Performance is critical (CLAUDE.md: "prioritize maximum throughput and minimal latency")
- The archiver shutdown already has proper timeout handling with graceful cancellation

## What to Optimize

1. **Reduce HTTP server shutdown timeout** — 30s is excessive; 5s is plenty for graceful drain
2. **Parallelize independent shutdown steps** — metricstore and archiver can shut down concurrently
3. **Add overall shutdown deadline** — ensure total shutdown time is bounded
4. **Keep ToCheckpointWAL as-is** — it's already parallelized with NumWorkers, and we need it for data integrity. The savings come from not waiting 30s for HTTP first.

Please provide a detailed implementation plan with specific code changes needed.


Design an implementation plan to reduce the shutdown time of this Go backend from ~40s to ~10s.

## Current Shutdown Flow (SEQUENTIAL, in server.go:418-444)

```go
func (s *Server) Shutdown(ctx context.Context) {
    shutdownCtx, cancel := context.WithTimeout(ctx, 30*time.Second)  // 30s timeout
    defer cancel()
    nc := nats.GetClient()
    if nc != nil { nc.Close() }

    // Step 1: HTTP server graceful shutdown — up to 30s timeout
    s.server.Shutdown(shutdownCtx)

    // Step 2: metricstore.Shutdown() — NO TIMEOUT, writes binary snapshots for ALL hosts
    metricstore.Shutdown()

    // Step 3: archiver.Shutdown(10s) — 10s timeout
    archiver.Shutdown(10 * time.Second)
}
```

Then in main.go:417, `taskmanager.Shutdown()` runs after.

## Root Cause Analysis

The shutdown is sequential with generous/missing timeouts:
- HTTP server gets up to 30s (server.go:420)
- metricstore.Shutdown() has NO timeout — calls ToCheckpointWAL() which writes binary snapshot files for every host using a worker pool (walCheckpoint.go:627-702). This can take 20+ seconds for large installations.
- archiver gets 10s (server.go:441)
- Worst case sequential: 30 + unbounded + 10 = 40s+

The key issue is NOT the WAL drain (that's fast) — it's the final binary checkpoint write in ToCheckpointWAL().

## metricstore.Shutdown() detail (metricstore.go:273-310)
1. Cancels context (stops background goroutines)
2. Closes WAL shard channels (triggers WAL staging goroutine drain)
3. Calls ToCheckpointWAL() — THIS IS THE BOTTLENECK: writes .bin files for every host

## Constraints
- The final checkpoint MUST still be written for data integrity — we can't skip it
- The WAL format means we have both WAL files AND periodic binary snapshots
- Performance is critical (CLAUDE.md: "prioritize maximum throughput and minimal latency")
- The archiver shutdown already has proper timeout handling with graceful cancellation

## What to Optimize

1. **Reduce HTTP server shutdown timeout** — 30s is excessive; 5s is plenty for graceful drain
2. **Parallelize independent shutdown steps** — metricstore and archiver can shut down concurrently
3. **Add overall shutdown deadline** — ensure total shutdown time is bounded
4. **Keep ToCheckpointWAL as-is** — it's already parallelized with NumWorkers, and we need it for data integrity. The savings come from not waiting 30s for HTTP first.

Please provide a detailed implementation plan with specific code changes needed.

Prompt:
Design an implementation plan to reduce the shutdown time of this Go backend from ~40s to ~10s.

## Current Shutdown Flow (SEQUENTIAL, in server.go:418-444)

```go
func (s *Server) Shutdown(ctx context.Context) {
    shutdownCtx, cancel := context.WithTimeout(ctx, 30*time.Second)  // 30s timeout
    defer cancel()
    nc := nats.GetClient()
    if nc != nil { nc.Close() }

    // Step 1: HTTP server graceful shutdown — up to 30s timeout
    s.server.Shutdown(shutdownCtx)

    // Step 2: metricstore.Shutdown() — NO TIMEOUT, writes binary snapshots for ALL hosts
    metricstore.Shutdown()

    // Step 3: archiver.Shutdown(10s) — 10s timeout
    archiver.Shutdown(10 * time.Second)
}
```

Then in main.go:417, `taskmanager.Shutdown()` runs after.

## Root Cause Analysis

The shutdown is sequential with generous/missing timeouts:
- HTTP server gets up to 30s (server.go:420)
- metricstore.Shutdown() has NO timeout — calls ToCheckpointWAL() which writes binary snapshot files for every host using a worker pool (walCheckpoint.go:627-702). This can take 20+ seconds for large installations.
- archiver gets 10s (server.go:441)
- Worst case sequential: 30 + unbounded + 10 = 40s+

The key issue is NOT the WAL drain (that's fast) — it's the final binary checkpoint write in ToCheckpointWAL().

## metricstore.Shutdown() detail (metricstore.go:273-310)
1. Cancels context (stops background goroutines)
2. Closes WAL shard channels (triggers WAL staging goroutine drain)
3. Calls ToCheckpointWAL() — THIS IS THE BOTTLENECK: writes .bin files for every host

## Constraints
- The final checkpoint MUST still be written for data integrity — we can't skip it
- The WAL format means we have both WAL files AND periodic binary snapshots
- Performance is critical (CLAUDE.md: "prioritize maximum throughput and minimal latency")
- The archiver shutdown already has proper timeout handling with graceful cancellation

## What to Optimize

1. **Reduce HTTP server shutdown timeout** — 30s is excessive; 5s is plenty for graceful drain
2. **Parallelize independent shutdown steps** — metricstore and archiver can shut down concurrently
3. **Add overall shutdown deadline** — ensure total shutdown time is bounded
4. **Keep ToCheckpointWAL as-is** — it's already parallelized with NumWorkers, and we need it for data integrity. The savings come from not waiting 30s for HTTP first.

Please provide a detailed implementation plan with specific code changes needed.


   400→	}
   401→
   402→	// Handle context cancellation for graceful shutdown
   403→	go func() {
   404→		<-ctx.Done()
   405→		shutdownCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
   406→		defer cancel()
   407→		if err := s.server.Shutdown(shutdownCtx); err != nil {
   408→			cclog.Errorf("Server shutdown error: %v", err)
   409→		}
   410→	}()
   411→
   412→	if err = s.server.Serve(listener); err != nil && err != http.ErrServerClosed {
   413→		return fmt.Errorf("server failed: %w", err)
   414→	}
   415→	return nil
   416→}
   417→
   418→func (s *Server) Shutdown(ctx context.Context) {
   419→	// Create a shutdown context with timeout
   420→	shutdownCtx, cancel := context.WithTimeout(ctx, 30*time.Second)
   421→	defer cancel()
   422→
   423→	nc := nats.GetClient()
   424→	if nc != nil {
   425→		nc.Close()
   426→	}
   427→
   428→	// First shut down the server gracefully (waiting for all ongoing requests)
   429→	if err := s.server.Shutdown(shutdownCtx); err != nil {
   430→		cclog.Errorf("Server shutdown error: %v", err)
   431→	}
   432→
   433→	// Archive all the metric store data
   434→	ms := metricstore.GetMemoryStore()
   435→
   436→	if ms != nil {
   437→		metricstore.Shutdown()
   438→	}
   439→
   440→	// Shutdown archiver with 10 second timeout for fast shutdown
   441→	if err := archiver.Shutdown(10 * time.Second); err != nil {
   442→		cclog.Warnf("Archiver shutdown: %v", err)
   443→	}
   444→}
   445→


   400→	})
   401→
   402→	// Handle shutdown signals
   403→	wg.Add(1)
   404→	sigs := make(chan os.Signal, 1)
   405→	signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
   406→	go func() {
   407→		defer wg.Done()
   408→		select {
   409→		case <-sigs:
   410→			cclog.Info("Shutdown signal received")
   411→		case <-ctx.Done():
   412→		}
   413→
   414→		runtime.SystemdNotify(false, "Shutting down ...")
   415→		srv.Shutdown(ctx)
   416→		util.FsWatcherShutdown()
   417→		taskmanager.Shutdown()
   418→	}()
   419→
   420→	// Set GC percent if not configured
   421→	if os.Getenv(envGOGC) == "" {
   422→		// trigger GC when heap grows 15% above the previous live set
   423→		debug.SetGCPercent(15)
   424→	}
   425→	runtime.SystemdNotify(true, "running")
   426→
   427→	waitDone := make(chan struct{})
   428→	go func() {
   429→		wg.Wait()
   430→		close(waitDone)
   431→	}()
   432→
   433→	go func() {
   434→		<-waitDone
   435→		close(errChan)
   436→	}()
   437→
   438→	// Wait for either:
   439→	// 1. An error from server startup


   260→// Shutdown performs a graceful shutdown of the metric store.
   261→//
   262→// This function cancels all background goroutines started by Init() and writes
   263→// a final checkpoint to disk before returning. It should be called when the
   264→// application receives a termination signal.
   265→//
   266→// The function will:
   267→//  1. Cancel the context to stop all background workers
   268→//  2. Close the WAL messages channel if using WAL format
   269→//  3. Write a final checkpoint to preserve in-memory data
   270→//  4. Log any errors encountered during shutdown
   271→//
   272→// Note: This function blocks until the final checkpoint is written.
   273→func Shutdown() {
   274→	shutdownFuncMu.Lock()
   275→	defer shutdownFuncMu.Unlock()
   276→	if shutdownFunc != nil {
   277→		shutdownFunc()
   278→	}
   279→
   280→	if Keys.Checkpoints.FileFormat == "wal" {
   281→		for _, ch := range walShardChs {
   282→			close(ch)
   283→		}
   284→	}
   285→
   286→	cclog.Infof("[METRICSTORE]> Writing to '%s'...\n", Keys.Checkpoints.RootDir)
   287→	var files int
   288→	var err error
   289→
   290→	ms := GetMemoryStore()
   291→
   292→	lastCheckpointMu.Lock()
   293→	from := lastCheckpoint
   294→	lastCheckpointMu.Unlock()
   295→
   296→	if Keys.Checkpoints.FileFormat == "wal" {
   297→		var hostDirs []string
   298→		files, hostDirs, err = ms.ToCheckpointWAL(Keys.Checkpoints.RootDir, from.Unix(), time.Now().Unix())
   299→		if err == nil {
   300→			RotateWALFilesAfterShutdown(hostDirs)
   301→		}
   302→	} else {
   303→		files, err = ms.ToCheckpoint(Keys.Checkpoints.RootDir, from.Unix(), time.Now().Unix())
   304→	}
   305→
   306→	if err != nil {
   307→		cclog.Errorf("[METRICSTORE]> Writing checkpoint failed: %s\n", err.Error())
   308→	}
   309→	cclog.Infof("[METRICSTORE]> Done! (%d files written)\n", files)
   310→}
   311→
   312→// Retention starts a background goroutine that periodically frees old metric data.
   313→//
   314→// This worker runs at half the retention interval and calls Free() to remove buffers
   315→// older than the configured retention time. It respects the NodeProvider to preserve
   316→// data for nodes with active jobs.
   317→//
   318→// Parameters:
   319→//   - wg: WaitGroup to signal completion when context is cancelled


   620→	}
   621→	return result.String()
   622→}
   623→
   624→// ToCheckpointWAL writes binary snapshot files for all hosts in parallel.
   625→// Returns the number of files written, the list of host directories that were
   626→// successfully checkpointed (for WAL rotation), and any errors.
   627→func (m *MemoryStore) ToCheckpointWAL(dir string, from, to int64) (int, []string, error) {
   628→	// Collect all cluster/host pairs.
   629→	m.root.lock.RLock()
   630→	totalHosts := 0
   631→	for _, l1 := range m.root.children {
   632→		l1.lock.RLock()
   633→		totalHosts += len(l1.children)
   634→		l1.lock.RUnlock()
   635→	}
   636→	m.root.lock.RUnlock()
   637→
   638→	levels := make([]*Level, 0, totalHosts)
   639→	selectors := make([][]string, 0, totalHosts)
   640→
   641→	m.root.lock.RLock()
   642→	for sel1, l1 := range m.root.children {
   643→		l1.lock.RLock()
   644→		for sel2, l2 := range l1.children {
   645→			levels = append(levels, l2)
   646→			selectors = append(selectors, []string{sel1, sel2})
   647→		}
   648→		l1.lock.RUnlock()
   649→	}
   650→	m.root.lock.RUnlock()
   651→
   652→	type workItem struct {
   653→		level    *Level
   654→		hostDir  string
   655→		selector []string
   656→	}
   657→
   658→	n, errs := int32(0), int32(0)
   659→	var successDirs []string
   660→	var successMu sync.Mutex
   661→
   662→	var wg sync.WaitGroup
   663→	wg.Add(Keys.NumWorkers)
   664→	work := make(chan workItem, Keys.NumWorkers*2)
   665→
   666→	for range Keys.NumWorkers {
   667→		go func() {
   668→			defer wg.Done()
   669→			for wi := range work {
   670→				err := wi.level.toCheckpointBinary(wi.hostDir, from, to, m)
   671→				if err != nil {
   672→					if err == ErrNoNewArchiveData {
   673→						continue
   674→					}
   675→					cclog.Errorf("[METRICSTORE]> binary checkpoint error for %s: %v", wi.hostDir, err)
   676→					atomic.AddInt32(&errs, 1)
   677→				} else {
   678→					atomic.AddInt32(&n, 1)
   679→					successMu.Lock()
   680→					successDirs = append(successDirs, wi.hostDir)
   681→					successMu.Unlock()
   682→				}
   683→			}
   684→		}()
   685→	}
   686→
   687→	for i := range levels {
   688→		hostDir := path.Join(dir, path.Join(selectors[i]...))
   689→		work <- workItem{
   690→			level:    levels[i],
   691→			hostDir:  hostDir,
   692→			selector: selectors[i],
   693→		}
   694→	}
   695→	close(work)
   696→	wg.Wait()
   697→
   698→	if errs > 0 {
   699→		return int(n), successDirs, fmt.Errorf("[METRICSTORE]> %d errors during binary checkpoint (%d successes)", errs, n)
   700→	}
   701→	return int(n), successDirs, nil
   702→}
   703→
   704→// toCheckpointBinary writes a binary snapshot file for a single host-level node.
   705→// Uses atomic rename (write to .tmp then rename) to avoid partial reads on crash.
   706→func (l *Level) toCheckpointBinary(dir string, from, to int64, m *MemoryStore) error {
   707→	cf, err := l.toCheckpointFile(from, to, m)
   708→	if err != nil {
   709→		return err


internal/archiver/archiveWorker.go:222:func Shutdown(timeout time.Duration) error {
/Users/jan/prg/ClusterCockpit/cc-backend/internal/archiver/archiveWorker.go-223-	cclog.Info("Initiating archiver shutdown...")
/Users/jan/prg/ClusterCockpit/cc-backend/internal/archiver/archiveWorker.go-224-
/Users/jan/prg/ClusterCockpit/cc-backend/internal/archiver/archiveWorker.go-225-	// Close channel to signal no more jobs will be accepted
/Users/jan/prg/ClusterCockpit/cc-backend/internal/archiver/archiveWorker.go-226-	close(archiveChannel)
/Users/jan/prg/ClusterCockpit/cc-backend/internal/archiver/archiveWorker.go-227-
/Users/jan/prg/ClusterCockpit/cc-backend/internal/archiver/archiveWorker.go-228-	// Create a channel to signal when all jobs are done
internal/archiver/archiveWorker.go-229-	done := make(chan struct{})
/Users/jan/prg/ClusterCockpit/cc-backend/internal/archiver/archiveWorker.go-230-	go func() {
/Users/jan/prg/ClusterCockpit/cc-backend/internal/archiver/archiveWorker.go-231-		archivePending.Wait()
/Users/jan/prg/ClusterCockpit/cc-backend/internal/archiver/archiveWorker.go-232-		close(done)
/Users/jan/prg/ClusterCockpit/cc-backend/internal/archiver/archiveWorker.go-233-	}()
/Users/jan/prg/ClusterCockpit/cc-backend/internal/archiver/archiveWorker.go-234-
/Users/jan/prg/ClusterCockpit/cc-backend/internal/archiver/archiveWorker.go-235-	// Wait for jobs to complete or timeout
/Users/jan/prg/ClusterCockpit/cc-backend/internal/archiver/archiveWorker.go-236-	select {
internal/archiver/archiveWorker.go-237-	case <-done:
/Users/jan/prg/ClusterCockpit/cc-backend/internal/archiver/archiveWorker.go-238-		cclog.Info("All archive jobs completed successfully")
/Users/jan/prg/ClusterCockpit/cc-backend/internal/archiver/archiveWorker.go-239-		// Wait for worker to exit
/Users/jan/prg/ClusterCockpit/cc-backend/internal/archiver/archiveWorker.go-240-		<-workerDone
/Users/jan/prg/ClusterCockpit/cc-backend/internal/archiver/archiveWorker.go-241-		return nil
internal/archiver/archiveWorker.go-242-	case <-time.After(timeout):


internal/taskmanager/taskManager.go:165:func Shutdown() {
/Users/jan/prg/ClusterCockpit/cc-backend/internal/taskmanager/taskManager.go-166-	if s != nil {
/Users/jan/prg/ClusterCockpit/cc-backend/internal/taskmanager/taskManager.go-167-		s.Shutdown()
/Users/jan/prg/ClusterCockpit/cc-backend/internal/taskmanager/taskManager.go-168-	}
/Users/jan/prg/ClusterCockpit/cc-backend/internal/taskmanager/taskManager.go-169-}


   380→		}
   381→		listener = tls.NewListener(listener, &tls.Config{
   382→			Certificates: []tls.Certificate{cert},
   383→			CipherSuites: []uint16{
   384→				tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
   385→				tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
   386→			},
   387→			MinVersion:               tls.VersionTLS12,
   388→			PreferServerCipherSuites: true,
   389→		})
   390→		cclog.Infof("HTTPS server listening at %s...", config.Keys.Addr)
   391→	} else {
   392→		cclog.Infof("HTTP server listening at %s...", config.Keys.Addr)
   393→	}
   394→	//
   395→	// Because this program will want to bind to a privileged port (like 80), the listener must
   396→	// be established first, then the user can be changed, and after that,
   397→	// the actual http server can be started.
   398→	if err := runtime.DropPrivileges(config.Keys.Group, config.Keys.User); err != nil {
   399→		return fmt.Errorf("dropping privileges: %w", err)
   400→	}
   401→
   402→	// Handle context cancellation for graceful shutdown
   403→	go func() {
   404→		<-ctx.Done()
   405→		shutdownCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
   406→		defer cancel()
   407→		if err := s.server.Shutdown(shutdownCtx); err != nil {
   408→			cclog.Errorf("Server shutdown error: %v", err)
   409→		}
   410→	}()
   411→
   412→	if err = s.server.Serve(listener); err != nil && err != http.ErrServerClosed {
   413→		return fmt.Errorf("server failed: %w", err)
   414→	}
   415→	return nil
   416→}
   417→
   418→func (s *Server) Shutdown(ctx context.Context) {
   419→	// Create a shutdown context with timeout
   420→	shutdownCtx, cancel := context.WithTimeout(ctx, 30*time.Second)
   421→	defer cancel()
   422→
   423→	nc := nats.GetClient()
   424→	if nc != nil {
   425→		nc.Close()
   426→	}
   427→
   428→	// First shut down the server gracefully (waiting for all ongoing requests)
   429→	if err := s.server.Shutdown(shutdownCtx); err != nil {
   430→		cclog.Errorf("Server shutdown error: %v", err)
   431→	}
   432→
   433→	// Archive all the metric store data
   434→	ms := metricstore.GetMemoryStore()
   435→
   436→	if ms != nil {
   437→		metricstore.Shutdown()
   438→	}
   439→
   440→	// Shutdown archiver with 10 second timeout for fast shutdown
   441→	if err := archiver.Shutdown(10 * time.Second); err != nil {
   442→		cclog.Warnf("Archiver shutdown: %v", err)
   443→	}
   444→}
   445→


   380→	// Initialize web UI
   381→	cfg := ccconf.GetPackageConfig("ui")
   382→	if err := web.Init(cfg); err != nil {
   383→		return fmt.Errorf("initializing web UI: %w", err)
   384→	}
   385→
   386→	// Initialize HTTP server
   387→	srv, err := NewServer(version, commit, date)
   388→	if err != nil {
   389→		return fmt.Errorf("creating server: %w", err)
   390→	}
   391→
   392→	// Channel to collect errors from server
   393→	errChan := make(chan error, 1)
   394→
   395→	// Start HTTP server
   396→	wg.Go(func() {
   397→		if err := srv.Start(ctx); err != nil {
   398→			errChan <- err
   399→		}
   400→	})
   401→
   402→	// Handle shutdown signals
   403→	wg.Add(1)
   404→	sigs := make(chan os.Signal, 1)
   405→	signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
   406→	go func() {
   407→		defer wg.Done()
   408→		select {
   409→		case <-sigs:
   410→			cclog.Info("Shutdown signal received")
   411→		case <-ctx.Done():
   412→		}
   413→
   414→		runtime.SystemdNotify(false, "Shutting down ...")
   415→		srv.Shutdown(ctx)
   416→		util.FsWatcherShutdown()
   417→		taskmanager.Shutdown()
   418→	}()
   419→
   420→	// Set GC percent if not configured
   421→	if os.Getenv(envGOGC) == "" {
   422→		// trigger GC when heap grows 15% above the previous live set
   423→		debug.SetGCPercent(15)
   424→	}
   425→	runtime.SystemdNotify(true, "running")
   426→
   427→	waitDone := make(chan struct{})
   428→	go func() {
   429→		wg.Wait()
   430→		close(waitDone)
   431→	}()
   432→
   433→	go func() {
   434→		<-waitDone
   435→		close(errChan)
   436→	}()
   437→
   438→	// Wait for either:
   439→	// 1. An error from server startup
   440→	// 2. Completion of all goroutines (normal shutdown or crash)
   441→	select {
   442→	case err := <-errChan:
   443→		// errChan will be closed when waitDone is closed, which happens
   444→		// when all goroutines complete (either from normal shutdown or error)
   445→		if err != nil {
   446→			return err
   447→		}
   448→	case <-time.After(100 * time.Millisecond):
   449→		// Give the server 100ms to start and report any immediate startup errors


351:func (s *Server) Start(ctx context.Context) error {
352-	// Use configurable timeouts with defaults
353-	readHeaderTimeout := time.Duration(defaultReadHeaderTimeout) * time.Second
354-	writeTimeout := time.Duration(defaultWriteTimeout) * time.Second
355-
356-	s.server = &http.Server{


No matches found


cmd/cc-backend/main.go:416:		util.FsWatcherShutdown()
/Users/jan/prg/ClusterCockpit/cc-backend/cmd/cc-backend/main.go-417-		taskmanager.Shutdown()
/Users/jan/prg/ClusterCockpit/cc-backend/cmd/cc-backend/main.go-418-	}()
/Users/jan/prg/ClusterCockpit/cc-backend/cmd/cc-backend/main.go-419-
/Users/jan/prg/ClusterCockpit/cc-backend/cmd/cc-backend/main.go-420-	// Set GC percent if not configured
/Users/jan/prg/ClusterCockpit/cc-backend/cmd/cc-backend/main.go-421-	if os.Getenv(envGOGC) == "" {
/Users/jan/prg/ClusterCockpit/cc-backend/cmd/cc-backend/main.go-422-		// trigger GC when heap grows 15% above the previous live set
/Users/jan/prg/ClusterCockpit/cc-backend/cmd/cc-backend/main.go-423-		debug.SetGCPercent(15)
/Users/jan/prg/ClusterCockpit/cc-backend/cmd/cc-backend/main.go-424-	}
/Users/jan/prg/ClusterCockpit/cc-backend/cmd/cc-backend/main.go-425-	runtime.SystemdNotify(true, "running")
/Users/jan/prg/ClusterCockpit/cc-backend/cmd/cc-backend/main.go-426-


Now I have a full picture. Let me also verify a subtle issue -- the double HTTP shutdown. `Start()` at line 402-410 spawns a goroutine that calls `s.server.Shutdown()` when `ctx` is cancelled, and then `Shutdown()` at line 429 calls `s.server.Shutdown()` again. Let me check if the `ctx` passed to `srv.Shutdown(ctx)` at main.go:415 is already cancelled at that point.

Looking at main.go:415 -- `srv.Shutdown(ctx)` is called in the signal handler goroutine. The `ctx` here is the context from the `run` function. When the signal arrives, `ctx` may or may not be cancelled. But the `Shutdown` method creates its own 30s timeout context from this `ctx`. Meanwhile, the goroutine in `Start()` at line 403 waits on `ctx.Done()` -- this fires when the parent context is cancelled.

The key observation: there may be a race where both paths try to shut down the HTTP server, but `http.Server.Shutdown` is safe to call multiple times. The real issue is that the `Shutdown()` method in server.go:418 calls `s.server.Shutdown(shutdownCtx)` with a fresh 30s timeout, blocking until the HTTP server is fully shut down or 30s passes.

Now I have enough information for the plan.

---

## Implementation Plan: Reduce Shutdown Time from ~40s to ~10s

### Problem Summary

The shutdown path in `server.go:Shutdown()` is fully sequential:

1. NATS close (fast, ~0s)
2. HTTP server shutdown with 30s timeout (blocks up to 30s)
3. `metricstore.Shutdown()` with NO timeout (blocks for binary checkpoint writes -- 10-20s for large installations)
4. `archiver.Shutdown()` with 10s timeout

Additionally, there is a duplicate HTTP shutdown path in `Start()` (line 402-410) that also uses a 30s timeout.

Worst case: 30 + unbounded + 10 = 40s+

### Solution: Three Changes

#### Change 1: Reduce HTTP server shutdown timeout from 30s to 5s

**File:** `/Users/jan/prg/ClusterCockpit/cc-backend/cmd/cc-backend/server.go`

In `Shutdown()` at line 420, change:
```go
shutdownCtx, cancel := context.WithTimeout(ctx, 30*time.Second)
```
to:
```go
shutdownCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
```

Also in `Start()` at line 405, change the goroutine's timeout from 30s to 5s:
```go
shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
```

Rationale: 5 seconds is more than enough for in-flight HTTP requests to complete. Most requests finish in milliseconds. The 30s timeout was overly generous and dominated shutdown time when the server was under load.

#### Change 2: Run metricstore and archiver shutdown concurrently

**File:** `/Users/jan/prg/ClusterCockpit/cc-backend/cmd/cc-backend/server.go`

Replace the sequential metricstore + archiver shutdown (lines 433-443) with concurrent execution using a `sync.WaitGroup`:

```go
func (s *Server) Shutdown(ctx context.Context) {
    shutdownCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
    defer cancel()

    nc := nats.GetClient()
    if nc != nil {
        nc.Close()
    }

    // Shut down the HTTP server first (max 5s)
    if err := s.server.Shutdown(shutdownCtx); err != nil {
        cclog.Errorf("Server shutdown error: %v", err)
    }

    // Shut down metricstore and archiver concurrently
    var wg sync.WaitGroup

    ms := metricstore.GetMemoryStore()
    if ms != nil {
        wg.Add(1)
        go func() {
            defer wg.Done()
            metricstore.Shutdown()
        }()
    }

    wg.Add(1)
    go func() {
        defer wg.Done()
        if err := archiver.Shutdown(10 * time.Second); err != nil {
            cclog.Warnf("Archiver shutdown: %v", err)
        }
    }()

    wg.Wait()
}
```

This means metricstore checkpoint writing and archiver drain happen at the same time instead of one after the other. Since they operate on completely independent resources (metricstore writes `.bin` files; archiver writes job archive files), there is no contention or ordering dependency.

Time savings: archiver's 10s timeout now overlaps with metricstore's checkpoint write instead of adding to it.

#### Change 3: Add an overall shutdown deadline

**File:** `/Users/jan/prg/ClusterCockpit/cc-backend/cmd/cc-backend/server.go`

Wrap the entire `Shutdown()` body with an overall 10s deadline. If the deadline is exceeded, log a warning and return (the process will exit anyway, and the OS will reclaim resources). The WAL files on disk guarantee that no data is lost even if the final binary snapshot is incomplete -- on next startup, the WAL will be replayed.

```go
func (s *Server) Shutdown(ctx context.Context) {
    deadline := time.After(10 * time.Second)
    done := make(chan struct{})

    go func() {
        defer close(done)
        // ... all shutdown logic from Change 2 goes here ...
    }()

    select {
    case <-done:
        cclog.Info("Shutdown completed successfully")
    case <-deadline:
        cclog.Warn("Shutdown deadline (10s) exceeded, forcing exit")
    }
}
```

This guarantees the function returns within 10 seconds regardless of what metricstore or archiver are doing. Data integrity is maintained because:
- The WAL format already provides crash recovery -- incomplete binary snapshots are handled by replaying WAL files on startup.
- `ToCheckpointWAL` uses atomic rename (write to `.tmp` then rename), so partial writes do not corrupt existing snapshots.

#### Change 4 (minor): Remove duplicate HTTP shutdown goroutine in Start()

**File:** `/Users/jan/prg/ClusterCockpit/cc-backend/cmd/cc-backend/server.go`

The goroutine at lines 402-410 in `Start()` also calls `s.server.Shutdown()` when the context is cancelled. This is redundant with `Shutdown()` at line 429, which is the intended shutdown path (called from main.go:415). While calling `Shutdown` twice is safe (it's idempotent), the duplicate goroutine adds confusion and a competing timeout.

Remove lines 402-410 entirely. The `Serve()` call at line 412 will return `http.ErrServerClosed` when `Shutdown()` is called from the signal handler, which is already handled correctly.

### Resulting Shutdown Flow

After all changes, the shutdown sequence becomes:

```
Signal received
  |
  v
HTTP server shutdown (max 5s)
  |
  v
[concurrent]
  +-- metricstore.Shutdown() (checkpoint write, typically 5-15s, hard cap 10s)
  +-- archiver.Shutdown()    (max 10s)
  |
  v (whichever finishes last, or 10s overall deadline)
  |
  v
FsWatcherShutdown() + taskmanager.Shutdown() (fast)
```

Worst case: 5s (HTTP) + max(metricstore, archiver) capped at 5s remaining = **10s total**.
Typical case: ~1s (HTTP) + ~5-8s (metricstore checkpoint) = **6-9s total**.

### Import Changes

**File:** `/Users/jan/prg/ClusterCockpit/cc-backend/cmd/cc-backend/server.go`

Add `"sync"` to imports if not already present.

### Testing

1. Run `go build ./cmd/cc-backend` to verify compilation.
2. Run `go test ./cmd/cc-backend/...` if there are tests for the server.
3. Manual test: start the server with sample data (`./startDemo.sh`), send SIGTERM, measure time to exit. Should be under 10s.
4. Verify data integrity: after shutdown, restart and confirm checkpoint data is loaded correctly.

### Critical Files for Implementation

- `/Users/jan/prg/ClusterCockpit/cc-backend/cmd/cc-backend/server.go` - Primary file: contains both `Start()` and `Shutdown()` methods that need modification
- `/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/metricstore.go` - Reference: `Shutdown()` function to understand what it blocks on (no changes needed)
- `/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go` - Reference: `ToCheckpointWAL()` to confirm atomic-rename safety for the overall deadline approach
- `/Users/jan/prg/ClusterCockpit/cc-backend/cmd/cc-backend/main.go` - Context: shutdown signal handler that calls `srv.Shutdown(ctx)` (no changes needed)
- `/Users/jan/prg/ClusterCockpit/cc-backend/internal/archiver/archiveWorker.go` - Reference: archiver `Shutdown()` to confirm no ordering dependency with metricstore

agentId: aaa23999739ba99a2 (use SendMessage with to: 'aaa23999739ba99a2' to continue this agent)
<usage>total_tokens: 22919
tool_uses: 11
duration_ms: 72039</usage>

Prompt:
Design an implementation plan to reduce the shutdown time of this Go backend from ~40s to ~10s.

## Current Shutdown Flow (SEQUENTIAL, in server.go:418-444)

```go
func (s *Server) Shutdown(ctx context.Context) {
    shutdownCtx, cancel := context.WithTimeout(ctx, 30*time.Second)  // 30s timeout
    defer cancel()
    nc := nats.GetClient()
    if nc != nil { nc.Close() }

    // Step 1: HTTP server graceful shutdown — up to 30s timeout
    s.server.Shutdown(shutdownCtx)

    // Step 2: metricstore.Shutdown() — NO TIMEOUT, writes binary snapshots for ALL hosts
    metricstore.Shutdown()

    // Step 3: archiver.Shutdown(10s) — 10s timeout
    archiver.Shutdown(10 * time.Second)
}
```

Then in main.go:417, `taskmanager.Shutdown()` runs after.

## Root Cause Analysis

The shutdown is sequential with generous/missing timeouts:
- HTTP server gets up to 30s (server.go:420)
- metricstore.Shutdown() has NO timeout — calls ToCheckpointWAL() which writes binary snapshot files for every host using a worker pool (walCheckpoint.go:627-702). This can take 20+ seconds for large installations.
- archiver gets 10s (server.go:441)
- Worst case sequential: 30 + unbounded + 10 = 40s+

The key issue is NOT the WAL drain (that's fast) — it's the final binary checkpoint write in ToCheckpointWAL().

## metricstore.Shutdown() detail (metricstore.go:273-310)
1. Cancels context (stops background goroutines)
2. Closes WAL shard channels (triggers WAL staging goroutine drain)
3. Calls ToCheckpointWAL() — THIS IS THE BOTTLENECK: writes .bin files for every host

## Constraints
- The final checkpoint MUST still be written for data integrity — we can't skip it
- The WAL format means we have both WAL files AND periodic binary snapshots
- Performance is critical (CLAUDE.md: "prioritize maximum throughput and minimal latency")
- The archiver shutdown already has proper timeout handling with graceful cancellation

## What to Optimize

1. **Reduce HTTP server shutdown timeout** — 30s is excessive; 5s is plenty for graceful drain
2. **Parallelize independent shutdown steps** — metricstore and archiver can shut down concurrently
3. **Add overall shutdown deadline** — ensure total shutdown time is bounded
4. **Keep ToCheckpointWAL as-is** — it's already parallelized with NumWorkers, and we need it for data integrity. The savings come from not waiting 30s for HTTP first.

Please provide a detailed implementation plan with specific code changes needed.


Let me verify the `Start()` method to confirm the duplicate shutdown goroutine.


   395→	// Because this program will want to bind to a privileged port (like 80), the listener must
   396→	// be established first, then the user can be changed, and after that,
   397→	// the actual http server can be started.
   398→	if err := runtime.DropPrivileges(config.Keys.Group, config.Keys.User); err != nil {
   399→		return fmt.Errorf("dropping privileges: %w", err)
   400→	}
   401→
   402→	// Handle context cancellation for graceful shutdown
   403→	go func() {
   404→		<-ctx.Done()
   405→		shutdownCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
   406→		defer cancel()
   407→		if err := s.server.Shutdown(shutdownCtx); err != nil {
   408→			cclog.Errorf("Server shutdown error: %v", err)
   409→		}
   410→	}()
   411→
   412→	if err = s.server.Serve(listener); err != nil && err != http.ErrServerClosed {
   413→		return fmt.Errorf("server failed: %w", err)
   414→	}
   415→	return nil
   416→}
   417→
   418→func (s *Server) Shutdown(ctx context.Context) {
   419→	// Create a shutdown context with timeout

	// Because this program will want to bind to a privileged port (like 80), the listener must
	// be established first, then the user can be changed, and after that,
	// the actual http server can be started.
	if err := runtime.DropPrivileges(config.Keys.Group, config.Keys.User); err != nil {
		return fmt.Errorf("dropping privileges: %w", err)
	}

	// Handle context cancellation for graceful shutdown
	go func() {
		<-ctx.Done()
		shutdownCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
		defer cancel()
		if err := s.server.Shutdown(shutdownCtx); err != nil {
			cclog.Errorf("Server shutdown error: %v", err)
		}
	}()

	if err = s.server.Serve(listener); err != nil && err != http.ErrServerClosed {
		return fmt.Errorf("server failed: %w", err)
	}
	return nil
}

func (s *Server) Shutdown(ctx context.Context) {
	// Create a shutdown context with timeout


   330→
   331→		if err := tagger.RunTaggers(); err != nil {
   332→			return fmt.Errorf("running job taggers: %w", err)
   333→		}
   334→	}
   335→
   336→	return nil
   337→}
   338→
   339→func runServer(ctx context.Context) error {
   340→	var wg sync.WaitGroup
   341→
   342→	// Initialize metric store if configuration is provided
   343→	haveMetricstore := false
   344→	mscfg := ccconf.GetPackageConfig("metric-store")
   345→	if mscfg != nil {
   346→		metrics := metricstore.BuildMetricList()
   347→		metricstore.Init(mscfg, metrics, &wg)
   348→
   349→		// Inject repository as NodeProvider to break import cycle
   350→		ms := metricstore.GetMemoryStore()
   351→		jobRepo := repository.GetJobRepository()
   352→		ms.SetNodeProvider(jobRepo)
   353→		metricstore.MetricStoreHandle = &metricstore.InternalMetricStore{}
   354→		haveMetricstore = true
   355→	} else {
   356→		metricstore.MetricStoreHandle = nil
   357→		cclog.Debug("missing internal metricstore configuration")
   358→	}
   359→

		if err := tagger.RunTaggers(); err != nil {
			return fmt.Errorf("running job taggers: %w", err)
		}
	}

	return nil
}

func runServer(ctx context.Context) error {
	var wg sync.WaitGroup

	// Initialize metric store if configuration is provided
	haveMetricstore := false
	mscfg := ccconf.GetPackageConfig("metric-store")
	if mscfg != nil {
		metrics := metricstore.BuildMetricList()
		metricstore.Init(mscfg, metrics, &wg)

		// Inject repository as NodeProvider to break import cycle
		ms := metricstore.GetMemoryStore()
		jobRepo := repository.GetJobRepository()
		ms.SetNodeProvider(jobRepo)
		metricstore.MetricStoreHandle = &metricstore.InternalMetricStore{}
		haveMetricstore = true
	} else {
		metricstore.MetricStoreHandle = nil
		cclog.Debug("missing internal metricstore configuration")
	}


334-	}
335-
336-	return nil
337-}
338-
339:func runServer(ctx context.Context) error {
--
581-
582-	// Run server with context
583-	ctx, cancel := context.WithCancel(context.Background())
584-	defer cancel()
585-
586:	return runServer(ctx)


So `ctx` is a `context.WithCancel(context.Background())` that is never explicitly cancelled during shutdown — only `defer cancel()` when the function returns. That means the goroutine in `Start()` (line 402-410) only fires when the process is already exiting. The `srv.Shutdown(ctx)` in server.go:429 is the primary shutdown path and fires first. The duplicate in `Start()` fires later when `ctx` gets cancelled by the deferred `cancel()`. Since `http.Server.Shutdown` is idempotent the second call is harmless but wasteful.

Now I have everything. Let me write the plan.


Write

# Plan: Reduce Shutdown Time from ~40s to ~10s

## Context

Shutdown has regressed from ~10s to ~40s. The root cause is the **fully sequential** shutdown in `server.go:Shutdown()`:

1. HTTP server graceful shutdown — **30s timeout** (line 420)
2. `metricstore.Shutdown()` — **no timeout**, writes binary snapshots for all hosts (line 437)
3. `archiver.Shutdown(10s)` — **10s timeout** (line 441)

Worst case sequential: 30 + unbounded + 10 = 40s+. Steps 2 and 3 are independent and can run concurrently. The HTTP timeout is also excessive.

Additionally, `Start()` (line 402-410) has a duplicate HTTP shutdown goroutine with its own 30s timeout that fires redundantly.

## Changes

### 1. Reduce HTTP server shutdown timeout: 30s -> 5s

**File:** `cmd/cc-backend/server.go:420`

5s is plenty for in-flight HTTP requests. Most complete in milliseconds.

### 2. Remove duplicate shutdown goroutine in `Start()`

**File:** `cmd/cc-backend/server.go:402-410`

Remove the `go func() { <-ctx.Done(); s.server.Shutdown(...) }()` block. The `Shutdown()` method (called from the signal handler in main.go:415) is the primary path. The duplicate only fires later when `ctx` is cancelled by the deferred `cancel()` in `main()` — after everything is already shut down.

After removal, `Start()` just calls `s.server.Serve(listener)` which returns `http.ErrServerClosed` when `Shutdown()` closes the server.

### 3. Run metricstore and archiver shutdown concurrently with overall deadline

**File:** `cmd/cc-backend/server.go:418-444`

Replace sequential metricstore + archiver shutdown with concurrent execution, wrapped in an overall 10s deadline:

```go
func (s *Server) Shutdown(ctx context.Context) {
    shutdownCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
    defer cancel()

    nc := nats.GetClient()
    if nc != nil {
        nc.Close()
    }

    if err := s.server.Shutdown(shutdownCtx); err != nil {
        cclog.Errorf("Server shutdown error: %v", err)
    }

    // Run metricstore and archiver shutdown concurrently.
    // They are independent: metricstore writes .bin snapshots,
    // archiver flushes pending job archives.
    done := make(chan struct{})
    go func() {
        defer close(done)
        var wg sync.WaitGroup

        ms := metricstore.GetMemoryStore()
        if ms != nil {
            wg.Add(1)
            go func() {
                defer wg.Done()
                metricstore.Shutdown()
            }()
        }

        wg.Add(1)
        go func() {
            defer wg.Done()
            if err := archiver.Shutdown(10 * time.Second); err != nil {
                cclog.Warnf("Archiver shutdown: %v", err)
            }
        }()

        wg.Wait()
    }()

    select {
    case <-done:
    case <-time.After(10 * time.Second):
        cclog.Warn("Shutdown deadline exceeded, forcing exit")
    }
}
```

Data integrity is safe because:
- WAL files on disk provide crash recovery — incomplete binary snapshots are handled by replaying WAL on next startup
- `toCheckpointBinary` uses atomic rename (write `.tmp` then rename), so partial writes don't corrupt existing snapshots

### Resulting Timeline

```
Signal → HTTP shutdown (max 5s) → concurrent(metricstore, archiver) → done
                                   ↑ overall 10s deadline from signal
```

Typical: ~1s (HTTP) + ~5-8s (checkpoint) = **6-9s total**
Worst case: capped at **~15s** (5s HTTP + 10s deadline)

## Files to Modify

- `cmd/cc-backend/server.go` — all changes here (add `"sync"` and `"time"` imports if needed)

## Verification

1. `go build ./cmd/cc-backend` — compiles
2. Start with `./startDemo.sh`, send SIGTERM, measure exit time
3. Restart and verify checkpoint data loads correctly (no data loss)


File created successfully at: /Users/jan/.claude/plans/whimsical-churning-platypus.md

# Plan: Reduce Shutdown Time from ~40s to ~10s

## Context

Shutdown has regressed from ~10s to ~40s. The root cause is the **fully sequential** shutdown in `server.go:Shutdown()`:

1. HTTP server graceful shutdown — **30s timeout** (line 420)
2. `metricstore.Shutdown()` — **no timeout**, writes binary snapshots for all hosts (line 437)
3. `archiver.Shutdown(10s)` — **10s timeout** (line 441)

Worst case sequential: 30 + unbounded + 10 = 40s+. Steps 2 and 3 are independent and can run concurrently. The HTTP timeout is also excessive.

Additionally, `Start()` (line 402-410) has a duplicate HTTP shutdown goroutine with its own 30s timeout that fires redundantly.

## Changes

### 1. Reduce HTTP server shutdown timeout: 30s -> 5s

**File:** `cmd/cc-backend/server.go:420`

5s is plenty for in-flight HTTP requests. Most complete in milliseconds.

### 2. Remove duplicate shutdown goroutine in `Start()`

**File:** `cmd/cc-backend/server.go:402-410`

Remove the `go func() { <-ctx.Done(); s.server.Shutdown(...) }()` block. The `Shutdown()` method (called from the signal handler in main.go:415) is the primary path. The duplicate only fires later when `ctx` is cancelled by the deferred `cancel()` in `main()` — after everything is already shut down.

After removal, `Start()` just calls `s.server.Serve(listener)` which returns `http.ErrServerClosed` when `Shutdown()` closes the server.

### 3. Run metricstore and archiver shutdown concurrently with overall deadline

**File:** `cmd/cc-backend/server.go:418-444`

Replace sequential metricstore + archiver shutdown with concurrent execution, wrapped in an overall 10s deadline:

```go
func (s *Server) Shutdown(ctx context.Context) {
    shutdownCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
    defer cancel()

    nc := nats.GetClient()
    if nc != nil {
        nc.Close()
    }

    if err := s.server.Shutdown(shutdownCtx); err != nil {
        cclog.Errorf("Server shutdown error: %v", err)
    }

    // Run metricstore and archiver shutdown concurrently.
    // They are independent: metricstore writes .bin snapshots,
    // archiver flushes pending job archives.
    done := make(chan struct{})
    go func() {
        defer close(done)
        var wg sync.WaitGroup

        ms := metricstore.GetMemoryStore()
        if ms != nil {
            wg.Add(1)
            go func() {
                defer wg.Done()
                metricstore.Shutdown()
            }()
        }

        wg.Add(1)
        go func() {
            defer wg.Done()
            if err := archiver.Shutdown(10 * time.Second); err != nil {
                cclog.Warnf("Archiver shutdown: %v", err)
            }
        }()

        wg.Wait()
    }()

    select {
    case <-done:
    case <-time.After(10 * time.Second):
        cclog.Warn("Shutdown deadline exceeded, forcing exit")
    }
}
```

Data integrity is safe because:
- WAL files on disk provide crash recovery — incomplete binary snapshots are handled by replaying WAL on next startup
- `toCheckpointBinary` uses atomic rename (write `.tmp` then rename), so partial writes don't corrupt existing snapshots

### Resulting Timeline

```
Signal → HTTP shutdown (max 5s) → concurrent(metricstore, archiver) → done
                                   ↑ overall 10s deadline from signal
```

Typical: ~1s (HTTP) + ~5-8s (checkpoint) = **6-9s total**
Worst case: capped at **~15s** (5s HTTP + 10s deadline)

## Files to Modify

- `cmd/cc-backend/server.go` — all changes here (add `"sync"` and `"time"` imports if needed)

## Verification

1. `go build ./cmd/cc-backend` — compiles
2. Start with `./startDemo.sh`, send SIGTERM, measure exit time
3. Restart and verify checkpoint data loads correctly (no data loss)


The plan is ready. Three targeted changes, all in `server.go`:

1. **Reduce HTTP timeout** from 30s to 5s
2. **Remove duplicate shutdown goroutine** in `Start()`
3. **Run metricstore + archiver concurrently** with a 10s overall deadline

The biggest win comes from parallelizing metricstore and archiver (they're independent) and not waiting 30s for HTTP drain. Data integrity is preserved because WAL replay handles incomplete snapshots on restart.


ToolSearch

Question: select:ExitPlanMode


Question: select:ExitPlanMode


ExitPlanMode

Prompt: build the Go backend

# Plan: Reduce Shutdown Time from ~40s to ~10s

## Context

Shutdown has regressed from ~10s to ~40s. The root cause is the **fully sequential** shutdown in `server.go:Shutdown()`:

1. HTTP server graceful shutdown — **30s timeout** (line 420)
2. `metricstore.Shutdown()` — **no timeout**, writes binary snapshots for all hosts (line 437)
3. `archiver.Shutdown(10s)` — **10s timeout** (line 441)

Worst case sequential: 30 + unbounded + 10 = 40s+. Steps 2 and 3 are independent and can run concurrently. The HTTP timeout is also excessive.

Additionally, `Start()` (line 402-410) has a duplicate HTTP shutdown goroutine with its own 30s timeout that fires redundantly.

## Changes

### 1. Reduce HTTP server shutdown timeout: 30s -> 5s

**File:** `cmd/cc-backend/server.go:420`

5s is plenty for in-flight HTTP requests. Most complete in milliseconds.

### 2. Remove duplicate shutdown goroutine in `Start()`

**File:** `cmd/cc-backend/server.go:402-410`

Remove the `go func() { <-ctx.Done(); s.server.Shutdown(...) }()` block. The `Shutdown()` method (called from the signal handler in main.go:415) is the primary path. The duplicate only fires later when `ctx` is cancelled by the deferred `cancel()` in `main()` — after everything is already shut down.

After removal, `Start()` just calls `s.server.Serve(listener)` which returns `http.ErrServerClosed` when `Shutdown()` closes the server.

### 3. Run metricstore and archiver shutdown concurrently with overall deadline

**File:** `cmd/cc-backend/server.go:418-444`

Replace sequential metricstore + archiver shutdown with concurrent execution, wrapped in an overall 10s deadline:

```go
func (s *Server) Shutdown(ctx context.Context) {
    shutdownCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
    defer cancel()

    nc := nats.GetClient()
    if nc != nil {
        nc.Close()
    }

    if err := s.server.Shutdown(shutdownCtx); err != nil {
        cclog.Errorf("Server shutdown error: %v", err)
    }

    // Run metricstore and archiver shutdown concurrently.
    // They are independent: metricstore writes .bin snapshots,
    // archiver flushes pending job archives.
    done := make(chan struct{})
    go func() {
        defer close(done)
        var wg sync.WaitGroup

        ms := metricstore.GetMemoryStore()
        if ms != nil {
            wg.Add(1)
            go func() {
                defer wg.Done()
                metricstore.Shutdown()
            }()
        }

        wg.Add(1)
        go func() {
            defer wg.Done()
            if err := archiver.Shutdown(10 * time.Second); err != nil {
                cclog.Warnf("Archiver shutdown: %v", err)
            }
        }()

        wg.Wait()
    }()

    select {
    case <-done:
    case <-time.After(10 * time.Second):
        cclog.Warn("Shutdown deadline exceeded, forcing exit")
    }
}
```

Data integrity is safe because:
- WAL files on disk provide crash recovery — incomplete binary snapshots are handled by replaying WAL on next startup
- `toCheckpointBinary` uses atomic rename (write `.tmp` then rename), so partial writes don't corrupt existing snapshots

### Resulting Timeline

```
Signal → HTTP shutdown (max 5s) → concurrent(metricstore, archiver) → done
                                   ↑ overall 10s deadline from signal
```

Typical: ~1s (HTTP) + ~5-8s (checkpoint) = **6-9s total**
Worst case: capped at **~15s** (5s HTTP + 10s deadline)

## Files to Modify

- `cmd/cc-backend/server.go` — all changes here (add `"sync"` and `"time"` imports if needed)

## Verification

1. `go build ./cmd/cc-backend` — compiles
2. Start with `./startDemo.sh`, send SIGTERM, measure exit time
3. Restart and verify checkpoint data loads correctly (no data loss)


User has approved your plan. You can now start coding. Start with updating your todo list if applicable

Your plan has been saved to: /Users/jan/.claude/plans/whimsical-churning-platypus.md
You can refer back to it if needed during implementation.

## Approved Plan:
# Plan: Reduce Shutdown Time from ~40s to ~10s

## Context

Shutdown has regressed from ~10s to ~40s. The root cause is the **fully sequential** shutdown in `server.go:Shutdown()`:

1. HTTP server graceful shutdown — **30s timeout** (line 420)
2. `metricstore.Shutdown()` — **no timeout**, writes binary snapshots for all hosts (line 437)
3. `archiver.Shutdown(10s)` — **10s timeout** (line 441)

Worst case sequential: 30 + unbounded + 10 = 40s+. Steps 2 and 3 are independent and can run concurrently. The HTTP timeout is also excessive.

Additionally, `Start()` (line 402-410) has a duplicate HTTP shutdown goroutine with its own 30s timeout that fires redundantly.

## Changes

### 1. Reduce HTTP server shutdown timeout: 30s -> 5s

**File:** `cmd/cc-backend/server.go:420`

5s is plenty for in-flight HTTP requests. Most complete in milliseconds.

### 2. Remove duplicate shutdown goroutine in `Start()`

**File:** `cmd/cc-backend/server.go:402-410`

Remove the `go func() { <-ctx.Done(); s.server.Shutdown(...) }()` block. The `Shutdown()` method (called from the signal handler in main.go:415) is the primary path. The duplicate only fires later when `ctx` is cancelled by the deferred `cancel()` in `main()` — after everything is already shut down.

After removal, `Start()` just calls `s.server.Serve(listener)` which returns `http.ErrServerClosed` when `Shutdown()` closes the server.

### 3. Run metricstore and archiver shutdown concurrently with overall deadline

**File:** `cmd/cc-backend/server.go:418-444`

Replace sequential metricstore + archiver shutdown with concurrent execution, wrapped in an overall 10s deadline:

```go
func (s *Server) Shutdown(ctx context.Context) {
    shutdownCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
    defer cancel()

    nc := nats.GetClient()
    if nc != nil {
        nc.Close()
    }

    if err := s.server.Shutdown(shutdownCtx); err != nil {
        cclog.Errorf("Server shutdown error: %v", err)
    }

    // Run metricstore and archiver shutdown concurrently.
    // They are independent: metricstore writes .bin snapshots,
    // archiver flushes pending job archives.
    done := make(chan struct{})
    go func() {
        defer close(done)
        var wg sync.WaitGroup

        ms := metricstore.GetMemoryStore()
        if ms != nil {
            wg.Add(1)
            go func() {
                defer wg.Done()
                metricstore.Shutdown()
            }()
        }

        wg.Add(1)
        go func() {
            defer wg.Done()
            if err := archiver.Shutdown(10 * time.Second); err != nil {
                cclog.Warnf("Archiver shutdown: %v", err)
            }
        }()

        wg.Wait()
    }()

    select {
    case <-done:
    case <-time.After(10 * time.Second):
        cclog.Warn("Shutdown deadline exceeded, forcing exit")
    }
}
```

Data integrity is safe because:
- WAL files on disk provide crash recovery — incomplete binary snapshots are handled by replaying WAL on next startup
- `toCheckpointBinary` uses atomic rename (write `.tmp` then rename), so partial writes don't corrupt existing snapshots

### Resulting Timeline

```
Signal → HTTP shutdown (max 5s) → concurrent(metricstore, archiver) → done
                                   ↑ overall 10s deadline from signal
```

Typical: ~1s (HTTP) + ~5-8s (checkpoint) = **6-9s total**
Worst case: capped at **~15s** (5s HTTP + 10s deadline)

## Files to Modify

- `cmd/cc-backend/server.go` — all changes here (add `"sync"` and `"time"` imports if needed)

## Verification

1. `go build ./cmd/cc-backend` — compiles
2. Start with `./startDemo.sh`, send SIGTERM, measure exit time
3. Restart and verify checkpoint data loads correctly (no data loss)


Let me implement the changes.


     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→// Package main provides the entry point for the ClusterCockpit backend server.
     7→// This file contains HTTP server setup, routing configuration, and
     8→// authentication middleware integration.
     9→package main
    10→
    11→import (
    12→	"context"
    13→	"crypto/tls"
    14→	"encoding/json"
    15→	"errors"
    16→	"fmt"
    17→	"net"
    18→	"net/http"
    19→	"os"
    20→	"strings"

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

// Package main provides the entry point for the ClusterCockpit backend server.
// This file contains HTTP server setup, routing configuration, and
// authentication middleware integration.
package main

import (
	"context"
	"crypto/tls"
	"encoding/json"
	"errors"
	"fmt"
	"net"
	"net/http"
	"os"
	"strings"


    20→	"strings"
    21→	"time"
    22→
    23→	"github.com/99designs/gqlgen/graphql"
    24→	"github.com/99designs/gqlgen/graphql/handler"
    25→	"github.com/99designs/gqlgen/graphql/handler/transport"
    26→	"github.com/99designs/gqlgen/graphql/playground"
    27→	"github.com/ClusterCockpit/cc-backend/internal/api"
    28→	"github.com/ClusterCockpit/cc-backend/internal/archiver"
    29→	"github.com/ClusterCockpit/cc-backend/internal/auth"
    30→	"github.com/ClusterCockpit/cc-backend/internal/config"
    31→	"github.com/ClusterCockpit/cc-backend/internal/graph"
    32→	"github.com/ClusterCockpit/cc-backend/internal/graph/generated"
    33→	"github.com/ClusterCockpit/cc-backend/internal/routerConfig"
    34→	"github.com/ClusterCockpit/cc-backend/pkg/metricstore"
    35→	"github.com/ClusterCockpit/cc-backend/web"
    36→	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
    37→	"github.com/ClusterCockpit/cc-lib/v2/nats"
    38→	"github.com/ClusterCockpit/cc-lib/v2/runtime"
    39→	"github.com/go-chi/chi/v5"
    40→	"github.com/go-chi/chi/v5/middleware"
    41→	"github.com/go-chi/cors"
    42→	httpSwagger "github.com/swaggo/http-swagger"
    43→)
    44→
    45→var buildInfo web.Build
    46→
    47→// Environment variable names
    48→const (
    49→	envDebug = "DEBUG"

	"strings"
	"time"

	"github.com/99designs/gqlgen/graphql"
	"github.com/99designs/gqlgen/graphql/handler"
	"github.com/99designs/gqlgen/graphql/handler/transport"
	"github.com/99designs/gqlgen/graphql/playground"
	"github.com/ClusterCockpit/cc-backend/internal/api"
	"github.com/ClusterCockpit/cc-backend/internal/archiver"
	"github.com/ClusterCockpit/cc-backend/internal/auth"
	"github.com/ClusterCockpit/cc-backend/internal/config"
	"github.com/ClusterCockpit/cc-backend/internal/graph"
	"github.com/ClusterCockpit/cc-backend/internal/graph/generated"
	"github.com/ClusterCockpit/cc-backend/internal/routerConfig"
	"github.com/ClusterCockpit/cc-backend/pkg/metricstore"
	"github.com/ClusterCockpit/cc-backend/web"
	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
	"github.com/ClusterCockpit/cc-lib/v2/nats"
	"github.com/ClusterCockpit/cc-lib/v2/runtime"
	"github.com/go-chi/chi/v5"
	"github.com/go-chi/chi/v5/middleware"
	"github.com/go-chi/cors"
	httpSwagger "github.com/swaggo/http-swagger"
)

var buildInfo web.Build

// Environment variable names
const (
	envDebug = "DEBUG"


Good — `"sync"` is not in the imports yet, and `"time"` is already there. Now let me apply all three changes:

**Change 1 & 2:** Remove duplicate shutdown goroutine in `Start()`:


Edit

	// Handle context cancellation for graceful shutdown
	go func() {
		<-ctx.Done()
		shutdownCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
		defer cancel()
		if err := s.server.Shutdown(shutdownCtx); err != nil {
			cclog.Errorf("Server shutdown error: %v", err)
		}
	}()

	if err = s.server.Serve(listener); err != nil && err != http.ErrServerClosed {

if err = s.server.Serve(listener); err != nil && err != http.ErrServerClosed {


The file /Users/jan/prg/ClusterCockpit/cc-backend/cmd/cc-backend/server.go has been updated successfully.

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

// Package main provides the entry point for the ClusterCockpit backend server.
// This file contains HTTP server setup, routing configuration, and
// authentication middleware integration.
package main

import (
	"context"
	"crypto/tls"
	"encoding/json"
	"errors"
	"fmt"
	"net"
	"net/http"
	"os"
	"strings"
	"time"

	"github.com/99designs/gqlgen/graphql"
	"github.com/99designs/gqlgen/graphql/handler"
	"github.com/99designs/gqlgen/graphql/handler/transport"
	"github.com/99designs/gqlgen/graphql/playground"
	"github.com/ClusterCockpit/cc-backend/internal/api"
	"github.com/ClusterCockpit/cc-backend/internal/archiver"
	"github.com/ClusterCockpit/cc-backend/internal/auth"
	"github.com/ClusterCockpit/cc-backend/internal/config"
	"github.com/ClusterCockpit/cc-backend/internal/graph"
	"github.com/ClusterCockpit/cc-backend/internal/graph/generated"
	"github.com/ClusterCockpit/cc-backend/internal/routerConfig"
	"github.com/ClusterCockpit/cc-backend/pkg/metricstore"
	"github.com/ClusterCockpit/cc-backend/web"
	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
	"github.com/ClusterCockpit/cc-lib/v2/nats"
	"github.com/ClusterCockpit/cc-lib/v2/runtime"
	"github.com/go-chi/chi/v5"
	"github.com/go-chi/chi/v5/middleware"
	"github.com/go-chi/cors"
	httpSwagger "github.com/swaggo/http-swagger"
)

var buildInfo web.Build

// Environment variable names
const (
	envDebug = "DEBUG"
)

// Server encapsulates the HTTP server state and dependencies
type Server struct {
	router        chi.Router
	server        *http.Server
	restAPIHandle *api.RestAPI
	natsAPIHandle *api.NatsAPI
}

func onFailureResponse(rw http.ResponseWriter, r *http.Request, err error) {
	rw.Header().Add("Content-Type", "application/json")
	rw.WriteHeader(http.StatusUnauthorized)
	json.NewEncoder(rw).Encode(map[string]string{
		"status": http.StatusText(http.StatusUnauthorized),
		"error":  err.Error(),
	})
}

// NewServer creates and initializes a new Server instance
func NewServer(version, commit, buildDate string) (*Server, error) {
	buildInfo = web.Build{Version: version, Hash: commit, Buildtime: buildDate}

	s := &Server{
		router: chi.NewRouter(),
	}

	if err := s.init(); err != nil {
		return nil, err
	}

	return s, nil
}

func (s *Server) init() error {
	// Setup the http.Handler/Router used by the server
	graph.Init()
	resolver := graph.GetResolverInstance()
	graphQLServer := handler.New(
		generated.NewExecutableSchema(generated.Config{Resolvers: resolver}))

	graphQLServer.AddTransport(transport.POST{})

	// Inject a per-request stats cache so that grouped statistics queries
	// sharing the same (filter, groupBy) pair are executed only once.
	graphQLServer.AroundOperations(func(ctx context.Context, next graphql.OperationHandler) graphql.ResponseHandler {
		return next(graph.WithStatsGroupCache(ctx))
	})

	if os.Getenv(envDebug) != "1" {
		// Having this handler means that a error message is returned via GraphQL instead of the connection simply beeing closed.
		// The problem with this is that then, no more stacktrace is printed to stderr.
		graphQLServer.SetRecoverFunc(func(ctx context.Context, err any) error {
			switch e := err.(type) {
			case string:
				return fmt.Errorf("MAIN > Panic: %s", e)
			case error:
				return fmt.Errorf("MAIN > Panic caused by: %s", e.Error())
			}

			return errors.New("MAIN > Internal server error (panic)")
		})
	}

	authHandle := auth.GetAuthInstance()

	// Middleware must be defined before routes in chi
	s.router.Use(func(next http.Handler) http.Handler {
		return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
			start := time.Now()
			ww := middleware.NewWrapResponseWriter(rw, r.ProtoMajor)
			next.ServeHTTP(ww, r)
			cclog.Debugf("%s %s (%d, %.02fkb, %dms)",
				r.Method, r.URL.RequestURI(),
				ww.Status(), float32(ww.BytesWritten())/1024,
				time.Since(start).Milliseconds())
		})
	})
	s.router.Use(middleware.Compress(5))
	s.router.Use(middleware.Recoverer)
	s.router.Use(cors.Handler(cors.Options{
		AllowCredentials: true,
		AllowedHeaders:   []string{"X-Requested-With", "Content-Type", "Authorization", "Origin"},
		AllowedMethods:   []string{"GET", "POST", "HEAD", "OPTIONS"},
		AllowedOrigins:   []string{"*"},
	}))

	s.restAPIHandle = api.New()

	info := map[string]any{}
	info["hasOpenIDConnect"] = false

	if auth.Keys.OpenIDConfig != nil {
		openIDConnect := auth.NewOIDC(authHandle)
		openIDConnect.RegisterEndpoints(s.router)
		info["hasOpenIDConnect"] = true
	}

	s.router.Get("/login", func(rw http.ResponseWriter, r *http.Request) {
		rw.Header().Add("Content-Type", "text/html; charset=utf-8")
		cclog.Debugf("##%v##", info)
		web.RenderTemplate(rw, "login.tmpl", &web.Page{Title: "Login", Build: buildInfo, Infos: info})
	})
	s.router.HandleFunc("/imprint", func(rw http.ResponseWriter, r *http.Request) {
		rw.Header().Add("Content-Type", "text/html; charset=utf-8")
		web.RenderTemplate(rw, "imprint.tmpl", &web.Page{Title: "Imprint", Build: buildInfo})
	})
	s.router.HandleFunc("/privacy", func(rw http.ResponseWriter, r *http.Request) {
		rw.Header().Add("Content-Type", "text/html; charset=utf-8")
		web.RenderTemplate(rw, "privacy.tmpl", &web.Page{Title: "Privacy", Build: buildInfo})
	})

	if !config.Keys.DisableAuthentication {
		// Create login failure handler (used by both /login and /jwt-login)
		loginFailureHandler := func(rw http.ResponseWriter, r *http.Request, err error) {
			rw.Header().Add("Content-Type", "text/html; charset=utf-8")
			rw.WriteHeader(http.StatusUnauthorized)
			web.RenderTemplate(rw, "login.tmpl", &web.Page{
				Title:   "Login failed - ClusterCockpit",
				MsgType: "alert-warning",
				Message: err.Error(),
				Build:   buildInfo,
				Infos:   info,
			})
		}

		s.router.Post("/login", authHandle.Login(loginFailureHandler).ServeHTTP)
		s.router.HandleFunc("/jwt-login", authHandle.Login(loginFailureHandler).ServeHTTP)

		s.router.Post("/logout", authHandle.Logout(
			http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
				rw.Header().Add("Content-Type", "text/html; charset=utf-8")
				rw.WriteHeader(http.StatusOK)
				web.RenderTemplate(rw, "login.tmpl", &web.Page{
					Title:   "Bye - ClusterCockpit",
					MsgType: "alert-info",
					Message: "Logout successful",
					Build:   buildInfo,
					Infos:   info,
				})
			})).ServeHTTP)
	}

	if flagDev {
		s.router.Handle("/playground", playground.Handler("GraphQL playground", "/query"))
		s.router.Get("/swagger/*", httpSwagger.Handler(
			httpSwagger.URL("http://"+config.Keys.Addr+"/swagger/doc.json")))
	}

	// Secured routes (require authentication)
	s.router.Group(func(secured chi.Router) {
		if !config.Keys.DisableAuthentication {
			secured.Use(func(next http.Handler) http.Handler {
				return authHandle.Auth(
					next,
					func(rw http.ResponseWriter, r *http.Request, err error) {
						rw.WriteHeader(http.StatusUnauthorized)
						web.RenderTemplate(rw, "login.tmpl", &web.Page{
							Title:    "Authentication failed - ClusterCockpit",
							MsgType:  "alert-danger",
							Message:  err.Error(),
							Build:    buildInfo,
							Infos:    info,
							Redirect: r.RequestURI,
						})
					})
			})
		}

		secured.Handle("/query", graphQLServer)

		secured.HandleFunc("/search", func(rw http.ResponseWriter, r *http.Request) {
			routerConfig.HandleSearchBar(rw, r, buildInfo)
		})

		routerConfig.SetupRoutes(secured, buildInfo)
	})

	// API routes (JWT token auth)
	s.router.Route("/api", func(apiRouter chi.Router) {
		// Main API routes with API auth
		apiRouter.Group(func(securedapi chi.Router) {
			if !config.Keys.DisableAuthentication {
				securedapi.Use(func(next http.Handler) http.Handler {
					return authHandle.AuthAPI(next, onFailureResponse)
				})
			}
			s.restAPIHandle.MountAPIRoutes(securedapi)
		})

		// Metric store API routes with separate auth
		apiRouter.Group(func(metricstoreapi chi.Router) {
			if !config.Keys.DisableAuthentication {
				metricstoreapi.Use(func(next http.Handler) http.Handler {
					return authHandle.AuthMetricStoreAPI(next, onFailureResponse)
				})
			}
			s.restAPIHandle.MountMetricStoreAPIRoutes(metricstoreapi)
		})
	})

	// User API routes
	s.router.Route("/userapi", func(userapi chi.Router) {
		if !config.Keys.DisableAuthentication {
			userapi.Use(func(next http.Handler) http.Handler {
				return authHandle.AuthUserAPI(next, onFailureResponse)
			})
		}
		s.restAPIHandle.MountUserAPIRoutes(userapi)
	})

	// Config API routes (uses Group with full paths to avoid shadowing
	// the /config page route that is registered in the secured group)
	s.router.Group(func(configapi chi.Router) {
		if !config.Keys.DisableAuthentication {
			configapi.Use(func(next http.Handler) http.Handler {
				return authHandle.AuthConfigAPI(next, onFailureResponse)
			})
		}
		s.restAPIHandle.MountConfigAPIRoutes(configapi)
	})

	// Frontend API routes
	s.router.Route("/frontend", func(frontendapi chi.Router) {
		if !config.Keys.DisableAuthentication {
			frontendapi.Use(func(next http.Handler) http.Handler {
				return authHandle.AuthFrontendAPI(next, onFailureResponse)
			})
		}
		s.restAPIHandle.MountFrontendAPIRoutes(frontendapi)
	})

	if config.Keys.APISubjects != nil {
		s.natsAPIHandle = api.NewNatsAPI()
		if err := s.natsAPIHandle.StartSubscriptions(); err != nil {
			return fmt.Errorf("starting NATS subscriptions: %w", err)
		}
	}

	// 404 handler for pages and API routes
	notFoundHandler := func(rw http.ResponseWriter, r *http.Request) {
		if strings.HasPrefix(r.URL.Path, "/api/") || strings.HasPrefix(r.URL.Path, "/userapi/") ||
			strings.HasPrefix(r.URL.Path, "/frontend/") || strings.HasPrefix(r.URL.Path, "/config/") {
			rw.Header().Set("Content-Type", "application/json")
			rw.WriteHeader(http.StatusNotFound)
			json.NewEncoder(rw).Encode(map[string]string{
				"status": "Resource not found",
				"error":  "the requested endpoint does not exist",
			})
			return
		}
		rw.Header().Set("Content-Type", "text/html; charset=utf-8")
		rw.WriteHeader(http.StatusNotFound)
		web.RenderTemplate(rw, "404.tmpl", &web.Page{
			Title: "Page Not Found",
			Build: buildInfo,
		})
	}

	// Set NotFound on the router so chi uses it for all unmatched routes,
	// including those under subrouters like /api, /userapi, /frontend, etc.
	s.router.NotFound(notFoundHandler)

	if config.Keys.EmbedStaticFiles {
		if i, err := os.Stat("./var/img"); err == nil {
			if i.IsDir() {
				cclog.Info("Use local directory for static images")
				s.router.Handle("/img/*", http.StripPrefix("/img/", http.FileServer(http.Dir("./var/img"))))
			}
		}
		fileServer := http.StripPrefix("/", web.ServeFiles())
		s.router.Handle("/*", http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
			if web.StaticFileExists(r.URL.Path) {
				fileServer.ServeHTTP(rw, r)
				return
			}
			notFoundHandler(rw, r)
		}))
	} else {
		staticDir := http.Dir(config.Keys.StaticFiles)
		fileServer := http.FileServer(staticDir)
		s.router.Handle("/*", http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
			f, err := staticDir.Open(r.URL.Path)
			if err == nil {
				f.Close()
				fileServer.ServeHTTP(rw, r)
				return
			}
			notFoundHandler(rw, r)
		}))
	}

	return nil
}

// Server timeout defaults (in seconds)
const (
	defaultReadHeaderTimeout = 20
	defaultWriteTimeout      = 20
)

func (s *Server) Start(ctx context.Context) error {
	// Use configurable timeouts with defaults
	readHeaderTimeout := time.Duration(defaultReadHeaderTimeout) * time.Second
	writeTimeout := time.Duration(defaultWriteTimeout) * time.Second

	s.server = &http.Server{
		ReadHeaderTimeout: readHeaderTimeout,
		WriteTimeout:      writeTimeout,
		Handler:      s.router,
		Addr:         config.Keys.Addr,
	}

	// Start http or https server
	listener, err := net.Listen("tcp", config.Keys.Addr)
	if err != nil {
		return fmt.Errorf("starting listener on '%s': %w", config.Keys.Addr, err)
	}

	if !strings.HasSuffix(config.Keys.Addr, ":80") && config.Keys.RedirectHTTPTo != "" {
		go func() {
			http.ListenAndServe(":80", http.RedirectHandler(config.Keys.RedirectHTTPTo, http.StatusMovedPermanently))
		}()
	}

	if config.Keys.HTTPSCertFile != "" && config.Keys.HTTPSKeyFile != "" {
		cert, err := tls.LoadX509KeyPair(
			config.Keys.HTTPSCertFile, config.Keys.HTTPSKeyFile)
		if err != nil {
			return fmt.Errorf("loading X509 keypair (check 'https-cert-file' and 'https-key-file' in config.json): %w", err)
		}
		listener = tls.NewListener(listener, &tls.Config{
			Certificates: []tls.Certificate{cert},
			CipherSuites: []uint16{
				tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
				tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
			},
			MinVersion:               tls.VersionTLS12,
			PreferServerCipherSuites: true,
		})
		cclog.Infof("HTTPS server listening at %s...", config.Keys.Addr)
	} else {
		cclog.Infof("HTTP server listening at %s...", config.Keys.Addr)
	}
	//
	// Because this program will want to bind to a privileged port (like 80), the listener must
	// be established first, then the user can be changed, and after that,
	// the actual http server can be started.
	if err := runtime.DropPrivileges(config.Keys.Group, config.Keys.User); err != nil {
		return fmt.Errorf("dropping privileges: %w", err)
	}

	// Handle context cancellation for graceful shutdown
	go func() {
		<-ctx.Done()
		shutdownCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
		defer cancel()
		if err := s.server.Shutdown(shutdownCtx); err != nil {
			cclog.Errorf("Server shutdown error: %v", err)
		}
	}()

	if err = s.server.Serve(listener); err != nil && err != http.ErrServerClosed {
		return fmt.Errorf("server failed: %w", err)
	}
	return nil
}

func (s *Server) Shutdown(ctx context.Context) {
	// Create a shutdown context with timeout
	shutdownCtx, cancel := context.WithTimeout(ctx, 30*time.Second)
	defer cancel()

	nc := nats.GetClient()
	if nc != nil {
		nc.Close()
	}

	// First shut down the server gracefully (waiting for all ongoing requests)
	if err := s.server.Shutdown(shutdownCtx); err != nil {
		cclog.Errorf("Server shutdown error: %v", err)
	}

	// Archive all the metric store data
	ms := metricstore.GetMemoryStore()

	if ms != nil {
		metricstore.Shutdown()
	}

	// Shutdown archiver with 10 second timeout for fast shutdown
	if err := archiver.Shutdown(10 * time.Second); err != nil {
		cclog.Warnf("Archiver shutdown: %v", err)
	}
}

-  // Handle context cancellation for graceful shutdown

-    shutdownCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second)

-    if err := s.server.Shutdown(shutdownCtx); err != nil {

-      cclog.Errorf("Server shutdown error: %v", err)


**Change 3:** Replace the `Shutdown()` method — reduce HTTP timeout to 5s, parallelize metricstore+archiver, add overall deadline:


func (s *Server) Shutdown(ctx context.Context) {
	// Create a shutdown context with timeout
	shutdownCtx, cancel := context.WithTimeout(ctx, 30*time.Second)
	defer cancel()

	nc := nats.GetClient()
	if nc != nil {
		nc.Close()
	}

	// First shut down the server gracefully (waiting for all ongoing requests)
	if err := s.server.Shutdown(shutdownCtx); err != nil {
		cclog.Errorf("Server shutdown error: %v", err)
	}

	// Archive all the metric store data
	ms := metricstore.GetMemoryStore()

	if ms != nil {
		metricstore.Shutdown()
	}

	// Shutdown archiver with 10 second timeout for fast shutdown
	if err := archiver.Shutdown(10 * time.Second); err != nil {
		cclog.Warnf("Archiver shutdown: %v", err)
	}
}

func (s *Server) Shutdown(ctx context.Context) {
	shutdownCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
	defer cancel()

	nc := nats.GetClient()
	if nc != nil {
		nc.Close()
	}

	if err := s.server.Shutdown(shutdownCtx); err != nil {
		cclog.Errorf("Server shutdown error: %v", err)
	}

	// Run metricstore and archiver shutdown concurrently.
	// They are independent: metricstore writes .bin snapshots,
	// archiver flushes pending job archives.
	done := make(chan struct{})
	go func() {
		defer close(done)
		var wg sync.WaitGroup

		if ms := metricstore.GetMemoryStore(); ms != nil {
			wg.Add(1)
			go func() {
				defer wg.Done()
				metricstore.Shutdown()
			}()
		}

		wg.Add(1)
		go func() {
			defer wg.Done()
			if err := archiver.Shutdown(10 * time.Second); err != nil {
				cclog.Warnf("Archiver shutdown: %v", err)
			}
		}()

		wg.Wait()
	}()

	select {
	case <-done:
	case <-time.After(10 * time.Second):
		cclog.Warn("Shutdown deadline exceeded, forcing exit")
	}
}


func (s *Server) Shutdown(ctx context.Context) {
	shutdownCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
	defer cancel()

	nc := nats.GetClient()
	if nc != nil {
		nc.Close()
	}

	if err := s.server.Shutdown(shutdownCtx); err != nil {
		cclog.Errorf("Server shutdown error: %v", err)
	}

	// Run metricstore and archiver shutdown concurrently.
	// They are independent: metricstore writes .bin snapshots,
	// archiver flushes pending job archives.
	done := make(chan struct{})
	go func() {
		defer close(done)
		var wg sync.WaitGroup

		if ms := metricstore.GetMemoryStore(); ms != nil {
			wg.Add(1)
			go func() {
				defer wg.Done()
				metricstore.Shutdown()
			}()
		}

		wg.Add(1)
		go func() {
			defer wg.Done()
			if err := archiver.Shutdown(10 * time.Second); err != nil {
				cclog.Warnf("Archiver shutdown: %v", err)
			}
		}()

		wg.Wait()
	}()

	select {
	case <-done:
	case <-time.After(10 * time.Second):
		cclog.Warn("Shutdown deadline exceeded, forcing exit")
	}
}

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

// Package main provides the entry point for the ClusterCockpit backend server.
// This file contains HTTP server setup, routing configuration, and
// authentication middleware integration.
package main

import (
	"context"
	"crypto/tls"
	"encoding/json"
	"errors"
	"fmt"
	"net"
	"net/http"
	"os"
	"strings"
	"time"

	"github.com/99designs/gqlgen/graphql"
	"github.com/99designs/gqlgen/graphql/handler"
	"github.com/99designs/gqlgen/graphql/handler/transport"
	"github.com/99designs/gqlgen/graphql/playground"
	"github.com/ClusterCockpit/cc-backend/internal/api"
	"github.com/ClusterCockpit/cc-backend/internal/archiver"
	"github.com/ClusterCockpit/cc-backend/internal/auth"
	"github.com/ClusterCockpit/cc-backend/internal/config"
	"github.com/ClusterCockpit/cc-backend/internal/graph"
	"github.com/ClusterCockpit/cc-backend/internal/graph/generated"
	"github.com/ClusterCockpit/cc-backend/internal/routerConfig"
	"github.com/ClusterCockpit/cc-backend/pkg/metricstore"
	"github.com/ClusterCockpit/cc-backend/web"
	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
	"github.com/ClusterCockpit/cc-lib/v2/nats"
	"github.com/ClusterCockpit/cc-lib/v2/runtime"
	"github.com/go-chi/chi/v5"
	"github.com/go-chi/chi/v5/middleware"
	"github.com/go-chi/cors"
	httpSwagger "github.com/swaggo/http-swagger"
)

var buildInfo web.Build

// Environment variable names
const (
	envDebug = "DEBUG"
)

// Server encapsulates the HTTP server state and dependencies
type Server struct {
	router        chi.Router
	server        *http.Server
	restAPIHandle *api.RestAPI
	natsAPIHandle *api.NatsAPI
}

func onFailureResponse(rw http.ResponseWriter, r *http.Request, err error) {
	rw.Header().Add("Content-Type", "application/json")
	rw.WriteHeader(http.StatusUnauthorized)
	json.NewEncoder(rw).Encode(map[string]string{
		"status": http.StatusText(http.StatusUnauthorized),
		"error":  err.Error(),
	})
}

// NewServer creates and initializes a new Server instance
func NewServer(version, commit, buildDate string) (*Server, error) {
	buildInfo = web.Build{Version: version, Hash: commit, Buildtime: buildDate}

	s := &Server{
		router: chi.NewRouter(),
	}

	if err := s.init(); err != nil {
		return nil, err
	}

	return s, nil
}

func (s *Server) init() error {
	// Setup the http.Handler/Router used by the server
	graph.Init()
	resolver := graph.GetResolverInstance()
	graphQLServer := handler.New(
		generated.NewExecutableSchema(generated.Config{Resolvers: resolver}))

	graphQLServer.AddTransport(transport.POST{})

	// Inject a per-request stats cache so that grouped statistics queries
	// sharing the same (filter, groupBy) pair are executed only once.
	graphQLServer.AroundOperations(func(ctx context.Context, next graphql.OperationHandler) graphql.ResponseHandler {
		return next(graph.WithStatsGroupCache(ctx))
	})

	if os.Getenv(envDebug) != "1" {
		// Having this handler means that a error message is returned via GraphQL instead of the connection simply beeing closed.
		// The problem with this is that then, no more stacktrace is printed to stderr.
		graphQLServer.SetRecoverFunc(func(ctx context.Context, err any) error {
			switch e := err.(type) {
			case string:
				return fmt.Errorf("MAIN > Panic: %s", e)
			case error:
				return fmt.Errorf("MAIN > Panic caused by: %s", e.Error())
			}

			return errors.New("MAIN > Internal server error (panic)")
		})
	}

	authHandle := auth.GetAuthInstance()

	// Middleware must be defined before routes in chi
	s.router.Use(func(next http.Handler) http.Handler {
		return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
			start := time.Now()
			ww := middleware.NewWrapResponseWriter(rw, r.ProtoMajor)
			next.ServeHTTP(ww, r)
			cclog.Debugf("%s %s (%d, %.02fkb, %dms)",
				r.Method, r.URL.RequestURI(),
				ww.Status(), float32(ww.BytesWritten())/1024,
				time.Since(start).Milliseconds())
		})
	})
	s.router.Use(middleware.Compress(5))
	s.router.Use(middleware.Recoverer)
	s.router.Use(cors.Handler(cors.Options{
		AllowCredentials: true,
		AllowedHeaders:   []string{"X-Requested-With", "Content-Type", "Authorization", "Origin"},
		AllowedMethods:   []string{"GET", "POST", "HEAD", "OPTIONS"},
		AllowedOrigins:   []string{"*"},
	}))

	s.restAPIHandle = api.New()

	info := map[string]any{}
	info["hasOpenIDConnect"] = false

	if auth.Keys.OpenIDConfig != nil {
		openIDConnect := auth.NewOIDC(authHandle)
		openIDConnect.RegisterEndpoints(s.router)
		info["hasOpenIDConnect"] = true
	}

	s.router.Get("/login", func(rw http.ResponseWriter, r *http.Request) {
		rw.Header().Add("Content-Type", "text/html; charset=utf-8")
		cclog.Debugf("##%v##", info)
		web.RenderTemplate(rw, "login.tmpl", &web.Page{Title: "Login", Build: buildInfo, Infos: info})
	})
	s.router.HandleFunc("/imprint", func(rw http.ResponseWriter, r *http.Request) {
		rw.Header().Add("Content-Type", "text/html; charset=utf-8")
		web.RenderTemplate(rw, "imprint.tmpl", &web.Page{Title: "Imprint", Build: buildInfo})
	})
	s.router.HandleFunc("/privacy", func(rw http.ResponseWriter, r *http.Request) {
		rw.Header().Add("Content-Type", "text/html; charset=utf-8")
		web.RenderTemplate(rw, "privacy.tmpl", &web.Page{Title: "Privacy", Build: buildInfo})
	})

	if !config.Keys.DisableAuthentication {
		// Create login failure handler (used by both /login and /jwt-login)
		loginFailureHandler := func(rw http.ResponseWriter, r *http.Request, err error) {
			rw.Header().Add("Content-Type", "text/html; charset=utf-8")
			rw.WriteHeader(http.StatusUnauthorized)
			web.RenderTemplate(rw, "login.tmpl", &web.Page{
				Title:   "Login failed - ClusterCockpit",
				MsgType: "alert-warning",
				Message: err.Error(),
				Build:   buildInfo,
				Infos:   info,
			})
		}

		s.router.Post("/login", authHandle.Login(loginFailureHandler).ServeHTTP)
		s.router.HandleFunc("/jwt-login", authHandle.Login(loginFailureHandler).ServeHTTP)

		s.router.Post("/logout", authHandle.Logout(
			http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
				rw.Header().Add("Content-Type", "text/html; charset=utf-8")
				rw.WriteHeader(http.StatusOK)
				web.RenderTemplate(rw, "login.tmpl", &web.Page{
					Title:   "Bye - ClusterCockpit",
					MsgType: "alert-info",
					Message: "Logout successful",
					Build:   buildInfo,
					Infos:   info,
				})
			})).ServeHTTP)
	}

	if flagDev {
		s.router.Handle("/playground", playground.Handler("GraphQL playground", "/query"))
		s.router.Get("/swagger/*", httpSwagger.Handler(
			httpSwagger.URL("http://"+config.Keys.Addr+"/swagger/doc.json")))
	}

	// Secured routes (require authentication)
	s.router.Group(func(secured chi.Router) {
		if !config.Keys.DisableAuthentication {
			secured.Use(func(next http.Handler) http.Handler {
				return authHandle.Auth(
					next,
					func(rw http.ResponseWriter, r *http.Request, err error) {
						rw.WriteHeader(http.StatusUnauthorized)
						web.RenderTemplate(rw, "login.tmpl", &web.Page{
							Title:    "Authentication failed - ClusterCockpit",
							MsgType:  "alert-danger",
							Message:  err.Error(),
							Build:    buildInfo,
							Infos:    info,
							Redirect: r.RequestURI,
						})
					})
			})
		}

		secured.Handle("/query", graphQLServer)

		secured.HandleFunc("/search", func(rw http.ResponseWriter, r *http.Request) {
			routerConfig.HandleSearchBar(rw, r, buildInfo)
		})

		routerConfig.SetupRoutes(secured, buildInfo)
	})

	// API routes (JWT token auth)
	s.router.Route("/api", func(apiRouter chi.Router) {
		// Main API routes with API auth
		apiRouter.Group(func(securedapi chi.Router) {
			if !config.Keys.DisableAuthentication {
				securedapi.Use(func(next http.Handler) http.Handler {
					return authHandle.AuthAPI(next, onFailureResponse)
				})
			}
			s.restAPIHandle.MountAPIRoutes(securedapi)
		})

		// Metric store API routes with separate auth
		apiRouter.Group(func(metricstoreapi chi.Router) {
			if !config.Keys.DisableAuthentication {
				metricstoreapi.Use(func(next http.Handler) http.Handler {
					return authHandle.AuthMetricStoreAPI(next, onFailureResponse)
				})
			}
			s.restAPIHandle.MountMetricStoreAPIRoutes(metricstoreapi)
		})
	})

	// User API routes
	s.router.Route("/userapi", func(userapi chi.Router) {
		if !config.Keys.DisableAuthentication {
			userapi.Use(func(next http.Handler) http.Handler {
				return authHandle.AuthUserAPI(next, onFailureResponse)
			})
		}
		s.restAPIHandle.MountUserAPIRoutes(userapi)
	})

	// Config API routes (uses Group with full paths to avoid shadowing
	// the /config page route that is registered in the secured group)
	s.router.Group(func(configapi chi.Router) {
		if !config.Keys.DisableAuthentication {
			configapi.Use(func(next http.Handler) http.Handler {
				return authHandle.AuthConfigAPI(next, onFailureResponse)
			})
		}
		s.restAPIHandle.MountConfigAPIRoutes(configapi)
	})

	// Frontend API routes
	s.router.Route("/frontend", func(frontendapi chi.Router) {
		if !config.Keys.DisableAuthentication {
			frontendapi.Use(func(next http.Handler) http.Handler {
				return authHandle.AuthFrontendAPI(next, onFailureResponse)
			})
		}
		s.restAPIHandle.MountFrontendAPIRoutes(frontendapi)
	})

	if config.Keys.APISubjects != nil {
		s.natsAPIHandle = api.NewNatsAPI()
		if err := s.natsAPIHandle.StartSubscriptions(); err != nil {
			return fmt.Errorf("starting NATS subscriptions: %w", err)
		}
	}

	// 404 handler for pages and API routes
	notFoundHandler := func(rw http.ResponseWriter, r *http.Request) {
		if strings.HasPrefix(r.URL.Path, "/api/") || strings.HasPrefix(r.URL.Path, "/userapi/") ||
			strings.HasPrefix(r.URL.Path, "/frontend/") || strings.HasPrefix(r.URL.Path, "/config/") {
			rw.Header().Set("Content-Type", "application/json")
			rw.WriteHeader(http.StatusNotFound)
			json.NewEncoder(rw).Encode(map[string]string{
				"status": "Resource not found",
				"error":  "the requested endpoint does not exist",
			})
			return
		}
		rw.Header().Set("Content-Type", "text/html; charset=utf-8")
		rw.WriteHeader(http.StatusNotFound)
		web.RenderTemplate(rw, "404.tmpl", &web.Page{
			Title: "Page Not Found",
			Build: buildInfo,
		})
	}

	// Set NotFound on the router so chi uses it for all unmatched routes,
	// including those under subrouters like /api, /userapi, /frontend, etc.
	s.router.NotFound(notFoundHandler)

	if config.Keys.EmbedStaticFiles {
		if i, err := os.Stat("./var/img"); err == nil {
			if i.IsDir() {
				cclog.Info("Use local directory for static images")
				s.router.Handle("/img/*", http.StripPrefix("/img/", http.FileServer(http.Dir("./var/img"))))
			}
		}
		fileServer := http.StripPrefix("/", web.ServeFiles())
		s.router.Handle("/*", http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
			if web.StaticFileExists(r.URL.Path) {
				fileServer.ServeHTTP(rw, r)
				return
			}
			notFoundHandler(rw, r)
		}))
	} else {
		staticDir := http.Dir(config.Keys.StaticFiles)
		fileServer := http.FileServer(staticDir)
		s.router.Handle("/*", http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
			f, err := staticDir.Open(r.URL.Path)
			if err == nil {
				f.Close()
				fileServer.ServeHTTP(rw, r)
				return
			}
			notFoundHandler(rw, r)
		}))
	}

	return nil
}

// Server timeout defaults (in seconds)
const (
	defaultReadHeaderTimeout = 20
	defaultWriteTimeout      = 20
)

func (s *Server) Start(ctx context.Context) error {
	// Use configurable timeouts with defaults
	readHeaderTimeout := time.Duration(defaultReadHeaderTimeout) * time.Second
	writeTimeout := time.Duration(defaultWriteTimeout) * time.Second

	s.server = &http.Server{
		ReadHeaderTimeout: readHeaderTimeout,
		WriteTimeout:      writeTimeout,
		Handler:      s.router,
		Addr:         config.Keys.Addr,
	}

	// Start http or https server
	listener, err := net.Listen("tcp", config.Keys.Addr)
	if err != nil {
		return fmt.Errorf("starting listener on '%s': %w", config.Keys.Addr, err)
	}

	if !strings.HasSuffix(config.Keys.Addr, ":80") && config.Keys.RedirectHTTPTo != "" {
		go func() {
			http.ListenAndServe(":80", http.RedirectHandler(config.Keys.RedirectHTTPTo, http.StatusMovedPermanently))
		}()
	}

	if config.Keys.HTTPSCertFile != "" && config.Keys.HTTPSKeyFile != "" {
		cert, err := tls.LoadX509KeyPair(
			config.Keys.HTTPSCertFile, config.Keys.HTTPSKeyFile)
		if err != nil {
			return fmt.Errorf("loading X509 keypair (check 'https-cert-file' and 'https-key-file' in config.json): %w", err)
		}
		listener = tls.NewListener(listener, &tls.Config{
			Certificates: []tls.Certificate{cert},
			CipherSuites: []uint16{
				tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
				tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
			},
			MinVersion:               tls.VersionTLS12,
			PreferServerCipherSuites: true,
		})
		cclog.Infof("HTTPS server listening at %s...", config.Keys.Addr)
	} else {
		cclog.Infof("HTTP server listening at %s...", config.Keys.Addr)
	}
	//
	// Because this program will want to bind to a privileged port (like 80), the listener must
	// be established first, then the user can be changed, and after that,
	// the actual http server can be started.
	if err := runtime.DropPrivileges(config.Keys.Group, config.Keys.User); err != nil {
		return fmt.Errorf("dropping privileges: %w", err)
	}

	if err = s.server.Serve(listener); err != nil && err != http.ErrServerClosed {
		return fmt.Errorf("server failed: %w", err)
	}
	return nil
}

func (s *Server) Shutdown(ctx context.Context) {
	// Create a shutdown context with timeout
	shutdownCtx, cancel := context.WithTimeout(ctx, 30*time.Second)
	defer cancel()

	nc := nats.GetClient()
	if nc != nil {
		nc.Close()
	}

	// First shut down the server gracefully (waiting for all ongoing requests)
	if err := s.server.Shutdown(shutdownCtx); err != nil {
		cclog.Errorf("Server shutdown error: %v", err)
	}

	// Archive all the metric store data
	ms := metricstore.GetMemoryStore()

	if ms != nil {
		metricstore.Shutdown()
	}

	// Shutdown archiver with 10 second timeout for fast shutdown
	if err := archiver.Shutdown(10 * time.Second); err != nil {
		cclog.Warnf("Archiver shutdown: %v", err)
	}
}

-  // Create a shutdown context with timeout

-  shutdownCtx, cancel := context.WithTimeout(ctx, 30*time.Second)

+  shutdownCtx, cancel := context.WithTimeout(ctx, 5*time.Second)

-  // First shut down the server gracefully (waiting for all ongoing requests)

+  // Run metricstore and archiver shutdown concurrently.

+  // They are independent: metricstore writes .bin snapshots,

+  // archiver flushes pending job archives.

+    if ms := metricstore.GetMemoryStore(); ms != nil {

-  // Shutdown archiver with 10 second timeout for fast shutdown

-  if err := archiver.Shutdown(10 * time.Second); err != nil {

-    cclog.Warnf("Archiver shutdown: %v", err)

+      if err := archiver.Shutdown(10 * time.Second); err != nil {

+        cclog.Warnf("Archiver shutdown: %v", err)

+    cclog.Warn("Shutdown deadline exceeded, forcing exit")


Need to add `"sync"` to imports: