/Users/jan/prg/ClusterCockpit/cc-lib/nats/client.go:150:func (c *Client) Subscribe(subject string, handler MessageHandler) error {
/Users/jan/prg/ClusterCockpit/cc-lib/nats/client.go:166:// SubscribeQueue registers a handler with queue group for load-balanced message processing.
/Users/jan/prg/ClusterCockpit/cc-lib/nats/client.go:167:func (c *Client) SubscribeQueue(subject, queue string, handler MessageHandler) error {
/Users/jan/prg/ClusterCockpit/cc-lib/nats/client.go:183:// SubscribeChan subscribes to a subject and delivers messages to the provided channel.
/Users/jan/prg/ClusterCockpit/cc-lib/nats/client.go:184:func (c *Client) SubscribeChan(subject string, ch chan *nats.Msg) error {


   140→
   141→	cclog.Infof("NATS connected to %s", cfg.Address)
   142→
   143→	return &Client{
   144→		conn:          nc,
   145→		subscriptions: make([]*nats.Subscription, 0),
   146→	}, nil
   147→}
   148→
   149→// Subscribe registers a handler for messages on the given subject.
   150→func (c *Client) Subscribe(subject string, handler MessageHandler) error {
   151→	c.mu.Lock()
   152→	defer c.mu.Unlock()
   153→
   154→	sub, err := c.conn.Subscribe(subject, func(msg *nats.Msg) {
   155→		handler(msg.Subject, msg.Data)
   156→	})
   157→	if err != nil {
   158→		return fmt.Errorf("NATS subscribe to '%s' failed: %w", subject, err)
   159→	}
   160→
   161→	c.subscriptions = append(c.subscriptions, sub)
   162→	cclog.Infof("NATS subscribed to '%s'", subject)
   163→	return nil
   164→}
   165→
   166→// SubscribeQueue registers a handler with queue group for load-balanced message processing.
   167→func (c *Client) SubscribeQueue(subject, queue string, handler MessageHandler) error {
   168→	c.mu.Lock()
   169→	defer c.mu.Unlock()
   170→
   171→	sub, err := c.conn.QueueSubscribe(subject, queue, func(msg *nats.Msg) {
   172→		handler(msg.Subject, msg.Data)
   173→	})
   174→	if err != nil {
   175→		return fmt.Errorf("NATS queue subscribe to '%s' (queue: %s) failed: %w", subject, queue, err)
   176→	}
   177→
   178→	c.subscriptions = append(c.subscriptions, sub)
   179→	cclog.Infof("NATS queue subscribed to '%s' (queue: %s)", subject, queue)
   180→	return nil
   181→}
   182→
   183→// SubscribeChan subscribes to a subject and delivers messages to the provided channel.
   184→func (c *Client) SubscribeChan(subject string, ch chan *nats.Msg) error {
   185→	c.mu.Lock()
   186→	defer c.mu.Unlock()
   187→
   188→	sub, err := c.conn.ChanSubscribe(subject, ch)
   189→	if err != nil {
   190→		return fmt.Errorf("NATS chan subscribe to '%s' failed: %w", subject, err)
   191→	}
   192→
   193→	c.subscriptions = append(c.subscriptions, sub)
   194→	cclog.Infof("NATS chan subscribed to '%s'", subject)
   195→	return nil
   196→}
   197→
   198→// Publish sends data to the specified subject.
   199→func (c *Client) Publish(subject string, data []byte) error {
   200→	if err := c.conn.Publish(subject, data); err != nil {
   201→		return fmt.Errorf("NATS publish to '%s' failed: %w", subject, err)
   202→	}
   203→	return nil
   204→}
   205→
   206→// Request sends a request and waits for a response with the given context timeout.
   207→func (c *Client) Request(subject string, data []byte, timeout context.Context) ([]byte, error) {
   208→	msg, err := c.conn.RequestWithContext(timeout, subject, data)
   209→	if err != nil {
   210→		return nil, fmt.Errorf("NATS request to '%s' failed: %w", subject, err)
   211→	}
   212→	return msg.Data, nil
   213→}
   214→
   215→// Flush flushes the connection buffer to ensure all published messages are sent.
   216→func (c *Client) Flush() error {
   217→	return c.conn.Flush()
   218→}
   219→

	cclog.Infof("NATS connected to %s", cfg.Address)

	return &Client{
		conn:          nc,
		subscriptions: make([]*nats.Subscription, 0),
	}, nil
}

// Subscribe registers a handler for messages on the given subject.
func (c *Client) Subscribe(subject string, handler MessageHandler) error {
	c.mu.Lock()
	defer c.mu.Unlock()

	sub, err := c.conn.Subscribe(subject, func(msg *nats.Msg) {
		handler(msg.Subject, msg.Data)
	})
	if err != nil {
		return fmt.Errorf("NATS subscribe to '%s' failed: %w", subject, err)
	}

	c.subscriptions = append(c.subscriptions, sub)
	cclog.Infof("NATS subscribed to '%s'", subject)
	return nil
}

// SubscribeQueue registers a handler with queue group for load-balanced message processing.
func (c *Client) SubscribeQueue(subject, queue string, handler MessageHandler) error {
	c.mu.Lock()
	defer c.mu.Unlock()

	sub, err := c.conn.QueueSubscribe(subject, queue, func(msg *nats.Msg) {
		handler(msg.Subject, msg.Data)
	})
	if err != nil {
		return fmt.Errorf("NATS queue subscribe to '%s' (queue: %s) failed: %w", subject, queue, err)
	}

	c.subscriptions = append(c.subscriptions, sub)
	cclog.Infof("NATS queue subscribed to '%s' (queue: %s)", subject, queue)
	return nil
}

// SubscribeChan subscribes to a subject and delivers messages to the provided channel.
func (c *Client) SubscribeChan(subject string, ch chan *nats.Msg) error {
	c.mu.Lock()
	defer c.mu.Unlock()

	sub, err := c.conn.ChanSubscribe(subject, ch)
	if err != nil {
		return fmt.Errorf("NATS chan subscribe to '%s' failed: %w", subject, err)
	}

	c.subscriptions = append(c.subscriptions, sub)
	cclog.Infof("NATS chan subscribed to '%s'", subject)
	return nil
}

// Publish sends data to the specified subject.
func (c *Client) Publish(subject string, data []byte) error {
	if err := c.conn.Publish(subject, data); err != nil {
		return fmt.Errorf("NATS publish to '%s' failed: %w", subject, err)
	}
	return nil
}

// Request sends a request and waits for a response with the given context timeout.
func (c *Client) Request(subject string, data []byte, timeout context.Context) ([]byte, error) {
	msg, err := c.conn.RequestWithContext(timeout, subject, data)
	if err != nil {
		return nil, fmt.Errorf("NATS request to '%s' failed: %w", subject, err)
	}
	return msg.Data, nil
}

// Flush flushes the connection buffer to ensure all published messages are sent.
func (c *Client) Flush() error {
	return c.conn.Flush()
}


Now I have a clear picture. The `SubscribeChan` doesn't support queue groups in cc-lib, so I'll use callback-based subscribe that pushes to buffered channels, with worker goroutines consuming from them. This cleanly replaces the semaphore pattern.

Let me implement all three changes.

**1. Add fields to `NATSConfig`:**


Edit

type NATSConfig struct {
	SubjectJobEvent  string `json:"subject-job-event"`
	SubjectNodeState string `json:"subject-node-state"`
}

type NATSConfig struct {
	SubjectJobEvent  string `json:"subject-job-event"`
	SubjectNodeState string `json:"subject-node-state"`
	QueueGroup       string `json:"queue-group"`
	JobConcurrency   int    `json:"job-concurrency"`
	NodeConcurrency  int    `json:"node-concurrency"`
}


The file /Users/jan/prg/ClusterCockpit/cc-backend/internal/config/config.go has been updated successfully.

type NATSConfig struct {
	SubjectJobEvent  string `json:"subject-job-event"`
	SubjectNodeState string `json:"subject-node-state"`
	QueueGroup       string `json:"queue-group"`
	JobConcurrency   int    `json:"job-concurrency"`
	NodeConcurrency  int    `json:"node-concurrency"`
}

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

// Package config implements the program configuration data structures, validation and parsing
package config

import (
	"bytes"
	"encoding/json"
	"time"

	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
	"github.com/ClusterCockpit/cc-lib/v2/resampler"
)

type ProgramConfig struct {
	// Address where the http (or https) server will listen on (for example: 'localhost:80').
	Addr string `json:"addr"`

	// Addresses from which secured admin API endpoints can be reached, can be wildcard "*"
	APIAllowedIPs []string `json:"api-allowed-ips"`

	APISubjects *NATSConfig `json:"api-subjects"`

	// Drop root permissions once .env was read and the port was taken.
	User  string `json:"user"`
	Group string `json:"group"`

	// Disable authentication (for everything: API, Web-UI, ...)
	DisableAuthentication bool `json:"disable-authentication"`

	// If `embed-static-files` is true (default), the frontend files are directly
	// embeded into the go binary and expected to be in web/frontend. Only if
	// it is false the files in `static-files` are served instead.
	EmbedStaticFiles bool   `json:"embed-static-files"`
	StaticFiles      string `json:"static-files"`

	// Path to SQLite database file
	DB string `json:"db"`

	EnableJobTaggers bool `json:"enable-job-taggers"`

	// Validate json input against schema
	Validate bool `json:"validate"`

	// If 0 or empty, the session does not expire!
	SessionMaxAge string `json:"session-max-age"`

	// If both those options are not empty, use HTTPS using those certificates.
	HTTPSCertFile string `json:"https-cert-file"`
	HTTPSKeyFile  string `json:"https-key-file"`

	// If not the empty string and `addr` does not end in ":80",
	// redirect every request incoming at port 80 to that url.
	RedirectHTTPTo string `json:"redirect-http-to"`

	// Where to store MachineState files
	MachineStateDir string `json:"machine-state-dir"`

	// If not zero, automatically mark jobs as stopped running X seconds longer than their walltime.
	StopJobsExceedingWalltime int `json:"stop-jobs-exceeding-walltime"`

	// Defines time X in seconds in which jobs are considered to be "short" and will be filtered in specific views.
	ShortRunningJobsDuration int `json:"short-running-jobs-duration"`

	// Energy Mix CO2 Emission Constant [g/kWh]
	// If entered, displays estimated CO2 emission for job based on jobs totalEnergy
	EmissionConstant int `json:"emission-constant"`

	// If exists, will enable dynamic zoom in frontend metric plots using the configured values
	EnableResampling *ResampleConfig `json:"resampling"`

	// Systemd unit name for log viewer (default: "clustercockpit")
	SystemdUnit string `json:"systemd-unit"`

	// Node state retention configuration
	NodeStateRetention *NodeStateRetention `json:"nodestate-retention"`

	// Database tuning configuration
	DbConfig *DbConfig `json:"db-config"`
}

type DbConfig struct {
	CacheSizeMB               int `json:"cache-size-mb"`
	SoftHeapLimitMB           int `json:"soft-heap-limit-mb"`
	MaxOpenConnections        int `json:"max-open-connections"`
	MaxIdleConnections        int `json:"max-idle-connections"`
	ConnectionMaxIdleTimeMins int `json:"max-idle-time-minutes"`
	BusyTimeoutMs             int `json:"busy-timeout-ms"`
}

type NodeStateRetention struct {
	Policy             string `json:"policy"`      // "delete" or "move"
	Age                int    `json:"age"`         // hours, default 24
	TargetKind         string `json:"target-kind"` // "file" or "s3"
	TargetPath         string `json:"target-path"`
	TargetEndpoint     string `json:"target-endpoint"`
	TargetBucket       string `json:"target-bucket"`
	TargetAccessKey    string `json:"target-access-key"`
	TargetSecretKey    string `json:"target-secret-key"`
	TargetRegion       string `json:"target-region"`
	TargetUsePathStyle bool   `json:"target-use-path-style"`
	MaxFileSizeMB      int    `json:"max-file-size-mb"`
}

type ResampleConfig struct {
	// Minimum number of points to trigger resampling of data
	MinimumPoints int `json:"minimum-points"`
	// Array of resampling target resolutions, in seconds; Example: [600,300,60]
	Resolutions []int `json:"resolutions"`
	// Trigger next zoom level at less than this many visible datapoints
	Trigger int `json:"trigger"`
}

type NATSConfig struct {
	SubjectJobEvent  string `json:"subject-job-event"`
	SubjectNodeState string `json:"subject-node-state"`
}

type IntRange struct {
	From int `json:"from"`
	To   int `json:"to"`
}

type TimeRange struct {
	From  *time.Time `json:"from"`
	To    *time.Time `json:"to"`
	Range string     `json:"range,omitempty"`
}

type FilterRanges struct {
	Duration  *IntRange  `json:"duration"`
	NumNodes  *IntRange  `json:"num-nodes"`
	StartTime *TimeRange `json:"start-time"`
}

var Keys ProgramConfig = ProgramConfig{
	Addr:                      "localhost:8080",
	EmbedStaticFiles:          true,
	DB:                        "./var/job.db",
	SessionMaxAge:             "168h",
	StopJobsExceedingWalltime: 0,
	ShortRunningJobsDuration:  5 * 60,
}

func Init(mainConfig json.RawMessage) {
	Validate(configSchema, mainConfig)
	dec := json.NewDecoder(bytes.NewReader(mainConfig))
	dec.DisallowUnknownFields()
	if err := dec.Decode(&Keys); err != nil {
		cclog.Abortf("Config Init: Could not decode config file '%s'.\nError: %s\n", mainConfig, err.Error())
	}

	if Keys.EnableResampling != nil && Keys.EnableResampling.MinimumPoints > 0 {
		resampler.SetMinimumRequiredPoints(Keys.EnableResampling.MinimumPoints)
	}
}

+  QueueGroup       string `json:"queue-group"`

+  JobConcurrency   int    `json:"job-concurrency"`

+  NodeConcurrency  int    `json:"node-concurrency"`


**2. Rework `nats.go` — replace semaphores with channel-based worker pool, add queue group support:**


// NatsAPI provides NATS subscription-based handlers for Job and Node operations.
// It mirrors the functionality of the REST API but uses NATS messaging with
// InfluxDB line protocol as the message format.
//
// # Message Format
//
// All NATS messages use InfluxDB line protocol format (https://docs.influxdata.com/influxdb/v2.0/reference/syntax/line-protocol/)
// with the following structure:
//
//	measurement,tag1=value1,tag2=value2 field1=value1,field2=value2 timestamp
//
// # Job Events
//
// Job start/stop events use the "job" measurement with a "function" tag to distinguish operations:
//
//	job,function=start_job event="{...JSON payload...}" <timestamp>
//	job,function=stop_job event="{...JSON payload...}" <timestamp>
//
// The JSON payload in the "event" field follows the schema.Job or StopJobAPIRequest structure.
//
// Example job start message:
//
//	job,function=start_job event="{\"jobId\":1001,\"user\":\"testuser\",\"cluster\":\"testcluster\",...}" 1234567890000000000
//
// # Node State Events
//
// Node state updates use the "nodestate" measurement with cluster information:
//
//	nodestate event="{...JSON payload...}" <timestamp>
//
// The JSON payload follows the UpdateNodeStatesRequest structure.
//
// Example node state message:
//
//	nodestate event="{\"cluster\":\"testcluster\",\"nodes\":[{\"hostname\":\"node01\",\"states\":[\"idle\"]}]}" 1234567890000000000
type NatsAPI struct {
	JobRepository *repository.JobRepository
	// RepositoryMutex protects job creation operations from race conditions
	// when checking for duplicate jobs during startJob calls.
	RepositoryMutex sync.Mutex
	// jobSem limits concurrent NATS job event processing to prevent goroutine accumulation
	// when the database is under contention.
	jobSem chan struct{}
	// nodeSem limits concurrent NATS node state processing.
	nodeSem chan struct{}
}

// NewNatsAPI creates a new NatsAPI instance with default dependencies.
func NewNatsAPI() *NatsAPI {
	return &NatsAPI{
		JobRepository: repository.GetJobRepository(),
		jobSem:        make(chan struct{}, 8),
		nodeSem:       make(chan struct{}, 2),
	}
}

// StartSubscriptions registers all NATS subscriptions for Job and Node APIs.
// Returns an error if the NATS client is not available or subscription fails.
func (api *NatsAPI) StartSubscriptions() error {
	client := nats.GetClient()
	if client == nil {
		cclog.Warn("NATS client not available, skipping API subscriptions")
		return nil
	}

	if config.Keys.APISubjects != nil {

		s := config.Keys.APISubjects

		if err := client.Subscribe(s.SubjectJobEvent, api.handleJobEvent); err != nil {
			return err
		}

		if err := client.Subscribe(s.SubjectNodeState, api.handleNodeState); err != nil {
			return err
		}

		cclog.Info("NATS API subscriptions started")
	}
	return nil
}

// natsMessage wraps a raw NATS message with its subject for channel-based processing.
type natsMessage struct {
	subject string
	data    []byte
}

// NatsAPI provides NATS subscription-based handlers for Job and Node operations.
// It mirrors the functionality of the REST API but uses NATS messaging with
// InfluxDB line protocol as the message format.
//
// # Message Format
//
// All NATS messages use InfluxDB line protocol format (https://docs.influxdata.com/influxdb/v2.0/reference/syntax/line-protocol/)
// with the following structure:
//
//	measurement,tag1=value1,tag2=value2 field1=value1,field2=value2 timestamp
//
// # Job Events
//
// Job start/stop events use the "job" measurement with a "function" tag to distinguish operations:
//
//	job,function=start_job event="{...JSON payload...}" <timestamp>
//	job,function=stop_job event="{...JSON payload...}" <timestamp>
//
// The JSON payload in the "event" field follows the schema.Job or StopJobAPIRequest structure.
//
// Example job start message:
//
//	job,function=start_job event="{\"jobId\":1001,\"user\":\"testuser\",\"cluster\":\"testcluster\",...}" 1234567890000000000
//
// # Node State Events
//
// Node state updates use the "nodestate" measurement with cluster information:
//
//	nodestate event="{...JSON payload...}" <timestamp>
//
// The JSON payload follows the UpdateNodeStatesRequest structure.
//
// Example node state message:
//
//	nodestate event="{\"cluster\":\"testcluster\",\"nodes\":[{\"hostname\":\"node01\",\"states\":[\"idle\"]}]}" 1234567890000000000
type NatsAPI struct {
	JobRepository *repository.JobRepository
	// RepositoryMutex protects job creation operations from race conditions
	// when checking for duplicate jobs during startJob calls.
	RepositoryMutex sync.Mutex
	// jobCh receives job event messages for processing by worker goroutines.
	jobCh chan natsMessage
	// nodeCh receives node state messages for processing by worker goroutines.
	nodeCh chan natsMessage
}

// NewNatsAPI creates a new NatsAPI instance with channel-based worker pools.
// Concurrency is configured via NATSConfig (defaults: JobConcurrency=8, NodeConcurrency=2).
func NewNatsAPI() *NatsAPI {
	jobConc := 8
	nodeConc := 2

	if s := config.Keys.APISubjects; s != nil {
		if s.JobConcurrency > 0 {
			jobConc = s.JobConcurrency
		}
		if s.NodeConcurrency > 0 {
			nodeConc = s.NodeConcurrency
		}
	}

	api := &NatsAPI{
		JobRepository: repository.GetJobRepository(),
		jobCh:         make(chan natsMessage, jobConc),
		nodeCh:        make(chan natsMessage, nodeConc),
	}

	// Start worker goroutines
	for range jobConc {
		go api.jobWorker()
	}
	for range nodeConc {
		go api.nodeWorker()
	}

	return api
}

// jobWorker processes job event messages from the job channel.
func (api *NatsAPI) jobWorker() {
	for msg := range api.jobCh {
		api.handleJobEvent(msg.subject, msg.data)
	}
}

// nodeWorker processes node state messages from the node channel.
func (api *NatsAPI) nodeWorker() {
	for msg := range api.nodeCh {
		api.handleNodeState(msg.subject, msg.data)
	}
}

// subscribe registers a NATS subscription using queue groups when configured.
func subscribe(client *nats.Client, subject string, queueGroup string, handler func(string, []byte)) error {
	if queueGroup != "" {
		return client.SubscribeQueue(subject, queueGroup, handler)
	}
	return client.Subscribe(subject, handler)
}

// StartSubscriptions registers all NATS subscriptions for Job and Node APIs.
// Messages are delivered to buffered channels and processed by worker goroutines.
// Returns an error if the NATS client is not available or subscription fails.
func (api *NatsAPI) StartSubscriptions() error {
	client := nats.GetClient()
	if client == nil {
		cclog.Warn("NATS client not available, skipping API subscriptions")
		return nil
	}

	if config.Keys.APISubjects != nil {
		s := config.Keys.APISubjects
		qg := s.QueueGroup

		if err := subscribe(client, s.SubjectJobEvent, qg, func(subject string, data []byte) {
			api.jobCh <- natsMessage{subject: subject, data: data}
		}); err != nil {
			return err
		}

		if err := subscribe(client, s.SubjectNodeState, qg, func(subject string, data []byte) {
			api.nodeCh <- natsMessage{subject: subject, data: data}
		}); err != nil {
			return err
		}

		cclog.Info("NATS API subscriptions started")
	}
	return nil
}


The file /Users/jan/prg/ClusterCockpit/cc-backend/internal/api/nats.go has been updated successfully.

// natsMessage wraps a raw NATS message with its subject for channel-based processing.
type natsMessage struct {
	subject string
	data    []byte
}

// NatsAPI provides NATS subscription-based handlers for Job and Node operations.
// It mirrors the functionality of the REST API but uses NATS messaging with
// InfluxDB line protocol as the message format.
//
// # Message Format
//
// All NATS messages use InfluxDB line protocol format (https://docs.influxdata.com/influxdb/v2.0/reference/syntax/line-protocol/)
// with the following structure:
//
//	measurement,tag1=value1,tag2=value2 field1=value1,field2=value2 timestamp
//
// # Job Events
//
// Job start/stop events use the "job" measurement with a "function" tag to distinguish operations:
//
//	job,function=start_job event="{...JSON payload...}" <timestamp>
//	job,function=stop_job event="{...JSON payload...}" <timestamp>
//
// The JSON payload in the "event" field follows the schema.Job or StopJobAPIRequest structure.
//
// Example job start message:
//
//	job,function=start_job event="{\"jobId\":1001,\"user\":\"testuser\",\"cluster\":\"testcluster\",...}" 1234567890000000000
//
// # Node State Events
//
// Node state updates use the "nodestate" measurement with cluster information:
//
//	nodestate event="{...JSON payload...}" <timestamp>
//
// The JSON payload follows the UpdateNodeStatesRequest structure.
//
// Example node state message:
//
//	nodestate event="{\"cluster\":\"testcluster\",\"nodes\":[{\"hostname\":\"node01\",\"states\":[\"idle\"]}]}" 1234567890000000000
type NatsAPI struct {
	JobRepository *repository.JobRepository
	// RepositoryMutex protects job creation operations from race conditions
	// when checking for duplicate jobs during startJob calls.
	RepositoryMutex sync.Mutex
	// jobCh receives job event messages for processing by worker goroutines.
	jobCh chan natsMessage
	// nodeCh receives node state messages for processing by worker goroutines.
	nodeCh chan natsMessage
}

// NewNatsAPI creates a new NatsAPI instance with channel-based worker pools.
// Concurrency is configured via NATSConfig (defaults: JobConcurrency=8, NodeConcurrency=2).
func NewNatsAPI() *NatsAPI {
	jobConc := 8
	nodeConc := 2

	if s := config.Keys.APISubjects; s != nil {
		if s.JobConcurrency > 0 {
			jobConc = s.JobConcurrency
		}
		if s.NodeConcurrency > 0 {
			nodeConc = s.NodeConcurrency
		}
	}

	api := &NatsAPI{
		JobRepository: repository.GetJobRepository(),
		jobCh:         make(chan natsMessage, jobConc),
		nodeCh:        make(chan natsMessage, nodeConc),
	}

	// Start worker goroutines
	for range jobConc {
		go api.jobWorker()
	}
	for range nodeConc {
		go api.nodeWorker()
	}

	return api
}

// jobWorker processes job event messages from the job channel.
func (api *NatsAPI) jobWorker() {
	for msg := range api.jobCh {
		api.handleJobEvent(msg.subject, msg.data)
	}
}

// nodeWorker processes node state messages from the node channel.
func (api *NatsAPI) nodeWorker() {
	for msg := range api.nodeCh {
		api.handleNodeState(msg.subject, msg.data)
	}
}

// subscribe registers a NATS subscription using queue groups when configured.
func subscribe(client *nats.Client, subject string, queueGroup string, handler func(string, []byte)) error {
	if queueGroup != "" {
		return client.SubscribeQueue(subject, queueGroup, handler)
	}
	return client.Subscribe(subject, handler)
}

// StartSubscriptions registers all NATS subscriptions for Job and Node APIs.
// Messages are delivered to buffered channels and processed by worker goroutines.
// Returns an error if the NATS client is not available or subscription fails.
func (api *NatsAPI) StartSubscriptions() error {
	client := nats.GetClient()
	if client == nil {
		cclog.Warn("NATS client not available, skipping API subscriptions")
		return nil
	}

	if config.Keys.APISubjects != nil {
		s := config.Keys.APISubjects
		qg := s.QueueGroup

		if err := subscribe(client, s.SubjectJobEvent, qg, func(subject string, data []byte) {
			api.jobCh <- natsMessage{subject: subject, data: data}
		}); err != nil {
			return err
		}

		if err := subscribe(client, s.SubjectNodeState, qg, func(subject string, data []byte) {
			api.nodeCh <- natsMessage{subject: subject, data: data}
		}); err != nil {
			return err
		}

		cclog.Info("NATS API subscriptions started")
	}
	return nil
}

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package api

import (
	"database/sql"
	"encoding/json"
	"strings"
	"sync"
	"time"

	"github.com/ClusterCockpit/cc-backend/internal/archiver"
	"github.com/ClusterCockpit/cc-backend/internal/config"
	"github.com/ClusterCockpit/cc-backend/internal/importer"
	"github.com/ClusterCockpit/cc-backend/internal/repository"
	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
	lp "github.com/ClusterCockpit/cc-lib/v2/ccMessage"
	"github.com/ClusterCockpit/cc-lib/v2/nats"
	"github.com/ClusterCockpit/cc-lib/v2/receivers"
	"github.com/ClusterCockpit/cc-lib/v2/schema"
	influx "github.com/ClusterCockpit/cc-line-protocol/v2/lineprotocol"
)

// NatsAPI provides NATS subscription-based handlers for Job and Node operations.
// It mirrors the functionality of the REST API but uses NATS messaging with
// InfluxDB line protocol as the message format.
//
// # Message Format
//
// All NATS messages use InfluxDB line protocol format (https://docs.influxdata.com/influxdb/v2.0/reference/syntax/line-protocol/)
// with the following structure:
//
//	measurement,tag1=value1,tag2=value2 field1=value1,field2=value2 timestamp
//
// # Job Events
//
// Job start/stop events use the "job" measurement with a "function" tag to distinguish operations:
//
//	job,function=start_job event="{...JSON payload...}" <timestamp>
//	job,function=stop_job event="{...JSON payload...}" <timestamp>
//
// The JSON payload in the "event" field follows the schema.Job or StopJobAPIRequest structure.
//
// Example job start message:
//
//	job,function=start_job event="{\"jobId\":1001,\"user\":\"testuser\",\"cluster\":\"testcluster\",...}" 1234567890000000000
//
// # Node State Events
//
// Node state updates use the "nodestate" measurement with cluster information:
//
//	nodestate event="{...JSON payload...}" <timestamp>
//
// The JSON payload follows the UpdateNodeStatesRequest structure.
//
// Example node state message:
//
//	nodestate event="{\"cluster\":\"testcluster\",\"nodes\":[{\"hostname\":\"node01\",\"states\":[\"idle\"]}]}" 1234567890000000000
type NatsAPI struct {
	JobRepository *repository.JobRepository
	// RepositoryMutex protects job creation operations from race conditions
	// when checking for duplicate jobs during startJob calls.
	RepositoryMutex sync.Mutex
	// jobSem limits concurrent NATS job event processing to prevent goroutine accumulation
	// when the database is under contention.
	jobSem chan struct{}
	// nodeSem limits concurrent NATS node state processing.
	nodeSem chan struct{}
}

// NewNatsAPI creates a new NatsAPI instance with default dependencies.
func NewNatsAPI() *NatsAPI {
	return &NatsAPI{
		JobRepository: repository.GetJobRepository(),
		jobSem:        make(chan struct{}, 8),
		nodeSem:       make(chan struct{}, 2),
	}
}

// StartSubscriptions registers all NATS subscriptions for Job and Node APIs.
// Returns an error if the NATS client is not available or subscription fails.
func (api *NatsAPI) StartSubscriptions() error {
	client := nats.GetClient()
	if client == nil {
		cclog.Warn("NATS client not available, skipping API subscriptions")
		return nil
	}

	if config.Keys.APISubjects != nil {

		s := config.Keys.APISubjects

		if err := client.Subscribe(s.SubjectJobEvent, api.handleJobEvent); err != nil {
			return err
		}

		if err := client.Subscribe(s.SubjectNodeState, api.handleNodeState); err != nil {
			return err
		}

		cclog.Info("NATS API subscriptions started")
	}
	return nil
}

// processJobEvent routes job event messages to the appropriate handler based on the "function" tag.
// Validates that required tags and fields are present before processing.
func (api *NatsAPI) processJobEvent(msg lp.CCMessage) {
	function, ok := msg.GetTag("function")
	if !ok {
		cclog.Errorf("Job event is missing required tag 'function': measurement=%s", msg.Name())
		return
	}

	switch function {
	case "start_job":
		v, ok := msg.GetEventValue()
		if !ok {
			cclog.Errorf("Job start event is missing event field with JSON payload")
			return
		}
		api.handleStartJob(v)

	case "stop_job":
		v, ok := msg.GetEventValue()
		if !ok {
			cclog.Errorf("Job stop event is missing event field with JSON payload")
			return
		}
		api.handleStopJob(v)

	default:
		cclog.Warnf("Unknown job event function '%s', expected 'start_job' or 'stop_job'", function)
	}
}

// handleJobEvent processes job-related messages received via NATS using InfluxDB line protocol.
// The message must be in line protocol format with measurement="job" and include:
//   - tag "function" with value "start_job" or "stop_job"
//   - field "event" containing JSON payload (schema.Job or StopJobAPIRequest)
//
// Example: job,function=start_job event="{\"jobId\":1001,...}" 1234567890000000000
func (api *NatsAPI) handleJobEvent(subject string, data []byte) {
	api.jobSem <- struct{}{}
	defer func() { <-api.jobSem }()

	if len(data) == 0 {
		cclog.Warnf("NATS %s: received empty message", subject)
		return
	}

	d := influx.NewDecoderWithBytes(data)

	for d.Next() {
		m, err := receivers.DecodeInfluxMessage(d)
		if err != nil {
			cclog.Errorf("NATS %s: failed to decode InfluxDB line protocol message: %v", subject, err)
			return
		}

		if !m.IsEvent() {
			cclog.Debugf("NATS %s: received non-event message, skipping", subject)
			continue
		}

		if m.Name() == "job" {
			api.processJobEvent(m)
		} else {
			cclog.Debugf("NATS %s: unexpected measurement name '%s', expected 'job'", subject, m.Name())
		}
	}
}

// handleStartJob processes job start messages received via NATS.
// The payload parameter contains JSON following the schema.Job structure.
// Jobs are validated, checked for duplicates, and inserted into the database.
func (api *NatsAPI) handleStartJob(payload string) {
	if payload == "" {
		cclog.Error("NATS start job: payload is empty")
		return
	}
	req := schema.Job{
		Shared:           "none",
		MonitoringStatus: schema.MonitoringStatusRunningOrArchiving,
	}

	dec := json.NewDecoder(strings.NewReader(payload))
	dec.DisallowUnknownFields()
	if err := dec.Decode(&req); err != nil {
		cclog.Errorf("NATS start job: parsing request failed: %v", err)
		return
	}

	cclog.Debugf("NATS start job: %s", req.GoString())
	req.State = schema.JobStateRunning

	if err := importer.SanityChecks(&req); err != nil {
		cclog.Errorf("NATS start job: sanity check failed: %v", err)
		return
	}

	var unlockOnce sync.Once
	api.RepositoryMutex.Lock()
	defer unlockOnce.Do(api.RepositoryMutex.Unlock)

	jobs, err := api.JobRepository.FindAll(&req.JobID, &req.Cluster, nil)
	if err != nil && err != sql.ErrNoRows {
		cclog.Errorf("NATS start job: checking for duplicate failed: %v", err)
		return
	}
	if err == nil {
		for _, job := range jobs {
			if (req.StartTime - job.StartTime) < secondsPerDay {
				cclog.Errorf("NATS start job: job with jobId %d, cluster %s already exists (dbid: %d)",
					req.JobID, req.Cluster, job.ID)
				return
			}
		}
	}

	// When tags are present, insert directly into the job table so that the
	// returned ID can be used with AddTagOrCreate (which queries the job table).
	var id int64
	if len(req.Tags) > 0 {
		id, err = api.JobRepository.StartDirect(&req)
	} else {
		id, err = api.JobRepository.Start(&req)
	}
	if err != nil {
		cclog.Errorf("NATS start job: insert into database failed: %v", err)
		return
	}
	unlockOnce.Do(api.RepositoryMutex.Unlock)

	for _, tag := range req.Tags {
		if _, err := api.JobRepository.AddTagOrCreate(nil, id, tag.Type, tag.Name, tag.Scope); err != nil {
			cclog.Errorf("NATS start job: adding tag to new job %d failed: %v", id, err)
			return
		}
	}

	cclog.Infof("NATS: new job (id: %d): cluster=%s, jobId=%d, user=%s, startTime=%d",
		id, req.Cluster, req.JobID, req.User, req.StartTime)
}

// handleStopJob processes job stop messages received via NATS.
// The payload parameter contains JSON following the StopJobAPIRequest structure.
// The job is marked as stopped in the database and archiving is triggered if monitoring is enabled.
func (api *NatsAPI) handleStopJob(payload string) {
	if payload == "" {
		cclog.Error("NATS stop job: payload is empty")
		return
	}
	var req StopJobAPIRequest

	dec := json.NewDecoder(strings.NewReader(payload))
	dec.DisallowUnknownFields()
	if err := dec.Decode(&req); err != nil {
		cclog.Errorf("NATS job stop: parsing request failed: %v", err)
		return
	}

	if req.JobID == nil {
		cclog.Errorf("NATS job stop: the field 'jobId' is required")
		return
	}

	isCached := false
	job, err := api.JobRepository.FindCached(req.JobID, req.Cluster, req.StartTime)
	if err != nil {
		// Not in cache, try main job table
		job, err = api.JobRepository.Find(req.JobID, req.Cluster, req.StartTime)
		if err != nil {
			cclog.Errorf("NATS job stop: finding job failed: %v", err)
			return
		}
	} else {
		isCached = true
	}

	if job.State != schema.JobStateRunning {
		cclog.Errorf("NATS job stop: jobId %d (id %d) on %s: job has already been stopped (state is: %s)",
			job.JobID, job.ID, job.Cluster, job.State)
		return
	}

	if job.StartTime > req.StopTime {
		cclog.Errorf("NATS job stop: jobId %d (id %d) on %s: stopTime %d must be >= startTime %d",
			job.JobID, job.ID, job.Cluster, req.StopTime, job.StartTime)
		return
	}

	if req.State != "" && !req.State.Valid() {
		cclog.Errorf("NATS job stop: jobId %d (id %d) on %s: invalid job state: %#v",
			job.JobID, job.ID, job.Cluster, req.State)
		return
	} else if req.State == "" {
		req.State = schema.JobStateCompleted
	}

	job.Duration = int32(req.StopTime - job.StartTime)
	job.State = req.State
	api.JobRepository.Mutex.Lock()
	defer api.JobRepository.Mutex.Unlock()

	// If the job is still in job_cache, transfer it to the job table first
	if isCached {
		newID, err := api.JobRepository.TransferCachedJobToMain(*job.ID)
		if err != nil {
			cclog.Errorf("NATS job stop: jobId %d (id %d) on %s: transferring cached job failed: %v",
				job.JobID, *job.ID, job.Cluster, err)
			return
		}
		cclog.Infof("NATS: transferred cached job to main table: old id %d -> new id %d (jobId=%d)", *job.ID, newID, job.JobID)
		job.ID = &newID
	}

	if err := api.JobRepository.Stop(*job.ID, job.Duration, job.State, job.MonitoringStatus); err != nil {
		cclog.Errorf("NATS job stop: jobId %d (id %d) on %s: marking job as '%s' failed: %v",
			job.JobID, *job.ID, job.Cluster, job.State, err)
		return
	}

	cclog.Infof("NATS: archiving job (dbid: %d): cluster=%s, jobId=%d, user=%s, startTime=%d, duration=%d, state=%s",
		*job.ID, job.Cluster, job.JobID, job.User, job.StartTime, job.Duration, job.State)

	if job.MonitoringStatus == schema.MonitoringStatusDisabled {
		return
	}

	archiver.TriggerArchiving(job)
}

// processNodestateEvent extracts and processes node state data from the InfluxDB message.
// Updates node states in the repository for all nodes in the payload.
func (api *NatsAPI) processNodestateEvent(msg lp.CCMessage) {
	v, ok := msg.GetEventValue()
	if !ok {
		cclog.Errorf("Nodestate event is missing event field with JSON payload")
		return
	}

	var req UpdateNodeStatesRequest

	dec := json.NewDecoder(strings.NewReader(v))
	dec.DisallowUnknownFields()
	if err := dec.Decode(&req); err != nil {
		cclog.Errorf("NATS nodestate: parsing request failed: %v", err)
		return
	}

	repo := repository.GetNodeRepository()
	requestReceived := time.Now().Unix()

	updates := make([]repository.NodeStateUpdate, 0, len(req.Nodes))
	for _, node := range req.Nodes {
		state := determineState(node.States)
		nodeState := schema.NodeStateDB{
			TimeStamp:       requestReceived,
			NodeState:       state,
			CpusAllocated:   node.CpusAllocated,
			MemoryAllocated: node.MemoryAllocated,
			GpusAllocated:   node.GpusAllocated,
			HealthState:     schema.MonitoringStateFull,
			JobsRunning:     node.JobsRunning,
		}
		updates = append(updates, repository.NodeStateUpdate{
			Hostname:  node.Hostname,
			Cluster:   req.Cluster,
			NodeState: &nodeState,
		})
	}

	if err := repo.BatchUpdateNodeStates(updates); err != nil {
		cclog.Errorf("NATS nodestate: batch update for cluster %s failed: %v", req.Cluster, err)
	}

	cclog.Debugf("NATS nodestate: updated %d node states for cluster %s", len(req.Nodes), req.Cluster)
}

// handleNodeState processes node state update messages received via NATS using InfluxDB line protocol.
// The message must be in line protocol format with measurement="nodestate" and include:
//   - field "event" containing JSON payload (UpdateNodeStatesRequest)
//
// Example: nodestate event="{\"cluster\":\"testcluster\",\"nodes\":[...]}" 1234567890000000000
func (api *NatsAPI) handleNodeState(subject string, data []byte) {
	api.nodeSem <- struct{}{}
	defer func() { <-api.nodeSem }()

	if len(data) == 0 {
		cclog.Warnf("NATS %s: received empty message", subject)
		return
	}

	d := influx.NewDecoderWithBytes(data)

	for d.Next() {
		m, err := receivers.DecodeInfluxMessage(d)
		if err != nil {
			cclog.Errorf("NATS %s: failed to decode InfluxDB line protocol message: %v", subject, err)
			return
		}

		if !m.IsEvent() {
			cclog.Warnf("NATS %s: received non-event message, skipping", subject)
			continue
		}

		if m.Name() == "nodestate" {
			api.processNodestateEvent(m)
		} else {
			cclog.Warnf("NATS %s: unexpected measurement name '%s', expected 'nodestate'", subject, m.Name())
		}
	}
}

+// natsMessage wraps a raw NATS message with its subject for channel-based processing.

-  // jobSem limits concurrent NATS job event processing to prevent goroutine accumulation

-  // when the database is under contention.

-  // nodeSem limits concurrent NATS node state processing.

+  // jobCh receives job event messages for processing by worker goroutines.

+  // nodeCh receives node state messages for processing by worker goroutines.

-// NewNatsAPI creates a new NatsAPI instance with default dependencies.

+// NewNatsAPI creates a new NatsAPI instance with channel-based worker pools.

+// Concurrency is configured via NATSConfig (defaults: JobConcurrency=8, NodeConcurrency=2).

+  if s := config.Keys.APISubjects; s != nil {

-    jobSem:        make(chan struct{}, 8),

-    nodeSem:       make(chan struct{}, 2),

+    jobCh:         make(chan natsMessage, jobConc),

+    nodeCh:        make(chan natsMessage, nodeConc),

+// jobWorker processes job event messages from the job channel.

+    api.handleJobEvent(msg.subject, msg.data)

+// nodeWorker processes node state messages from the node channel.

+    api.handleNodeState(msg.subject, msg.data)

+// subscribe registers a NATS subscription using queue groups when configured.

+func subscribe(client *nats.Client, subject string, queueGroup string, handler func(string, []byte)) error {

+    return client.SubscribeQueue(subject, queueGroup, handler)

+  return client.Subscribe(subject, handler)

+// Messages are delivered to buffered channels and processed by worker goroutines.

-    if err := client.Subscribe(s.SubjectJobEvent, api.handleJobEvent); err != nil {

+    if err := subscribe(client, s.SubjectJobEvent, qg, func(subject string, data []byte) {

+      api.jobCh <- natsMessage{subject: subject, data: data}

-    if err := client.Subscribe(s.SubjectNodeState, api.handleNodeState); err != nil {

+    if err := subscribe(client, s.SubjectNodeState, qg, func(subject string, data []byte) {

+      api.nodeCh <- natsMessage{subject: subject, data: data}


Now remove the semaphore acquire/release from `handleJobEvent` and `handleNodeState` since workers handle concurrency:


func (api *NatsAPI) handleJobEvent(subject string, data []byte) {
	api.jobSem <- struct{}{}
	defer func() { <-api.jobSem }()

	if len(data) == 0 {

func (api *NatsAPI) handleJobEvent(subject string, data []byte) {
	if len(data) == 0 {


func (api *NatsAPI) handleJobEvent(subject string, data []byte) {
	if len(data) == 0 {

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package api

import (
	"database/sql"
	"encoding/json"
	"strings"
	"sync"
	"time"

	"github.com/ClusterCockpit/cc-backend/internal/archiver"
	"github.com/ClusterCockpit/cc-backend/internal/config"
	"github.com/ClusterCockpit/cc-backend/internal/importer"
	"github.com/ClusterCockpit/cc-backend/internal/repository"
	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
	lp "github.com/ClusterCockpit/cc-lib/v2/ccMessage"
	"github.com/ClusterCockpit/cc-lib/v2/nats"
	"github.com/ClusterCockpit/cc-lib/v2/receivers"
	"github.com/ClusterCockpit/cc-lib/v2/schema"
	influx "github.com/ClusterCockpit/cc-line-protocol/v2/lineprotocol"
)

// natsMessage wraps a raw NATS message with its subject for channel-based processing.
type natsMessage struct {
	subject string
	data    []byte
}

// NatsAPI provides NATS subscription-based handlers for Job and Node operations.
// It mirrors the functionality of the REST API but uses NATS messaging with
// InfluxDB line protocol as the message format.
//
// # Message Format
//
// All NATS messages use InfluxDB line protocol format (https://docs.influxdata.com/influxdb/v2.0/reference/syntax/line-protocol/)
// with the following structure:
//
//	measurement,tag1=value1,tag2=value2 field1=value1,field2=value2 timestamp
//
// # Job Events
//
// Job start/stop events use the "job" measurement with a "function" tag to distinguish operations:
//
//	job,function=start_job event="{...JSON payload...}" <timestamp>
//	job,function=stop_job event="{...JSON payload...}" <timestamp>
//
// The JSON payload in the "event" field follows the schema.Job or StopJobAPIRequest structure.
//
// Example job start message:
//
//	job,function=start_job event="{\"jobId\":1001,\"user\":\"testuser\",\"cluster\":\"testcluster\",...}" 1234567890000000000
//
// # Node State Events
//
// Node state updates use the "nodestate" measurement with cluster information:
//
//	nodestate event="{...JSON payload...}" <timestamp>
//
// The JSON payload follows the UpdateNodeStatesRequest structure.
//
// Example node state message:
//
//	nodestate event="{\"cluster\":\"testcluster\",\"nodes\":[{\"hostname\":\"node01\",\"states\":[\"idle\"]}]}" 1234567890000000000
type NatsAPI struct {
	JobRepository *repository.JobRepository
	// RepositoryMutex protects job creation operations from race conditions
	// when checking for duplicate jobs during startJob calls.
	RepositoryMutex sync.Mutex
	// jobCh receives job event messages for processing by worker goroutines.
	jobCh chan natsMessage
	// nodeCh receives node state messages for processing by worker goroutines.
	nodeCh chan natsMessage
}

// NewNatsAPI creates a new NatsAPI instance with channel-based worker pools.
// Concurrency is configured via NATSConfig (defaults: JobConcurrency=8, NodeConcurrency=2).
func NewNatsAPI() *NatsAPI {
	jobConc := 8
	nodeConc := 2

	if s := config.Keys.APISubjects; s != nil {
		if s.JobConcurrency > 0 {
			jobConc = s.JobConcurrency
		}
		if s.NodeConcurrency > 0 {
			nodeConc = s.NodeConcurrency
		}
	}

	api := &NatsAPI{
		JobRepository: repository.GetJobRepository(),
		jobCh:         make(chan natsMessage, jobConc),
		nodeCh:        make(chan natsMessage, nodeConc),
	}

	// Start worker goroutines
	for range jobConc {
		go api.jobWorker()
	}
	for range nodeConc {
		go api.nodeWorker()
	}

	return api
}

// jobWorker processes job event messages from the job channel.
func (api *NatsAPI) jobWorker() {
	for msg := range api.jobCh {
		api.handleJobEvent(msg.subject, msg.data)
	}
}

// nodeWorker processes node state messages from the node channel.
func (api *NatsAPI) nodeWorker() {
	for msg := range api.nodeCh {
		api.handleNodeState(msg.subject, msg.data)
	}
}

// subscribe registers a NATS subscription using queue groups when configured.
func subscribe(client *nats.Client, subject string, queueGroup string, handler func(string, []byte)) error {
	if queueGroup != "" {
		return client.SubscribeQueue(subject, queueGroup, handler)
	}
	return client.Subscribe(subject, handler)
}

// StartSubscriptions registers all NATS subscriptions for Job and Node APIs.
// Messages are delivered to buffered channels and processed by worker goroutines.
// Returns an error if the NATS client is not available or subscription fails.
func (api *NatsAPI) StartSubscriptions() error {
	client := nats.GetClient()
	if client == nil {
		cclog.Warn("NATS client not available, skipping API subscriptions")
		return nil
	}

	if config.Keys.APISubjects != nil {
		s := config.Keys.APISubjects
		qg := s.QueueGroup

		if err := subscribe(client, s.SubjectJobEvent, qg, func(subject string, data []byte) {
			api.jobCh <- natsMessage{subject: subject, data: data}
		}); err != nil {
			return err
		}

		if err := subscribe(client, s.SubjectNodeState, qg, func(subject string, data []byte) {
			api.nodeCh <- natsMessage{subject: subject, data: data}
		}); err != nil {
			return err
		}

		cclog.Info("NATS API subscriptions started")
	}
	return nil
}

// processJobEvent routes job event messages to the appropriate handler based on the "function" tag.
// Validates that required tags and fields are present before processing.
func (api *NatsAPI) processJobEvent(msg lp.CCMessage) {
	function, ok := msg.GetTag("function")
	if !ok {
		cclog.Errorf("Job event is missing required tag 'function': measurement=%s", msg.Name())
		return
	}

	switch function {
	case "start_job":
		v, ok := msg.GetEventValue()
		if !ok {
			cclog.Errorf("Job start event is missing event field with JSON payload")
			return
		}
		api.handleStartJob(v)

	case "stop_job":
		v, ok := msg.GetEventValue()
		if !ok {
			cclog.Errorf("Job stop event is missing event field with JSON payload")
			return
		}
		api.handleStopJob(v)

	default:
		cclog.Warnf("Unknown job event function '%s', expected 'start_job' or 'stop_job'", function)
	}
}

// handleJobEvent processes job-related messages received via NATS using InfluxDB line protocol.
// The message must be in line protocol format with measurement="job" and include:
//   - tag "function" with value "start_job" or "stop_job"
//   - field "event" containing JSON payload (schema.Job or StopJobAPIRequest)
//
// Example: job,function=start_job event="{\"jobId\":1001,...}" 1234567890000000000
func (api *NatsAPI) handleJobEvent(subject string, data []byte) {
	api.jobSem <- struct{}{}
	defer func() { <-api.jobSem }()

	if len(data) == 0 {
		cclog.Warnf("NATS %s: received empty message", subject)
		return
	}

	d := influx.NewDecoderWithBytes(data)

	for d.Next() {
		m, err := receivers.DecodeInfluxMessage(d)
		if err != nil {
			cclog.Errorf("NATS %s: failed to decode InfluxDB line protocol message: %v", subject, err)
			return
		}

		if !m.IsEvent() {
			cclog.Debugf("NATS %s: received non-event message, skipping", subject)
			continue
		}

		if m.Name() == "job" {
			api.processJobEvent(m)
		} else {
			cclog.Debugf("NATS %s: unexpected measurement name '%s', expected 'job'", subject, m.Name())
		}
	}
}

// handleStartJob processes job start messages received via NATS.
// The payload parameter contains JSON following the schema.Job structure.
// Jobs are validated, checked for duplicates, and inserted into the database.
func (api *NatsAPI) handleStartJob(payload string) {
	if payload == "" {
		cclog.Error("NATS start job: payload is empty")
		return
	}
	req := schema.Job{
		Shared:           "none",
		MonitoringStatus: schema.MonitoringStatusRunningOrArchiving,
	}

	dec := json.NewDecoder(strings.NewReader(payload))
	dec.DisallowUnknownFields()
	if err := dec.Decode(&req); err != nil {
		cclog.Errorf("NATS start job: parsing request failed: %v", err)
		return
	}

	cclog.Debugf("NATS start job: %s", req.GoString())
	req.State = schema.JobStateRunning

	if err := importer.SanityChecks(&req); err != nil {
		cclog.Errorf("NATS start job: sanity check failed: %v", err)
		return
	}

	var unlockOnce sync.Once
	api.RepositoryMutex.Lock()
	defer unlockOnce.Do(api.RepositoryMutex.Unlock)

	jobs, err := api.JobRepository.FindAll(&req.JobID, &req.Cluster, nil)
	if err != nil && err != sql.ErrNoRows {
		cclog.Errorf("NATS start job: checking for duplicate failed: %v", err)
		return
	}
	if err == nil {
		for _, job := range jobs {
			if (req.StartTime - job.StartTime) < secondsPerDay {
				cclog.Errorf("NATS start job: job with jobId %d, cluster %s already exists (dbid: %d)",
					req.JobID, req.Cluster, job.ID)
				return
			}
		}
	}

	// When tags are present, insert directly into the job table so that the
	// returned ID can be used with AddTagOrCreate (which queries the job table).
	var id int64
	if len(req.Tags) > 0 {
		id, err = api.JobRepository.StartDirect(&req)
	} else {
		id, err = api.JobRepository.Start(&req)
	}
	if err != nil {
		cclog.Errorf("NATS start job: insert into database failed: %v", err)
		return
	}
	unlockOnce.Do(api.RepositoryMutex.Unlock)

	for _, tag := range req.Tags {
		if _, err := api.JobRepository.AddTagOrCreate(nil, id, tag.Type, tag.Name, tag.Scope); err != nil {
			cclog.Errorf("NATS start job: adding tag to new job %d failed: %v", id, err)
			return
		}
	}

	cclog.Infof("NATS: new job (id: %d): cluster=%s, jobId=%d, user=%s, startTime=%d",
		id, req.Cluster, req.JobID, req.User, req.StartTime)
}

// handleStopJob processes job stop messages received via NATS.
// The payload parameter contains JSON following the StopJobAPIRequest structure.
// The job is marked as stopped in the database and archiving is triggered if monitoring is enabled.
func (api *NatsAPI) handleStopJob(payload string) {
	if payload == "" {
		cclog.Error("NATS stop job: payload is empty")
		return
	}
	var req StopJobAPIRequest

	dec := json.NewDecoder(strings.NewReader(payload))
	dec.DisallowUnknownFields()
	if err := dec.Decode(&req); err != nil {
		cclog.Errorf("NATS job stop: parsing request failed: %v", err)
		return
	}

	if req.JobID == nil {
		cclog.Errorf("NATS job stop: the field 'jobId' is required")
		return
	}

	isCached := false
	job, err := api.JobRepository.FindCached(req.JobID, req.Cluster, req.StartTime)
	if err != nil {
		// Not in cache, try main job table
		job, err = api.JobRepository.Find(req.JobID, req.Cluster, req.StartTime)
		if err != nil {
			cclog.Errorf("NATS job stop: finding job failed: %v", err)
			return
		}
	} else {
		isCached = true
	}

	if job.State != schema.JobStateRunning {
		cclog.Errorf("NATS job stop: jobId %d (id %d) on %s: job has already been stopped (state is: %s)",
			job.JobID, job.ID, job.Cluster, job.State)
		return
	}

	if job.StartTime > req.StopTime {
		cclog.Errorf("NATS job stop: jobId %d (id %d) on %s: stopTime %d must be >= startTime %d",
			job.JobID, job.ID, job.Cluster, req.StopTime, job.StartTime)
		return
	}

	if req.State != "" && !req.State.Valid() {
		cclog.Errorf("NATS job stop: jobId %d (id %d) on %s: invalid job state: %#v",
			job.JobID, job.ID, job.Cluster, req.State)
		return
	} else if req.State == "" {
		req.State = schema.JobStateCompleted
	}

	job.Duration = int32(req.StopTime - job.StartTime)
	job.State = req.State
	api.JobRepository.Mutex.Lock()
	defer api.JobRepository.Mutex.Unlock()

	// If the job is still in job_cache, transfer it to the job table first
	if isCached {
		newID, err := api.JobRepository.TransferCachedJobToMain(*job.ID)
		if err != nil {
			cclog.Errorf("NATS job stop: jobId %d (id %d) on %s: transferring cached job failed: %v",
				job.JobID, *job.ID, job.Cluster, err)
			return
		}
		cclog.Infof("NATS: transferred cached job to main table: old id %d -> new id %d (jobId=%d)", *job.ID, newID, job.JobID)
		job.ID = &newID
	}

	if err := api.JobRepository.Stop(*job.ID, job.Duration, job.State, job.MonitoringStatus); err != nil {
		cclog.Errorf("NATS job stop: jobId %d (id %d) on %s: marking job as '%s' failed: %v",
			job.JobID, *job.ID, job.Cluster, job.State, err)
		return
	}

	cclog.Infof("NATS: archiving job (dbid: %d): cluster=%s, jobId=%d, user=%s, startTime=%d, duration=%d, state=%s",
		*job.ID, job.Cluster, job.JobID, job.User, job.StartTime, job.Duration, job.State)

	if job.MonitoringStatus == schema.MonitoringStatusDisabled {
		return
	}

	archiver.TriggerArchiving(job)
}

// processNodestateEvent extracts and processes node state data from the InfluxDB message.
// Updates node states in the repository for all nodes in the payload.
func (api *NatsAPI) processNodestateEvent(msg lp.CCMessage) {
	v, ok := msg.GetEventValue()
	if !ok {
		cclog.Errorf("Nodestate event is missing event field with JSON payload")
		return
	}

	var req UpdateNodeStatesRequest

	dec := json.NewDecoder(strings.NewReader(v))
	dec.DisallowUnknownFields()
	if err := dec.Decode(&req); err != nil {
		cclog.Errorf("NATS nodestate: parsing request failed: %v", err)
		return
	}

	repo := repository.GetNodeRepository()
	requestReceived := time.Now().Unix()

	updates := make([]repository.NodeStateUpdate, 0, len(req.Nodes))
	for _, node := range req.Nodes {
		state := determineState(node.States)
		nodeState := schema.NodeStateDB{
			TimeStamp:       requestReceived,
			NodeState:       state,
			CpusAllocated:   node.CpusAllocated,
			MemoryAllocated: node.MemoryAllocated,
			GpusAllocated:   node.GpusAllocated,
			HealthState:     schema.MonitoringStateFull,
			JobsRunning:     node.JobsRunning,
		}
		updates = append(updates, repository.NodeStateUpdate{
			Hostname:  node.Hostname,
			Cluster:   req.Cluster,
			NodeState: &nodeState,
		})
	}

	if err := repo.BatchUpdateNodeStates(updates); err != nil {
		cclog.Errorf("NATS nodestate: batch update for cluster %s failed: %v", req.Cluster, err)
	}

	cclog.Debugf("NATS nodestate: updated %d node states for cluster %s", len(req.Nodes), req.Cluster)
}

// handleNodeState processes node state update messages received via NATS using InfluxDB line protocol.
// The message must be in line protocol format with measurement="nodestate" and include:
//   - field "event" containing JSON payload (UpdateNodeStatesRequest)
//
// Example: nodestate event="{\"cluster\":\"testcluster\",\"nodes\":[...]}" 1234567890000000000
func (api *NatsAPI) handleNodeState(subject string, data []byte) {
	api.nodeSem <- struct{}{}
	defer func() { <-api.nodeSem }()

	if len(data) == 0 {
		cclog.Warnf("NATS %s: received empty message", subject)
		return
	}

	d := influx.NewDecoderWithBytes(data)

	for d.Next() {
		m, err := receivers.DecodeInfluxMessage(d)
		if err != nil {
			cclog.Errorf("NATS %s: failed to decode InfluxDB line protocol message: %v", subject, err)
			return
		}

		if !m.IsEvent() {
			cclog.Warnf("NATS %s: received non-event message, skipping", subject)
			continue
		}

		if m.Name() == "nodestate" {
			api.processNodestateEvent(m)
		} else {
			cclog.Warnf("NATS %s: unexpected measurement name '%s', expected 'nodestate'", subject, m.Name())
		}
	}
}


func (api *NatsAPI) handleNodeState(subject string, data []byte) {
	api.nodeSem <- struct{}{}
	defer func() { <-api.nodeSem }()

	if len(data) == 0 {

func (api *NatsAPI) handleNodeState(subject string, data []byte) {
	if len(data) == 0 {


func (api *NatsAPI) handleNodeState(subject string, data []byte) {
	if len(data) == 0 {

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package api

import (
	"database/sql"
	"encoding/json"
	"strings"
	"sync"
	"time"

	"github.com/ClusterCockpit/cc-backend/internal/archiver"
	"github.com/ClusterCockpit/cc-backend/internal/config"
	"github.com/ClusterCockpit/cc-backend/internal/importer"
	"github.com/ClusterCockpit/cc-backend/internal/repository"
	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
	lp "github.com/ClusterCockpit/cc-lib/v2/ccMessage"
	"github.com/ClusterCockpit/cc-lib/v2/nats"
	"github.com/ClusterCockpit/cc-lib/v2/receivers"
	"github.com/ClusterCockpit/cc-lib/v2/schema"
	influx "github.com/ClusterCockpit/cc-line-protocol/v2/lineprotocol"
)

// natsMessage wraps a raw NATS message with its subject for channel-based processing.
type natsMessage struct {
	subject string
	data    []byte
}

// NatsAPI provides NATS subscription-based handlers for Job and Node operations.
// It mirrors the functionality of the REST API but uses NATS messaging with
// InfluxDB line protocol as the message format.
//
// # Message Format
//
// All NATS messages use InfluxDB line protocol format (https://docs.influxdata.com/influxdb/v2.0/reference/syntax/line-protocol/)
// with the following structure:
//
//	measurement,tag1=value1,tag2=value2 field1=value1,field2=value2 timestamp
//
// # Job Events
//
// Job start/stop events use the "job" measurement with a "function" tag to distinguish operations:
//
//	job,function=start_job event="{...JSON payload...}" <timestamp>
//	job,function=stop_job event="{...JSON payload...}" <timestamp>
//
// The JSON payload in the "event" field follows the schema.Job or StopJobAPIRequest structure.
//
// Example job start message:
//
//	job,function=start_job event="{\"jobId\":1001,\"user\":\"testuser\",\"cluster\":\"testcluster\",...}" 1234567890000000000
//
// # Node State Events
//
// Node state updates use the "nodestate" measurement with cluster information:
//
//	nodestate event="{...JSON payload...}" <timestamp>
//
// The JSON payload follows the UpdateNodeStatesRequest structure.
//
// Example node state message:
//
//	nodestate event="{\"cluster\":\"testcluster\",\"nodes\":[{\"hostname\":\"node01\",\"states\":[\"idle\"]}]}" 1234567890000000000
type NatsAPI struct {
	JobRepository *repository.JobRepository
	// RepositoryMutex protects job creation operations from race conditions
	// when checking for duplicate jobs during startJob calls.
	RepositoryMutex sync.Mutex
	// jobCh receives job event messages for processing by worker goroutines.
	jobCh chan natsMessage
	// nodeCh receives node state messages for processing by worker goroutines.
	nodeCh chan natsMessage
}

// NewNatsAPI creates a new NatsAPI instance with channel-based worker pools.
// Concurrency is configured via NATSConfig (defaults: JobConcurrency=8, NodeConcurrency=2).
func NewNatsAPI() *NatsAPI {
	jobConc := 8
	nodeConc := 2

	if s := config.Keys.APISubjects; s != nil {
		if s.JobConcurrency > 0 {
			jobConc = s.JobConcurrency
		}
		if s.NodeConcurrency > 0 {
			nodeConc = s.NodeConcurrency
		}
	}

	api := &NatsAPI{
		JobRepository: repository.GetJobRepository(),
		jobCh:         make(chan natsMessage, jobConc),
		nodeCh:        make(chan natsMessage, nodeConc),
	}

	// Start worker goroutines
	for range jobConc {
		go api.jobWorker()
	}
	for range nodeConc {
		go api.nodeWorker()
	}

	return api
}

// jobWorker processes job event messages from the job channel.
func (api *NatsAPI) jobWorker() {
	for msg := range api.jobCh {
		api.handleJobEvent(msg.subject, msg.data)
	}
}

// nodeWorker processes node state messages from the node channel.
func (api *NatsAPI) nodeWorker() {
	for msg := range api.nodeCh {
		api.handleNodeState(msg.subject, msg.data)
	}
}

// subscribe registers a NATS subscription using queue groups when configured.
func subscribe(client *nats.Client, subject string, queueGroup string, handler func(string, []byte)) error {
	if queueGroup != "" {
		return client.SubscribeQueue(subject, queueGroup, handler)
	}
	return client.Subscribe(subject, handler)
}

// StartSubscriptions registers all NATS subscriptions for Job and Node APIs.
// Messages are delivered to buffered channels and processed by worker goroutines.
// Returns an error if the NATS client is not available or subscription fails.
func (api *NatsAPI) StartSubscriptions() error {
	client := nats.GetClient()
	if client == nil {
		cclog.Warn("NATS client not available, skipping API subscriptions")
		return nil
	}

	if config.Keys.APISubjects != nil {
		s := config.Keys.APISubjects
		qg := s.QueueGroup

		if err := subscribe(client, s.SubjectJobEvent, qg, func(subject string, data []byte) {
			api.jobCh <- natsMessage{subject: subject, data: data}
		}); err != nil {
			return err
		}

		if err := subscribe(client, s.SubjectNodeState, qg, func(subject string, data []byte) {
			api.nodeCh <- natsMessage{subject: subject, data: data}
		}); err != nil {
			return err
		}

		cclog.Info("NATS API subscriptions started")
	}
	return nil
}

// processJobEvent routes job event messages to the appropriate handler based on the "function" tag.
// Validates that required tags and fields are present before processing.
func (api *NatsAPI) processJobEvent(msg lp.CCMessage) {
	function, ok := msg.GetTag("function")
	if !ok {
		cclog.Errorf("Job event is missing required tag 'function': measurement=%s", msg.Name())
		return
	}

	switch function {
	case "start_job":
		v, ok := msg.GetEventValue()
		if !ok {
			cclog.Errorf("Job start event is missing event field with JSON payload")
			return
		}
		api.handleStartJob(v)

	case "stop_job":
		v, ok := msg.GetEventValue()
		if !ok {
			cclog.Errorf("Job stop event is missing event field with JSON payload")
			return
		}
		api.handleStopJob(v)

	default:
		cclog.Warnf("Unknown job event function '%s', expected 'start_job' or 'stop_job'", function)
	}
}

// handleJobEvent processes job-related messages received via NATS using InfluxDB line protocol.
// The message must be in line protocol format with measurement="job" and include:
//   - tag "function" with value "start_job" or "stop_job"
//   - field "event" containing JSON payload (schema.Job or StopJobAPIRequest)
//
// Example: job,function=start_job event="{\"jobId\":1001,...}" 1234567890000000000
func (api *NatsAPI) handleJobEvent(subject string, data []byte) {
	if len(data) == 0 {
		cclog.Warnf("NATS %s: received empty message", subject)
		return
	}

	d := influx.NewDecoderWithBytes(data)

	for d.Next() {
		m, err := receivers.DecodeInfluxMessage(d)
		if err != nil {
			cclog.Errorf("NATS %s: failed to decode InfluxDB line protocol message: %v", subject, err)
			return
		}

		if !m.IsEvent() {
			cclog.Debugf("NATS %s: received non-event message, skipping", subject)
			continue
		}

		if m.Name() == "job" {
			api.processJobEvent(m)
		} else {
			cclog.Debugf("NATS %s: unexpected measurement name '%s', expected 'job'", subject, m.Name())
		}
	}
}

// handleStartJob processes job start messages received via NATS.
// The payload parameter contains JSON following the schema.Job structure.
// Jobs are validated, checked for duplicates, and inserted into the database.
func (api *NatsAPI) handleStartJob(payload string) {
	if payload == "" {
		cclog.Error("NATS start job: payload is empty")
		return
	}
	req := schema.Job{
		Shared:           "none",
		MonitoringStatus: schema.MonitoringStatusRunningOrArchiving,
	}

	dec := json.NewDecoder(strings.NewReader(payload))
	dec.DisallowUnknownFields()
	if err := dec.Decode(&req); err != nil {
		cclog.Errorf("NATS start job: parsing request failed: %v", err)
		return
	}

	cclog.Debugf("NATS start job: %s", req.GoString())
	req.State = schema.JobStateRunning

	if err := importer.SanityChecks(&req); err != nil {
		cclog.Errorf("NATS start job: sanity check failed: %v", err)
		return
	}

	var unlockOnce sync.Once
	api.RepositoryMutex.Lock()
	defer unlockOnce.Do(api.RepositoryMutex.Unlock)

	jobs, err := api.JobRepository.FindAll(&req.JobID, &req.Cluster, nil)
	if err != nil && err != sql.ErrNoRows {
		cclog.Errorf("NATS start job: checking for duplicate failed: %v", err)
		return
	}
	if err == nil {
		for _, job := range jobs {
			if (req.StartTime - job.StartTime) < secondsPerDay {
				cclog.Errorf("NATS start job: job with jobId %d, cluster %s already exists (dbid: %d)",
					req.JobID, req.Cluster, job.ID)
				return
			}
		}
	}

	// When tags are present, insert directly into the job table so that the
	// returned ID can be used with AddTagOrCreate (which queries the job table).
	var id int64
	if len(req.Tags) > 0 {
		id, err = api.JobRepository.StartDirect(&req)
	} else {
		id, err = api.JobRepository.Start(&req)
	}
	if err != nil {
		cclog.Errorf("NATS start job: insert into database failed: %v", err)
		return
	}
	unlockOnce.Do(api.RepositoryMutex.Unlock)

	for _, tag := range req.Tags {
		if _, err := api.JobRepository.AddTagOrCreate(nil, id, tag.Type, tag.Name, tag.Scope); err != nil {
			cclog.Errorf("NATS start job: adding tag to new job %d failed: %v", id, err)
			return
		}
	}

	cclog.Infof("NATS: new job (id: %d): cluster=%s, jobId=%d, user=%s, startTime=%d",
		id, req.Cluster, req.JobID, req.User, req.StartTime)
}

// handleStopJob processes job stop messages received via NATS.
// The payload parameter contains JSON following the StopJobAPIRequest structure.
// The job is marked as stopped in the database and archiving is triggered if monitoring is enabled.
func (api *NatsAPI) handleStopJob(payload string) {
	if payload == "" {
		cclog.Error("NATS stop job: payload is empty")
		return
	}
	var req StopJobAPIRequest

	dec := json.NewDecoder(strings.NewReader(payload))
	dec.DisallowUnknownFields()
	if err := dec.Decode(&req); err != nil {
		cclog.Errorf("NATS job stop: parsing request failed: %v", err)
		return
	}

	if req.JobID == nil {
		cclog.Errorf("NATS job stop: the field 'jobId' is required")
		return
	}

	isCached := false
	job, err := api.JobRepository.FindCached(req.JobID, req.Cluster, req.StartTime)
	if err != nil {
		// Not in cache, try main job table
		job, err = api.JobRepository.Find(req.JobID, req.Cluster, req.StartTime)
		if err != nil {
			cclog.Errorf("NATS job stop: finding job failed: %v", err)
			return
		}
	} else {
		isCached = true
	}

	if job.State != schema.JobStateRunning {
		cclog.Errorf("NATS job stop: jobId %d (id %d) on %s: job has already been stopped (state is: %s)",
			job.JobID, job.ID, job.Cluster, job.State)
		return
	}

	if job.StartTime > req.StopTime {
		cclog.Errorf("NATS job stop: jobId %d (id %d) on %s: stopTime %d must be >= startTime %d",
			job.JobID, job.ID, job.Cluster, req.StopTime, job.StartTime)
		return
	}

	if req.State != "" && !req.State.Valid() {
		cclog.Errorf("NATS job stop: jobId %d (id %d) on %s: invalid job state: %#v",
			job.JobID, job.ID, job.Cluster, req.State)
		return
	} else if req.State == "" {
		req.State = schema.JobStateCompleted
	}

	job.Duration = int32(req.StopTime - job.StartTime)
	job.State = req.State
	api.JobRepository.Mutex.Lock()
	defer api.JobRepository.Mutex.Unlock()

	// If the job is still in job_cache, transfer it to the job table first
	if isCached {
		newID, err := api.JobRepository.TransferCachedJobToMain(*job.ID)
		if err != nil {
			cclog.Errorf("NATS job stop: jobId %d (id %d) on %s: transferring cached job failed: %v",
				job.JobID, *job.ID, job.Cluster, err)
			return
		}
		cclog.Infof("NATS: transferred cached job to main table: old id %d -> new id %d (jobId=%d)", *job.ID, newID, job.JobID)
		job.ID = &newID
	}

	if err := api.JobRepository.Stop(*job.ID, job.Duration, job.State, job.MonitoringStatus); err != nil {
		cclog.Errorf("NATS job stop: jobId %d (id %d) on %s: marking job as '%s' failed: %v",
			job.JobID, *job.ID, job.Cluster, job.State, err)
		return
	}

	cclog.Infof("NATS: archiving job (dbid: %d): cluster=%s, jobId=%d, user=%s, startTime=%d, duration=%d, state=%s",
		*job.ID, job.Cluster, job.JobID, job.User, job.StartTime, job.Duration, job.State)

	if job.MonitoringStatus == schema.MonitoringStatusDisabled {
		return
	}

	archiver.TriggerArchiving(job)
}

// processNodestateEvent extracts and processes node state data from the InfluxDB message.
// Updates node states in the repository for all nodes in the payload.
func (api *NatsAPI) processNodestateEvent(msg lp.CCMessage) {
	v, ok := msg.GetEventValue()
	if !ok {
		cclog.Errorf("Nodestate event is missing event field with JSON payload")
		return
	}

	var req UpdateNodeStatesRequest

	dec := json.NewDecoder(strings.NewReader(v))
	dec.DisallowUnknownFields()
	if err := dec.Decode(&req); err != nil {
		cclog.Errorf("NATS nodestate: parsing request failed: %v", err)
		return
	}

	repo := repository.GetNodeRepository()
	requestReceived := time.Now().Unix()

	updates := make([]repository.NodeStateUpdate, 0, len(req.Nodes))
	for _, node := range req.Nodes {
		state := determineState(node.States)
		nodeState := schema.NodeStateDB{
			TimeStamp:       requestReceived,
			NodeState:       state,
			CpusAllocated:   node.CpusAllocated,
			MemoryAllocated: node.MemoryAllocated,
			GpusAllocated:   node.GpusAllocated,
			HealthState:     schema.MonitoringStateFull,
			JobsRunning:     node.JobsRunning,
		}
		updates = append(updates, repository.NodeStateUpdate{
			Hostname:  node.Hostname,
			Cluster:   req.Cluster,
			NodeState: &nodeState,
		})
	}

	if err := repo.BatchUpdateNodeStates(updates); err != nil {
		cclog.Errorf("NATS nodestate: batch update for cluster %s failed: %v", req.Cluster, err)
	}

	cclog.Debugf("NATS nodestate: updated %d node states for cluster %s", len(req.Nodes), req.Cluster)
}

// handleNodeState processes node state update messages received via NATS using InfluxDB line protocol.
// The message must be in line protocol format with measurement="nodestate" and include:
//   - field "event" containing JSON payload (UpdateNodeStatesRequest)
//
// Example: nodestate event="{\"cluster\":\"testcluster\",\"nodes\":[...]}" 1234567890000000000
func (api *NatsAPI) handleNodeState(subject string, data []byte) {
	api.nodeSem <- struct{}{}
	defer func() { <-api.nodeSem }()

	if len(data) == 0 {
		cclog.Warnf("NATS %s: received empty message", subject)
		return
	}

	d := influx.NewDecoderWithBytes(data)

	for d.Next() {
		m, err := receivers.DecodeInfluxMessage(d)
		if err != nil {
			cclog.Errorf("NATS %s: failed to decode InfluxDB line protocol message: %v", subject, err)
			return
		}

		if !m.IsEvent() {
			cclog.Warnf("NATS %s: received non-event message, skipping", subject)
			continue
		}

		if m.Name() == "nodestate" {
			api.processNodestateEvent(m)
		} else {
			cclog.Warnf("NATS %s: unexpected measurement name '%s', expected 'nodestate'", subject, m.Name())
		}
	}
}


There are still references to the old semaphore fields. Let me check what's on those lines.


No matches found


Good, no more references. The diagnostics must have been stale. Now let me build and test.


Bash

Build all packages


Vet api package


(Bash completed with no output)


Build and vet pass. Now run the tests.


go test ./internal/api/... -v -count=1 -timeout 120s 2>&1 | tail -80

Run NATS API tests


<6>[INFO]     2026/03/16 11:52:39 archiveWorker.go:238: All archive jobs completed successfully
<6>[INFO]     2026/03/16 11:52:39 archiveWorker.go:121: Archive channel closed, worker exiting
--- PASS: TestNatsHandleNodeStateEdgeCases (0.30s)
    --- PASS: TestNatsHandleNodeStateEdgeCases/missing_cluster_field_in_JSON (0.05s)
    --- PASS: TestNatsHandleNodeStateEdgeCases/malformed_JSON_with_unescaped_quotes (0.05s)
    --- PASS: TestNatsHandleNodeStateEdgeCases/unicode_characters_in_hostname (0.05s)
    --- PASS: TestNatsHandleNodeStateEdgeCases/very_large_node_count (0.05s)
    --- PASS: TestNatsHandleNodeStateEdgeCases/timestamp_in_past (0.05s)
=== RUN   TestNatsHandleStartJobDuplicatePrevention
<6>[INFO]     2026/03/16 11:52:39 migration.go:122: Legacy database without version or missing database file!
<6>[INFO]     2026/03/16 11:52:39 migration.go:129: unsupported database version 0, need 11.
Please backup your database file and run cc-backend -migrate-db
<6>[INFO]     2026/03/16 11:52:39 dbConnection.go:96: SQLite config: cache_size=2048MB/conn, soft_heap_limit=16384MB, max_conns=4
<6>[INFO]     2026/03/16 11:52:39 nats.go:297: NATS: new job (id: 1): cluster=testcluster, jobId=5001, user=testuser, startTime=1234567890
<3>[ERROR]    2026/03/16 11:52:40 /Users/jan/prg/ClusterCockpit/cc-backend/internal/api/nats.go:269: NATS start job: job with jobId 5001, cluster testcluster already exists (dbid: 113234315163880)
<6>[INFO]     2026/03/16 11:52:40 archiveWorker.go:223: Initiating archiver shutdown...
<6>[INFO]     2026/03/16 11:52:40 archiveWorker.go:238: All archive jobs completed successfully
<6>[INFO]     2026/03/16 11:52:40 archiveWorker.go:121: Archive channel closed, worker exiting
--- PASS: TestNatsHandleStartJobDuplicatePrevention (0.25s)
=== RUN   TestRestApi
<6>[INFO]     2026/03/16 11:52:40 migration.go:122: Legacy database without version or missing database file!
<6>[INFO]     2026/03/16 11:52:40 migration.go:129: unsupported database version 0, need 11.
Please backup your database file and run cc-backend -migrate-db
<6>[INFO]     2026/03/16 11:52:40 dbConnection.go:96: SQLite config: cache_size=2048MB/conn, soft_heap_limit=16384MB, max_conns=4
<6>[INFO]     2026/03/16 11:52:40 rest.go:88: Enabling REST start/stop job API
=== RUN   TestRestApi/StartJob
<6>[INFO]     2026/03/16 11:52:40 job.go:779: new job (id: 1): cluster=testcluster, jobId=123, user=testuser, startTime=123456789
=== RUN   TestRestApi/StopJob
<4>[WARNING]  2026/03/16 11:52:40 job.go:162: Error while scanning rows (Job): sql: no rows in result set
<6>[INFO]     2026/03/16 11:52:40 job.go:1035: archiving job... (dbid: 1): cluster=testcluster, jobId=123, user=testuser, startTime=123456789, duration=1000, state=completed
=== RUN   TestRestApi/CheckArchive
=== RUN   TestRestApi/CheckDoubleStart
<4>[WARNING]  2026/03/16 11:52:40 rest.go:187: REST ERROR : a job with that jobId, cluster and startTime already exists: dbid: 1, jobid: 123
=== RUN   TestRestApi/StartJobFailed
<6>[INFO]     2026/03/16 11:52:40 job.go:779: new job (id: 1): cluster=testcluster, jobId=12345, user=testuser, startTime=12345678
<6>[INFO]     2026/03/16 11:52:40 archiveWorker.go:165: archiving job (dbid: 1) successful
=== RUN   TestRestApi/StopJobFailed
<4>[WARNING]  2026/03/16 11:52:41 job.go:162: Error while scanning rows (Job): sql: no rows in result set
<6>[INFO]     2026/03/16 11:52:41 job.go:1035: archiving job... (dbid: 2): cluster=testcluster, jobId=12345, user=testuser, startTime=12345678, duration=10000, state=failed
=== RUN   TestRestApi/GetUsedNodesNoRunning
<6>[INFO]     2026/03/16 11:52:41 archiveWorker.go:223: Initiating archiver shutdown...
<6>[INFO]     2026/03/16 11:52:41 archiveWorker.go:165: archiving job (dbid: 2) successful
<6>[INFO]     2026/03/16 11:52:41 archiveWorker.go:121: Archive channel closed, worker exiting
<6>[INFO]     2026/03/16 11:52:41 archiveWorker.go:238: All archive jobs completed successfully
--- PASS: TestRestApi (1.05s)
    --- PASS: TestRestApi/StartJob (0.00s)
    --- PASS: TestRestApi/StopJob (0.00s)
    --- PASS: TestRestApi/CheckArchive (0.00s)
    --- PASS: TestRestApi/CheckDoubleStart (0.00s)
    --- PASS: TestRestApi/StartJobFailed (0.00s)
    --- PASS: TestRestApi/StopJobFailed (0.00s)
    --- PASS: TestRestApi/GetUsedNodesNoRunning (0.00s)
=== RUN   TestStopJobWithReusedJobId
<6>[INFO]     2026/03/16 11:52:41 migration.go:122: Legacy database without version or missing database file!
<6>[INFO]     2026/03/16 11:52:41 migration.go:129: unsupported database version 0, need 11.
Please backup your database file and run cc-backend -migrate-db
<6>[INFO]     2026/03/16 11:52:41 dbConnection.go:96: SQLite config: cache_size=2048MB/conn, soft_heap_limit=16384MB, max_conns=4
<6>[INFO]     2026/03/16 11:52:41 rest.go:88: Enabling REST start/stop job API
=== RUN   TestStopJobWithReusedJobId/StartFirstJob
<6>[INFO]     2026/03/16 11:52:41 job.go:779: new job (id: 1): cluster=testcluster, jobId=999, user=testuser, startTime=200000000
=== RUN   TestStopJobWithReusedJobId/StopFirstJobAsFailed
<4>[WARNING]  2026/03/16 11:52:42 job.go:162: Error while scanning rows (Job): sql: no rows in result set
<6>[INFO]     2026/03/16 11:52:42 job.go:1035: archiving job... (dbid: 1): cluster=testcluster, jobId=999, user=testuser, startTime=200000000, duration=1000, state=failed
<6>[INFO]     2026/03/16 11:52:42 archiveWorker.go:165: archiving job (dbid: 1) successful
=== RUN   TestStopJobWithReusedJobId/StartSecondJob
<6>[INFO]     2026/03/16 11:52:43 job.go:779: new job (id: 1): cluster=testcluster, jobId=999, user=testuser, startTime=300000000
=== RUN   TestStopJobWithReusedJobId/StopSecondJobBeforeSync
<6>[INFO]     2026/03/16 11:52:43 job.go:1026: transferred cached job to main table: old id 1 -> new id 2 (jobId=999)
<6>[INFO]     2026/03/16 11:52:43 job.go:1035: archiving job... (dbid: 2): cluster=testcluster, jobId=999, user=testuser, startTime=300000000, duration=1000, state=completed
<6>[INFO]     2026/03/16 11:52:43 archiveWorker.go:223: Initiating archiver shutdown...
<6>[INFO]     2026/03/16 11:52:43 archiveWorker.go:165: archiving job (dbid: 2) successful
<6>[INFO]     2026/03/16 11:52:43 archiveWorker.go:121: Archive channel closed, worker exiting
<6>[INFO]     2026/03/16 11:52:43 archiveWorker.go:238: All archive jobs completed successfully
--- PASS: TestStopJobWithReusedJobId (2.04s)
    --- PASS: TestStopJobWithReusedJobId/StartFirstJob (0.00s)
    --- PASS: TestStopJobWithReusedJobId/StopFirstJobAsFailed (0.00s)
    --- PASS: TestStopJobWithReusedJobId/StartSecondJob (0.00s)
    --- PASS: TestStopJobWithReusedJobId/StopSecondJobBeforeSync (0.00s)
PASS
ok  	github.com/ClusterCockpit/cc-backend/internal/api	7.036s


All tests pass. Here's a summary of the changes:

### `internal/config/config.go`
- Added `QueueGroup`, `JobConcurrency`, and `NodeConcurrency` fields to `NATSConfig`

### `internal/api/nats.go`
- **Replaced semaphore pattern with channel-based worker pool**: `jobSem`/`nodeSem` channels replaced by `jobCh`/`nodeCh` message channels with dedicated worker goroutines
- **Configurable concurrency**: `NewNatsAPI()` reads `JobConcurrency` (default 8) and `NodeConcurrency` (default 2) from config to set channel buffer size and worker count
- **Queue group support**: New `subscribe()` helper uses `SubscribeQueue()` when `QueueGroup` is configured, falls back to `Subscribe()` otherwise
- **Architecture**: NATS callbacks now push to buffered channels; N worker goroutines per channel consume and process messages, decoupling delivery from processing
- Removed semaphore acquire/release from `handleJobEvent` and `handleNodeState`


Remove Queue group support again


     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→package api
     7→
     8→import (
     9→	"database/sql"
    10→	"encoding/json"
    11→	"strings"
    12→	"sync"
    13→	"time"
    14→
    15→	"github.com/ClusterCockpit/cc-backend/internal/archiver"
    16→	"github.com/ClusterCockpit/cc-backend/internal/config"
    17→	"github.com/ClusterCockpit/cc-backend/internal/importer"
    18→	"github.com/ClusterCockpit/cc-backend/internal/repository"
    19→	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
    20→	lp "github.com/ClusterCockpit/cc-lib/v2/ccMessage"
    21→	"github.com/ClusterCockpit/cc-lib/v2/nats"
    22→	"github.com/ClusterCockpit/cc-lib/v2/receivers"
    23→	"github.com/ClusterCockpit/cc-lib/v2/schema"
    24→	influx "github.com/ClusterCockpit/cc-line-protocol/v2/lineprotocol"
    25→)
    26→
    27→// natsMessage wraps a raw NATS message with its subject for channel-based processing.
    28→type natsMessage struct {
    29→	subject string
    30→	data    []byte
    31→}
    32→
    33→// NatsAPI provides NATS subscription-based handlers for Job and Node operations.
    34→// It mirrors the functionality of the REST API but uses NATS messaging with
    35→// InfluxDB line protocol as the message format.
    36→//
    37→// # Message Format
    38→//
    39→// All NATS messages use InfluxDB line protocol format (https://docs.influxdata.com/influxdb/v2.0/reference/syntax/line-protocol/)
    40→// with the following structure:
    41→//
    42→//	measurement,tag1=value1,tag2=value2 field1=value1,field2=value2 timestamp
    43→//
    44→// # Job Events
    45→//
    46→// Job start/stop events use the "job" measurement with a "function" tag to distinguish operations:
    47→//
    48→//	job,function=start_job event="{...JSON payload...}" <timestamp>
    49→//	job,function=stop_job event="{...JSON payload...}" <timestamp>
    50→//
    51→// The JSON payload in the "event" field follows the schema.Job or StopJobAPIRequest structure.
    52→//
    53→// Example job start message:
    54→//
    55→//	job,function=start_job event="{\"jobId\":1001,\"user\":\"testuser\",\"cluster\":\"testcluster\",...}" 1234567890000000000
    56→//
    57→// # Node State Events
    58→//
    59→// Node state updates use the "nodestate" measurement with cluster information:
    60→//
    61→//	nodestate event="{...JSON payload...}" <timestamp>
    62→//
    63→// The JSON payload follows the UpdateNodeStatesRequest structure.
    64→//
    65→// Example node state message:
    66→//
    67→//	nodestate event="{\"cluster\":\"testcluster\",\"nodes\":[{\"hostname\":\"node01\",\"states\":[\"idle\"]}]}" 1234567890000000000
    68→type NatsAPI struct {
    69→	JobRepository *repository.JobRepository
    70→	// RepositoryMutex protects job creation operations from race conditions
    71→	// when checking for duplicate jobs during startJob calls.
    72→	RepositoryMutex sync.Mutex
    73→	// jobCh receives job event messages for processing by worker goroutines.
    74→	jobCh chan natsMessage
    75→	// nodeCh receives node state messages for processing by worker goroutines.
    76→	nodeCh chan natsMessage
    77→}
    78→
    79→// NewNatsAPI creates a new NatsAPI instance with channel-based worker pools.
    80→// Concurrency is configured via NATSConfig (defaults: JobConcurrency=8, NodeConcurrency=2).
    81→func NewNatsAPI() *NatsAPI {
    82→	jobConc := 8
    83→	nodeConc := 2
    84→
    85→	if s := config.Keys.APISubjects; s != nil {
    86→		if s.JobConcurrency > 0 {
    87→			jobConc = s.JobConcurrency
    88→		}
    89→		if s.NodeConcurrency > 0 {
    90→			nodeConc = s.NodeConcurrency
    91→		}
    92→	}
    93→
    94→	api := &NatsAPI{
    95→		JobRepository: repository.GetJobRepository(),
    96→		jobCh:         make(chan natsMessage, jobConc),
    97→		nodeCh:        make(chan natsMessage, nodeConc),
    98→	}
    99→
   100→	// Start worker goroutines
   101→	for range jobConc {
   102→		go api.jobWorker()
   103→	}
   104→	for range nodeConc {
   105→		go api.nodeWorker()
   106→	}
   107→
   108→	return api
   109→}
   110→
   111→// jobWorker processes job event messages from the job channel.
   112→func (api *NatsAPI) jobWorker() {
   113→	for msg := range api.jobCh {
   114→		api.handleJobEvent(msg.subject, msg.data)
   115→	}
   116→}
   117→
   118→// nodeWorker processes node state messages from the node channel.
   119→func (api *NatsAPI) nodeWorker() {
   120→	for msg := range api.nodeCh {
   121→		api.handleNodeState(msg.subject, msg.data)
   122→	}
   123→}
   124→
   125→// subscribe registers a NATS subscription using queue groups when configured.
   126→func subscribe(client *nats.Client, subject string, queueGroup string, handler func(string, []byte)) error {
   127→	if queueGroup != "" {
   128→		return client.SubscribeQueue(subject, queueGroup, handler)
   129→	}
   130→	return client.Subscribe(subject, handler)
   131→}
   132→
   133→// StartSubscriptions registers all NATS subscriptions for Job and Node APIs.
   134→// Messages are delivered to buffered channels and processed by worker goroutines.
   135→// Returns an error if the NATS client is not available or subscription fails.
   136→func (api *NatsAPI) StartSubscriptions() error {
   137→	client := nats.GetClient()
   138→	if client == nil {
   139→		cclog.Warn("NATS client not available, skipping API subscriptions")
   140→		return nil
   141→	}
   142→
   143→	if config.Keys.APISubjects != nil {
   144→		s := config.Keys.APISubjects
   145→		qg := s.QueueGroup
   146→
   147→		if err := subscribe(client, s.SubjectJobEvent, qg, func(subject string, data []byte) {
   148→			api.jobCh <- natsMessage{subject: subject, data: data}
   149→		}); err != nil {
   150→			return err
   151→		}
   152→
   153→		if err := subscribe(client, s.SubjectNodeState, qg, func(subject string, data []byte) {
   154→			api.nodeCh <- natsMessage{subject: subject, data: data}
   155→		}); err != nil {
   156→			return err
   157→		}
   158→
   159→		cclog.Info("NATS API subscriptions started")
   160→	}

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package api

import (
	"database/sql"
	"encoding/json"
	"strings"
	"sync"
	"time"

	"github.com/ClusterCockpit/cc-backend/internal/archiver"
	"github.com/ClusterCockpit/cc-backend/internal/config"
	"github.com/ClusterCockpit/cc-backend/internal/importer"
	"github.com/ClusterCockpit/cc-backend/internal/repository"
	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
	lp "github.com/ClusterCockpit/cc-lib/v2/ccMessage"
	"github.com/ClusterCockpit/cc-lib/v2/nats"
	"github.com/ClusterCockpit/cc-lib/v2/receivers"
	"github.com/ClusterCockpit/cc-lib/v2/schema"
	influx "github.com/ClusterCockpit/cc-line-protocol/v2/lineprotocol"
)

// natsMessage wraps a raw NATS message with its subject for channel-based processing.
type natsMessage struct {
	subject string
	data    []byte
}

// NatsAPI provides NATS subscription-based handlers for Job and Node operations.
// It mirrors the functionality of the REST API but uses NATS messaging with
// InfluxDB line protocol as the message format.
//
// # Message Format
//
// All NATS messages use InfluxDB line protocol format (https://docs.influxdata.com/influxdb/v2.0/reference/syntax/line-protocol/)
// with the following structure:
//
//	measurement,tag1=value1,tag2=value2 field1=value1,field2=value2 timestamp
//
// # Job Events
//
// Job start/stop events use the "job" measurement with a "function" tag to distinguish operations:
//
//	job,function=start_job event="{...JSON payload...}" <timestamp>
//	job,function=stop_job event="{...JSON payload...}" <timestamp>
//
// The JSON payload in the "event" field follows the schema.Job or StopJobAPIRequest structure.
//
// Example job start message:
//
//	job,function=start_job event="{\"jobId\":1001,\"user\":\"testuser\",\"cluster\":\"testcluster\",...}" 1234567890000000000
//
// # Node State Events
//
// Node state updates use the "nodestate" measurement with cluster information:
//
//	nodestate event="{...JSON payload...}" <timestamp>
//
// The JSON payload follows the UpdateNodeStatesRequest structure.
//
// Example node state message:
//
//	nodestate event="{\"cluster\":\"testcluster\",\"nodes\":[{\"hostname\":\"node01\",\"states\":[\"idle\"]}]}" 1234567890000000000
type NatsAPI struct {
	JobRepository *repository.JobRepository
	// RepositoryMutex protects job creation operations from race conditions
	// when checking for duplicate jobs during startJob calls.
	RepositoryMutex sync.Mutex
	// jobCh receives job event messages for processing by worker goroutines.
	jobCh chan natsMessage
	// nodeCh receives node state messages for processing by worker goroutines.
	nodeCh chan natsMessage
}

// NewNatsAPI creates a new NatsAPI instance with channel-based worker pools.
// Concurrency is configured via NATSConfig (defaults: JobConcurrency=8, NodeConcurrency=2).
func NewNatsAPI() *NatsAPI {
	jobConc := 8
	nodeConc := 2

	if s := config.Keys.APISubjects; s != nil {
		if s.JobConcurrency > 0 {
			jobConc = s.JobConcurrency
		}
		if s.NodeConcurrency > 0 {
			nodeConc = s.NodeConcurrency
		}
	}

	api := &NatsAPI{
		JobRepository: repository.GetJobRepository(),
		jobCh:         make(chan natsMessage, jobConc),
		nodeCh:        make(chan natsMessage, nodeConc),
	}

	// Start worker goroutines
	for range jobConc {
		go api.jobWorker()
	}
	for range nodeConc {
		go api.nodeWorker()
	}

	return api
}

// jobWorker processes job event messages from the job channel.
func (api *NatsAPI) jobWorker() {
	for msg := range api.jobCh {
		api.handleJobEvent(msg.subject, msg.data)
	}
}

// nodeWorker processes node state messages from the node channel.
func (api *NatsAPI) nodeWorker() {
	for msg := range api.nodeCh {
		api.handleNodeState(msg.subject, msg.data)
	}
}

// subscribe registers a NATS subscription using queue groups when configured.
func subscribe(client *nats.Client, subject string, queueGroup string, handler func(string, []byte)) error {
	if queueGroup != "" {
		return client.SubscribeQueue(subject, queueGroup, handler)
	}
	return client.Subscribe(subject, handler)
}

// StartSubscriptions registers all NATS subscriptions for Job and Node APIs.
// Messages are delivered to buffered channels and processed by worker goroutines.
// Returns an error if the NATS client is not available or subscription fails.
func (api *NatsAPI) StartSubscriptions() error {
	client := nats.GetClient()
	if client == nil {
		cclog.Warn("NATS client not available, skipping API subscriptions")
		return nil
	}

	if config.Keys.APISubjects != nil {
		s := config.Keys.APISubjects
		qg := s.QueueGroup

		if err := subscribe(client, s.SubjectJobEvent, qg, func(subject string, data []byte) {
			api.jobCh <- natsMessage{subject: subject, data: data}
		}); err != nil {
			return err
		}

		if err := subscribe(client, s.SubjectNodeState, qg, func(subject string, data []byte) {
			api.nodeCh <- natsMessage{subject: subject, data: data}
		}); err != nil {
			return err
		}

		cclog.Info("NATS API subscriptions started")
	}


Edit

	QueueGroup       string `json:"queue-group"`
	JobConcurrency   int    `json:"job-concurrency"`

JobConcurrency   int    `json:"job-concurrency"`


// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

// Package config implements the program configuration data structures, validation and parsing
package config

import (
	"bytes"
	"encoding/json"
	"time"

	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
	"github.com/ClusterCockpit/cc-lib/v2/resampler"
)

type ProgramConfig struct {
	// Address where the http (or https) server will listen on (for example: 'localhost:80').
	Addr string `json:"addr"`

	// Addresses from which secured admin API endpoints can be reached, can be wildcard "*"
	APIAllowedIPs []string `json:"api-allowed-ips"`

	APISubjects *NATSConfig `json:"api-subjects"`

	// Drop root permissions once .env was read and the port was taken.
	User  string `json:"user"`
	Group string `json:"group"`

	// Disable authentication (for everything: API, Web-UI, ...)
	DisableAuthentication bool `json:"disable-authentication"`

	// If `embed-static-files` is true (default), the frontend files are directly
	// embeded into the go binary and expected to be in web/frontend. Only if
	// it is false the files in `static-files` are served instead.
	EmbedStaticFiles bool   `json:"embed-static-files"`
	StaticFiles      string `json:"static-files"`

	// Path to SQLite database file
	DB string `json:"db"`

	EnableJobTaggers bool `json:"enable-job-taggers"`

	// Validate json input against schema
	Validate bool `json:"validate"`

	// If 0 or empty, the session does not expire!
	SessionMaxAge string `json:"session-max-age"`

	// If both those options are not empty, use HTTPS using those certificates.
	HTTPSCertFile string `json:"https-cert-file"`
	HTTPSKeyFile  string `json:"https-key-file"`

	// If not the empty string and `addr` does not end in ":80",
	// redirect every request incoming at port 80 to that url.
	RedirectHTTPTo string `json:"redirect-http-to"`

	// Where to store MachineState files
	MachineStateDir string `json:"machine-state-dir"`

	// If not zero, automatically mark jobs as stopped running X seconds longer than their walltime.
	StopJobsExceedingWalltime int `json:"stop-jobs-exceeding-walltime"`

	// Defines time X in seconds in which jobs are considered to be "short" and will be filtered in specific views.
	ShortRunningJobsDuration int `json:"short-running-jobs-duration"`

	// Energy Mix CO2 Emission Constant [g/kWh]
	// If entered, displays estimated CO2 emission for job based on jobs totalEnergy
	EmissionConstant int `json:"emission-constant"`

	// If exists, will enable dynamic zoom in frontend metric plots using the configured values
	EnableResampling *ResampleConfig `json:"resampling"`

	// Systemd unit name for log viewer (default: "clustercockpit")
	SystemdUnit string `json:"systemd-unit"`

	// Node state retention configuration
	NodeStateRetention *NodeStateRetention `json:"nodestate-retention"`

	// Database tuning configuration
	DbConfig *DbConfig `json:"db-config"`
}

type DbConfig struct {
	CacheSizeMB               int `json:"cache-size-mb"`
	SoftHeapLimitMB           int `json:"soft-heap-limit-mb"`
	MaxOpenConnections        int `json:"max-open-connections"`
	MaxIdleConnections        int `json:"max-idle-connections"`
	ConnectionMaxIdleTimeMins int `json:"max-idle-time-minutes"`
	BusyTimeoutMs             int `json:"busy-timeout-ms"`
}

type NodeStateRetention struct {
	Policy             string `json:"policy"`      // "delete" or "move"
	Age                int    `json:"age"`         // hours, default 24
	TargetKind         string `json:"target-kind"` // "file" or "s3"
	TargetPath         string `json:"target-path"`
	TargetEndpoint     string `json:"target-endpoint"`
	TargetBucket       string `json:"target-bucket"`
	TargetAccessKey    string `json:"target-access-key"`
	TargetSecretKey    string `json:"target-secret-key"`
	TargetRegion       string `json:"target-region"`
	TargetUsePathStyle bool   `json:"target-use-path-style"`
	MaxFileSizeMB      int    `json:"max-file-size-mb"`
}

type ResampleConfig struct {
	// Minimum number of points to trigger resampling of data
	MinimumPoints int `json:"minimum-points"`
	// Array of resampling target resolutions, in seconds; Example: [600,300,60]
	Resolutions []int `json:"resolutions"`
	// Trigger next zoom level at less than this many visible datapoints
	Trigger int `json:"trigger"`
}

type NATSConfig struct {
	SubjectJobEvent  string `json:"subject-job-event"`
	SubjectNodeState string `json:"subject-node-state"`
	QueueGroup       string `json:"queue-group"`
	JobConcurrency   int    `json:"job-concurrency"`
	NodeConcurrency  int    `json:"node-concurrency"`
}

type IntRange struct {
	From int `json:"from"`
	To   int `json:"to"`
}

type TimeRange struct {
	From  *time.Time `json:"from"`
	To    *time.Time `json:"to"`
	Range string     `json:"range,omitempty"`
}

type FilterRanges struct {
	Duration  *IntRange  `json:"duration"`
	NumNodes  *IntRange  `json:"num-nodes"`
	StartTime *TimeRange `json:"start-time"`
}

var Keys ProgramConfig = ProgramConfig{
	Addr:                      "localhost:8080",
	EmbedStaticFiles:          true,
	DB:                        "./var/job.db",
	SessionMaxAge:             "168h",
	StopJobsExceedingWalltime: 0,
	ShortRunningJobsDuration:  5 * 60,
}

func Init(mainConfig json.RawMessage) {
	Validate(configSchema, mainConfig)
	dec := json.NewDecoder(bytes.NewReader(mainConfig))
	dec.DisallowUnknownFields()
	if err := dec.Decode(&Keys); err != nil {
		cclog.Abortf("Config Init: Could not decode config file '%s'.\nError: %s\n", mainConfig, err.Error())
	}

	if Keys.EnableResampling != nil && Keys.EnableResampling.MinimumPoints > 0 {
		resampler.SetMinimumRequiredPoints(Keys.EnableResampling.MinimumPoints)
	}
}

-  QueueGroup       string `json:"queue-group"`


// subscribe registers a NATS subscription using queue groups when configured.
func subscribe(client *nats.Client, subject string, queueGroup string, handler func(string, []byte)) error {
	if queueGroup != "" {
		return client.SubscribeQueue(subject, queueGroup, handler)
	}
	return client.Subscribe(subject, handler)
}

// StartSubscriptions registers all NATS subscriptions for Job and Node APIs.
// Messages are delivered to buffered channels and processed by worker goroutines.
// Returns an error if the NATS client is not available or subscription fails.
func (api *NatsAPI) StartSubscriptions() error {
	client := nats.GetClient()
	if client == nil {
		cclog.Warn("NATS client not available, skipping API subscriptions")
		return nil
	}

	if config.Keys.APISubjects != nil {
		s := config.Keys.APISubjects
		qg := s.QueueGroup

		if err := subscribe(client, s.SubjectJobEvent, qg, func(subject string, data []byte) {
			api.jobCh <- natsMessage{subject: subject, data: data}
		}); err != nil {
			return err
		}

		if err := subscribe(client, s.SubjectNodeState, qg, func(subject string, data []byte) {
			api.nodeCh <- natsMessage{subject: subject, data: data}
		}); err != nil {
			return err
		}

		cclog.Info("NATS API subscriptions started")
	}

// StartSubscriptions registers all NATS subscriptions for Job and Node APIs.
// Messages are delivered to buffered channels and processed by worker goroutines.
// Returns an error if the NATS client is not available or subscription fails.
func (api *NatsAPI) StartSubscriptions() error {
	client := nats.GetClient()
	if client == nil {
		cclog.Warn("NATS client not available, skipping API subscriptions")
		return nil
	}

	if config.Keys.APISubjects != nil {
		s := config.Keys.APISubjects

		if err := client.Subscribe(s.SubjectJobEvent, func(subject string, data []byte) {
			api.jobCh <- natsMessage{subject: subject, data: data}
		}); err != nil {
			return err
		}

		if err := client.Subscribe(s.SubjectNodeState, func(subject string, data []byte) {
			api.nodeCh <- natsMessage{subject: subject, data: data}
		}); err != nil {
			return err
		}

		cclog.Info("NATS API subscriptions started")
	}