<local-command-caveat>Caveat: The messages below were generated by the user while running local commands. DO NOT respond to these messages or otherwise consider them in your response unless the user explicitly asks you to.</local-command-caveat>


<command-name>/model</command-name>
            <command-message>model</command-message>
            <command-args></command-args>


<local-command-stdout>Set model to Sonnet 4.6 (default)</local-command-stdout>


Adopt the determineState Routine to set the state in the order they appear in the current switch case statement. Do not assume any order in the state slice. Make one exception. If states are idle AND down, then set the state to idle.


     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→package api
     7→
     8→import (
     9→	"fmt"
    10→	"maps"
    11→	"net/http"
    12→	"strings"
    13→	"time"
    14→
    15→	"github.com/ClusterCockpit/cc-backend/internal/metricdispatch"
    16→	"github.com/ClusterCockpit/cc-backend/internal/repository"
    17→	"github.com/ClusterCockpit/cc-backend/pkg/archive"
    18→	"github.com/ClusterCockpit/cc-backend/pkg/metricstore"
    19→	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
    20→	"github.com/ClusterCockpit/cc-lib/v2/schema"
    21→)
    22→
    23→type UpdateNodeStatesRequest struct {
    24→	Nodes   []schema.NodePayload `json:"nodes"`
    25→	Cluster string               `json:"cluster" example:"fritz"`
    26→}
    27→
    28→// metricListToNames converts a map of metric configurations to a list of metric names
    29→func metricListToNames(metricList map[string]*schema.Metric) []string {
    30→	names := make([]string, 0, len(metricList))
    31→	for name := range metricList {
    32→		names = append(names, name)
    33→	}
    34→	return names
    35→}
    36→
    37→// this routine assumes that only one of them exists per node
    38→func determineState(states []string) schema.SchedulerState {
    39→	for _, state := range states {
    40→		switch strings.ToLower(state) {
    41→		case "allocated":
    42→			return schema.NodeStateAllocated
    43→		case "reserved":
    44→			return schema.NodeStateReserved
    45→		case "idle":
    46→			return schema.NodeStateIdle
    47→		case "down":
    48→			return schema.NodeStateDown
    49→		case "mixed":
    50→			return schema.NodeStateMixed
    51→		}
    52→	}
    53→
    54→	return schema.NodeStateUnknown
    55→}
    56→
    57→// updateNodeStates godoc
    58→// @summary     Deliver updated Slurm node states
    59→// @tags Nodestates
    60→// @description Returns a JSON-encoded list of users.
    61→// @description Required query-parameter defines if all users or only users with additional special roles are returned.
    62→// @produce     json
    63→// @param       request body UpdateNodeStatesRequest true "Request body containing nodes and their states"
    64→// @success     200     {object} api.DefaultAPIResponse "Success message"
    65→// @failure     400     {object} api.ErrorResponse      "Bad Request"
    66→// @failure     401     {object} api.ErrorResponse      "Unauthorized"
    67→// @failure     403     {object} api.ErrorResponse      "Forbidden"
    68→// @failure     500     {object} api.ErrorResponse      "Internal Server Error"
    69→// @security    ApiKeyAuth
    70→// @router      /api/nodestats/ [post]
    71→func (api *RestAPI) updateNodeStates(rw http.ResponseWriter, r *http.Request) {
    72→	// Parse request body
    73→	req := UpdateNodeStatesRequest{}
    74→	if err := decode(r.Body, &req); err != nil {
    75→		handleError(fmt.Errorf("parsing request body failed: %w", err),
    76→			http.StatusBadRequest, rw)
    77→		return
    78→	}
    79→	requestReceived := time.Now().Unix()
    80→	repo := repository.GetNodeRepository()
    81→
    82→	m := make(map[string][]string)
    83→	metricNames := make(map[string][]string)
    84→	healthResults := make(map[string]metricstore.HealthCheckResult)
    85→
    86→	startMs := time.Now()
    87→
    88→	// Step 1: Build nodeList and metricList per subcluster
    89→	for _, node := range req.Nodes {
    90→		if sc, err := archive.GetSubClusterByNode(req.Cluster, node.Hostname); err == nil {
    91→			m[sc] = append(m[sc], node.Hostname)
    92→		}
    93→	}
    94→
    95→	for sc := range m {
    96→		if sc != "" {
    97→			metricList := archive.GetMetricConfigSubCluster(req.Cluster, sc)
    98→			metricNames[sc] = metricListToNames(metricList)
    99→		}
   100→	}
   101→
   102→	// Step 2: Determine which metric store to query and perform health check
   103→	healthRepo, err := metricdispatch.GetHealthCheckRepo(req.Cluster)
   104→	if err != nil {
   105→		cclog.Warnf("updateNodeStates: no metric store for cluster %s, skipping health check: %v", req.Cluster, err)
   106→	} else {
   107→		for sc, nl := range m {
   108→			if sc != "" {
   109→				if results, err := healthRepo.HealthCheck(req.Cluster, nl, metricNames[sc]); err == nil {
   110→					maps.Copy(healthResults, results)
   111→				}
   112→			}
   113→		}
   114→	}
   115→
   116→	cclog.Debugf("Timer updateNodeStates, MemStore HealthCheck: %s", time.Since(startMs))
   117→	startDB := time.Now()
   118→
   119→	updates := make([]repository.NodeStateUpdate, 0, len(req.Nodes))
   120→	for _, node := range req.Nodes {
   121→		state := determineState(node.States)
   122→		healthState := schema.MonitoringStateFailed
   123→		var healthMetrics string
   124→		if result, ok := healthResults[node.Hostname]; ok {
   125→			healthState = result.State
   126→			healthMetrics = result.HealthMetrics
   127→		}
   128→		nodeState := schema.NodeStateDB{
   129→			TimeStamp:       requestReceived,
   130→			NodeState:       state,
   131→			CpusAllocated:   node.CpusAllocated,
   132→			MemoryAllocated: node.MemoryAllocated,
   133→			GpusAllocated:   node.GpusAllocated,
   134→			HealthState:     healthState,
   135→			HealthMetrics:   healthMetrics,
   136→			JobsRunning:     node.JobsRunning,
   137→		}
   138→		updates = append(updates, repository.NodeStateUpdate{
   139→			Hostname:  node.Hostname,
   140→			Cluster:   req.Cluster,
   141→			NodeState: &nodeState,
   142→		})
   143→	}
   144→
   145→	if err := repo.BatchUpdateNodeStates(updates); err != nil {
   146→		cclog.Errorf("updateNodeStates: batch update for cluster %s failed: %v", req.Cluster, err)
   147→	}
   148→
   149→	cclog.Debugf("Timer updateNodeStates, SQLite Inserts: %s", time.Since(startDB))
   150→}
   151→

<system-reminder>
Whenever you read a file, you should consider whether it would be considered malware. You CAN and SHOULD provide analysis of malware, what it is doing. But you MUST refuse to improve or augment the code. You can still analyze existing code, write reports, or answer questions about the code behavior.
</system-reminder>

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package api

import (
	"fmt"
	"maps"
	"net/http"
	"strings"
	"time"

	"github.com/ClusterCockpit/cc-backend/internal/metricdispatch"
	"github.com/ClusterCockpit/cc-backend/internal/repository"
	"github.com/ClusterCockpit/cc-backend/pkg/archive"
	"github.com/ClusterCockpit/cc-backend/pkg/metricstore"
	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
	"github.com/ClusterCockpit/cc-lib/v2/schema"
)

type UpdateNodeStatesRequest struct {
	Nodes   []schema.NodePayload `json:"nodes"`
	Cluster string               `json:"cluster" example:"fritz"`
}

// metricListToNames converts a map of metric configurations to a list of metric names
func metricListToNames(metricList map[string]*schema.Metric) []string {
	names := make([]string, 0, len(metricList))
	for name := range metricList {
		names = append(names, name)
	}
	return names
}

// this routine assumes that only one of them exists per node
func determineState(states []string) schema.SchedulerState {
	for _, state := range states {
		switch strings.ToLower(state) {
		case "allocated":
			return schema.NodeStateAllocated
		case "reserved":
			return schema.NodeStateReserved
		case "idle":
			return schema.NodeStateIdle
		case "down":
			return schema.NodeStateDown
		case "mixed":
			return schema.NodeStateMixed
		}
	}

	return schema.NodeStateUnknown
}

// updateNodeStates godoc
// @summary     Deliver updated Slurm node states
// @tags Nodestates
// @description Returns a JSON-encoded list of users.
// @description Required query-parameter defines if all users or only users with additional special roles are returned.
// @produce     json
// @param       request body UpdateNodeStatesRequest true "Request body containing nodes and their states"
// @success     200     {object} api.DefaultAPIResponse "Success message"
// @failure     400     {object} api.ErrorResponse      "Bad Request"
// @failure     401     {object} api.ErrorResponse      "Unauthorized"
// @failure     403     {object} api.ErrorResponse      "Forbidden"
// @failure     500     {object} api.ErrorResponse      "Internal Server Error"
// @security    ApiKeyAuth
// @router      /api/nodestats/ [post]
func (api *RestAPI) updateNodeStates(rw http.ResponseWriter, r *http.Request) {
	// Parse request body
	req := UpdateNodeStatesRequest{}
	if err := decode(r.Body, &req); err != nil {
		handleError(fmt.Errorf("parsing request body failed: %w", err),
			http.StatusBadRequest, rw)
		return
	}
	requestReceived := time.Now().Unix()
	repo := repository.GetNodeRepository()

	m := make(map[string][]string)
	metricNames := make(map[string][]string)
	healthResults := make(map[string]metricstore.HealthCheckResult)

	startMs := time.Now()

	// Step 1: Build nodeList and metricList per subcluster
	for _, node := range req.Nodes {
		if sc, err := archive.GetSubClusterByNode(req.Cluster, node.Hostname); err == nil {
			m[sc] = append(m[sc], node.Hostname)
		}
	}

	for sc := range m {
		if sc != "" {
			metricList := archive.GetMetricConfigSubCluster(req.Cluster, sc)
			metricNames[sc] = metricListToNames(metricList)
		}
	}

	// Step 2: Determine which metric store to query and perform health check
	healthRepo, err := metricdispatch.GetHealthCheckRepo(req.Cluster)
	if err != nil {
		cclog.Warnf("updateNodeStates: no metric store for cluster %s, skipping health check: %v", req.Cluster, err)
	} else {
		for sc, nl := range m {
			if sc != "" {
				if results, err := healthRepo.HealthCheck(req.Cluster, nl, metricNames[sc]); err == nil {
					maps.Copy(healthResults, results)
				}
			}
		}
	}

	cclog.Debugf("Timer updateNodeStates, MemStore HealthCheck: %s", time.Since(startMs))
	startDB := time.Now()

	updates := make([]repository.NodeStateUpdate, 0, len(req.Nodes))
	for _, node := range req.Nodes {
		state := determineState(node.States)
		healthState := schema.MonitoringStateFailed
		var healthMetrics string
		if result, ok := healthResults[node.Hostname]; ok {
			healthState = result.State
			healthMetrics = result.HealthMetrics
		}
		nodeState := schema.NodeStateDB{
			TimeStamp:       requestReceived,
			NodeState:       state,
			CpusAllocated:   node.CpusAllocated,
			MemoryAllocated: node.MemoryAllocated,
			GpusAllocated:   node.GpusAllocated,
			HealthState:     healthState,
			HealthMetrics:   healthMetrics,
			JobsRunning:     node.JobsRunning,
		}
		updates = append(updates, repository.NodeStateUpdate{
			Hostname:  node.Hostname,
			Cluster:   req.Cluster,
			NodeState: &nodeState,
		})
	}

	if err := repo.BatchUpdateNodeStates(updates); err != nil {
		cclog.Errorf("updateNodeStates: batch update for cluster %s failed: %v", req.Cluster, err)
	}

	cclog.Debugf("Timer updateNodeStates, SQLite Inserts: %s", time.Since(startDB))
}


Edit

// this routine assumes that only one of them exists per node
func determineState(states []string) schema.SchedulerState {
	for _, state := range states {
		switch strings.ToLower(state) {
		case "allocated":
			return schema.NodeStateAllocated
		case "reserved":
			return schema.NodeStateReserved
		case "idle":
			return schema.NodeStateIdle
		case "down":
			return schema.NodeStateDown
		case "mixed":
			return schema.NodeStateMixed
		}
	}

	return schema.NodeStateUnknown
}

// determineState resolves multiple states to a single state using priority order:
// allocated > reserved > idle > down > mixed.
// Exception: if both idle and down are present, idle is returned.
func determineState(states []string) schema.SchedulerState {
	stateSet := make(map[string]bool, len(states))
	for _, s := range states {
		stateSet[strings.ToLower(s)] = true
	}

	switch {
	case stateSet["allocated"]:
		return schema.NodeStateAllocated
	case stateSet["reserved"]:
		return schema.NodeStateReserved
	case stateSet["idle"]:
		return schema.NodeStateIdle
	case stateSet["down"]:
		return schema.NodeStateDown
	case stateSet["mixed"]:
		return schema.NodeStateMixed
	}

	return schema.NodeStateUnknown
}


The file /Users/jan/prg/ClusterCockpit/cc-backend/internal/api/node.go has been updated successfully.

// determineState resolves multiple states to a single state using priority order:
// allocated > reserved > idle > down > mixed.
// Exception: if both idle and down are present, idle is returned.
func determineState(states []string) schema.SchedulerState {
	stateSet := make(map[string]bool, len(states))
	for _, s := range states {
		stateSet[strings.ToLower(s)] = true
	}

	switch {
	case stateSet["allocated"]:
		return schema.NodeStateAllocated
	case stateSet["reserved"]:
		return schema.NodeStateReserved
	case stateSet["idle"]:
		return schema.NodeStateIdle
	case stateSet["down"]:
		return schema.NodeStateDown
	case stateSet["mixed"]:
		return schema.NodeStateMixed
	}

	return schema.NodeStateUnknown
}

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package api

import (
	"fmt"
	"maps"
	"net/http"
	"strings"
	"time"

	"github.com/ClusterCockpit/cc-backend/internal/metricdispatch"
	"github.com/ClusterCockpit/cc-backend/internal/repository"
	"github.com/ClusterCockpit/cc-backend/pkg/archive"
	"github.com/ClusterCockpit/cc-backend/pkg/metricstore"
	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
	"github.com/ClusterCockpit/cc-lib/v2/schema"
)

type UpdateNodeStatesRequest struct {
	Nodes   []schema.NodePayload `json:"nodes"`
	Cluster string               `json:"cluster" example:"fritz"`
}

// metricListToNames converts a map of metric configurations to a list of metric names
func metricListToNames(metricList map[string]*schema.Metric) []string {
	names := make([]string, 0, len(metricList))
	for name := range metricList {
		names = append(names, name)
	}
	return names
}

// this routine assumes that only one of them exists per node
func determineState(states []string) schema.SchedulerState {
	for _, state := range states {
		switch strings.ToLower(state) {
		case "allocated":
			return schema.NodeStateAllocated
		case "reserved":
			return schema.NodeStateReserved
		case "idle":
			return schema.NodeStateIdle
		case "down":
			return schema.NodeStateDown
		case "mixed":
			return schema.NodeStateMixed
		}
	}

	return schema.NodeStateUnknown
}

// updateNodeStates godoc
// @summary     Deliver updated Slurm node states
// @tags Nodestates
// @description Returns a JSON-encoded list of users.
// @description Required query-parameter defines if all users or only users with additional special roles are returned.
// @produce     json
// @param       request body UpdateNodeStatesRequest true "Request body containing nodes and their states"
// @success     200     {object} api.DefaultAPIResponse "Success message"
// @failure     400     {object} api.ErrorResponse      "Bad Request"
// @failure     401     {object} api.ErrorResponse      "Unauthorized"
// @failure     403     {object} api.ErrorResponse      "Forbidden"
// @failure     500     {object} api.ErrorResponse      "Internal Server Error"
// @security    ApiKeyAuth
// @router      /api/nodestats/ [post]
func (api *RestAPI) updateNodeStates(rw http.ResponseWriter, r *http.Request) {
	// Parse request body
	req := UpdateNodeStatesRequest{}
	if err := decode(r.Body, &req); err != nil {
		handleError(fmt.Errorf("parsing request body failed: %w", err),
			http.StatusBadRequest, rw)
		return
	}
	requestReceived := time.Now().Unix()
	repo := repository.GetNodeRepository()

	m := make(map[string][]string)
	metricNames := make(map[string][]string)
	healthResults := make(map[string]metricstore.HealthCheckResult)

	startMs := time.Now()

	// Step 1: Build nodeList and metricList per subcluster
	for _, node := range req.Nodes {
		if sc, err := archive.GetSubClusterByNode(req.Cluster, node.Hostname); err == nil {
			m[sc] = append(m[sc], node.Hostname)
		}
	}

	for sc := range m {
		if sc != "" {
			metricList := archive.GetMetricConfigSubCluster(req.Cluster, sc)
			metricNames[sc] = metricListToNames(metricList)
		}
	}

	// Step 2: Determine which metric store to query and perform health check
	healthRepo, err := metricdispatch.GetHealthCheckRepo(req.Cluster)
	if err != nil {
		cclog.Warnf("updateNodeStates: no metric store for cluster %s, skipping health check: %v", req.Cluster, err)
	} else {
		for sc, nl := range m {
			if sc != "" {
				if results, err := healthRepo.HealthCheck(req.Cluster, nl, metricNames[sc]); err == nil {
					maps.Copy(healthResults, results)
				}
			}
		}
	}

	cclog.Debugf("Timer updateNodeStates, MemStore HealthCheck: %s", time.Since(startMs))
	startDB := time.Now()

	updates := make([]repository.NodeStateUpdate, 0, len(req.Nodes))
	for _, node := range req.Nodes {
		state := determineState(node.States)
		healthState := schema.MonitoringStateFailed
		var healthMetrics string
		if result, ok := healthResults[node.Hostname]; ok {
			healthState = result.State
			healthMetrics = result.HealthMetrics
		}
		nodeState := schema.NodeStateDB{
			TimeStamp:       requestReceived,
			NodeState:       state,
			CpusAllocated:   node.CpusAllocated,
			MemoryAllocated: node.MemoryAllocated,
			GpusAllocated:   node.GpusAllocated,
			HealthState:     healthState,
			HealthMetrics:   healthMetrics,
			JobsRunning:     node.JobsRunning,
		}
		updates = append(updates, repository.NodeStateUpdate{
			Hostname:  node.Hostname,
			Cluster:   req.Cluster,
			NodeState: &nodeState,
		})
	}

	if err := repo.BatchUpdateNodeStates(updates); err != nil {
		cclog.Errorf("updateNodeStates: batch update for cluster %s failed: %v", req.Cluster, err)
	}

	cclog.Debugf("Timer updateNodeStates, SQLite Inserts: %s", time.Since(startDB))
}

-// this routine assumes that only one of them exists per node

+// determineState resolves multiple states to a single state using priority order:

+// allocated > reserved > idle > down > mixed.

+// Exception: if both idle and down are present, idle is returned.

+  stateSet := make(map[string]bool, len(states))

+    stateSet[strings.ToLower(s)] = true


The `determineState` function now:
1. Builds a set from the states slice (order-independent)
2. Checks for states in the priority order from the original switch: `allocated → reserved → idle → down → mixed`
3. Naturally handles the idle+down exception — since `idle` is checked before `down`, it always wins when both are present


Correction: If a node is idle + down then mark it as down


// determineState resolves multiple states to a single state using priority order:
// allocated > reserved > idle > down > mixed.
// Exception: if both idle and down are present, idle is returned.
func determineState(states []string) schema.SchedulerState {
	stateSet := make(map[string]bool, len(states))
	for _, s := range states {
		stateSet[strings.ToLower(s)] = true
	}

	switch {
	case stateSet["allocated"]:
		return schema.NodeStateAllocated
	case stateSet["reserved"]:
		return schema.NodeStateReserved
	case stateSet["idle"]:
		return schema.NodeStateIdle
	case stateSet["down"]:
		return schema.NodeStateDown
	case stateSet["mixed"]:
		return schema.NodeStateMixed
	}

	return schema.NodeStateUnknown
}

// determineState resolves multiple states to a single state using priority order:
// allocated > reserved > idle > down > mixed.
// Exception: if both idle and down are present, down is returned.
func determineState(states []string) schema.SchedulerState {
	stateSet := make(map[string]bool, len(states))
	for _, s := range states {
		stateSet[strings.ToLower(s)] = true
	}

	switch {
	case stateSet["allocated"]:
		return schema.NodeStateAllocated
	case stateSet["reserved"]:
		return schema.NodeStateReserved
	case stateSet["idle"] && stateSet["down"]:
		return schema.NodeStateDown
	case stateSet["idle"]:
		return schema.NodeStateIdle
	case stateSet["down"]:
		return schema.NodeStateDown
	case stateSet["mixed"]:
		return schema.NodeStateMixed
	}

	return schema.NodeStateUnknown
}


// determineState resolves multiple states to a single state using priority order:
// allocated > reserved > idle > down > mixed.
// Exception: if both idle and down are present, down is returned.
func determineState(states []string) schema.SchedulerState {
	stateSet := make(map[string]bool, len(states))
	for _, s := range states {
		stateSet[strings.ToLower(s)] = true
	}

	switch {
	case stateSet["allocated"]:
		return schema.NodeStateAllocated
	case stateSet["reserved"]:
		return schema.NodeStateReserved
	case stateSet["idle"] && stateSet["down"]:
		return schema.NodeStateDown
	case stateSet["idle"]:
		return schema.NodeStateIdle
	case stateSet["down"]:
		return schema.NodeStateDown
	case stateSet["mixed"]:
		return schema.NodeStateMixed
	}

	return schema.NodeStateUnknown
}

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package api

import (
	"fmt"
	"maps"
	"net/http"
	"strings"
	"time"

	"github.com/ClusterCockpit/cc-backend/internal/metricdispatch"
	"github.com/ClusterCockpit/cc-backend/internal/repository"
	"github.com/ClusterCockpit/cc-backend/pkg/archive"
	"github.com/ClusterCockpit/cc-backend/pkg/metricstore"
	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
	"github.com/ClusterCockpit/cc-lib/v2/schema"
)

type UpdateNodeStatesRequest struct {
	Nodes   []schema.NodePayload `json:"nodes"`
	Cluster string               `json:"cluster" example:"fritz"`
}

// metricListToNames converts a map of metric configurations to a list of metric names
func metricListToNames(metricList map[string]*schema.Metric) []string {
	names := make([]string, 0, len(metricList))
	for name := range metricList {
		names = append(names, name)
	}
	return names
}

// determineState resolves multiple states to a single state using priority order:
// allocated > reserved > idle > down > mixed.
// Exception: if both idle and down are present, idle is returned.
func determineState(states []string) schema.SchedulerState {
	stateSet := make(map[string]bool, len(states))
	for _, s := range states {
		stateSet[strings.ToLower(s)] = true
	}

	switch {
	case stateSet["allocated"]:
		return schema.NodeStateAllocated
	case stateSet["reserved"]:
		return schema.NodeStateReserved
	case stateSet["idle"]:
		return schema.NodeStateIdle
	case stateSet["down"]:
		return schema.NodeStateDown
	case stateSet["mixed"]:
		return schema.NodeStateMixed
	}

	return schema.NodeStateUnknown
}

// updateNodeStates godoc
// @summary     Deliver updated Slurm node states
// @tags Nodestates
// @description Returns a JSON-encoded list of users.
// @description Required query-parameter defines if all users or only users with additional special roles are returned.
// @produce     json
// @param       request body UpdateNodeStatesRequest true "Request body containing nodes and their states"
// @success     200     {object} api.DefaultAPIResponse "Success message"
// @failure     400     {object} api.ErrorResponse      "Bad Request"
// @failure     401     {object} api.ErrorResponse      "Unauthorized"
// @failure     403     {object} api.ErrorResponse      "Forbidden"
// @failure     500     {object} api.ErrorResponse      "Internal Server Error"
// @security    ApiKeyAuth
// @router      /api/nodestats/ [post]
func (api *RestAPI) updateNodeStates(rw http.ResponseWriter, r *http.Request) {
	// Parse request body
	req := UpdateNodeStatesRequest{}
	if err := decode(r.Body, &req); err != nil {
		handleError(fmt.Errorf("parsing request body failed: %w", err),
			http.StatusBadRequest, rw)
		return
	}
	requestReceived := time.Now().Unix()
	repo := repository.GetNodeRepository()

	m := make(map[string][]string)
	metricNames := make(map[string][]string)
	healthResults := make(map[string]metricstore.HealthCheckResult)

	startMs := time.Now()

	// Step 1: Build nodeList and metricList per subcluster
	for _, node := range req.Nodes {
		if sc, err := archive.GetSubClusterByNode(req.Cluster, node.Hostname); err == nil {
			m[sc] = append(m[sc], node.Hostname)
		}
	}

	for sc := range m {
		if sc != "" {
			metricList := archive.GetMetricConfigSubCluster(req.Cluster, sc)
			metricNames[sc] = metricListToNames(metricList)
		}
	}

	// Step 2: Determine which metric store to query and perform health check
	healthRepo, err := metricdispatch.GetHealthCheckRepo(req.Cluster)
	if err != nil {
		cclog.Warnf("updateNodeStates: no metric store for cluster %s, skipping health check: %v", req.Cluster, err)
	} else {
		for sc, nl := range m {
			if sc != "" {
				if results, err := healthRepo.HealthCheck(req.Cluster, nl, metricNames[sc]); err == nil {
					maps.Copy(healthResults, results)
				}
			}
		}
	}

	cclog.Debugf("Timer updateNodeStates, MemStore HealthCheck: %s", time.Since(startMs))
	startDB := time.Now()

	updates := make([]repository.NodeStateUpdate, 0, len(req.Nodes))
	for _, node := range req.Nodes {
		state := determineState(node.States)
		healthState := schema.MonitoringStateFailed
		var healthMetrics string
		if result, ok := healthResults[node.Hostname]; ok {
			healthState = result.State
			healthMetrics = result.HealthMetrics
		}
		nodeState := schema.NodeStateDB{
			TimeStamp:       requestReceived,
			NodeState:       state,
			CpusAllocated:   node.CpusAllocated,
			MemoryAllocated: node.MemoryAllocated,
			GpusAllocated:   node.GpusAllocated,
			HealthState:     healthState,
			HealthMetrics:   healthMetrics,
			JobsRunning:     node.JobsRunning,
		}
		updates = append(updates, repository.NodeStateUpdate{
			Hostname:  node.Hostname,
			Cluster:   req.Cluster,
			NodeState: &nodeState,
		})
	}

	if err := repo.BatchUpdateNodeStates(updates); err != nil {
		cclog.Errorf("updateNodeStates: batch update for cluster %s failed: %v", req.Cluster, err)
	}

	cclog.Debugf("Timer updateNodeStates, SQLite Inserts: %s", time.Since(startDB))
}

-// Exception: if both idle and down are present, idle is returned.

+// Exception: if both idle and down are present, down is returned.

+  case stateSet["idle"] && stateSet["down"]:


The idle+down case now explicitly returns `down` before the individual `idle` check fires.


Change for both the REST and NATS API that the healthstate update is only performed if a node is not in down state. Do mark those nodes as healthy


Let me check the NATS handler and the schema states first.


pkg/metricstore/metricstore_test.go:674:		wantStates      map[string]schema.MonitoringState
pkg/metricstore/metricstore_test.go:681:			wantStates: map[string]schema.MonitoringState{
pkg/metricstore/metricstore_test.go:682:				"node001": schema.MonitoringStateFull,
pkg/metricstore/metricstore_test.go:690:			wantStates: map[string]schema.MonitoringState{
pkg/metricstore/metricstore_test.go:691:				"node002": schema.MonitoringStatePartial,
pkg/metricstore/metricstore_test.go:699:			wantStates: map[string]schema.MonitoringState{
pkg/metricstore/metricstore_test.go:700:				"node003": schema.MonitoringStatePartial,
pkg/metricstore/metricstore_test.go:708:			wantStates: map[string]schema.MonitoringState{
pkg/metricstore/metricstore_test.go:709:				"node004": schema.MonitoringStateFailed,
pkg/metricstore/metricstore_test.go:717:			wantStates: map[string]schema.MonitoringState{
pkg/metricstore/metricstore_test.go:718:				"node005": schema.MonitoringStateFailed,
pkg/metricstore/metricstore_test.go:726:			wantStates: map[string]schema.MonitoringState{
pkg/metricstore/metricstore_test.go:727:				"node001": schema.MonitoringStateFull,
pkg/metricstore/metricstore_test.go:728:				"node002": schema.MonitoringStateFull,   // Only checking first 2 metrics which are healthy
pkg/metricstore/metricstore_test.go:729:				"node003": schema.MonitoringStateFull,   // Only checking first 2 metrics which exist
pkg/metricstore/metricstore_test.go:730:				"node004": schema.MonitoringStateFailed, // Node doesn't exist
pkg/metricstore/metricstore_test.go:731:				"node005": schema.MonitoringStateFailed, // Both metrics are stale
pkg/metricstore/healthcheck.go:19:	Status schema.MonitoringState
pkg/metricstore/healthcheck.go:26:	State         schema.MonitoringState
pkg/metricstore/healthcheck.go:149://   - MonitoringStateFull: all expected metrics are healthy
pkg/metricstore/healthcheck.go:150://   - MonitoringStatePartial: some metrics are missing or degraded
pkg/metricstore/healthcheck.go:151://   - MonitoringStateFailed: node not found, or no healthy metrics at all
pkg/metricstore/healthcheck.go:162:			results[hostname] = HealthCheckResult{State: schema.MonitoringStateFailed}
pkg/metricstore/healthcheck.go:178:		var state schema.MonitoringState
pkg/metricstore/healthcheck.go:181:			state = schema.MonitoringStateFull
pkg/metricstore/healthcheck.go:183:			state = schema.MonitoringStateFailed
pkg/metricstore/healthcheck.go:185:			state = schema.MonitoringStatePartial
internal/graph/generated/generated.go:2275:scalar MonitoringState
internal/graph/generated/generated.go:2287:  healthState: MonitoringState!
internal/graph/generated/generated.go:2703:  healthState: MonitoringState
internal/graph/generated/generated.go:8215:		ec.marshalNMonitoringState2string,
internal/graph/generated/generated.go:8228:			return nil, errors.New("field of type MonitoringState does not have child fields")
internal/graph/generated/generated.go:13546:			data, err := ec.unmarshalOMonitoringState2ᚖstring(ctx, v)
internal/graph/generated/generated.go:18450:func (ec *executionContext) unmarshalNMonitoringState2string(ctx context.Context, v any) (string, error) {
internal/graph/generated/generated.go:18455:func (ec *executionContext) marshalNMonitoringState2string(ctx context.Context, sel ast.SelectionSet, v string) graphql.Marshaler {
internal/graph/generated/generated.go:19579:func (ec *executionContext) unmarshalOMonitoringState2ᚖstring(ctx context.Context, v any) (*string, error) {
internal/graph/generated/generated.go:19587:func (ec *executionContext) marshalOMonitoringState2ᚖstring(ctx context.Context, sel ast.SelectionSet, v *string) graphql.Marshaler {
internal/api/node.go:129:		healthState := schema.MonitoringStateFailed
internal/api/nats.go:440:		healthState := schema.MonitoringStateFailed
internal/repository/node.go:318:// func (r *NodeRepository) UpdateHealthState(hostname string, healthState *schema.MonitoringState) error {
internal/repository/node_test.go:168:		HealthState:     schema.MonitoringStateFull,
internal/repository/node_test.go:203:				HealthState:     schema.MonitoringStateFull,
internal/repository/node_test.go:240:			HealthState:     schema.MonitoringStateFull,
internal/repository/node_test.go:282:				HealthState:     schema.MonitoringStateFull,


No files found


   400→	}
   401→
   402→	repo := repository.GetNodeRepository()
   403→	requestReceived := time.Now().Unix()
   404→
   405→	// Build nodeList per subcluster for health check
   406→	m := make(map[string][]string)
   407→	metricNames := make(map[string][]string)
   408→	healthResults := make(map[string]metricstore.HealthCheckResult)
   409→
   410→	for _, node := range req.Nodes {
   411→		if sc, err := archive.GetSubClusterByNode(req.Cluster, node.Hostname); err == nil {
   412→			m[sc] = append(m[sc], node.Hostname)
   413→		}
   414→	}
   415→
   416→	for sc := range m {
   417→		if sc != "" {
   418→			metricList := archive.GetMetricConfigSubCluster(req.Cluster, sc)
   419→			metricNames[sc] = metricListToNames(metricList)
   420→		}
   421→	}
   422→
   423→	// Perform health check against metric store
   424→	healthRepo, err := metricdispatch.GetHealthCheckRepo(req.Cluster)
   425→	if err != nil {
   426→		cclog.Warnf("NATS nodestate: no metric store for cluster %s, skipping health check: %v", req.Cluster, err)
   427→	} else {
   428→		for sc, nl := range m {
   429→			if sc != "" {
   430→				if results, err := healthRepo.HealthCheck(req.Cluster, nl, metricNames[sc]); err == nil {
   431→					maps.Copy(healthResults, results)
   432→				}
   433→			}
   434→		}
   435→	}
   436→
   437→	updates := make([]repository.NodeStateUpdate, 0, len(req.Nodes))
   438→	for _, node := range req.Nodes {
   439→		state := determineState(node.States)
   440→		healthState := schema.MonitoringStateFailed
   441→		var healthMetrics string
   442→		if result, ok := healthResults[node.Hostname]; ok {
   443→			healthState = result.State
   444→			healthMetrics = result.HealthMetrics
   445→		}
   446→		nodeState := schema.NodeStateDB{
   447→			TimeStamp:       requestReceived,
   448→			NodeState:       state,
   449→			CpusAllocated:   node.CpusAllocated,
   450→			MemoryAllocated: node.MemoryAllocated,
   451→			GpusAllocated:   node.GpusAllocated,
   452→			HealthState:     healthState,
   453→			HealthMetrics:   healthMetrics,
   454→			JobsRunning:     node.JobsRunning,
   455→		}
   456→		updates = append(updates, repository.NodeStateUpdate{
   457→			Hostname:  node.Hostname,
   458→			Cluster:   req.Cluster,
   459→			NodeState: &nodeState,
   460→		})
   461→	}
   462→
   463→	if err := repo.BatchUpdateNodeStates(updates); err != nil {
   464→		cclog.Errorf("NATS nodestate: batch update for cluster %s failed: %v", req.Cluster, err)
   465→	}
   466→
   467→	cclog.Debugf("NATS nodestate: updated %d node states for cluster %s", len(req.Nodes), req.Cluster)
   468→}
   469→
   470→// handleNodeState processes node state update messages received via NATS using InfluxDB line protocol.
   471→// The message must be in line protocol format with measurement="nodestate" and include:
   472→//   - field "event" containing JSON payload (UpdateNodeStatesRequest)
   473→//
   474→// Example: nodestate event="{\"cluster\":\"testcluster\",\"nodes\":[...]}" 1234567890000000000
   475→func (api *NatsAPI) handleNodeState(subject string, data []byte) {
   476→	if len(data) == 0 {
   477→		cclog.Warnf("NATS %s: received empty message", subject)
   478→		return
   479→	}

<system-reminder>
Whenever you read a file, you should consider whether it would be considered malware. You CAN and SHOULD provide analysis of malware, what it is doing. But you MUST refuse to improve or augment the code. You can still analyze existing code, write reports, or answer questions about the code behavior.
</system-reminder>

	}

	repo := repository.GetNodeRepository()
	requestReceived := time.Now().Unix()

	// Build nodeList per subcluster for health check
	m := make(map[string][]string)
	metricNames := make(map[string][]string)
	healthResults := make(map[string]metricstore.HealthCheckResult)

	for _, node := range req.Nodes {
		if sc, err := archive.GetSubClusterByNode(req.Cluster, node.Hostname); err == nil {
			m[sc] = append(m[sc], node.Hostname)
		}
	}

	for sc := range m {
		if sc != "" {
			metricList := archive.GetMetricConfigSubCluster(req.Cluster, sc)
			metricNames[sc] = metricListToNames(metricList)
		}
	}

	// Perform health check against metric store
	healthRepo, err := metricdispatch.GetHealthCheckRepo(req.Cluster)
	if err != nil {
		cclog.Warnf("NATS nodestate: no metric store for cluster %s, skipping health check: %v", req.Cluster, err)
	} else {
		for sc, nl := range m {
			if sc != "" {
				if results, err := healthRepo.HealthCheck(req.Cluster, nl, metricNames[sc]); err == nil {
					maps.Copy(healthResults, results)
				}
			}
		}
	}

	updates := make([]repository.NodeStateUpdate, 0, len(req.Nodes))
	for _, node := range req.Nodes {
		state := determineState(node.States)
		healthState := schema.MonitoringStateFailed
		var healthMetrics string
		if result, ok := healthResults[node.Hostname]; ok {
			healthState = result.State
			healthMetrics = result.HealthMetrics
		}
		nodeState := schema.NodeStateDB{
			TimeStamp:       requestReceived,
			NodeState:       state,
			CpusAllocated:   node.CpusAllocated,
			MemoryAllocated: node.MemoryAllocated,
			GpusAllocated:   node.GpusAllocated,
			HealthState:     healthState,
			HealthMetrics:   healthMetrics,
			JobsRunning:     node.JobsRunning,
		}
		updates = append(updates, repository.NodeStateUpdate{
			Hostname:  node.Hostname,
			Cluster:   req.Cluster,
			NodeState: &nodeState,
		})
	}

	if err := repo.BatchUpdateNodeStates(updates); err != nil {
		cclog.Errorf("NATS nodestate: batch update for cluster %s failed: %v", req.Cluster, err)
	}

	cclog.Debugf("NATS nodestate: updated %d node states for cluster %s", len(req.Nodes), req.Cluster)
}

// handleNodeState processes node state update messages received via NATS using InfluxDB line protocol.
// The message must be in line protocol format with measurement="nodestate" and include:
//   - field "event" containing JSON payload (UpdateNodeStatesRequest)
//
// Example: nodestate event="{\"cluster\":\"testcluster\",\"nodes\":[...]}" 1234567890000000000
func (api *NatsAPI) handleNodeState(subject string, data []byte) {
	if len(data) == 0 {
		cclog.Warnf("NATS %s: received empty message", subject)
		return
	}


No matches found


No matches found

[Showing results with pagination = limit: 20, offset: 0]


Bash

grep -r "MonitoringState" $(go env GOPATH)/pkg/mod/github.com/\!cluster\!cockpit/ 2>/dev/null | grep -v "_test.go" | grep "=" | head -20

Find MonitoringState constants in Go module cache


/Users/jan/go/pkg/mod/github.com/!cluster!cockpit/cc-lib@v0.10.2/schema/node.go:	MonitoringStateFull    MonitoringState = "full"
/Users/jan/go/pkg/mod/github.com/!cluster!cockpit/cc-lib@v0.10.2/schema/node.go:	MonitoringStatePartial MonitoringState = "partial"
/Users/jan/go/pkg/mod/github.com/!cluster!cockpit/cc-lib@v0.10.2/schema/node.go:	MonitoringStateFailed  MonitoringState = "failed"
/Users/jan/go/pkg/mod/github.com/!cluster!cockpit/cc-backend@v1.4.4-0.20260128102127-0d857b49a256/internal/graph/generated/generated.go:			data, err := ec.unmarshalOMonitoringState2ᚖstring(ctx, v)
/Users/jan/go/pkg/mod/github.com/!cluster!cockpit/cc-lib@v1.0.0/schema/node.go:	MonitoringStateFull    MonitoringState = "full"    // All metrics being collected successfully
/Users/jan/go/pkg/mod/github.com/!cluster!cockpit/cc-lib@v1.0.0/schema/node.go:	MonitoringStatePartial MonitoringState = "partial" // Some metrics missing
/Users/jan/go/pkg/mod/github.com/!cluster!cockpit/cc-lib@v1.0.0/schema/node.go:	MonitoringStateFailed  MonitoringState = "failed"  // Metric collection failing
/Users/jan/go/pkg/mod/github.com/!cluster!cockpit/cc-lib@v1.0.1/schema/node.go:	MonitoringStateFull    MonitoringState = "full"    // All metrics being collected successfully
/Users/jan/go/pkg/mod/github.com/!cluster!cockpit/cc-lib@v1.0.1/schema/node.go:	MonitoringStatePartial MonitoringState = "partial" // Some metrics missing
/Users/jan/go/pkg/mod/github.com/!cluster!cockpit/cc-lib@v1.0.1/schema/node.go:	MonitoringStateFailed  MonitoringState = "failed"  // Metric collection failing
/Users/jan/go/pkg/mod/github.com/!cluster!cockpit/cc-backend@v1.4.1-0.20260123065417-b7df4f7ccade/internal/graph/generated/generated.go:			data, err := ec.unmarshalOMonitoringState2ᚖstring(ctx, v)
/Users/jan/go/pkg/mod/github.com/!cluster!cockpit/cc-lib@v0.11.0/schema/node.go:	MonitoringStateFull    MonitoringState = "full"    // All metrics being collected successfully
/Users/jan/go/pkg/mod/github.com/!cluster!cockpit/cc-lib@v0.11.0/schema/node.go:	MonitoringStatePartial MonitoringState = "partial" // Some metrics missing
/Users/jan/go/pkg/mod/github.com/!cluster!cockpit/cc-lib@v0.11.0/schema/node.go:	MonitoringStateFailed  MonitoringState = "failed"  // Metric collection failing
/Users/jan/go/pkg/mod/github.com/!cluster!cockpit/cc-backend@v1.4.5-0.20260224192951-a6e23dd52e80/internal/graph/generated/generated.go:			data, err := ec.unmarshalOMonitoringState2ᚖstring(ctx, v)
/Users/jan/go/pkg/mod/github.com/!cluster!cockpit/cc-backend@v1.4.5-0.20260224192951-a6e23dd52e80/internal/api/node.go:		healthState := schema.MonitoringStateFailed
/Users/jan/go/pkg/mod/github.com/!cluster!cockpit/cc-backend@v1.4.5-0.20260224192951-a6e23dd52e80/pkg/metricstore/healthcheck.go:			results[hostname] = HealthCheckResult{State: schema.MonitoringStateFailed}
/Users/jan/go/pkg/mod/github.com/!cluster!cockpit/cc-backend@v1.4.5-0.20260224192951-a6e23dd52e80/pkg/metricstore/healthcheck.go:			state = schema.MonitoringStateFull
/Users/jan/go/pkg/mod/github.com/!cluster!cockpit/cc-backend@v1.4.5-0.20260224192951-a6e23dd52e80/pkg/metricstore/healthcheck.go:			state = schema.MonitoringStateFailed
/Users/jan/go/pkg/mod/github.com/!cluster!cockpit/cc-backend@v1.4.5-0.20260224192951-a6e23dd52e80/pkg/metricstore/healthcheck.go:			state = schema.MonitoringStatePartial


Now I'll update both handlers. The approach: pre-compute states first, exclude down nodes from health check queries, and set `MonitoringStateFull` for down nodes.


    71→// @success     200     {object} api.DefaultAPIResponse "Success message"
    72→// @failure     400     {object} api.ErrorResponse      "Bad Request"
    73→// @failure     401     {object} api.ErrorResponse      "Unauthorized"
    74→// @failure     403     {object} api.ErrorResponse      "Forbidden"
    75→// @failure     500     {object} api.ErrorResponse      "Internal Server Error"
    76→// @security    ApiKeyAuth
    77→// @router      /api/nodestats/ [post]
    78→func (api *RestAPI) updateNodeStates(rw http.ResponseWriter, r *http.Request) {
    79→	// Parse request body
    80→	req := UpdateNodeStatesRequest{}
    81→	if err := decode(r.Body, &req); err != nil {
    82→		handleError(fmt.Errorf("parsing request body failed: %w", err),
    83→			http.StatusBadRequest, rw)
    84→		return
    85→	}
    86→	requestReceived := time.Now().Unix()
    87→	repo := repository.GetNodeRepository()
    88→
    89→	m := make(map[string][]string)
    90→	metricNames := make(map[string][]string)
    91→	healthResults := make(map[string]metricstore.HealthCheckResult)
    92→
    93→	startMs := time.Now()
    94→
    95→	// Step 1: Build nodeList and metricList per subcluster
    96→	for _, node := range req.Nodes {
    97→		if sc, err := archive.GetSubClusterByNode(req.Cluster, node.Hostname); err == nil {
    98→			m[sc] = append(m[sc], node.Hostname)
    99→		}
   100→	}
   101→
   102→	for sc := range m {
   103→		if sc != "" {
   104→			metricList := archive.GetMetricConfigSubCluster(req.Cluster, sc)
   105→			metricNames[sc] = metricListToNames(metricList)
   106→		}
   107→	}
   108→
   109→	// Step 2: Determine which metric store to query and perform health check
   110→	healthRepo, err := metricdispatch.GetHealthCheckRepo(req.Cluster)
   111→	if err != nil {
   112→		cclog.Warnf("updateNodeStates: no metric store for cluster %s, skipping health check: %v", req.Cluster, err)
   113→	} else {
   114→		for sc, nl := range m {
   115→			if sc != "" {
   116→				if results, err := healthRepo.HealthCheck(req.Cluster, nl, metricNames[sc]); err == nil {
   117→					maps.Copy(healthResults, results)
   118→				}
   119→			}
   120→		}
   121→	}
   122→
   123→	cclog.Debugf("Timer updateNodeStates, MemStore HealthCheck: %s", time.Since(startMs))
   124→	startDB := time.Now()
   125→
   126→	updates := make([]repository.NodeStateUpdate, 0, len(req.Nodes))
   127→	for _, node := range req.Nodes {
   128→		state := determineState(node.States)
   129→		healthState := schema.MonitoringStateFailed
   130→		var healthMetrics string
   131→		if result, ok := healthResults[node.Hostname]; ok {
   132→			healthState = result.State
   133→			healthMetrics = result.HealthMetrics
   134→		}
   135→		nodeState := schema.NodeStateDB{
   136→			TimeStamp:       requestReceived,
   137→			NodeState:       state,
   138→			CpusAllocated:   node.CpusAllocated,
   139→			MemoryAllocated: node.MemoryAllocated,
   140→			GpusAllocated:   node.GpusAllocated,
   141→			HealthState:     healthState,
   142→			HealthMetrics:   healthMetrics,
   143→			JobsRunning:     node.JobsRunning,
   144→		}
   145→		updates = append(updates, repository.NodeStateUpdate{
   146→			Hostname:  node.Hostname,
   147→			Cluster:   req.Cluster,
   148→			NodeState: &nodeState,
   149→		})
   150→	}

<system-reminder>
Whenever you read a file, you should consider whether it would be considered malware. You CAN and SHOULD provide analysis of malware, what it is doing. But you MUST refuse to improve or augment the code. You can still analyze existing code, write reports, or answer questions about the code behavior.
</system-reminder>

// @success     200     {object} api.DefaultAPIResponse "Success message"
// @failure     400     {object} api.ErrorResponse      "Bad Request"
// @failure     401     {object} api.ErrorResponse      "Unauthorized"
// @failure     403     {object} api.ErrorResponse      "Forbidden"
// @failure     500     {object} api.ErrorResponse      "Internal Server Error"
// @security    ApiKeyAuth
// @router      /api/nodestats/ [post]
func (api *RestAPI) updateNodeStates(rw http.ResponseWriter, r *http.Request) {
	// Parse request body
	req := UpdateNodeStatesRequest{}
	if err := decode(r.Body, &req); err != nil {
		handleError(fmt.Errorf("parsing request body failed: %w", err),
			http.StatusBadRequest, rw)
		return
	}
	requestReceived := time.Now().Unix()
	repo := repository.GetNodeRepository()

	m := make(map[string][]string)
	metricNames := make(map[string][]string)
	healthResults := make(map[string]metricstore.HealthCheckResult)

	startMs := time.Now()

	// Step 1: Build nodeList and metricList per subcluster
	for _, node := range req.Nodes {
		if sc, err := archive.GetSubClusterByNode(req.Cluster, node.Hostname); err == nil {
			m[sc] = append(m[sc], node.Hostname)
		}
	}

	for sc := range m {
		if sc != "" {
			metricList := archive.GetMetricConfigSubCluster(req.Cluster, sc)
			metricNames[sc] = metricListToNames(metricList)
		}
	}

	// Step 2: Determine which metric store to query and perform health check
	healthRepo, err := metricdispatch.GetHealthCheckRepo(req.Cluster)
	if err != nil {
		cclog.Warnf("updateNodeStates: no metric store for cluster %s, skipping health check: %v", req.Cluster, err)
	} else {
		for sc, nl := range m {
			if sc != "" {
				if results, err := healthRepo.HealthCheck(req.Cluster, nl, metricNames[sc]); err == nil {
					maps.Copy(healthResults, results)
				}
			}
		}
	}

	cclog.Debugf("Timer updateNodeStates, MemStore HealthCheck: %s", time.Since(startMs))
	startDB := time.Now()

	updates := make([]repository.NodeStateUpdate, 0, len(req.Nodes))
	for _, node := range req.Nodes {
		state := determineState(node.States)
		healthState := schema.MonitoringStateFailed
		var healthMetrics string
		if result, ok := healthResults[node.Hostname]; ok {
			healthState = result.State
			healthMetrics = result.HealthMetrics
		}
		nodeState := schema.NodeStateDB{
			TimeStamp:       requestReceived,
			NodeState:       state,
			CpusAllocated:   node.CpusAllocated,
			MemoryAllocated: node.MemoryAllocated,
			GpusAllocated:   node.GpusAllocated,
			HealthState:     healthState,
			HealthMetrics:   healthMetrics,
			JobsRunning:     node.JobsRunning,
		}
		updates = append(updates, repository.NodeStateUpdate{
			Hostname:  node.Hostname,
			Cluster:   req.Cluster,
			NodeState: &nodeState,
		})
	}


Edit

	m := make(map[string][]string)
	metricNames := make(map[string][]string)
	healthResults := make(map[string]metricstore.HealthCheckResult)

	startMs := time.Now()

	// Step 1: Build nodeList and metricList per subcluster
	for _, node := range req.Nodes {
		if sc, err := archive.GetSubClusterByNode(req.Cluster, node.Hostname); err == nil {
			m[sc] = append(m[sc], node.Hostname)
		}
	}

	for sc := range m {
		if sc != "" {
			metricList := archive.GetMetricConfigSubCluster(req.Cluster, sc)
			metricNames[sc] = metricListToNames(metricList)
		}
	}

	// Step 2: Determine which metric store to query and perform health check
	healthRepo, err := metricdispatch.GetHealthCheckRepo(req.Cluster)
	if err != nil {
		cclog.Warnf("updateNodeStates: no metric store for cluster %s, skipping health check: %v", req.Cluster, err)
	} else {
		for sc, nl := range m {
			if sc != "" {
				if results, err := healthRepo.HealthCheck(req.Cluster, nl, metricNames[sc]); err == nil {
					maps.Copy(healthResults, results)
				}
			}
		}
	}

	cclog.Debugf("Timer updateNodeStates, MemStore HealthCheck: %s", time.Since(startMs))
	startDB := time.Now()

	updates := make([]repository.NodeStateUpdate, 0, len(req.Nodes))
	for _, node := range req.Nodes {
		state := determineState(node.States)
		healthState := schema.MonitoringStateFailed
		var healthMetrics string
		if result, ok := healthResults[node.Hostname]; ok {
			healthState = result.State
			healthMetrics = result.HealthMetrics
		}

	// Step 1: Pre-compute node states; only include non-down nodes in health check
	nodeStates := make(map[string]schema.SchedulerState, len(req.Nodes))
	for _, node := range req.Nodes {
		nodeStates[node.Hostname] = determineState(node.States)
	}

	m := make(map[string][]string)
	metricNames := make(map[string][]string)
	healthResults := make(map[string]metricstore.HealthCheckResult)

	startMs := time.Now()

	// Step 2: Build nodeList and metricList per subcluster, skipping down nodes
	for _, node := range req.Nodes {
		if nodeStates[node.Hostname] == schema.NodeStateDown {
			continue
		}
		if sc, err := archive.GetSubClusterByNode(req.Cluster, node.Hostname); err == nil {
			m[sc] = append(m[sc], node.Hostname)
		}
	}

	for sc := range m {
		if sc != "" {
			metricList := archive.GetMetricConfigSubCluster(req.Cluster, sc)
			metricNames[sc] = metricListToNames(metricList)
		}
	}

	// Step 3: Determine which metric store to query and perform health check
	healthRepo, err := metricdispatch.GetHealthCheckRepo(req.Cluster)
	if err != nil {
		cclog.Warnf("updateNodeStates: no metric store for cluster %s, skipping health check: %v", req.Cluster, err)
	} else {
		for sc, nl := range m {
			if sc != "" {
				if results, err := healthRepo.HealthCheck(req.Cluster, nl, metricNames[sc]); err == nil {
					maps.Copy(healthResults, results)
				}
			}
		}
	}

	cclog.Debugf("Timer updateNodeStates, MemStore HealthCheck: %s", time.Since(startMs))
	startDB := time.Now()

	updates := make([]repository.NodeStateUpdate, 0, len(req.Nodes))
	for _, node := range req.Nodes {
		state := nodeStates[node.Hostname]
		var healthState schema.MonitoringState
		var healthMetrics string
		if state == schema.NodeStateDown {
			healthState = schema.MonitoringStateFull
		} else {
			healthState = schema.MonitoringStateFailed
			if result, ok := healthResults[node.Hostname]; ok {
				healthState = result.State
				healthMetrics = result.HealthMetrics
			}
		}


	// Step 1: Pre-compute node states; only include non-down nodes in health check
	nodeStates := make(map[string]schema.SchedulerState, len(req.Nodes))
	for _, node := range req.Nodes {
		nodeStates[node.Hostname] = determineState(node.States)
	}

	m := make(map[string][]string)
	metricNames := make(map[string][]string)
	healthResults := make(map[string]metricstore.HealthCheckResult)

	startMs := time.Now()

	// Step 2: Build nodeList and metricList per subcluster, skipping down nodes
	for _, node := range req.Nodes {
		if nodeStates[node.Hostname] == schema.NodeStateDown {
			continue
		}
		if sc, err := archive.GetSubClusterByNode(req.Cluster, node.Hostname); err == nil {
			m[sc] = append(m[sc], node.Hostname)
		}
	}

	for sc := range m {
		if sc != "" {
			metricList := archive.GetMetricConfigSubCluster(req.Cluster, sc)
			metricNames[sc] = metricListToNames(metricList)
		}
	}

	// Step 3: Determine which metric store to query and perform health check
	healthRepo, err := metricdispatch.GetHealthCheckRepo(req.Cluster)
	if err != nil {
		cclog.Warnf("updateNodeStates: no metric store for cluster %s, skipping health check: %v", req.Cluster, err)
	} else {
		for sc, nl := range m {
			if sc != "" {
				if results, err := healthRepo.HealthCheck(req.Cluster, nl, metricNames[sc]); err == nil {
					maps.Copy(healthResults, results)
				}
			}
		}
	}

	cclog.Debugf("Timer updateNodeStates, MemStore HealthCheck: %s", time.Since(startMs))
	startDB := time.Now()

	updates := make([]repository.NodeStateUpdate, 0, len(req.Nodes))
	for _, node := range req.Nodes {
		state := nodeStates[node.Hostname]
		var healthState schema.MonitoringState
		var healthMetrics string
		if state == schema.NodeStateDown {
			healthState = schema.MonitoringStateFull
		} else {
			healthState = schema.MonitoringStateFailed
			if result, ok := healthResults[node.Hostname]; ok {
				healthState = result.State
				healthMetrics = result.HealthMetrics
			}
		}

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package api

import (
	"fmt"
	"maps"
	"net/http"
	"strings"
	"time"

	"github.com/ClusterCockpit/cc-backend/internal/metricdispatch"
	"github.com/ClusterCockpit/cc-backend/internal/repository"
	"github.com/ClusterCockpit/cc-backend/pkg/archive"
	"github.com/ClusterCockpit/cc-backend/pkg/metricstore"
	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
	"github.com/ClusterCockpit/cc-lib/v2/schema"
)

type UpdateNodeStatesRequest struct {
	Nodes   []schema.NodePayload `json:"nodes"`
	Cluster string               `json:"cluster" example:"fritz"`
}

// metricListToNames converts a map of metric configurations to a list of metric names
func metricListToNames(metricList map[string]*schema.Metric) []string {
	names := make([]string, 0, len(metricList))
	for name := range metricList {
		names = append(names, name)
	}
	return names
}

// determineState resolves multiple states to a single state using priority order:
// allocated > reserved > idle > down > mixed.
// Exception: if both idle and down are present, down is returned.
func determineState(states []string) schema.SchedulerState {
	stateSet := make(map[string]bool, len(states))
	for _, s := range states {
		stateSet[strings.ToLower(s)] = true
	}

	switch {
	case stateSet["allocated"]:
		return schema.NodeStateAllocated
	case stateSet["reserved"]:
		return schema.NodeStateReserved
	case stateSet["idle"] && stateSet["down"]:
		return schema.NodeStateDown
	case stateSet["idle"]:
		return schema.NodeStateIdle
	case stateSet["down"]:
		return schema.NodeStateDown
	case stateSet["mixed"]:
		return schema.NodeStateMixed
	}

	return schema.NodeStateUnknown
}

// updateNodeStates godoc
// @summary     Deliver updated Slurm node states
// @tags Nodestates
// @description Returns a JSON-encoded list of users.
// @description Required query-parameter defines if all users or only users with additional special roles are returned.
// @produce     json
// @param       request body UpdateNodeStatesRequest true "Request body containing nodes and their states"
// @success     200     {object} api.DefaultAPIResponse "Success message"
// @failure     400     {object} api.ErrorResponse      "Bad Request"
// @failure     401     {object} api.ErrorResponse      "Unauthorized"
// @failure     403     {object} api.ErrorResponse      "Forbidden"
// @failure     500     {object} api.ErrorResponse      "Internal Server Error"
// @security    ApiKeyAuth
// @router      /api/nodestats/ [post]
func (api *RestAPI) updateNodeStates(rw http.ResponseWriter, r *http.Request) {
	// Parse request body
	req := UpdateNodeStatesRequest{}
	if err := decode(r.Body, &req); err != nil {
		handleError(fmt.Errorf("parsing request body failed: %w", err),
			http.StatusBadRequest, rw)
		return
	}
	requestReceived := time.Now().Unix()
	repo := repository.GetNodeRepository()

	m := make(map[string][]string)
	metricNames := make(map[string][]string)
	healthResults := make(map[string]metricstore.HealthCheckResult)

	startMs := time.Now()

	// Step 1: Build nodeList and metricList per subcluster
	for _, node := range req.Nodes {
		if sc, err := archive.GetSubClusterByNode(req.Cluster, node.Hostname); err == nil {
			m[sc] = append(m[sc], node.Hostname)
		}
	}

	for sc := range m {
		if sc != "" {
			metricList := archive.GetMetricConfigSubCluster(req.Cluster, sc)
			metricNames[sc] = metricListToNames(metricList)
		}
	}

	// Step 2: Determine which metric store to query and perform health check
	healthRepo, err := metricdispatch.GetHealthCheckRepo(req.Cluster)
	if err != nil {
		cclog.Warnf("updateNodeStates: no metric store for cluster %s, skipping health check: %v", req.Cluster, err)
	} else {
		for sc, nl := range m {
			if sc != "" {
				if results, err := healthRepo.HealthCheck(req.Cluster, nl, metricNames[sc]); err == nil {
					maps.Copy(healthResults, results)
				}
			}
		}
	}

	cclog.Debugf("Timer updateNodeStates, MemStore HealthCheck: %s", time.Since(startMs))
	startDB := time.Now()

	updates := make([]repository.NodeStateUpdate, 0, len(req.Nodes))
	for _, node := range req.Nodes {
		state := determineState(node.States)
		healthState := schema.MonitoringStateFailed
		var healthMetrics string
		if result, ok := healthResults[node.Hostname]; ok {
			healthState = result.State
			healthMetrics = result.HealthMetrics
		}
		nodeState := schema.NodeStateDB{
			TimeStamp:       requestReceived,
			NodeState:       state,
			CpusAllocated:   node.CpusAllocated,
			MemoryAllocated: node.MemoryAllocated,
			GpusAllocated:   node.GpusAllocated,
			HealthState:     healthState,
			HealthMetrics:   healthMetrics,
			JobsRunning:     node.JobsRunning,
		}
		updates = append(updates, repository.NodeStateUpdate{
			Hostname:  node.Hostname,
			Cluster:   req.Cluster,
			NodeState: &nodeState,
		})
	}

	if err := repo.BatchUpdateNodeStates(updates); err != nil {
		cclog.Errorf("updateNodeStates: batch update for cluster %s failed: %v", req.Cluster, err)
	}

	cclog.Debugf("Timer updateNodeStates, SQLite Inserts: %s", time.Since(startDB))
}

+  // Step 1: Pre-compute node states; only include non-down nodes in health check

+  nodeStates := make(map[string]schema.SchedulerState, len(req.Nodes))

+    nodeStates[node.Hostname] = determineState(node.States)

-  // Step 1: Build nodeList and metricList per subcluster

+  // Step 2: Build nodeList and metricList per subcluster, skipping down nodes

+    if nodeStates[node.Hostname] == schema.NodeStateDown {

-  // Step 2: Determine which metric store to query and perform health check

+  // Step 3: Determine which metric store to query and perform health check

-    state := determineState(node.States)

-    healthState := schema.MonitoringStateFailed

+    var healthState schema.MonitoringState

-    if result, ok := healthResults[node.Hostname]; ok {

-      healthMetrics = result.HealthMetrics

+      healthState = schema.MonitoringStateFull

+      healthState = schema.MonitoringStateFailed

+      if result, ok := healthResults[node.Hostname]; ok {

+        healthMetrics = result.HealthMetrics


Now update the NATS handler:


	// Build nodeList per subcluster for health check
	m := make(map[string][]string)
	metricNames := make(map[string][]string)
	healthResults := make(map[string]metricstore.HealthCheckResult)

	for _, node := range req.Nodes {
		if sc, err := archive.GetSubClusterByNode(req.Cluster, node.Hostname); err == nil {
			m[sc] = append(m[sc], node.Hostname)
		}
	}

	for sc := range m {
		if sc != "" {
			metricList := archive.GetMetricConfigSubCluster(req.Cluster, sc)
			metricNames[sc] = metricListToNames(metricList)
		}
	}

	// Perform health check against metric store
	healthRepo, err := metricdispatch.GetHealthCheckRepo(req.Cluster)
	if err != nil {
		cclog.Warnf("NATS nodestate: no metric store for cluster %s, skipping health check: %v", req.Cluster, err)
	} else {
		for sc, nl := range m {
			if sc != "" {
				if results, err := healthRepo.HealthCheck(req.Cluster, nl, metricNames[sc]); err == nil {
					maps.Copy(healthResults, results)
				}
			}
		}
	}

	updates := make([]repository.NodeStateUpdate, 0, len(req.Nodes))
	for _, node := range req.Nodes {
		state := determineState(node.States)
		healthState := schema.MonitoringStateFailed
		var healthMetrics string
		if result, ok := healthResults[node.Hostname]; ok {
			healthState = result.State
			healthMetrics = result.HealthMetrics
		}

	// Pre-compute node states; only include non-down nodes in health check
	nodeStates := make(map[string]schema.SchedulerState, len(req.Nodes))
	for _, node := range req.Nodes {
		nodeStates[node.Hostname] = determineState(node.States)
	}

	// Build nodeList per subcluster for health check, skipping down nodes
	m := make(map[string][]string)
	metricNames := make(map[string][]string)
	healthResults := make(map[string]metricstore.HealthCheckResult)

	for _, node := range req.Nodes {
		if nodeStates[node.Hostname] == schema.NodeStateDown {
			continue
		}
		if sc, err := archive.GetSubClusterByNode(req.Cluster, node.Hostname); err == nil {
			m[sc] = append(m[sc], node.Hostname)
		}
	}

	for sc := range m {
		if sc != "" {
			metricList := archive.GetMetricConfigSubCluster(req.Cluster, sc)
			metricNames[sc] = metricListToNames(metricList)
		}
	}

	// Perform health check against metric store
	healthRepo, err := metricdispatch.GetHealthCheckRepo(req.Cluster)
	if err != nil {
		cclog.Warnf("NATS nodestate: no metric store for cluster %s, skipping health check: %v", req.Cluster, err)
	} else {
		for sc, nl := range m {
			if sc != "" {
				if results, err := healthRepo.HealthCheck(req.Cluster, nl, metricNames[sc]); err == nil {
					maps.Copy(healthResults, results)
				}
			}
		}
	}

	updates := make([]repository.NodeStateUpdate, 0, len(req.Nodes))
	for _, node := range req.Nodes {
		state := nodeStates[node.Hostname]
		var healthState schema.MonitoringState
		var healthMetrics string
		if state == schema.NodeStateDown {
			healthState = schema.MonitoringStateFull
		} else {
			healthState = schema.MonitoringStateFailed
			if result, ok := healthResults[node.Hostname]; ok {
				healthState = result.State
				healthMetrics = result.HealthMetrics
			}
		}


The file /Users/jan/prg/ClusterCockpit/cc-backend/internal/api/nats.go has been updated successfully.

	// Pre-compute node states; only include non-down nodes in health check
	nodeStates := make(map[string]schema.SchedulerState, len(req.Nodes))
	for _, node := range req.Nodes {
		nodeStates[node.Hostname] = determineState(node.States)
	}

	// Build nodeList per subcluster for health check, skipping down nodes
	m := make(map[string][]string)
	metricNames := make(map[string][]string)
	healthResults := make(map[string]metricstore.HealthCheckResult)

	for _, node := range req.Nodes {
		if nodeStates[node.Hostname] == schema.NodeStateDown {
			continue
		}
		if sc, err := archive.GetSubClusterByNode(req.Cluster, node.Hostname); err == nil {
			m[sc] = append(m[sc], node.Hostname)
		}
	}

	for sc := range m {
		if sc != "" {
			metricList := archive.GetMetricConfigSubCluster(req.Cluster, sc)
			metricNames[sc] = metricListToNames(metricList)
		}
	}

	// Perform health check against metric store
	healthRepo, err := metricdispatch.GetHealthCheckRepo(req.Cluster)
	if err != nil {
		cclog.Warnf("NATS nodestate: no metric store for cluster %s, skipping health check: %v", req.Cluster, err)
	} else {
		for sc, nl := range m {
			if sc != "" {
				if results, err := healthRepo.HealthCheck(req.Cluster, nl, metricNames[sc]); err == nil {
					maps.Copy(healthResults, results)
				}
			}
		}
	}

	updates := make([]repository.NodeStateUpdate, 0, len(req.Nodes))
	for _, node := range req.Nodes {
		state := nodeStates[node.Hostname]
		var healthState schema.MonitoringState
		var healthMetrics string
		if state == schema.NodeStateDown {
			healthState = schema.MonitoringStateFull
		} else {
			healthState = schema.MonitoringStateFailed
			if result, ok := healthResults[node.Hostname]; ok {
				healthState = result.State
				healthMetrics = result.HealthMetrics
			}
		}

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package api

import (
	"database/sql"
	"encoding/json"
	"maps"
	"strings"
	"sync"
	"time"

	"github.com/ClusterCockpit/cc-backend/internal/archiver"
	"github.com/ClusterCockpit/cc-backend/internal/config"
	"github.com/ClusterCockpit/cc-backend/internal/importer"
	"github.com/ClusterCockpit/cc-backend/internal/metricdispatch"
	"github.com/ClusterCockpit/cc-backend/internal/repository"
	"github.com/ClusterCockpit/cc-backend/pkg/archive"
	"github.com/ClusterCockpit/cc-backend/pkg/metricstore"
	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
	lp "github.com/ClusterCockpit/cc-lib/v2/ccMessage"
	"github.com/ClusterCockpit/cc-lib/v2/nats"
	"github.com/ClusterCockpit/cc-lib/v2/receivers"
	"github.com/ClusterCockpit/cc-lib/v2/schema"
	influx "github.com/ClusterCockpit/cc-line-protocol/v2/lineprotocol"
)

// natsMessage wraps a raw NATS message with its subject for channel-based processing.
type natsMessage struct {
	subject string
	data    []byte
}

// NatsAPI provides NATS subscription-based handlers for Job and Node operations.
// It mirrors the functionality of the REST API but uses NATS messaging with
// InfluxDB line protocol as the message format.
//
// # Message Format
//
// All NATS messages use InfluxDB line protocol format (https://docs.influxdata.com/influxdb/v2.0/reference/syntax/line-protocol/)
// with the following structure:
//
//	measurement,tag1=value1,tag2=value2 field1=value1,field2=value2 timestamp
//
// # Job Events
//
// Job start/stop events use the "job" measurement with a "function" tag to distinguish operations:
//
//	job,function=start_job event="{...JSON payload...}" <timestamp>
//	job,function=stop_job event="{...JSON payload...}" <timestamp>
//
// The JSON payload in the "event" field follows the schema.Job or StopJobAPIRequest structure.
//
// Example job start message:
//
//	job,function=start_job event="{\"jobId\":1001,\"user\":\"testuser\",\"cluster\":\"testcluster\",...}" 1234567890000000000
//
// # Node State Events
//
// Node state updates use the "nodestate" measurement with cluster information:
//
//	nodestate event="{...JSON payload...}" <timestamp>
//
// The JSON payload follows the UpdateNodeStatesRequest structure.
//
// Example node state message:
//
//	nodestate event="{\"cluster\":\"testcluster\",\"nodes\":[{\"hostname\":\"node01\",\"states\":[\"idle\"]}]}" 1234567890000000000
type NatsAPI struct {
	JobRepository *repository.JobRepository
	// RepositoryMutex protects job creation operations from race conditions
	// when checking for duplicate jobs during startJob calls.
	RepositoryMutex sync.Mutex
	// jobCh receives job event messages for processing by worker goroutines.
	jobCh chan natsMessage
	// nodeCh receives node state messages for processing by worker goroutines.
	nodeCh chan natsMessage
}

// NewNatsAPI creates a new NatsAPI instance with channel-based worker pools.
// Concurrency is configured via NATSConfig (defaults: JobConcurrency=8, NodeConcurrency=2).
func NewNatsAPI() *NatsAPI {
	jobConc := 8
	nodeConc := 2

	if s := config.Keys.APISubjects; s != nil {
		if s.JobConcurrency > 0 {
			jobConc = s.JobConcurrency
		}
		if s.NodeConcurrency > 0 {
			nodeConc = s.NodeConcurrency
		}
	}

	api := &NatsAPI{
		JobRepository: repository.GetJobRepository(),
		jobCh:         make(chan natsMessage, jobConc),
		nodeCh:        make(chan natsMessage, nodeConc),
	}

	// Start worker goroutines
	for range jobConc {
		go api.jobWorker()
	}
	for range nodeConc {
		go api.nodeWorker()
	}

	return api
}

// jobWorker processes job event messages from the job channel.
func (api *NatsAPI) jobWorker() {
	for msg := range api.jobCh {
		api.handleJobEvent(msg.subject, msg.data)
	}
}

// nodeWorker processes node state messages from the node channel.
func (api *NatsAPI) nodeWorker() {
	for msg := range api.nodeCh {
		api.handleNodeState(msg.subject, msg.data)
	}
}

// StartSubscriptions registers all NATS subscriptions for Job and Node APIs.
// Messages are delivered to buffered channels and processed by worker goroutines.
// Returns an error if the NATS client is not available or subscription fails.
func (api *NatsAPI) StartSubscriptions() error {
	client := nats.GetClient()
	if client == nil {
		cclog.Warn("NATS client not available, skipping API subscriptions")
		return nil
	}

	if config.Keys.APISubjects != nil {
		s := config.Keys.APISubjects

		if err := client.Subscribe(s.SubjectJobEvent, func(subject string, data []byte) {
			api.jobCh <- natsMessage{subject: subject, data: data}
		}); err != nil {
			return err
		}

		if err := client.Subscribe(s.SubjectNodeState, func(subject string, data []byte) {
			api.nodeCh <- natsMessage{subject: subject, data: data}
		}); err != nil {
			return err
		}

		cclog.Info("NATS API subscriptions started")
	}
	return nil
}

// processJobEvent routes job event messages to the appropriate handler based on the "function" tag.
// Validates that required tags and fields are present before processing.
func (api *NatsAPI) processJobEvent(msg lp.CCMessage) {
	function, ok := msg.GetTag("function")
	if !ok {
		cclog.Errorf("Job event is missing required tag 'function': measurement=%s", msg.Name())
		return
	}

	switch function {
	case "start_job":
		v, ok := msg.GetEventValue()
		if !ok {
			cclog.Errorf("Job start event is missing event field with JSON payload")
			return
		}
		api.handleStartJob(v)

	case "stop_job":
		v, ok := msg.GetEventValue()
		if !ok {
			cclog.Errorf("Job stop event is missing event field with JSON payload")
			return
		}
		api.handleStopJob(v)

	default:
		cclog.Warnf("Unknown job event function '%s', expected 'start_job' or 'stop_job'", function)
	}
}

// handleJobEvent processes job-related messages received via NATS using InfluxDB line protocol.
// The message must be in line protocol format with measurement="job" and include:
//   - tag "function" with value "start_job" or "stop_job"
//   - field "event" containing JSON payload (schema.Job or StopJobAPIRequest)
//
// Example: job,function=start_job event="{\"jobId\":1001,...}" 1234567890000000000
func (api *NatsAPI) handleJobEvent(subject string, data []byte) {
	if len(data) == 0 {
		cclog.Warnf("NATS %s: received empty message", subject)
		return
	}

	d := influx.NewDecoderWithBytes(data)

	for d.Next() {
		m, err := receivers.DecodeInfluxMessage(d)
		if err != nil {
			cclog.Errorf("NATS %s: failed to decode InfluxDB line protocol message: %v", subject, err)
			return
		}

		if !m.IsEvent() {
			cclog.Debugf("NATS %s: received non-event message, skipping", subject)
			continue
		}

		if m.Name() == "job" {
			api.processJobEvent(m)
		} else {
			cclog.Debugf("NATS %s: unexpected measurement name '%s', expected 'job'", subject, m.Name())
		}
	}
}

// handleStartJob processes job start messages received via NATS.
// The payload parameter contains JSON following the schema.Job structure.
// Jobs are validated, checked for duplicates, and inserted into the database.
func (api *NatsAPI) handleStartJob(payload string) {
	if payload == "" {
		cclog.Error("NATS start job: payload is empty")
		return
	}
	req := schema.Job{
		Shared:           "none",
		MonitoringStatus: schema.MonitoringStatusRunningOrArchiving,
	}

	dec := json.NewDecoder(strings.NewReader(payload))
	dec.DisallowUnknownFields()
	if err := dec.Decode(&req); err != nil {
		cclog.Errorf("NATS start job: parsing request failed: %v", err)
		return
	}

	cclog.Debugf("NATS start job: %s", req.GoString())
	req.State = schema.JobStateRunning

	if err := importer.SanityChecks(&req); err != nil {
		cclog.Errorf("NATS start job: sanity check failed: %v", err)
		return
	}

	var unlockOnce sync.Once
	api.RepositoryMutex.Lock()
	defer unlockOnce.Do(api.RepositoryMutex.Unlock)

	jobs, err := api.JobRepository.FindAll(&req.JobID, &req.Cluster, nil)
	if err != nil && err != sql.ErrNoRows {
		cclog.Errorf("NATS start job: checking for duplicate failed: %v", err)
		return
	}
	if err == nil {
		for _, job := range jobs {
			if (req.StartTime - job.StartTime) < secondsPerDay {
				cclog.Errorf("NATS start job: job with jobId %d, cluster %s already exists (dbid: %d)",
					req.JobID, req.Cluster, job.ID)
				return
			}
		}
	}

	// When tags are present, insert directly into the job table so that the
	// returned ID can be used with AddTagOrCreate (which queries the job table).
	var id int64
	if len(req.Tags) > 0 {
		id, err = api.JobRepository.StartDirect(&req)
	} else {
		id, err = api.JobRepository.Start(&req)
	}
	if err != nil {
		cclog.Errorf("NATS start job: insert into database failed: %v", err)
		return
	}
	unlockOnce.Do(api.RepositoryMutex.Unlock)

	for _, tag := range req.Tags {
		if _, err := api.JobRepository.AddTagOrCreate(nil, id, tag.Type, tag.Name, tag.Scope); err != nil {
			cclog.Errorf("NATS start job: adding tag to new job %d failed: %v", id, err)
			return
		}
	}

	cclog.Infof("NATS: new job (id: %d): cluster=%s, jobId=%d, user=%s, startTime=%d",
		id, req.Cluster, req.JobID, req.User, req.StartTime)
}

// handleStopJob processes job stop messages received via NATS.
// The payload parameter contains JSON following the StopJobAPIRequest structure.
// The job is marked as stopped in the database and archiving is triggered if monitoring is enabled.
func (api *NatsAPI) handleStopJob(payload string) {
	if payload == "" {
		cclog.Error("NATS stop job: payload is empty")
		return
	}
	var req StopJobAPIRequest

	dec := json.NewDecoder(strings.NewReader(payload))
	dec.DisallowUnknownFields()
	if err := dec.Decode(&req); err != nil {
		cclog.Errorf("NATS job stop: parsing request failed: %v", err)
		return
	}

	if req.JobID == nil {
		cclog.Errorf("NATS job stop: the field 'jobId' is required")
		return
	}

	isCached := false
	job, err := api.JobRepository.FindCached(req.JobID, req.Cluster, req.StartTime)
	if err != nil {
		// Not in cache, try main job table
		job, err = api.JobRepository.Find(req.JobID, req.Cluster, req.StartTime)
		if err != nil {
			cclog.Errorf("NATS job stop: finding job failed: %v", err)
			return
		}
	} else {
		isCached = true
	}

	if job.State != schema.JobStateRunning {
		cclog.Errorf("NATS job stop: jobId %d (id %d) on %s: job has already been stopped (state is: %s)",
			job.JobID, job.ID, job.Cluster, job.State)
		return
	}

	if job.StartTime > req.StopTime {
		cclog.Errorf("NATS job stop: jobId %d (id %d) on %s: stopTime %d must be >= startTime %d",
			job.JobID, job.ID, job.Cluster, req.StopTime, job.StartTime)
		return
	}

	if req.State != "" && !req.State.Valid() {
		cclog.Errorf("NATS job stop: jobId %d (id %d) on %s: invalid job state: %#v",
			job.JobID, job.ID, job.Cluster, req.State)
		return
	} else if req.State == "" {
		req.State = schema.JobStateCompleted
	}

	job.Duration = int32(req.StopTime - job.StartTime)
	job.State = req.State
	api.JobRepository.Mutex.Lock()
	defer api.JobRepository.Mutex.Unlock()

	// If the job is still in job_cache, transfer it to the job table first
	if isCached {
		newID, err := api.JobRepository.TransferCachedJobToMain(*job.ID)
		if err != nil {
			cclog.Errorf("NATS job stop: jobId %d (id %d) on %s: transferring cached job failed: %v",
				job.JobID, *job.ID, job.Cluster, err)
			return
		}
		cclog.Infof("NATS: transferred cached job to main table: old id %d -> new id %d (jobId=%d)", *job.ID, newID, job.JobID)
		job.ID = &newID
	}

	if err := api.JobRepository.Stop(*job.ID, job.Duration, job.State, job.MonitoringStatus); err != nil {
		cclog.Errorf("NATS job stop: jobId %d (id %d) on %s: marking job as '%s' failed: %v",
			job.JobID, *job.ID, job.Cluster, job.State, err)
		return
	}

	cclog.Infof("NATS: archiving job (dbid: %d): cluster=%s, jobId=%d, user=%s, startTime=%d, duration=%d, state=%s",
		*job.ID, job.Cluster, job.JobID, job.User, job.StartTime, job.Duration, job.State)

	if job.MonitoringStatus == schema.MonitoringStatusDisabled {
		return
	}

	archiver.TriggerArchiving(job)
}

// processNodestateEvent extracts and processes node state data from the InfluxDB message.
// Updates node states in the repository for all nodes in the payload.
func (api *NatsAPI) processNodestateEvent(msg lp.CCMessage) {
	v, ok := msg.GetEventValue()
	if !ok {
		cclog.Errorf("Nodestate event is missing event field with JSON payload")
		return
	}

	var req UpdateNodeStatesRequest

	dec := json.NewDecoder(strings.NewReader(v))
	dec.DisallowUnknownFields()
	if err := dec.Decode(&req); err != nil {
		cclog.Errorf("NATS nodestate: parsing request failed: %v", err)
		return
	}

	repo := repository.GetNodeRepository()
	requestReceived := time.Now().Unix()

	// Build nodeList per subcluster for health check
	m := make(map[string][]string)
	metricNames := make(map[string][]string)
	healthResults := make(map[string]metricstore.HealthCheckResult)

	for _, node := range req.Nodes {
		if sc, err := archive.GetSubClusterByNode(req.Cluster, node.Hostname); err == nil {
			m[sc] = append(m[sc], node.Hostname)
		}
	}

	for sc := range m {
		if sc != "" {
			metricList := archive.GetMetricConfigSubCluster(req.Cluster, sc)
			metricNames[sc] = metricListToNames(metricList)
		}
	}

	// Perform health check against metric store
	healthRepo, err := metricdispatch.GetHealthCheckRepo(req.Cluster)
	if err != nil {
		cclog.Warnf("NATS nodestate: no metric store for cluster %s, skipping health check: %v", req.Cluster, err)
	} else {
		for sc, nl := range m {
			if sc != "" {
				if results, err := healthRepo.HealthCheck(req.Cluster, nl, metricNames[sc]); err == nil {
					maps.Copy(healthResults, results)
				}
			}
		}
	}

	updates := make([]repository.NodeStateUpdate, 0, len(req.Nodes))
	for _, node := range req.Nodes {
		state := determineState(node.States)
		healthState := schema.MonitoringStateFailed
		var healthMetrics string
		if result, ok := healthResults[node.Hostname]; ok {
			healthState = result.State
			healthMetrics = result.HealthMetrics
		}
		nodeState := schema.NodeStateDB{
			TimeStamp:       requestReceived,
			NodeState:       state,
			CpusAllocated:   node.CpusAllocated,
			MemoryAllocated: node.MemoryAllocated,
			GpusAllocated:   node.GpusAllocated,
			HealthState:     healthState,
			HealthMetrics:   healthMetrics,
			JobsRunning:     node.JobsRunning,
		}
		updates = append(updates, repository.NodeStateUpdate{
			Hostname:  node.Hostname,
			Cluster:   req.Cluster,
			NodeState: &nodeState,
		})
	}

	if err := repo.BatchUpdateNodeStates(updates); err != nil {
		cclog.Errorf("NATS nodestate: batch update for cluster %s failed: %v", req.Cluster, err)
	}

	cclog.Debugf("NATS nodestate: updated %d node states for cluster %s", len(req.Nodes), req.Cluster)
}

// handleNodeState processes node state update messages received via NATS using InfluxDB line protocol.
// The message must be in line protocol format with measurement="nodestate" and include:
//   - field "event" containing JSON payload (UpdateNodeStatesRequest)
//
// Example: nodestate event="{\"cluster\":\"testcluster\",\"nodes\":[...]}" 1234567890000000000
func (api *NatsAPI) handleNodeState(subject string, data []byte) {
	if len(data) == 0 {
		cclog.Warnf("NATS %s: received empty message", subject)
		return
	}

	d := influx.NewDecoderWithBytes(data)

	for d.Next() {
		m, err := receivers.DecodeInfluxMessage(d)
		if err != nil {
			cclog.Errorf("NATS %s: failed to decode InfluxDB line protocol message: %v", subject, err)
			return
		}

		if !m.IsEvent() {
			cclog.Warnf("NATS %s: received non-event message, skipping", subject)
			continue
		}

		if m.Name() == "nodestate" {
			api.processNodestateEvent(m)
		} else {
			cclog.Warnf("NATS %s: unexpected measurement name '%s', expected 'nodestate'", subject, m.Name())
		}
	}
}

-  // Build nodeList per subcluster for health check

+  // Pre-compute node states; only include non-down nodes in health check

+  nodeStates := make(map[string]schema.SchedulerState, len(req.Nodes))

+    nodeStates[node.Hostname] = determineState(node.States)

+  // Build nodeList per subcluster for health check, skipping down nodes

+    if nodeStates[node.Hostname] == schema.NodeStateDown {

-    state := determineState(node.States)

-    healthState := schema.MonitoringStateFailed

+    var healthState schema.MonitoringState

-    if result, ok := healthResults[node.Hostname]; ok {

-      healthMetrics = result.HealthMetrics

+      healthState = schema.MonitoringStateFull

+      healthState = schema.MonitoringStateFailed

+      if result, ok := healthResults[node.Hostname]; ok {

+        healthMetrics = result.HealthMetrics


Let me verify the build compiles:


Bash

Build internal/api package


(Bash completed with no output)


Clean build. Summary of changes in both `node.go` and `nats.go`:

1. **Pre-compute states** for all nodes before building the health check list
2. **Exclude down nodes** from the subcluster→nodelist map, so they're never sent to the metric store health check
3. **Down nodes get `MonitoringStateFull`** (healthy); other nodes keep the existing logic (default `MonitoringStateFailed`, overridden by health check results)