     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→package api
     6→
     7→import (
     8→	"context"
     9→	"database/sql"
    10→	"encoding/json"
    11→	"fmt"
    12→	"os"
    13→	"path/filepath"
    14→	"testing"
    15→	"time"
    16→
    17→	"github.com/ClusterCockpit/cc-backend/internal/archiver"
    18→	"github.com/ClusterCockpit/cc-backend/internal/auth"
    19→	"github.com/ClusterCockpit/cc-backend/internal/config"
    20→	"github.com/ClusterCockpit/cc-backend/internal/graph"
    21→	"github.com/ClusterCockpit/cc-backend/internal/repository"
    22→	"github.com/ClusterCockpit/cc-backend/pkg/archive"
    23→	"github.com/ClusterCockpit/cc-backend/pkg/metricstore"
    24→	ccconf "github.com/ClusterCockpit/cc-lib/v2/ccConfig"
    25→	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
    26→	lp "github.com/ClusterCockpit/cc-lib/v2/ccMessage"
    27→	"github.com/ClusterCockpit/cc-lib/v2/schema"
    28→
    29→	_ "github.com/mattn/go-sqlite3"
    30→)
    31→
    32→func setupNatsTest(t *testing.T) *NatsAPI {
    33→	repository.ResetConnection()
    34→
    35→	const testconfig = `{
    36→		"main": {
    37→	"addr":            "0.0.0.0:8080",
    38→	"validate": false,
    39→  "api-allowed-ips": [
    40→    "*"
    41→  ]
    42→	},
    43→	"archive": {
    44→		"kind": "file",
    45→		"path": "./var/job-archive"
    46→	},
    47→	"auth": {
    48→  "jwts": {
    49→      "max-age": "2m"
    50→  }
    51→	}
    52→}`
    53→	const testclusterJSON = `{
    54→        "name": "testcluster",
    55→		"subClusters": [
    56→			{
    57→				"name": "sc1",
    58→				"nodes": "host123,host124,host125",
    59→				"processorType": "Intel Core i7-4770",
    60→				"socketsPerNode": 1,
    61→				"coresPerSocket": 4,
    62→				"threadsPerCore": 2,
    63→                "flopRateScalar": {
    64→                  "unit": {
    65→                    "prefix": "G",
    66→                    "base": "F/s"
    67→                  },
    68→                  "value": 14
    69→                },
    70→                "flopRateSimd": {
    71→                  "unit": {
    72→                    "prefix": "G",
    73→                    "base": "F/s"
    74→                  },
    75→                  "value": 112
    76→                },
    77→                "memoryBandwidth": {
    78→                  "unit": {
    79→                    "prefix": "G",
    80→                    "base": "B/s"
    81→                  },
    82→                  "value": 24
    83→                },
    84→                "numberOfNodes": 70,
    85→				"topology": {
    86→					"node": [0, 1, 2, 3, 4, 5, 6, 7],
    87→					"socket": [[0, 1, 2, 3, 4, 5, 6, 7]],
    88→					"memoryDomain": [[0, 1, 2, 3, 4, 5, 6, 7]],
    89→					"die": [[0, 1, 2, 3, 4, 5, 6, 7]],
    90→					"core": [[0], [1], [2], [3], [4], [5], [6], [7]]
    91→				}
    92→			}
    93→		],
    94→		"metricConfig": [
    95→			{
    96→				"name": "load_one",
    97→			    "unit": { "base": ""},
    98→				"scope": "node",
    99→				"timestep": 60,
   100→                "aggregation": "avg",
   101→				"peak": 8,
   102→				"normal": 0,
   103→				"caution": 0,
   104→				"alert": 0
   105→			}
   106→		]
   107→	}`
   108→
   109→	cclog.Init("info", true)
   110→	tmpdir := t.TempDir()
   111→	jobarchive := filepath.Join(tmpdir, "job-archive")
   112→	if err := os.Mkdir(jobarchive, 0o777); err != nil {
   113→		t.Fatal(err)
   114→	}
   115→
   116→	if err := os.WriteFile(filepath.Join(jobarchive, "version.txt"), fmt.Appendf(nil, "%d", 3), 0o666); err != nil {
   117→		t.Fatal(err)
   118→	}
   119→
   120→	if err := os.Mkdir(filepath.Join(jobarchive, "testcluster"), 0o777); err != nil {
   121→		t.Fatal(err)
   122→	}
   123→
   124→	if err := os.WriteFile(filepath.Join(jobarchive, "testcluster", "cluster.json"), []byte(testclusterJSON), 0o666); err != nil {
   125→		t.Fatal(err)
   126→	}
   127→
   128→	dbfilepath := filepath.Join(tmpdir, "test.db")
   129→	err := repository.MigrateDB(dbfilepath)
   130→	if err != nil {
   131→		t.Fatal(err)
   132→	}
   133→
   134→	cfgFilePath := filepath.Join(tmpdir, "config.json")
   135→	if err := os.WriteFile(cfgFilePath, []byte(testconfig), 0o666); err != nil {
   136→		t.Fatal(err)
   137→	}
   138→
   139→	ccconf.Init(cfgFilePath)
   140→
   141→	// Load and check main configuration
   142→	if cfg := ccconf.GetPackageConfig("main"); cfg != nil {
   143→		config.Init(cfg)
   144→	} else {
   145→		cclog.Abort("Main configuration must be present")
   146→	}
   147→	archiveCfg := fmt.Sprintf("{\"kind\": \"file\",\"path\": \"%s\"}", jobarchive)
   148→
   149→	repository.Connect(dbfilepath)
   150→
   151→	if err := archive.Init(json.RawMessage(archiveCfg)); err != nil {
   152→		t.Fatal(err)
   153→	}
   154→
   155→	// metricstore initialization removed - it's initialized via callback in tests
   156→
   157→	archiver.Start(repository.GetJobRepository(), context.Background())
   158→
   159→	if cfg := ccconf.GetPackageConfig("auth"); cfg != nil {
   160→		auth.Init(&cfg)
   161→	} else {
   162→		cclog.Warn("Authentication disabled due to missing configuration")
   163→		auth.Init(nil)
   164→	}
   165→
   166→	graph.Init()
   167→
   168→	return NewNatsAPI()
   169→}
   170→
   171→func cleanupNatsTest() {
   172→	if err := archiver.Shutdown(5 * time.Second); err != nil {
   173→		cclog.Warnf("Archiver shutdown timeout in tests: %v", err)
   174→	}
   175→}
   176→
   177→func TestNatsHandleStartJob(t *testing.T) {
   178→	natsAPI := setupNatsTest(t)
   179→	t.Cleanup(cleanupNatsTest)
   180→
   181→	tests := []struct {
   182→		name          string
   183→		payload       string
   184→		expectError   bool
   185→		validateJob   func(t *testing.T, job *schema.Job)
   186→		shouldFindJob bool
   187→	}{
   188→		{
   189→			name: "valid job start",
   190→			payload: `{
   191→				"jobId": 1001,
   192→				"user": "testuser1",
   193→				"project": "testproj1",
   194→				"cluster": "testcluster",
   195→				"partition": "main",
   196→				"walltime": 7200,
   197→				"numNodes": 1,
   198→				"numHwthreads": 8,
   199→				"numAcc": 0,
   200→				"shared": "none",
   201→				"monitoringStatus": 1,
   202→				"smt": 1,
   203→				"resources": [
   204→					{
   205→						"hostname": "host123",
   206→						"hwthreads": [0, 1, 2, 3, 4, 5, 6, 7]
   207→					}
   208→				],
   209→				"startTime": 1234567890
   210→			}`,
   211→			expectError:   false,
   212→			shouldFindJob: true,
   213→			validateJob: func(t *testing.T, job *schema.Job) {
   214→				if job.JobID != 1001 {
   215→					t.Errorf("expected JobID 1001, got %d", job.JobID)
   216→				}
   217→				if job.User != "testuser1" {
   218→					t.Errorf("expected user testuser1, got %s", job.User)
   219→				}
   220→				if job.State != schema.JobStateRunning {
   221→					t.Errorf("expected state running, got %s", job.State)
   222→				}
   223→			},
   224→		},
   225→		{
   226→			name: "invalid JSON",
   227→			payload: `{
   228→				"jobId": "not a number",
   229→				"user": "testuser2"
   230→			}`,
   231→			expectError:   true,
   232→			shouldFindJob: false,
   233→		},
   234→		{
   235→			name: "missing required fields",
   236→			payload: `{
   237→				"jobId": 1002
   238→			}`,
   239→			expectError:   true,
   240→			shouldFindJob: false,
   241→		},
   242→		{
   243→			name: "job with unknown fields (should fail due to DisallowUnknownFields)",
   244→			payload: `{
   245→				"jobId": 1003,
   246→				"user": "testuser3",
   247→				"project": "testproj3",
   248→				"cluster": "testcluster",
   249→				"partition": "main",
   250→				"walltime": 3600,
   251→				"numNodes": 1,
   252→				"numHwthreads": 8,
   253→				"unknownField": "should cause error",
   254→				"startTime": 1234567900
   255→			}`,
   256→			expectError:   true,
   257→			shouldFindJob: false,
   258→		},
   259→		{
   260→			name: "job with tags",
   261→			payload: `{
   262→				"jobId": 1004,
   263→				"user": "testuser4",
   264→				"project": "testproj4",
   265→				"cluster": "testcluster",
   266→				"partition": "main",
   267→				"walltime": 3600,
   268→				"numNodes": 1,
   269→				"numHwthreads": 8,
   270→				"numAcc": 0,
   271→				"shared": "none",
   272→				"monitoringStatus": 1,
   273→				"smt": 1,
   274→				"resources": [
   275→					{
   276→						"hostname": "host123",
   277→						"hwthreads": [0, 1, 2, 3]
   278→					}
   279→				],
   280→				"tags": [
   281→					{
   282→						"type": "test",
   283→						"name": "testtag",
   284→						"scope": "testuser4"
   285→					}
   286→				],
   287→				"startTime": 1234567910
   288→			}`,
   289→			expectError:   false,
   290→			shouldFindJob: true,
   291→			validateJob: func(t *testing.T, job *schema.Job) {
   292→				if job.JobID != 1004 {
   293→					t.Errorf("expected JobID 1004, got %d", job.JobID)
   294→				}
   295→			},
   296→		},
   297→	}
   298→
   299→	for _, tt := range tests {
   300→		t.Run(tt.name, func(t *testing.T) {
   301→			natsAPI.handleStartJob(tt.payload)
   302→			natsAPI.JobRepository.SyncJobs()
   303→
   304→			// Allow some time for async operations
   305→			time.Sleep(100 * time.Millisecond)
   306→
   307→			if tt.shouldFindJob {
   308→				// Extract jobId from payload
   309→				var payloadMap map[string]any
   310→				json.Unmarshal([]byte(tt.payload), &payloadMap)
   311→				jobID := int64(payloadMap["jobId"].(float64))
   312→				cluster := payloadMap["cluster"].(string)
   313→				startTime := int64(payloadMap["startTime"].(float64))
   314→
   315→				job, err := natsAPI.JobRepository.Find(&jobID, &cluster, &startTime)
   316→				if err != nil {
   317→					if !tt.expectError {
   318→						t.Fatalf("expected to find job, but got error: %v", err)
   319→					}
   320→					return
   321→				}
   322→
   323→				if tt.validateJob != nil {
   324→					tt.validateJob(t, job)
   325→				}
   326→			}
   327→		})
   328→	}
   329→}
   330→
   331→func TestNatsHandleStopJob(t *testing.T) {
   332→	natsAPI := setupNatsTest(t)
   333→	t.Cleanup(cleanupNatsTest)
   334→
   335→	// First, create a running job
   336→	startPayload := `{
   337→		"jobId": 2001,
   338→		"user": "testuser",
   339→		"project": "testproj",
   340→		"cluster": "testcluster",
   341→		"partition": "main",
   342→		"walltime": 3600,
   343→		"numNodes": 1,
   344→		"numHwthreads": 8,
   345→		"numAcc": 0,
   346→		"shared": "none",
   347→		"monitoringStatus": 1,
   348→		"smt": 1,
   349→		"resources": [
   350→			{
   351→				"hostname": "host123",
   352→				"hwthreads": [0, 1, 2, 3, 4, 5, 6, 7]
   353→			}
   354→		],
   355→		"startTime": 1234567890
   356→	}`
   357→
   358→	natsAPI.handleStartJob(startPayload)
   359→	natsAPI.JobRepository.SyncJobs()
   360→	time.Sleep(100 * time.Millisecond)
   361→
   362→	tests := []struct {
   363→		name         string
   364→		payload      string
   365→		expectError  bool
   366→		validateJob  func(t *testing.T, job *schema.Job)
   367→		setupJobFunc func() // Optional: create specific test job
   368→	}{
   369→		{
   370→			name: "valid job stop - completed",
   371→			payload: `{
   372→				"jobId": 2001,
   373→				"cluster": "testcluster",
   374→				"startTime": 1234567890,
   375→				"jobState": "completed",
   376→				"stopTime": 1234571490
   377→			}`,
   378→			expectError: false,
   379→			validateJob: func(t *testing.T, job *schema.Job) {
   380→				if job.State != schema.JobStateCompleted {
   381→					t.Errorf("expected state completed, got %s", job.State)
   382→				}
   383→				expectedDuration := int32(1234571490 - 1234567890)
   384→				if job.Duration != expectedDuration {
   385→					t.Errorf("expected duration %d, got %d", expectedDuration, job.Duration)
   386→				}
   387→			},
   388→		},
   389→		{
   390→			name: "valid job stop - failed",
   391→			setupJobFunc: func() {
   392→				startPayloadFailed := `{
   393→					"jobId": 2002,
   394→					"user": "testuser",
   395→					"project": "testproj",
   396→					"cluster": "testcluster",
   397→					"partition": "main",
   398→					"walltime": 3600,
   399→					"numNodes": 1,
   400→					"numHwthreads": 8,
   401→					"numAcc": 0,
   402→					"shared": "none",
   403→					"monitoringStatus": 1,
   404→					"smt": 1,
   405→					"resources": [
   406→						{
   407→							"hostname": "host123",
   408→							"hwthreads": [0, 1, 2, 3]
   409→						}
   410→					],
   411→					"startTime": 1234567900
   412→				}`
   413→				natsAPI.handleStartJob(startPayloadFailed)
   414→				natsAPI.JobRepository.SyncJobs()
   415→				time.Sleep(100 * time.Millisecond)
   416→			},
   417→			payload: `{
   418→				"jobId": 2002,
   419→				"cluster": "testcluster",
   420→				"startTime": 1234567900,
   421→				"jobState": "failed",
   422→				"stopTime": 1234569900
   423→			}`,
   424→			expectError: false,
   425→			validateJob: func(t *testing.T, job *schema.Job) {
   426→				if job.State != schema.JobStateFailed {
   427→					t.Errorf("expected state failed, got %s", job.State)
   428→				}
   429→			},
   430→		},
   431→		{
   432→			name: "invalid JSON",
   433→			payload: `{
   434→				"jobId": "not a number"
   435→			}`,
   436→			expectError: true,
   437→		},
   438→		{
   439→			name: "missing jobId",
   440→			payload: `{
   441→				"cluster": "testcluster",
   442→				"jobState": "completed",
   443→				"stopTime": 1234571490
   444→			}`,
   445→			expectError: true,
   446→		},
   447→		{
   448→			name: "invalid job state",
   449→			setupJobFunc: func() {
   450→				startPayloadInvalid := `{
   451→					"jobId": 2003,
   452→					"user": "testuser",
   453→					"project": "testproj",
   454→					"cluster": "testcluster",
   455→					"partition": "main",
   456→					"walltime": 3600,
   457→					"numNodes": 1,
   458→					"numHwthreads": 8,
   459→					"numAcc": 0,
   460→					"shared": "none",
   461→					"monitoringStatus": 1,
   462→					"smt": 1,
   463→					"resources": [
   464→						{
   465→							"hostname": "host123",
   466→							"hwthreads": [0, 1]
   467→						}
   468→					],
   469→					"startTime": 1234567910
   470→				}`
   471→				natsAPI.handleStartJob(startPayloadInvalid)
   472→				natsAPI.JobRepository.SyncJobs()
   473→				time.Sleep(100 * time.Millisecond)
   474→			},
   475→			payload: `{
   476→				"jobId": 2003,
   477→				"cluster": "testcluster",
   478→				"startTime": 1234567910,
   479→				"jobState": "invalid_state",
   480→				"stopTime": 1234571510
   481→			}`,
   482→			expectError: true,
   483→		},
   484→		{
   485→			name: "stopTime before startTime",
   486→			setupJobFunc: func() {
   487→				startPayloadTime := `{
   488→					"jobId": 2004,
   489→					"user": "testuser",
   490→					"project": "testproj",
   491→					"cluster": "testcluster",
   492→					"partition": "main",
   493→					"walltime": 3600,
   494→					"numNodes": 1,
   495→					"numHwthreads": 8,
   496→					"numAcc": 0,
   497→					"shared": "none",
   498→					"monitoringStatus": 1,
   499→					"smt": 1,
   500→					"resources": [
   501→						{
   502→							"hostname": "host123",
   503→							"hwthreads": [0]
   504→						}
   505→					],
   506→					"startTime": 1234567920
   507→				}`
   508→				natsAPI.handleStartJob(startPayloadTime)
   509→				natsAPI.JobRepository.SyncJobs()
   510→				time.Sleep(100 * time.Millisecond)
   511→			},
   512→			payload: `{
   513→				"jobId": 2004,
   514→				"cluster": "testcluster",
   515→				"startTime": 1234567920,
   516→				"jobState": "completed",
   517→				"stopTime": 1234567900
   518→			}`,
   519→			expectError: true,
   520→		},
   521→		{
   522→			name: "job not found",
   523→			payload: `{
   524→				"jobId": 99999,
   525→				"cluster": "testcluster",
   526→				"startTime": 1234567890,
   527→				"jobState": "completed",
   528→				"stopTime": 1234571490
   529→			}`,
   530→			expectError: true,
   531→		},
   532→	}
   533→
   534→	testData := schema.JobData{
   535→		"load_one": map[schema.MetricScope]*schema.JobMetric{
   536→			schema.MetricScopeNode: {
   537→				Unit:     schema.Unit{Base: "load"},
   538→				Timestep: 60,
   539→				Series: []schema.Series{
   540→					{
   541→						Hostname:   "host123",
   542→						Statistics: schema.MetricStatistics{Min: 0.1, Avg: 0.2, Max: 0.3},
   543→						Data:       []schema.Float{0.1, 0.1, 0.1, 0.2, 0.2, 0.2, 0.3, 0.3, 0.3},
   544→					},
   545→				},
   546→			},
   547→		},
   548→	}
   549→
   550→	metricstore.TestLoadDataCallback = func(job *schema.Job, metrics []string, scopes []schema.MetricScope, ctx context.Context, resolution int) (schema.JobData, error) {
   551→		return testData, nil
   552→	}
   553→
   554→	for _, tt := range tests {
   555→		t.Run(tt.name, func(t *testing.T) {
   556→			if tt.setupJobFunc != nil {
   557→				tt.setupJobFunc()
   558→			}
   559→
   560→			natsAPI.handleStopJob(tt.payload)
   561→
   562→			// Allow some time for async operations
   563→			time.Sleep(100 * time.Millisecond)
   564→
   565→			if !tt.expectError && tt.validateJob != nil {
   566→				// Extract job details from payload
   567→				var payloadMap map[string]any
   568→				json.Unmarshal([]byte(tt.payload), &payloadMap)
   569→				jobID := int64(payloadMap["jobId"].(float64))
   570→				cluster := payloadMap["cluster"].(string)
   571→
   572→				var startTime *int64
   573→				if st, ok := payloadMap["startTime"]; ok {
   574→					t := int64(st.(float64))
   575→					startTime = &t
   576→				}
   577→
   578→				job, err := natsAPI.JobRepository.Find(&jobID, &cluster, startTime)
   579→				if err != nil {
   580→					t.Fatalf("expected to find job, but got error: %v", err)
   581→				}
   582→
   583→				tt.validateJob(t, job)
   584→			}
   585→		})
   586→	}
   587→}
   588→
   589→func TestNatsHandleNodeState(t *testing.T) {
   590→	natsAPI := setupNatsTest(t)
   591→	t.Cleanup(cleanupNatsTest)
   592→
   593→	tests := []struct {
   594→		name        string
   595→		data        []byte
   596→		expectError bool
   597→		validateFn  func(t *testing.T)
   598→	}{
   599→		{
   600→			name:        "valid node state update",
   601→			data:        []byte(`nodestate event="{\"cluster\":\"testcluster\",\"nodes\":[{\"hostname\":\"host123\",\"states\":[\"allocated\"],\"cpusAllocated\":8,\"memoryAllocated\":16384,\"gpusAllocated\":0,\"jobsRunning\":1}]}" 1234567890000000000`),
   602→			expectError: false,
   603→			validateFn: func(t *testing.T) {
   604→				// In a full test, we would verify the node state was updated in the database
   605→				// For now, just ensure no error occurred
   606→			},
   607→		},
   608→		{
   609→			name:        "multiple nodes",
   610→			data:        []byte(`nodestate event="{\"cluster\":\"testcluster\",\"nodes\":[{\"hostname\":\"host123\",\"states\":[\"idle\"],\"cpusAllocated\":0,\"memoryAllocated\":0,\"gpusAllocated\":0,\"jobsRunning\":0},{\"hostname\":\"host124\",\"states\":[\"allocated\"],\"cpusAllocated\":4,\"memoryAllocated\":8192,\"gpusAllocated\":1,\"jobsRunning\":1}]}" 1234567890000000000`),
   611→			expectError: false,
   612→		},
   613→		{
   614→			name:        "invalid JSON in event field",
   615→			data:        []byte(`nodestate event="{\"cluster\":\"testcluster\",\"nodes\":\"not an array\"}" 1234567890000000000`),
   616→			expectError: true,
   617→		},
   618→		{
   619→			name:        "empty nodes array",
   620→			data:        []byte(`nodestate event="{\"cluster\":\"testcluster\",\"nodes\":[]}" 1234567890000000000`),
   621→			expectError: false, // Empty array should not cause error
   622→		},
   623→		{
   624→			name:        "invalid line protocol format",
   625→			data:        []byte(`invalid line protocol format`),
   626→			expectError: true,
   627→		},
   628→		{
   629→			name:        "empty data",
   630→			data:        []byte(``),
   631→			expectError: false, // Should be handled gracefully with warning
   632→		},
   633→	}
   634→
   635→	for _, tt := range tests {
   636→		t.Run(tt.name, func(t *testing.T) {
   637→			natsAPI.handleNodeState("test.subject", tt.data)
   638→
   639→			// Allow some time for async operations
   640→			time.Sleep(50 * time.Millisecond)
   641→
   642→			if tt.validateFn != nil {
   643→				tt.validateFn(t)
   644→			}
   645→		})
   646→	}
   647→}
   648→
   649→func TestNatsProcessJobEvent(t *testing.T) {
   650→	natsAPI := setupNatsTest(t)
   651→	t.Cleanup(cleanupNatsTest)
   652→
   653→	msgStartJob, err := lp.NewMessage(
   654→		"job",
   655→		map[string]string{"function": "start_job"},
   656→		nil,
   657→		map[string]any{
   658→			"event": `{
   659→				"jobId": 3001,
   660→				"user": "testuser",
   661→				"project": "testproj",
   662→				"cluster": "testcluster",
   663→				"partition": "main",
   664→				"walltime": 3600,
   665→				"numNodes": 1,
   666→				"numHwthreads": 8,
   667→				"numAcc": 0,
   668→				"shared": "none",
   669→				"monitoringStatus": 1,
   670→				"smt": 1,
   671→				"resources": [
   672→					{
   673→						"hostname": "host123",
   674→						"hwthreads": [0, 1, 2, 3]
   675→					}
   676→				],
   677→				"startTime": 1234567890
   678→			}`,
   679→		},
   680→		time.Now(),
   681→	)
   682→	if err != nil {
   683→		t.Fatalf("failed to create test message: %v", err)
   684→	}
   685→
   686→	msgMissingTag, err := lp.NewMessage(
   687→		"job",
   688→		map[string]string{},
   689→		nil,
   690→		map[string]any{
   691→			"event": `{}`,
   692→		},
   693→		time.Now(),
   694→	)
   695→	if err != nil {
   696→		t.Fatalf("failed to create test message: %v", err)
   697→	}
   698→
   699→	msgUnknownFunc, err := lp.NewMessage(
   700→		"job",
   701→		map[string]string{"function": "unknown_function"},
   702→		nil,
   703→		map[string]any{
   704→			"event": `{}`,
   705→		},
   706→		time.Now(),
   707→	)
   708→	if err != nil {
   709→		t.Fatalf("failed to create test message: %v", err)
   710→	}
   711→
   712→	tests := []struct {
   713→		name        string
   714→		message     lp.CCMessage
   715→		expectError bool
   716→	}{
   717→		{
   718→			name:        "start_job function",
   719→			message:     msgStartJob,
   720→			expectError: false,
   721→		},
   722→		{
   723→			name:        "missing function tag",
   724→			message:     msgMissingTag,
   725→			expectError: true,
   726→		},
   727→		{
   728→			name:        "unknown function",
   729→			message:     msgUnknownFunc,
   730→			expectError: false,
   731→		},
   732→	}
   733→
   734→	for _, tt := range tests {
   735→		t.Run(tt.name, func(t *testing.T) {
   736→			natsAPI.processJobEvent(tt.message)
   737→			time.Sleep(50 * time.Millisecond)
   738→		})
   739→	}
   740→}
   741→
   742→func TestNatsHandleJobEvent(t *testing.T) {
   743→	natsAPI := setupNatsTest(t)
   744→	t.Cleanup(cleanupNatsTest)
   745→
   746→	tests := []struct {
   747→		name        string
   748→		data        []byte
   749→		expectError bool
   750→	}{
   751→		{
   752→			name:        "valid influx line protocol",
   753→			data:        []byte(`job,function=start_job event="{\"jobId\":4001,\"user\":\"testuser\",\"project\":\"testproj\",\"cluster\":\"testcluster\",\"partition\":\"main\",\"walltime\":3600,\"numNodes\":1,\"numHwthreads\":8,\"numAcc\":0,\"shared\":\"none\",\"monitoringStatus\":1,\"smt\":1,\"resources\":[{\"hostname\":\"host123\",\"hwthreads\":[0,1,2,3]}],\"startTime\":1234567890}" 1234567890000000000`),
   754→			expectError: false,
   755→		},
   756→		{
   757→			name:        "invalid influx line protocol",
   758→			data:        []byte(`invalid line protocol format`),
   759→			expectError: true,
   760→		},
   761→		{
   762→			name:        "empty data",
   763→			data:        []byte(``),
   764→			expectError: false, // Decoder should handle empty input gracefully
   765→		},
   766→	}
   767→
   768→	for _, tt := range tests {
   769→		t.Run(tt.name, func(t *testing.T) {
   770→			// HandleJobEvent doesn't return errors, it logs them
   771→			// We're just ensuring it doesn't panic
   772→			natsAPI.handleJobEvent("test.subject", tt.data)
   773→			time.Sleep(50 * time.Millisecond)
   774→		})
   775→	}
   776→}
   777→
   778→func TestNatsHandleJobEventEdgeCases(t *testing.T) {
   779→	natsAPI := setupNatsTest(t)
   780→	t.Cleanup(cleanupNatsTest)
   781→
   782→	tests := []struct {
   783→		name        string
   784→		data        []byte
   785→		expectError bool
   786→		description string
   787→	}{
   788→		{
   789→			name:        "non-event message (metric data)",
   790→			data:        []byte(`job,function=start_job value=123.45 1234567890000000000`),
   791→			expectError: false,
   792→			description: "Should skip non-event messages gracefully",
   793→		},
   794→		{
   795→			name:        "wrong measurement name",
   796→			data:        []byte(`wrongmeasurement,function=start_job event="{}" 1234567890000000000`),
   797→			expectError: false,
   798→			description: "Should warn about unexpected measurement but not fail",
   799→		},
   800→		{
   801→			name:        "missing event field",
   802→			data:        []byte(`job,function=start_job other_field="value" 1234567890000000000`),
   803→			expectError: true,
   804→			description: "Should error when event field is missing",
   805→		},
   806→		{
   807→			name:        "multiple measurements in one message",
   808→			data:        []byte("job,function=start_job event=\"{}\" 1234567890000000000\njob,function=stop_job event=\"{}\" 1234567890000000000"),
   809→			expectError: false,
   810→			description: "Should process multiple lines",
   811→		},
   812→		{
   813→			name:        "escaped quotes in JSON payload",
   814→			data:        []byte(`job,function=start_job event="{\"jobId\":6001,\"user\":\"test\\\"user\",\"cluster\":\"test\"}" 1234567890000000000`),
   815→			expectError: true,
   816→			description: "Should handle escaped quotes (though JSON parsing may fail)",
   817→		},
   818→	}
   819→
   820→	for _, tt := range tests {
   821→		t.Run(tt.name, func(t *testing.T) {
   822→			natsAPI.handleJobEvent("test.subject", tt.data)
   823→			time.Sleep(50 * time.Millisecond)
   824→		})
   825→	}
   826→}
   827→
   828→func TestNatsHandleNodeStateEdgeCases(t *testing.T) {
   829→	natsAPI := setupNatsTest(t)
   830→	t.Cleanup(cleanupNatsTest)
   831→
   832→	tests := []struct {
   833→		name        string
   834→		data        []byte
   835→		expectError bool
   836→		description string
   837→	}{
   838→		{
   839→			name:        "missing cluster field in JSON",
   840→			data:        []byte(`nodestate event="{\"nodes\":[]}" 1234567890000000000`),
   841→			expectError: true,
   842→			description: "Should fail when cluster is missing",
   843→		},
   844→		{
   845→			name:        "malformed JSON with unescaped quotes",
   846→			data:        []byte(`nodestate event="{\"cluster\":\"test"cluster\",\"nodes\":[]}" 1234567890000000000`),
   847→			expectError: true,
   848→			description: "Should fail on malformed JSON",
   849→		},
   850→		{
   851→			name:        "unicode characters in hostname",
   852→			data:        []byte(`nodestate event="{\"cluster\":\"testcluster\",\"nodes\":[{\"hostname\":\"host-ñ123\",\"states\":[\"idle\"],\"cpusAllocated\":0,\"memoryAllocated\":0,\"gpusAllocated\":0,\"jobsRunning\":0}]}" 1234567890000000000`),
   853→			expectError: false,
   854→			description: "Should handle unicode characters",
   855→		},
   856→		{
   857→			name:        "very large node count",
   858→			data:        []byte(`nodestate event="{\"cluster\":\"testcluster\",\"nodes\":[{\"hostname\":\"node1\",\"states\":[\"idle\"],\"cpusAllocated\":0,\"memoryAllocated\":0,\"gpusAllocated\":0,\"jobsRunning\":0},{\"hostname\":\"node2\",\"states\":[\"idle\"],\"cpusAllocated\":0,\"memoryAllocated\":0,\"gpusAllocated\":0,\"jobsRunning\":0},{\"hostname\":\"node3\",\"states\":[\"idle\"],\"cpusAllocated\":0,\"memoryAllocated\":0,\"gpusAllocated\":0,\"jobsRunning\":0}]}" 1234567890000000000`),
   859→			expectError: false,
   860→			description: "Should handle multiple nodes efficiently",
   861→		},
   862→		{
   863→			name:        "timestamp in past",
   864→			data:        []byte(`nodestate event="{\"cluster\":\"testcluster\",\"nodes\":[]}" 1000000000000000000`),
   865→			expectError: false,
   866→			description: "Should accept any valid timestamp",
   867→		},
   868→	}
   869→
   870→	for _, tt := range tests {
   871→		t.Run(tt.name, func(t *testing.T) {
   872→			natsAPI.handleNodeState("test.subject", tt.data)
   873→			time.Sleep(50 * time.Millisecond)
   874→		})
   875→	}
   876→}
   877→
   878→func TestNatsHandleStartJobDuplicatePrevention(t *testing.T) {
   879→	natsAPI := setupNatsTest(t)
   880→	t.Cleanup(cleanupNatsTest)
   881→
   882→	// Start a job
   883→	payload := `{
   884→		"jobId": 5001,
   885→		"user": "testuser",
   886→		"project": "testproj",
   887→		"cluster": "testcluster",
   888→		"partition": "main",
   889→		"walltime": 3600,
   890→		"numNodes": 1,
   891→		"numHwthreads": 8,
   892→		"numAcc": 0,
   893→		"shared": "none",
   894→		"monitoringStatus": 1,
   895→		"smt": 1,
   896→		"resources": [
   897→			{
   898→				"hostname": "host123",
   899→				"hwthreads": [0, 1, 2, 3]
   900→			}
   901→		],
   902→		"startTime": 1234567890
   903→	}`
   904→
   905→	natsAPI.handleStartJob(payload)
   906→	natsAPI.JobRepository.SyncJobs()
   907→	time.Sleep(100 * time.Millisecond)
   908→
   909→	// Try to start the same job again (within 24 hours)
   910→	duplicatePayload := `{
   911→		"jobId": 5001,
   912→		"user": "testuser",
   913→		"project": "testproj",
   914→		"cluster": "testcluster",
   915→		"partition": "main",
   916→		"walltime": 3600,
   917→		"numNodes": 1,
   918→		"numHwthreads": 8,
   919→		"numAcc": 0,
   920→		"shared": "none",
   921→		"monitoringStatus": 1,
   922→		"smt": 1,
   923→		"resources": [
   924→			{
   925→				"hostname": "host123",
   926→				"hwthreads": [0, 1, 2, 3]
   927→			}
   928→		],
   929→		"startTime": 1234567900
   930→	}`
   931→
   932→	natsAPI.handleStartJob(duplicatePayload)
   933→	natsAPI.JobRepository.SyncJobs()
   934→	time.Sleep(100 * time.Millisecond)
   935→
   936→	// Verify only one job exists
   937→	jobID := int64(5001)
   938→	cluster := "testcluster"
   939→	jobs, err := natsAPI.JobRepository.FindAll(&jobID, &cluster, nil)
   940→	if err != nil && err != sql.ErrNoRows {
   941→		t.Fatalf("unexpected error: %v", err)
   942→	}
   943→
   944→	if len(jobs) != 1 {
   945→		t.Errorf("expected 1 job, got %d", len(jobs))
   946→	}
   947→}
   948→


Bash

grep -n "type NodePayload\|type NodeStateDB" /Users/jan/prg/ClusterCockpit/cc-lib/schema/*.go 2>/dev/null | head -20


/Users/jan/prg/ClusterCockpit/cc-lib/schema/node.go:51:type NodePayload struct {
/Users/jan/prg/ClusterCockpit/cc-lib/schema/node.go:72:type NodeStateDB struct {


find /Users/jan/prg/ClusterCockpit -name "*.go" -type f -exec grep -l "type NodePayload" {} \;


Found 3 files
internal/api/nats.go
internal/api/node.go
internal/repository/node.go


Found 2 files
internal/api/nats.go
internal/api/node.go


internal/repository/node.go:248:type NodeStateUpdate struct {
internal/repository/node.go:257:func (r *NodeRepository) BatchUpdateNodeStates(updates []NodeStateUpdate) error {


     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→package repository
     7→
     8→import (
     9→	"context"
    10→	"database/sql"
    11→	"encoding/json"
    12→	"fmt"
    13→	"slices"
    14→	"sort"
    15→	"strings"
    16→	"sync"
    17→	"time"
    18→
    19→	"github.com/ClusterCockpit/cc-backend/internal/graph/model"
    20→	"github.com/ClusterCockpit/cc-backend/pkg/archive"
    21→	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
    22→	"github.com/ClusterCockpit/cc-lib/v2/lrucache"
    23→	"github.com/ClusterCockpit/cc-lib/v2/schema"
    24→	sq "github.com/Masterminds/squirrel"
    25→	"github.com/jmoiron/sqlx"
    26→)
    27→
    28→var (
    29→	nodeRepoOnce     sync.Once
    30→	nodeRepoInstance *NodeRepository
    31→)
    32→
    33→type NodeRepository struct {
    34→	DB        *sqlx.DB
    35→	stmtCache *sq.StmtCache
    36→	cache     *lrucache.Cache
    37→	driver    string
    38→}
    39→
    40→func GetNodeRepository() *NodeRepository {
    41→	nodeRepoOnce.Do(func() {
    42→		db := GetConnection()
    43→
    44→		nodeRepoInstance = &NodeRepository{
    45→			DB:     db.DB,
    46→			driver: db.Driver,
    47→
    48→			stmtCache: sq.NewStmtCache(db.DB),
    49→			cache:     lrucache.New(repoConfig.CacheSize),
    50→		}
    51→	})
    52→	return nodeRepoInstance
    53→}
    54→
    55→// latestStateCondition returns a squirrel expression that restricts node_state
    56→// rows to the latest per node_id using a correlated subquery.
    57→// Requires the query to join node and node_state tables.
    58→func latestStateCondition() sq.Sqlizer {
    59→	return sq.Expr(
    60→		"node_state.id = (SELECT ns2.id FROM node_state ns2 WHERE ns2.node_id = node.id ORDER BY ns2.time_stamp DESC LIMIT 1)",
    61→	)
    62→}
    63→
    64→// applyNodeFilters applies common NodeFilter conditions to a query that joins
    65→// the node and node_state tables with latestStateCondition.
    66→func applyNodeFilters(query sq.SelectBuilder, filters []*model.NodeFilter) sq.SelectBuilder {
    67→	for _, f := range filters {
    68→		if f.Cluster != nil {
    69→			query = buildStringCondition("node.cluster", f.Cluster, query)
    70→		}
    71→		if f.SubCluster != nil {
    72→			query = buildStringCondition("node.subcluster", f.SubCluster, query)
    73→		}
    74→		if f.Hostname != nil {
    75→			query = buildStringCondition("node.hostname", f.Hostname, query)
    76→		}
    77→		if f.SchedulerState != nil {
    78→			query = query.Where("node_state.node_state = ?", f.SchedulerState)
    79→		}
    80→		if f.HealthState != nil {
    81→			query = query.Where("node_state.health_state = ?", f.HealthState)
    82→		}
    83→	}
    84→	return query
    85→}
    86→
    87→func (r *NodeRepository) FetchMetadata(hostname string, cluster string) (map[string]string, error) {
    88→	start := time.Now()
    89→
    90→	RawMetaData := make([]byte, 0)
    91→
    92→	if err := sq.Select("node.meta_data").From("node").
    93→		Where("node.hostname = ?", hostname).
    94→		Where("node.cluster = ?", cluster).
    95→		RunWith(r.stmtCache).QueryRow().Scan(&RawMetaData); err != nil {
    96→		cclog.Warn("Error while scanning for node metadata")
    97→		return nil, err
    98→	}
    99→
   100→	if len(RawMetaData) == 0 {
   101→		return nil, nil
   102→	}
   103→
   104→	MetaData := make(map[string]string)
   105→
   106→	if err := json.Unmarshal(RawMetaData, &MetaData); err != nil {
   107→		cclog.Warn("Error while unmarshaling raw metadata json")
   108→		return nil, err
   109→	}
   110→
   111→	cclog.Debugf("Timer FetchMetadata %s", time.Since(start))
   112→	return MetaData, nil
   113→}
   114→
   115→func (r *NodeRepository) GetNode(hostname string, cluster string, withMeta bool) (*schema.Node, error) {
   116→	node := &schema.Node{}
   117→	if err := sq.Select("node.hostname", "node.cluster", "node.subcluster",
   118→		"node_state.node_state", "node_state.health_state").
   119→		From("node").
   120→		Join("node_state ON node_state.node_id = node.id").
   121→		Where(latestStateCondition()).
   122→		Where("node.hostname = ?", hostname).
   123→		Where("node.cluster = ?", cluster).
   124→		RunWith(r.DB).
   125→		QueryRow().Scan(&node.Hostname, &node.Cluster, &node.SubCluster, &node.NodeState, &node.HealthState); err != nil {
   126→		cclog.Warnf("Error while querying node '%s' from database: %v", hostname, err)
   127→		return nil, err
   128→	}
   129→
   130→	if withMeta {
   131→		var err error
   132→		var meta map[string]string
   133→		if meta, err = r.FetchMetadata(hostname, cluster); err != nil {
   134→			cclog.Warnf("Error while fetching metadata for node '%s'", hostname)
   135→			return nil, err
   136→		}
   137→		node.MetaData = meta
   138→	}
   139→
   140→	return node, nil
   141→}
   142→
   143→func (r *NodeRepository) GetNodeByID(id int64, withMeta bool) (*schema.Node, error) {
   144→	node := &schema.Node{}
   145→	if err := sq.Select("node.hostname", "node.cluster", "node.subcluster",
   146→		"node_state.node_state", "node_state.health_state").
   147→		From("node").
   148→		Join("node_state ON node_state.node_id = node.id").
   149→		Where(latestStateCondition()).
   150→		Where("node.id = ?", id).


   245→}
   246→
   247→// NodeStateUpdate holds the data needed to update one node's state in a batch operation.
   248→type NodeStateUpdate struct {
   249→	Hostname  string
   250→	Cluster   string
   251→	NodeState *schema.NodeStateDB
   252→}
   253→
   254→// BatchUpdateNodeStates inserts node state rows for multiple nodes in a single transaction.
   255→// For each node, it looks up (or creates) the node row, then inserts the state row.
   256→// This reduces lock acquisitions from 2*N to 1 for N nodes.
   257→func (r *NodeRepository) BatchUpdateNodeStates(updates []NodeStateUpdate) error {
   258→	tx, err := r.DB.Beginx()
   259→	if err != nil {
   260→		return fmt.Errorf("BatchUpdateNodeStates: begin transaction: %w", err)
   261→	}
   262→	defer tx.Rollback()
   263→
   264→	stmtLookup, err := tx.Preparex("SELECT id FROM node WHERE hostname = ? AND cluster = ?")
   265→	if err != nil {
   266→		return fmt.Errorf("BatchUpdateNodeStates: prepare lookup: %w", err)
   267→	}
   268→	defer stmtLookup.Close()
   269→
   270→	stmtInsertNode, err := tx.PrepareNamed(NamedNodeInsert)
   271→	if err != nil {
   272→		return fmt.Errorf("BatchUpdateNodeStates: prepare node insert: %w", err)
   273→	}
   274→	defer stmtInsertNode.Close()
   275→
   276→	stmtInsertState, err := tx.PrepareNamed(NamedNodeStateInsert)
   277→	if err != nil {
   278→		return fmt.Errorf("BatchUpdateNodeStates: prepare state insert: %w", err)
   279→	}
   280→	defer stmtInsertState.Close()
   281→
   282→	for _, u := range updates {
   283→		var id int64
   284→		if err := stmtLookup.QueryRow(u.Hostname, u.Cluster).Scan(&id); err != nil {
   285→			if err == sql.ErrNoRows {
   286→				subcluster, scErr := archive.GetSubClusterByNode(u.Cluster, u.Hostname)
   287→				if scErr != nil {
   288→					cclog.Errorf("BatchUpdateNodeStates: subcluster lookup for '%s' in '%s': %v", u.Hostname, u.Cluster, scErr)
   289→					continue
   290→				}
   291→				node := schema.NodeDB{
   292→					Hostname: u.Hostname, Cluster: u.Cluster, SubCluster: subcluster,
   293→				}
   294→				res, insertErr := stmtInsertNode.Exec(&node)
   295→				if insertErr != nil {
   296→					cclog.Errorf("BatchUpdateNodeStates: insert node '%s': %v", u.Hostname, insertErr)
   297→					continue
   298→				}
   299→				id, _ = res.LastInsertId()
   300→			} else {
   301→				cclog.Errorf("BatchUpdateNodeStates: lookup node '%s': %v", u.Hostname, err)
   302→				continue
   303→			}
   304→		}
   305→
   306→		u.NodeState.NodeID = id
   307→		if _, err := stmtInsertState.Exec(u.NodeState); err != nil {
   308→			cclog.Errorf("BatchUpdateNodeStates: insert state for '%s': %v", u.Hostname, err)
   309→		}
   310→	}
   311→
   312→	if err := tx.Commit(); err != nil {
   313→		return fmt.Errorf("BatchUpdateNodeStates: commit: %w", err)
   314→	}
   315→	return nil
   316→}
   317→
   318→// func (r *NodeRepository) UpdateHealthState(hostname string, healthState *schema.MonitoringState) error {
   319→// 	if _, err := sq.Update("node").Set("health_state", healthState).Where("node.id = ?", id).RunWith(r.DB).Exec(); err != nil {
   320→// 		cclog.Errorf("error while updating node '%d'", id)
   321→// 		return err
   322→// 	}
   323→//
   324→// 	return nil
   325→// }
   326→
   327→// NodeStateWithNode combines a node state row with denormalized node info.
   328→type NodeStateWithNode struct {
   329→	ID              int64  `db:"id"`
   330→	TimeStamp       int64  `db:"time_stamp"`
   331→	NodeState       string `db:"node_state"`
   332→	HealthState     string `db:"health_state"`
   333→	HealthMetrics   string `db:"health_metrics"`
   334→	CpusAllocated   int    `db:"cpus_allocated"`
   335→	MemoryAllocated int64  `db:"memory_allocated"`
   336→	GpusAllocated   int    `db:"gpus_allocated"`
   337→	JobsRunning     int    `db:"jobs_running"`
   338→	Hostname        string `db:"hostname"`
   339→	Cluster         string `db:"cluster"`
   340→	SubCluster      string `db:"subcluster"`
   341→}
   342→
   343→// FindNodeStatesBefore returns all node_state rows with time_stamp < cutoff,
   344→// joined with node info for denormalized archiving.


No files found


cd /Users/jan/prg/ClusterCockpit && find . -name "*.go" -type f | xargs grep -l "type NodePayload" 2>/dev/null | head -5


     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-lib.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→package schema
     7→
     8→// SchedulerState represents the current state of a node in the HPC job scheduler.
     9→// States typically reflect SLURM/PBS node states.
    10→type SchedulerState string
    11→
    12→const (
    13→	NodeStateAllocated SchedulerState = "allocated" // Node is fully allocated to jobs
    14→	NodeStateReserved  SchedulerState = "reserved"  // Node is reserved but not yet allocated
    15→	NodeStateIdle      SchedulerState = "idle"      // Node is available for jobs
    16→	NodeStateMixed     SchedulerState = "mixed"     // Node is partially allocated
    17→	NodeStateDown      SchedulerState = "down"      // Node is down/offline
    18→	NodeStateUnknown   SchedulerState = "unknown"   // Node state unknown
    19→)
    20→
    21→// MonitoringState indicates the health monitoring status of a node.
    22→// Reflects whether metric collection is working correctly.
    23→type MonitoringState string
    24→
    25→const (
    26→	MonitoringStateFull    MonitoringState = "full"    // All metrics being collected successfully
    27→	MonitoringStatePartial MonitoringState = "partial" // Some metrics missing
    28→	MonitoringStateFailed  MonitoringState = "failed"  // Metric collection failing
    29→)
    30→
    31→// Node represents the current state and resource utilization of a compute node.
    32→//
    33→// Combines scheduler state with monitoring health and current resource allocation.
    34→// Used for displaying node status in dashboards and tracking node utilization.
    35→type Node struct {
    36→	Hostname        string              `json:"hostname"`        // Node hostname
    37→	Cluster         string              `json:"cluster"`         // Cluster name
    38→	SubCluster      string              `json:"subCluster"`      // Subcluster name
    39→	MetaData        map[string]string   `json:"metaData"`        // Additional metadata
    40→	HealthData      map[string][]string `json:"healthData"`      // Details about available metrics per node
    41→	NodeState       SchedulerState      `json:"nodeState"`       // Scheduler/resource manager state
    42→	HealthState     MonitoringState     `json:"healthState"`     // Monitoring system health
    43→	CpusAllocated   int                 `json:"cpusAllocated"`   // Number of allocated CPUs
    44→	MemoryAllocated int                 `json:"memoryAllocated"` // Allocated memory in MB
    45→	GpusAllocated   int                 `json:"gpusAllocated"`   // Number of allocated GPUs
    46→	JobsRunning     int                 `json:"jobsRunning"`     // Number of jobs running on this node
    47→}
    48→
    49→// NodePayload is the request body format for the node state REST API.
    50→// Used when updateing node states from external monitoring or scheduler systems.
    51→type NodePayload struct {
    52→	Hostname        string   `json:"hostname"`        // Node hostname
    53→	States          []string `json:"states"`          // State strings (flexible format)
    54→	CpusAllocated   int      `json:"cpusAllocated"`   // Number of allocated CPUs
    55→	MemoryAllocated int64    `json:"memoryAllocated"` // Allocated memory in MB
    56→	GpusAllocated   int      `json:"gpusAllocated"`   // Number of allocated GPUs
    57→	JobsRunning     int      `json:"jobsRunning"`     // Number of running jobs
    58→}
    59→
    60→// NodeDB is the database model for the node table.
    61→// Stores static node configuration and metadata.
    62→type NodeDB struct {
    63→	ID          int64  `json:"id" db:"id"`                                // Database ID
    64→	Hostname    string `json:"hostname" db:"hostname" example:"fritz"`    // Node hostname
    65→	Cluster     string `json:"cluster" db:"cluster" example:"fritz"`      // Cluster name
    66→	SubCluster  string `json:"subCluster" db:"subcluster" example:"main"` // Subcluster name
    67→	RawMetaData []byte `json:"-" db:"meta_data"`                          // Metadata as JSON blob
    68→}
    69→
    70→// NodeStateDB is the database model for the node_state table.
    71→// Stores time-stamped snapshots of node state and resource allocation.
    72→type NodeStateDB struct {
    73→	ID              int64           `json:"id" db:"id"`                                                                                // Database ID
    74→	TimeStamp       int64           `json:"timeStamp" db:"time_stamp" example:"1649723812"`                                                                     // Unix timestamp
    75→	NodeState       SchedulerState  `json:"nodeState" db:"node_state" example:"completed" enums:"completed,failed,cancelled,stopped,timeout,out_of_memory"`     // Scheduler state
    76→	HealthState     MonitoringState `json:"healthState" db:"health_state" example:"completed" enums:"completed,failed,cancelled,stopped,timeout,out_of_memory"` // Monitoring health
    77→	HealthMetrics   string          `json:"healthMetrics" db:"health_metrics"`                                                                                // JSON: {"missing":[...],"degraded":[...]}
    78→	CpusAllocated   int             `json:"cpusAllocated" db:"cpus_allocated"`                                                                                // Allocated CPUs
    79→	MemoryAllocated int64           `json:"memoryAllocated" db:"memory_allocated"`                                                                              // Allocated memory (MB)
    80→	GpusAllocated   int             `json:"gpusAllocated" db:"gpus_allocated"`                                                                                // Allocated GPUs
    81→	JobsRunning     int             `json:"jobsRunning" db:"jobs_running" example:"12"`                                                                         // Running jobs
    82→	NodeID          int64           `json:"_" db:"node_id"`                                                                                // Foreign key to NodeDB
    83→}
    84→


    71→func (api *RestAPI) updateNodeStates(rw http.ResponseWriter, r *http.Request) {
    72→	// Parse request body
    73→	req := UpdateNodeStatesRequest{}
    74→	if err := decode(r.Body, &req); err != nil {
    75→		handleError(fmt.Errorf("parsing request body failed: %w", err),
    76→			http.StatusBadRequest, rw)
    77→		return
    78→	}
    79→	requestReceived := time.Now().Unix()
    80→	repo := repository.GetNodeRepository()
    81→
    82→	m := make(map[string][]string)
    83→	metricNames := make(map[string][]string)
    84→	healthResults := make(map[string]metricstore.HealthCheckResult)
    85→
    86→	startMs := time.Now()
    87→
    88→	// Step 1: Build nodeList and metricList per subcluster
    89→	for _, node := range req.Nodes {
    90→		if sc, err := archive.GetSubClusterByNode(req.Cluster, node.Hostname); err == nil {
    91→			m[sc] = append(m[sc], node.Hostname)
    92→		}
    93→	}
    94→
    95→	for sc := range m {
    96→		if sc != "" {
    97→			metricList := archive.GetMetricConfigSubCluster(req.Cluster, sc)
    98→			metricNames[sc] = metricListToNames(metricList)
    99→		}
   100→	}
   101→
   102→	// Step 2: Determine which metric store to query and perform health check
   103→	healthRepo, err := metricdispatch.GetHealthCheckRepo(req.Cluster)
   104→	if err != nil {
   105→		cclog.Warnf("updateNodeStates: no metric store for cluster %s, skipping health check: %v", req.Cluster, err)
   106→	} else {
   107→		for sc, nl := range m {
   108→			if sc != "" {
   109→				if results, err := healthRepo.HealthCheck(req.Cluster, nl, metricNames[sc]); err == nil {
   110→					maps.Copy(healthResults, results)
   111→				}
   112→			}
   113→		}
   114→	}
   115→
   116→	cclog.Debugf("Timer updateNodeStates, MemStore HealthCheck: %s", time.Since(startMs))
   117→	startDB := time.Now()
   118→
   119→	updates := make([]repository.NodeStateUpdate, 0, len(req.Nodes))
   120→	for _, node := range req.Nodes {
   121→		state := determineState(node.States)
   122→		healthState := schema.MonitoringStateFailed
   123→		var healthMetrics string
   124→		if result, ok := healthResults[node.Hostname]; ok {
   125→			healthState = result.State
   126→			healthMetrics = result.HealthMetrics
   127→		}
   128→		nodeState := schema.NodeStateDB{
   129→			TimeStamp:       requestReceived,
   130→			NodeState:       state,
   131→			CpusAllocated:   node.CpusAllocated,
   132→			MemoryAllocated: node.MemoryAllocated,
   133→			GpusAllocated:   node.GpusAllocated,
   134→			HealthState:     healthState,
   135→			HealthMetrics:   healthMetrics,
   136→			JobsRunning:     node.JobsRunning,
   137→		}
   138→		updates = append(updates, repository.NodeStateUpdate{
   139→			Hostname:  node.Hostname,
   140→			Cluster:   req.Cluster,
   141→			NodeState: &nodeState,
   142→		})
   143→	}
   144→
   145→	if err := repo.BatchUpdateNodeStates(updates); err != nil {
   146→		cclog.Errorf("updateNodeStates: batch update for cluster %s failed: %v", req.Cluster, err)
   147→	}
   148→
   149→	cclog.Debugf("Timer updateNodeStates, SQLite Inserts: %s", time.Since(startDB))
   150→}


28:// metricListToNames converts a map of metric configurations to a list of metric names
29:func metricListToNames(metricList map[string]*schema.Metric) []string {
38:func determineState(states []string) schema.SchedulerState {
98:			metricNames[sc] = metricListToNames(metricList)
103:	healthRepo, err := metricdispatch.GetHealthCheckRepo(req.Cluster)
121:		state := determineState(node.States)


find /Users/jan/prg/ClusterCockpit/cc-backend -name "metricdispatch.go" -type f


(Bash completed with no output)


find /Users/jan/prg/ClusterCockpit/cc-backend -name "*.go" -type f | xargs grep -l "GetHealthCheckRepo" | head -5


/Users/jan/prg/ClusterCockpit/cc-backend/internal/api/node.go
/Users/jan/prg/ClusterCockpit/cc-backend/internal/metricdispatch/metricdata.go


     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→package metricdispatch
     6→
     7→import (
     8→	"bytes"
     9→	"context"
    10→	"encoding/json"
    11→	"fmt"
    12→	"time"
    13→
    14→	"github.com/ClusterCockpit/cc-backend/internal/config"
    15→	ccms "github.com/ClusterCockpit/cc-backend/internal/metricstoreclient"
    16→	"github.com/ClusterCockpit/cc-backend/pkg/metricstore"
    17→	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
    18→	"github.com/ClusterCockpit/cc-lib/v2/schema"
    19→)
    20→
    21→type MetricDataRepository interface {
    22→	// Return the JobData for the given job, only with the requested metrics.
    23→	LoadData(job *schema.Job,
    24→		metrics []string,
    25→		scopes []schema.MetricScope,
    26→		ctx context.Context,
    27→		resolution int) (schema.JobData, error)
    28→
    29→	// Return a map of metrics to a map of nodes to the metric statistics of the job. node scope only.
    30→	LoadStats(job *schema.Job,
    31→		metrics []string,
    32→		ctx context.Context) (map[string]map[string]schema.MetricStatistics, error)
    33→
    34→	// Return a map of metrics to a map of scopes to the scoped metric statistics of the job.
    35→	LoadScopedStats(job *schema.Job,
    36→		metrics []string,
    37→		scopes []schema.MetricScope,
    38→		ctx context.Context) (schema.ScopedJobStats, error)
    39→
    40→	// Return a map of hosts to a map of metrics at the requested scopes (currently only node) for that node.
    41→	LoadNodeData(cluster string,
    42→		metrics, nodes []string,
    43→		scopes []schema.MetricScope,
    44→		from, to time.Time,
    45→		ctx context.Context) (map[string]map[string][]*schema.JobMetric, error)
    46→
    47→	// Return a map of hosts to a map of metrics to a map of scopes for multiple nodes.
    48→	LoadNodeListData(cluster, subCluster string,
    49→		nodes []string,
    50→		metrics []string,
    51→		scopes []schema.MetricScope,
    52→		resolution int,
    53→		from, to time.Time,
    54→		ctx context.Context) (map[string]schema.JobData, error)
    55→
    56→	// HealthCheck evaluates the monitoring state for a set of nodes against expected metrics.
    57→	HealthCheck(cluster string,
    58→		nodes []string,
    59→		metrics []string) (map[string]metricstore.HealthCheckResult, error)
    60→}
    61→
    62→type CCMetricStoreConfig struct {
    63→	Scope string `json:"scope"`
    64→	URL   string `json:"url"`
    65→	Token string `json:"token"`
    66→}
    67→
    68→var metricDataRepos map[string]MetricDataRepository = map[string]MetricDataRepository{}
    69→
    70→func Init(rawConfig json.RawMessage) error {
    71→	if rawConfig != nil {
    72→		var configs []CCMetricStoreConfig
    73→		config.Validate(configSchema, rawConfig)
    74→		dec := json.NewDecoder(bytes.NewReader(rawConfig))
    75→		dec.DisallowUnknownFields()
    76→		if err := dec.Decode(&configs); err != nil {
    77→			return fmt.Errorf("[METRICDISPATCH]> External Metric Store Config Init: Could not decode config file '%s' Error: %s", rawConfig, err.Error())
    78→		}
    79→
    80→		if len(configs) == 0 {
    81→			return fmt.Errorf("[METRICDISPATCH]> No external metric store configurations found in config file")
    82→		}
    83→
    84→		for _, config := range configs {
    85→			metricDataRepos[config.Scope] = ccms.NewCCMetricStore(config.URL, config.Token)
    86→		}
    87→	}
    88→
    89→	return nil
    90→}
    91→
    92→func GetMetricDataRepo(cluster string, subcluster string) (MetricDataRepository, error) {
    93→	var repo MetricDataRepository
    94→	var ok bool
    95→
    96→	key := cluster + "-" + subcluster
    97→	repo, ok = metricDataRepos[key]
    98→
    99→	if !ok {
   100→		repo, ok = metricDataRepos[cluster]


grep -n "GetHealthCheckRepo" /Users/jan/prg/ClusterCockpit/cc-backend/internal/metricdispatch/metricdata.go


119:// GetHealthCheckRepo returns the MetricDataRepository for performing health checks on a cluster.
121:func GetHealthCheckRepo(cluster string) (MetricDataRepository, error) {


   115→
   116→	return repo, nil
   117→}
   118→
   119→// GetHealthCheckRepo returns the MetricDataRepository for performing health checks on a cluster.
   120→// It uses the same fallback logic as GetMetricDataRepo: cluster → wildcard → internal.
   121→func GetHealthCheckRepo(cluster string) (MetricDataRepository, error) {
   122→	return GetMetricDataRepo(cluster, "")
   123→}
   124→


     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→// Package api provides the REST API layer for ClusterCockpit.
     7→// It handles HTTP requests for job management, user administration,
     8→// cluster queries, node state updates, and metrics storage operations.
     9→// The API supports both JWT token authentication and session-based authentication.
    10→package api
    11→
    12→import (
    13→	"encoding/json"
    14→	"fmt"
    15→	"io"
    16→	"net/http"
    17→	"os"
    18→	"path/filepath"
    19→	"strings"
    20→	"sync"
    21→
    22→	"github.com/ClusterCockpit/cc-backend/internal/auth"
    23→	"github.com/ClusterCockpit/cc-backend/internal/config"
    24→	"github.com/ClusterCockpit/cc-backend/internal/repository"
    25→	"github.com/ClusterCockpit/cc-backend/internal/tagger"
    26→	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
    27→	"github.com/ClusterCockpit/cc-lib/v2/schema"
    28→	"github.com/ClusterCockpit/cc-lib/v2/util"
    29→	"github.com/go-chi/chi/v5"
    30→)
    31→
    32→// @title                      ClusterCockpit REST API
    33→// @version                    1.0.0
    34→// @description                API for batch job control.
    35→
    36→// @contact.name               ClusterCockpit Project
    37→// @contact.url                https://github.com/ClusterCockpit
    38→// @contact.email              support@clustercockpit.org
    39→
    40→// @license.name               MIT License
    41→// @license.url                https://opensource.org/licenses/MIT
    42→
    43→// @host                       localhost:8080
    44→
    45→// @securityDefinitions.apikey ApiKeyAuth
    46→// @in                         header
    47→// @name                       X-Auth-Token
    48→
    49→const (
    50→	noticeFilePath  = "./var/notice.txt"
    51→	noticeFilePerms = 0o644
    52→	maxNoticeLength = 10000 // Maximum allowed notice content length in characters
    53→)
    54→
    55→type RestAPI struct {
    56→	JobRepository   *repository.JobRepository
    57→	Authentication  *auth.Authentication
    58→	MachineStateDir string
    59→	// RepositoryMutex protects job creation operations from race conditions
    60→	// when checking for duplicate jobs during startJob API calls.
    61→	// It prevents concurrent job starts with the same jobId/cluster/startTime
    62→	// from creating duplicate entries in the database.
    63→	RepositoryMutex sync.Mutex
    64→}
    65→
    66→// New creates and initializes a new RestAPI instance with configured dependencies.
    67→func New() *RestAPI {
    68→	return &RestAPI{
    69→		JobRepository:   repository.GetJobRepository(),
    70→		MachineStateDir: config.Keys.MachineStateDir,
    71→		Authentication:  auth.GetAuthInstance(),
    72→	}
    73→}
    74→
    75→// MountAPIRoutes registers REST API endpoints for job and cluster management.
    76→// These routes use JWT token authentication via the X-Auth-Token header.
    77→func (api *RestAPI) MountAPIRoutes(r chi.Router) {
    78→	// REST API Uses TokenAuth
    79→	// User List
    80→	r.Get("/users/", api.getUsers)
    81→	// Cluster List
    82→	r.Get("/clusters/", api.getClusters)
    83→	// Slurm node state
    84→	r.Post("/nodestate/", api.updateNodeStates)
    85→	r.Put("/nodestate/", api.updateNodeStates)
    86→	// Job Handler
    87→	if config.Keys.APISubjects == nil {
    88→		cclog.Info("Enabling REST start/stop job API")
    89→		r.Post("/jobs/start_job/", api.startJob)
    90→		r.Put("/jobs/start_job/", api.startJob)
    91→		r.Post("/jobs/stop_job/", api.stopJobByRequest)
    92→		r.Put("/jobs/stop_job/", api.stopJobByRequest)
    93→	}
    94→	r.Get("/jobs/", api.getJobs)
    95→	r.Get("/jobs/used_nodes", api.getUsedNodes)
    96→	r.Post("/jobs/tag_job/{id}", api.tagJob)
    97→	r.Patch("/jobs/tag_job/{id}", api.tagJob)
    98→	r.Delete("/jobs/tag_job/{id}", api.removeTagJob)
    99→	r.Patch("/jobs/edit_meta/{id}", api.editMeta)
   100→	r.Patch("/jobs/edit_meta/", api.editMetaByRequest)
   101→	r.Get("/jobs/metrics/{id}", api.getJobMetrics)
   102→	r.Delete("/jobs/delete_job/", api.deleteJobByRequest)
   103→	r.Delete("/jobs/delete_job/{id}", api.deleteJobByID)
   104→	r.Delete("/jobs/delete_job_before/{ts}", api.deleteJobBefore)
   105→	r.Post("/jobs/{id}", api.getJobByID)
   106→	r.Get("/jobs/{id}", api.getCompleteJobByID)
   107→
   108→	r.Delete("/tags/", api.removeTags)
   109→
   110→	if api.MachineStateDir != "" {
   111→		r.Get("/machine_state/{cluster}/{host}", api.getMachineState)
   112→		r.Put("/machine_state/{cluster}/{host}", api.putMachineState)
   113→		r.Post("/machine_state/{cluster}/{host}", api.putMachineState)
   114→	}
   115→}
   116→
   117→// MountUserAPIRoutes registers user-accessible REST API endpoints.
   118→// These are limited endpoints for regular users with JWT token authentication.
   119→func (api *RestAPI) MountUserAPIRoutes(r chi.Router) {
   120→	// REST API Uses TokenAuth
   121→	r.Get("/jobs/", api.getJobs)
   122→	r.Post("/jobs/{id}", api.getJobByID)
   123→	r.Get("/jobs/{id}", api.getCompleteJobByID)
   124→	r.Get("/jobs/metrics/{id}", api.getJobMetrics)
   125→}
   126→
   127→// MountMetricStoreAPIRoutes registers metric storage API endpoints.
   128→// These endpoints handle metric data ingestion and health checks with JWT token authentication.
   129→func (api *RestAPI) MountMetricStoreAPIRoutes(r chi.Router) {
   130→	// REST API Uses TokenAuth
   131→	r.Post("/free", freeMetrics)
   132→	r.Post("/write", writeMetrics)
   133→	r.Get("/debug", debugMetrics)
   134→	r.Post("/healthcheck", api.updateNodeStates)
   135→	// Same endpoints but with trailing slash
   136→	r.Post("/free/", freeMetrics)
   137→	r.Post("/write/", writeMetrics)
   138→	r.Get("/debug/", debugMetrics)
   139→	r.Post("/healthcheck/", api.updateNodeStates)
   140→}
   141→
   142→// MountConfigAPIRoutes registers configuration and user management endpoints.
   143→// These routes use session-based authentication and require admin privileges.
   144→// Routes use full paths (including /config prefix) to avoid conflicting with
   145→// the /config page route when registered via Group instead of Route.
   146→func (api *RestAPI) MountConfigAPIRoutes(r chi.Router) {
   147→	// Settings Frontend Uses SessionAuth
   148→	if api.Authentication != nil {
   149→		r.Get("/config/roles/", api.getRoles)
   150→		r.Post("/config/users/", api.createUser)


grep -n "GetSubClusterByNode\|GetMetricConfigSubCluster" /Users/jan/prg/ClusterCockpit/cc-backend/pkg/archive/*.go | head -20


/Users/jan/prg/ClusterCockpit/cc-backend/pkg/archive/clusterConfig.go:198:func GetMetricConfigSubCluster(cluster, subcluster string) map[string]*schema.Metric {
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/archive/clusterConfig.go:287:func GetSubClusterByNode(cluster, hostname string) (string, error) {


   195→	return nil, fmt.Errorf("subcluster '%v' not found for cluster '%v', or cluster '%v' not configured", subcluster, cluster, cluster)
   196→}
   197→
   198→func GetMetricConfigSubCluster(cluster, subcluster string) map[string]*schema.Metric {
   199→	metrics := make(map[string]*schema.Metric)
   200→
   201→	for _, c := range Clusters {
   202→		if c.Name == cluster {
   203→			for _, m := range c.MetricConfig {
   204→				for _, s := range m.SubClusters {
   205→					if s.Name == subcluster {
   206→						metrics[m.Name] = &schema.Metric{
   207→							Name:    m.Name,
   208→							Unit:    s.Unit,
   209→							Peak:    s.Peak,
   210→							Normal:  s.Normal,
   211→							Caution: s.Caution,
   212→							Alert:   s.Alert,
   213→						}
   214→						break
   215→					}
   216→				}
   217→
   218→				_, ok := metrics[m.Name]
   219→				if !ok {
   220→					metrics[m.Name] = &schema.Metric{
   221→						Name:    m.Name,
   222→						Unit:    m.Unit,
   223→						Peak:    m.Peak,
   224→						Normal:  m.Normal,
   225→						Caution: m.Caution,
   226→						Alert:   m.Alert,
   227→					}
   228→				}
   229→			}
   230→			break
   231→		}
   232→	}
   233→
   234→	return metrics
   235→}
   236→
   237→func GetMetricConfig(cluster, metric string) *schema.MetricConfig {
   238→	for _, c := range Clusters {
   239→		if c.Name == cluster {
   240→			for _, m := range c.MetricConfig {
   241→				if m.Name == metric {
   242→					return m
   243→				}
   244→			}


<tool_use_error>InputValidationError: Read failed due to the following issue:
The parameter `offset` type is expected as `number` but provided as `string`</tool_use_error>

InputValidationError: [
  {
    "expected": "number",
    "code": "invalid_type",
    "path": [
      "offset"
    ],
    "message": "Invalid input: expected number, received string"
  }
]


   285→}
   286→
   287→func GetSubClusterByNode(cluster, hostname string) (string, error) {
   288→	for sc, nl := range NodeLists[cluster] {
   289→		if nl != nil && nl.Contains(hostname) {
   290→			return sc, nil
   291→		}
   292→	}
   293→
   294→	c := GetCluster(cluster)
   295→	if c == nil {
   296→		return "", fmt.Errorf("ARCHIVE/CLUSTERCONFIG > unkown cluster: %v", cluster)
   297→	}
   298→
   299→	if c.SubClusters[0].Nodes == "" {
   300→		return c.SubClusters[0].Name, nil
   301→	}
   302→
   303→	return "", fmt.Errorf("ARCHIVE/CLUSTERCONFIG > no subcluster found for cluster %v and host %v", cluster, hostname)
   304→}
   305→
   306→func MetricIndex(mc []*schema.MetricConfig, name string) (int, error) {
   307→	for i, m := range mc {
   308→		if m.Name == name {
   309→			return i, nil


find /Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/migrations -name "*.up.sql" | head -10


     1→CREATE TABLE "node" (
     2→    id INTEGER PRIMARY KEY,
     3→    hostname VARCHAR(255) NOT NULL,
     4→    cluster VARCHAR(255) NOT NULL,
     5→    subcluster VARCHAR(255) NOT NULL,
     6→    meta_data TEXT,          -- JSON
     7→    UNIQUE (hostname, cluster)
     8→);
     9→
    10→CREATE TABLE "node_state" (
    11→    id INTEGER PRIMARY KEY,
    12→    time_stamp INTEGER NOT NULL,
    13→    jobs_running INTEGER DEFAULT 0 NOT NULL,
    14→    cpus_allocated INTEGER DEFAULT 0 NOT NULL,
    15→    memory_allocated INTEGER DEFAULT 0 NOT NULL,
    16→    gpus_allocated INTEGER DEFAULT 0 NOT NULL,
    17→    node_state VARCHAR(255) NOT NULL
    18→    CHECK (node_state IN (
    19→        'allocated', 'reserved', 'idle', 'mixed',
    20→        'down', 'unknown'
    21→    )),
    22→    health_state VARCHAR(255) NOT NULL
    23→    CHECK (health_state IN (
    24→        'full', 'partial', 'failed'
    25→    )),
    26→    health_metrics TEXT,          -- JSON array of strings
    27→    node_id INTEGER,
    28→    FOREIGN KEY (node_id) REFERENCES node (id)
    29→);
    30→
    31→-- Add NEW Indices For New Node Table VARCHAR Fields
    32→CREATE INDEX IF NOT EXISTS nodes_cluster ON node (cluster);
    33→CREATE INDEX IF NOT EXISTS nodes_cluster_subcluster ON node (cluster, subcluster);
    34→
    35→-- Add NEW Indices For New Node_State Table Fields
    36→CREATE INDEX IF NOT EXISTS nodestates_timestamp ON node_state (time_stamp);
    37→CREATE INDEX IF NOT EXISTS nodestates_state_timestamp ON node_state (node_state, time_stamp);
    38→CREATE INDEX IF NOT EXISTS nodestates_health_timestamp ON node_state (health_state, time_stamp);
    39→CREATE INDEX IF NOT EXISTS nodestates_nodeid_state ON node_state (node_id, node_state);
    40→CREATE INDEX IF NOT EXISTS nodestates_nodeid_health ON node_state (node_id, health_state);
    41→CREATE INDEX IF NOT EXISTS nodestates_nodeid_timestamp ON node_state (node_id, time_stamp DESC);
    42→
    43→-- Add NEW Indices For Increased Amounts of Tags
    44→CREATE INDEX IF NOT EXISTS tags_jobid ON jobtag (job_id);
    45→CREATE INDEX IF NOT EXISTS tags_tagid ON jobtag (tag_id);
    46→
    47→-- Optimize DB index usage
    48→PRAGMA optimize;


grep -n "decode\|handleError\|DefaultAPIResponse\|ErrorResponse" /Users/jan/prg/ClusterCockpit/cc-backend/internal/api/rest.go | head -20


172:// ErrorResponse model
173:type ErrorResponse struct {
179:// DefaultAPIResponse model
180:type DefaultAPIResponse struct {
184:// handleError writes a standardized JSON error response with the given status code.
186:func handleError(err error, statusCode int, rw http.ResponseWriter) {
190:	if err := json.NewEncoder(rw).Encode(ErrorResponse{
198:// decode reads JSON from r into val with strict validation that rejects unknown fields.
199:func decode(r io.Reader, val any) error {
224:// @failure     400 {object} ErrorResponse "Bad Request"
225:// @failure     403 {object} ErrorResponse "Forbidden"
226:// @failure     500 {object} ErrorResponse "Internal Server Error"
231:		handleError(fmt.Errorf("only admins are allowed to update the notice.txt file"), http.StatusForbidden, rw)
239:		handleError(fmt.Errorf("notice content exceeds maximum length of %d characters", maxNoticeLength), http.StatusBadRequest, rw)
248:			handleError(fmt.Errorf("creating notice file failed: %w", err), http.StatusInternalServerError, rw)
257:		handleError(fmt.Errorf("writing to notice file failed: %w", err), http.StatusInternalServerError, rw)
276:		handleError(fmt.Errorf("only admins are allowed to list taggers"), http.StatusForbidden, rw)
288:		handleError(fmt.Errorf("only admins are allowed to run taggers"), http.StatusForbidden, rw)
294:		handleError(fmt.Errorf("missing required parameter: name"), http.StatusBadRequest, rw)
299:		handleError(err, http.StatusConflict, rw)


   170→}
   171→
   172→// ErrorResponse model
   173→type ErrorResponse struct {
   174→	// Statustext of Errorcode
   175→	Status string `json:"status"`
   176→	Error  string `json:"error"` // Error Message
   177→}
   178→
   179→// DefaultAPIResponse model
   180→type DefaultAPIResponse struct {
   181→	Message string `json:"msg"`
   182→}
   183→
   184→// handleError writes a standardized JSON error response with the given status code.
   185→// It logs the error at WARN level and ensures proper Content-Type headers are set.
   186→func handleError(err error, statusCode int, rw http.ResponseWriter) {
   187→	cclog.Warnf("REST ERROR : %s", err.Error())
   188→	rw.Header().Add("Content-Type", "application/json")
   189→	rw.WriteHeader(statusCode)
   190→	if err := json.NewEncoder(rw).Encode(ErrorResponse{
   191→		Status: http.StatusText(statusCode),
   192→		Error:  err.Error(),
   193→	}); err != nil {
   194→		cclog.Errorf("Failed to encode error response: %v", err)
   195→	}
   196→}
   197→
   198→// decode reads JSON from r into val with strict validation that rejects unknown fields.
   199→func decode(r io.Reader, val any) error {
   200→	dec := json.NewDecoder(r)
   201→	dec.DisallowUnknownFields()
   202→	return dec.Decode(val)
   203→}
   204→


Perfect! Now I have all the information needed. Let me create a comprehensive analysis document.

## Comprehensive Analysis of NATS Node State Handler Implementation

Based on my thorough examination of the codebase, here's the complete analysis of the NATS node state update implementation:

---

### 1. FULL NATS NODE STATE HANDLER IMPLEMENTATION

**Location:** `/Users/jan/prg/ClusterCockpit/cc-backend/internal/api/nats.go` (lines 380-458)

The handler consists of two main functions:

#### **handleNodeState** (lines 427-458)
```
Purpose: Entry point for NATS node state messages
Protocol: Receives InfluxDB line protocol encoded messages
Message Format: nodestate event="{...JSON...}" <timestamp>
Processing: Decodes line protocol, validates message type, routes to processNodestateEvent
```

#### **processNodestateEvent** (lines 380-425)
```
Purpose: Processes the actual node state update logic
Input: CCMessage object from line protocol decoder
Validation: Extracts event field, parses JSON
Database Op: Batch updates node states via repository
Logging: Debug-level logging of update counts
```

---

### 2. MESSAGE FORMAT SPECIFICATION

**Line Protocol Format:**
```
nodestate event="{...JSON payload...}" <timestamp>
```

**JSON Payload Structure (UpdateNodeStatesRequest):**
Location: `/Users/jan/prg/ClusterCockpit/cc-backend/internal/api/node.go` (lines 23-26)

```go
type UpdateNodeStatesRequest struct {
    Nodes   []schema.NodePayload `json:"nodes"`
    Cluster string               `json:"cluster" example:"fritz"`
}
```

**NodePayload Structure (schema.NodePayload):**
Location: `/Users/jan/prg/ClusterCockpit/cc-lib/schema/node.go` (lines 49-58)

```go
type NodePayload struct {
    Hostname        string   `json:"hostname"`        // Node hostname
    States          []string `json:"states"`          // State strings (flexible format)
    CpusAllocated   int      `json:"cpusAllocated"`   // Number of allocated CPUs
    MemoryAllocated int64    `json:"memoryAllocated"` // Allocated memory in MB
    GpusAllocated   int      `json:"gpusAllocated"`   // Number of allocated GPUs
    JobsRunning     int      `json:"jobsRunning"`     // Number of running jobs
}
```

**Example Message:**
```
nodestate event="{\"cluster\":\"testcluster\",\"nodes\":[{\"hostname\":\"node001\",\"states\":[\"allocated\"],\"cpusAllocated\":8,\"memoryAllocated\":16384,\"gpusAllocated\":0,\"jobsRunning\":1}]}" 1234567890000000000
```

---

### 3. VALIDATION AND BUSINESS LOGIC

#### **State Determination (determineState function)**
Location: `/Users/jan/prg/ClusterCockpit/cc-backend/internal/api/node.go` (lines 37-55)

Maps string state values to SchedulerState enum:
- `"allocated"` → `NodeStateAllocated`
- `"reserved"` → `NodeStateReserved`
- `"idle"` → `NodeStateIdle`
- `"down"` → `NodeStateDown`
- `"mixed"` → `NodeStateMixed`
- Default → `NodeStateUnknown` (case-insensitive matching)

#### **Validation Steps in processNodestateEvent**
1. **Message Type Check:** Validates message is an "event" type (line 383-386)
2. **JSON Parsing:** Uses `json.NewDecoder` with `DisallowUnknownFields` (line 391-396)
3. **Line Protocol Validation:** Checks measurement name is "nodestate" (line 452-455)
4. **Error Handling:** Non-fatal errors are logged; processing continues for other nodes

#### **No Health Check Validation**
Unlike REST API, NATS handler:
- **Does NOT perform health checks** against metric store
- **Sets HealthState to `MonitoringStateFull`** unconditionally (line 410)
- **Does NOT include HealthMetrics** (always nil)

---

### 4. DATABASE OPERATIONS

All database operations are delegated to `repository.NodeRepository.BatchUpdateNodeStates()`.

**Location:** `/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/node.go` (lines 254-316)

#### **NodeStateUpdate Structure:**
```go
type NodeStateUpdate struct {
    Hostname  string
    Cluster   string
    NodeState *schema.NodeStateDB
}
```

#### **NodeStateDB Structure (Database Model):**
Location: `/Users/jan/prg/ClusterCockpit/cc-lib/schema/node.go` (lines 70-83)

```go
type NodeStateDB struct {
    ID              int64           `db:"id"`                // Database ID
    TimeStamp       int64           `db:"time_stamp"`        // Unix timestamp
    NodeState       SchedulerState  `db:"node_state"`        // Scheduler state
    HealthState     MonitoringState `db:"health_state"`      // Monitoring health
    HealthMetrics   string          `db:"health_metrics"`    // JSON health data
    CpusAllocated   int             `db:"cpus_allocated"`    // Allocated CPUs
    MemoryAllocated int64           `db:"memory_allocated"`  // Allocated memory
    GpusAllocated   int             `db:"gpus_allocated"`    // Allocated GPUs
    JobsRunning     int             `db:"jobs_running"`      // Running jobs
    NodeID          int64           `db:"node_id"`           // Foreign key to node table
}
```

#### **BatchUpdateNodeStates Transaction Flow:**

1. **Begin Transaction** (line 258)
2. **Prepare Statements** (lines 264-280):
   - Node lookup query
   - Node insert statement
   - Node state insert statement
3. **For Each Node Update** (lines 282-310):
   - **Lookup node** by hostname + cluster
   - **If not found:**
     - Get subcluster via `archive.GetSubClusterByNode()`
     - Create new node row with `NamedNodeInsert`
   - **If found or created:**
     - Insert node state row with `NamedNodeStateInsert`
4. **Commit Transaction** (line 312)

#### **Error Handling:**
- Individual node errors are logged but don't prevent processing of other nodes
- Errors logged at `ERROR` level (cclog.Errorf)
- Transaction continues despite individual node failures

#### **Timestamp Assignment:**
- Uses `time.Now().Unix()` as the request received time (line 399)
- This becomes the `TimeStamp` for all node state records in the batch

---

### 5. KEY DIFFERENCES FROM REST IMPLEMENTATION

| Aspect | REST API | NATS API |
|--------|----------|----------|
| **Input Protocol** | HTTP POST/PUT with JSON body | NATS message with InfluxDB line protocol |
| **Health Checks** | YES - performs health checks against metric store | NO - skips health checks entirely |
| **Health State Value** | Determined from health check results or defaults to "failed" | Always set to `MonitoringStateFull` |
| **Health Metrics** | Populated from health check results | Always empty/nil |
| **Message Parsing** | Direct JSON decode from HTTP body | Line protocol decode → JSON extraction |
| **Concurrency** | Single handler | Worker pool (default 2 workers, configurable) |
| **Response** | HTTP response sent back to client | No response (async, fire-and-forget) |
| **Error Handling** | Errors returned in HTTP response | Errors logged only, no client notification |
| **Request Timestamp** | `time.Now().Unix()` at processing time | `time.Now().Unix()` at processing time (same) |
| **Timeout Handling** | HTTP timeout from HTTP client | Channel-based with configurable concurrency |

---

### 6. CONFIGURATION

**NATS Subject Configuration:**
Location: `/Users/jan/prg/ClusterCockpit/cc-backend/internal/config/config.go` (lines 117-122)

```go
type NATSConfig struct {
    SubjectJobEvent  string `json:"subject-job-event"`
    SubjectNodeState string `json:"subject-node-state"`
    JobConcurrency   int    `json:"job-concurrency"`
    NodeConcurrency  int    `json:"node-concurrency"`
}
```

**config.json Example:**
```json
{
  "main": {
    "api-subjects": {
      "subject-node-state": "cc.node.state",
      "node-concurrency": 2
    }
  }
}
```

**Worker Pool Setup:**
Location: `/Users/jan/prg/ClusterCockpit/cc-backend/internal/api/nats.go` (lines 79-109)

```
Default Concurrency: 2 node workers, 8 job workers
Configurable via: config.Keys.APISubjects.NodeConcurrency
Buffering: Messages buffered in channels (size = concurrency level)
```

---

### 7. INTEGRATION POINTS

#### **Subscription Setup:**
Location: `/Users/jan/prg/ClusterCockpit/cc-backend/internal/api/nats.go` (lines 125-153)

```
StartSubscriptions():
  - Gets NATS client
  - Subscribes to SubjectNodeState
  - Routes messages to nodeCh channel
  - Workers consume messages asynchronously
```

#### **REST vs NATS Toggle:**
Location: `/Users/jan/prg/ClusterCockpit/cc-backend/internal/api/rest.go` (lines 87-93)

The REST job start/stop endpoints are disabled when NATS subjects are configured:
```go
if config.Keys.APISubjects == nil {
    cclog.Info("Enabling REST start/stop job API")
    r.Post("/jobs/start_job/", api.startJob)
    r.Put("/jobs/start_job/", api.startJob)
    r.Post("/jobs/stop_job/", api.stopJobByRequest)
    r.Put("/jobs/stop_job/", api.stopJobByRequest)
}
```

The node state endpoint is **always enabled** (not conditional):
```go
r.Post("/nodestate/", api.updateNodeStates)
r.Put("/nodestate/", api.updateNodeStates)
```

---

### 8. TESTING COVERAGE

Comprehensive test suite in `/Users/jan/prg/ClusterCockpit/cc-backend/internal/api/nats_test.go`:

- **TestNatsHandleNodeState** (lines 589-647): Valid updates, multiple nodes, invalid JSON, empty data
- **TestNatsHandleNodeStateEdgeCases** (lines 828-876): Missing fields, malformed JSON, unicode, large batches, timestamp handling
- **TestNatsProcessJobEvent** (lines 649-740): Message routing and validation
- **TestNatsHandleStartJob** & **TestNatsHandleStopJob**: Job lifecycle tests

---

### SUMMARY

The NATS node state handler is a **simplified, async version** of the REST endpoint that:
- Trades **health check coverage** for **performance** (no metric store queries)
- Uses **worker pool concurrency** for throughput
- Applies **the same state determination logic** as REST
- Performs **batch database operations** with transactional safety
- Logs all errors but continues processing (fire-and-forget semantics)
- Always marks nodes as `MonitoringStateFull` without validation

This design is suitable for trusted, internal monitoring systems where the health check is performed elsewhere before sending state updates.

agentId: a23b8dfea9f3e3990 (for resuming to continue this agent's work if needed)
<usage>total_tokens: 57779
tool_uses: 18
duration_ms: 73543</usage>

Prompt:
Find and thoroughly analyze the NATS implementation for handling node state updates in the cc-backend project. Look in internal/api/nats.go. I need:
1. The full implementation of the NATS node state handler
2. The message format it expects
3. All validation and business logic
4. Database operations performed
5. How it differs from the REST implementation

Also check the config for NATS subjects and any related types/structs.


Perfect! Now I have all the information I need. Let me create a comprehensive analysis document:

## Comprehensive Analysis of UpdateNodeStates REST Implementation

Based on my thorough analysis of the cc-backend codebase, here is the complete documentation of the REST implementation for updating node states:

### 1. ENDPOINT REGISTRATION

**File:** `/Users/jan/prg/ClusterCockpit/cc-backend/internal/api/rest.go`

The UpdateNodeStates endpoint is registered in two places:

1. **MountAPIRoutes()** (line 84-85) - Primary API routes:
   ```go
   r.Post("/nodestate/", api.updateNodeStates)
   r.Put("/nodestate/", api.updateNodeStates)
   ```

2. **MountMetricStoreAPIRoutes()** (line 134, 139) - Metric store health check:
   ```go
   r.Post("/healthcheck", api.updateNodeStates)
   r.Post("/healthcheck/", api.updateNodeStates)
   ```

Both POST and PUT methods are supported with identical behavior.

---

### 2. REQUEST FORMAT / STRUCT

**File:** `/Users/jan/prg/ClusterCockpit/cc-backend/internal/api/node.go` (lines 23-26)

```go
type UpdateNodeStatesRequest struct {
    Nodes   []schema.NodePayload `json:"nodes"`
    Cluster string               `json:"cluster" example:"fritz"`
}
```

**NodePayload structure** (from `/Users/jan/prg/ClusterCockpit/cc-lib/schema/node.go`, lines 51-58):

```go
type NodePayload struct {
    Hostname        string   `json:"hostname"`        // Node hostname
    States          []string `json:"states"`          // State strings (flexible format)
    CpusAllocated   int      `json:"cpusAllocated"`   // Number of allocated CPUs
    MemoryAllocated int64    `json:"memoryAllocated"` // Allocated memory in MB
    GpusAllocated   int      `json:"gpusAllocated"`   // Number of allocated GPUs
    JobsRunning     int      `json:"jobsRunning"`     // Number of running jobs
}
```

**Example Request:**
```json
{
  "cluster": "fritz",
  "nodes": [
    {
      "hostname": "node001",
      "states": ["allocated"],
      "cpusAllocated": 8,
      "memoryAllocated": 16384,
      "gpusAllocated": 0,
      "jobsRunning": 1
    }
  ]
}
```

---

### 3. FULL HANDLER IMPLEMENTATION

**File:** `/Users/jan/prg/ClusterCockpit/cc-backend/internal/api/node.go` (lines 57-150)

```go
// updateNodeStates godoc
// @summary     Deliver updated Slurm node states
// @tags Nodestates
// @description Returns a JSON-encoded list of users.
// @description Required query-parameter defines if all users or only users with additional special roles are returned.
// @produce     json
// @param       request body UpdateNodeStatesRequest true "Request body containing nodes and their states"
// @success     200     {object} api.DefaultAPIResponse "Success message"
// @failure     400     {object} api.ErrorResponse      "Bad Request"
// @failure     401     {object} api.ErrorResponse      "Unauthorized"
// @failure     403     {object} api.ErrorResponse      "Forbidden"
// @failure     500     {object} api.ErrorResponse      "Internal Server Error"
// @security    ApiKeyAuth
// @router      /api/nodestats/ [post]
func (api *RestAPI) updateNodeStates(rw http.ResponseWriter, r *http.Request) {
    // Parse request body
    req := UpdateNodeStatesRequest{}
    if err := decode(r.Body, &req); err != nil {
        handleError(fmt.Errorf("parsing request body failed: %w", err),
            http.StatusBadRequest, rw)
        return
    }
    requestReceived := time.Now().Unix()
    repo := repository.GetNodeRepository()

    m := make(map[string][]string)
    metricNames := make(map[string][]string)
    healthResults := make(map[string]metricstore.HealthCheckResult)

    startMs := time.Now()

    // Step 1: Build nodeList and metricList per subcluster
    for _, node := range req.Nodes {
        if sc, err := archive.GetSubClusterByNode(req.Cluster, node.Hostname); err == nil {
            m[sc] = append(m[sc], node.Hostname)
        }
    }

    for sc := range m {
        if sc != "" {
            metricList := archive.GetMetricConfigSubCluster(req.Cluster, sc)
            metricNames[sc] = metricListToNames(metricList)
        }
    }

    // Step 2: Determine which metric store to query and perform health check
    healthRepo, err := metricdispatch.GetHealthCheckRepo(req.Cluster)
    if err != nil {
        cclog.Warnf("updateNodeStates: no metric store for cluster %s, skipping health check: %v", req.Cluster, err)
    } else {
        for sc, nl := range m {
            if sc != "" {
                if results, err := healthRepo.HealthCheck(req.Cluster, nl, metricNames[sc]); err == nil {
                    maps.Copy(healthResults, results)
                }
            }
        }
    }

    cclog.Debugf("Timer updateNodeStates, MemStore HealthCheck: %s", time.Since(startMs))
    startDB := time.Now()

    updates := make([]repository.NodeStateUpdate, 0, len(req.Nodes))
    for _, node := range req.Nodes {
        state := determineState(node.States)
        healthState := schema.MonitoringStateFailed
        var healthMetrics string
        if result, ok := healthResults[node.Hostname]; ok {
            healthState = result.State
            healthMetrics = result.HealthMetrics
        }
        nodeState := schema.NodeStateDB{
            TimeStamp:       requestReceived,
            NodeState:       state,
            CpusAllocated:   node.CpusAllocated,
            MemoryAllocated: node.MemoryAllocated,
            GpusAllocated:   node.GpusAllocated,
            HealthState:     healthState,
            HealthMetrics:   healthMetrics,
            JobsRunning:     node.JobsRunning,
        }
        updates = append(updates, repository.NodeStateUpdate{
            Hostname:  node.Hostname,
            Cluster:   req.Cluster,
            NodeState: &nodeState,
        })
    }

    if err := repo.BatchUpdateNodeStates(updates); err != nil {
        cclog.Errorf("updateNodeStates: batch update for cluster %s failed: %v", req.Cluster, err)
    }

    cclog.Debugf("Timer updateNodeStates, SQLite Inserts: %s", time.Since(startDB))
}
```

---

### 4. VALIDATION AND BUSINESS LOGIC

**Step 1: Request Parsing (lines 72-78)**
- Strict JSON parsing using `decode()` which rejects unknown fields
- Returns `HTTP 400 BadRequest` if JSON is malformed
- Uses `http.StatusBadRequest` error handling

**Step 2: Subcluster Mapping (lines 88-93)**
- For each node in request, determines its subcluster using `archive.GetSubClusterByNode()`
- Builds map of `subcluster → []hostname` for batch processing
- Silently skips nodes that can't be resolved to a subcluster

**Step 3: Metric Configuration Lookup (lines 95-100)**
- For each subcluster, retrieves metric configurations using `archive.GetMetricConfigSubCluster()`
- Converts metric config map to list of metric names using `metricListToNames()`
- Builds `metricNames` map for health check queries

**Step 4: Health Check (lines 102-114)**
- Retrieves health check repository using `metricdispatch.GetHealthCheckRepo()`
- If no metric store is configured, continues gracefully with warning log
- For each subcluster, performs health check against all nodes and metrics
- Results are merged into `healthResults` map by hostname

**Step 5: State Determination (lines 119-127)**
- For each node, calls `determineState(node.States)` to convert states array to SchedulerState
- Defaults health state to `schema.MonitoringStateFailed`
- If health check results exist for the node, uses those health states/metrics

**Helper Functions:**

**determineState()** (lines 38-55):
```go
func determineState(states []string) schema.SchedulerState {
    for _, state := range states {
        switch strings.ToLower(state) {
        case "allocated":
            return schema.NodeStateAllocated
        case "reserved":
            return schema.NodeStateReserved
        case "idle":
            return schema.NodeStateIdle
        case "down":
            return schema.NodeStateDown
        case "mixed":
            return schema.NodeStateMixed
        }
    }
    return schema.NodeStateUnknown
}
```
- Case-insensitive matching of state strings
- Returns first matching known state
- Defaults to `NodeStateUnknown` if none match
- Assumes only one state per node

**metricListToNames()** (lines 28-35):
```go
func metricListToNames(metricList map[string]*schema.Metric) []string {
    names := make([]string, 0, len(metricList))
    for name := range metricList {
        names = append(names, name)
    }
    return names
}
```
- Extracts metric names from metric config map
- Returns unordered list suitable for health check queries

---

### 5. DATABASE OPERATIONS

**NodeStateUpdate Structure** (`/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/node.go`, lines 247-252):

```go
type NodeStateUpdate struct {
    Hostname  string
    Cluster   string
    NodeState *schema.NodeStateDB
}
```

**BatchUpdateNodeStates()** (lines 254-316 of node.go):

Implements transactional batch insert with the following flow:

```go
func (r *NodeRepository) BatchUpdateNodeStates(updates []NodeStateUpdate) error {
    // 1. Begin transaction
    tx, err := r.DB.Beginx()

    // 2. Prepare three statements within transaction:
    stmtLookup, err := tx.Preparex("SELECT id FROM node WHERE hostname = ? AND cluster = ?")
    stmtInsertNode, err := tx.PrepareNamed(NamedNodeInsert)
    stmtInsertState, err := tx.PrepareNamed(NamedNodeStateInsert)

    // 3. For each node update:
    for _, u := range updates {
        // a. Try to lookup existing node by hostname+cluster
        var id int64
        if err := stmtLookup.QueryRow(u.Hostname, u.Cluster).Scan(&id); err != nil {
            if err == sql.ErrNoRows {
                // b. If node doesn't exist, look up subcluster and insert new node
                subcluster, scErr := archive.GetSubClusterByNode(u.Cluster, u.Hostname)
                if scErr != nil {
                    cclog.Errorf("BatchUpdateNodeStates: subcluster lookup for '%s' in '%s': %v", u.Hostname, u.Cluster, scErr)
                    continue
                }
                node := schema.NodeDB{
                    Hostname: u.Hostname, Cluster: u.Cluster, SubCluster: subcluster,
                }
                res, insertErr := stmtInsertNode.Exec(&node)
                if insertErr != nil {
                    cclog.Errorf("BatchUpdateNodeStates: insert node '%s': %v", u.Hostname, insertErr)
                    continue
                }
                id, _ = res.LastInsertId()
            } else {
                cclog.Errorf("BatchUpdateNodeStates: lookup node '%s': %v", u.Hostname, err)
                continue
            }
        }

        // c. Insert node state row with the node_id
        u.NodeState.NodeID = id
        if _, err := stmtInsertState.Exec(u.NodeState); err != nil {
            cclog.Errorf("BatchUpdateNodeStates: insert state for '%s': %v", u.Hostname, err)
        }
    }

    // 4. Commit transaction
    if err := tx.Commit(); err != nil {
        return fmt.Errorf("BatchUpdateNodeStates: commit: %w", err)
    }
    return nil
}
```

**SQL Statements Used:**

**NamedNodeInsert** (lines 176-178):
```sql
INSERT INTO node (hostname, cluster, subcluster)
    VALUES (:hostname, :cluster, :subcluster);
```

**NamedNodeStateInsert** (lines 199-203):
```sql
INSERT INTO node_state (time_stamp, node_state, health_state, health_metrics,
    cpus_allocated, memory_allocated, gpus_allocated, jobs_running, node_id)
    VALUES (:time_stamp, :node_state, :health_state, :health_metrics,
    :cpus_allocated, :memory_allocated, :gpus_allocated, :jobs_running, :node_id);
```

**Database Schema:**

**node table** (migration 10_node-table.up.sql):
```sql
CREATE TABLE "node" (
    id INTEGER PRIMARY KEY,
    hostname VARCHAR(255) NOT NULL,
    cluster VARCHAR(255) NOT NULL,
    subcluster VARCHAR(255) NOT NULL,
    meta_data TEXT,          -- JSON
    UNIQUE (hostname, cluster)
);
```

**node_state table:**
```sql
CREATE TABLE "node_state" (
    id INTEGER PRIMARY KEY,
    time_stamp INTEGER NOT NULL,
    jobs_running INTEGER DEFAULT 0 NOT NULL,
    cpus_allocated INTEGER DEFAULT 0 NOT NULL,
    memory_allocated INTEGER DEFAULT 0 NOT NULL,
    gpus_allocated INTEGER DEFAULT 0 NOT NULL,
    node_state VARCHAR(255) NOT NULL
        CHECK (node_state IN ('allocated', 'reserved', 'idle', 'mixed', 'down', 'unknown')),
    health_state VARCHAR(255) NOT NULL
        CHECK (health_state IN ('full', 'partial', 'failed')),
    health_metrics TEXT,          -- JSON array of strings
    node_id INTEGER,
    FOREIGN KEY (node_id) REFERENCES node (id)
);
```

**Key Transaction Features:**
- All node lookups and inserts/state inserts happen within a single transaction
- Reduces lock acquisitions from 2*N to 1 for N nodes
- Errors during individual node updates are logged but don't abort the batch
- Uses prepared statements for efficiency
- Defers rollback in case of errors

**NodeStateDB Structure** (from schema/node.go, lines 72-83):
```go
type NodeStateDB struct {
    ID              int64           `db:"id"`
    TimeStamp       int64           `db:"time_stamp"` // Unix timestamp (set to requestReceived)
    NodeState       SchedulerState  `db:"node_state"` // allocated/reserved/idle/mixed/down/unknown
    HealthState     MonitoringState `db:"health_state"` // full/partial/failed
    HealthMetrics   string          `db:"health_metrics"` // JSON: {"missing":[...],"degraded":[...]}
    CpusAllocated   int             `db:"cpus_allocated"`
    MemoryAllocated int64           `db:"memory_allocated"`
    GpusAllocated   int             `db:"gpus_allocated"`
    JobsRunning     int             `db:"jobs_running"`
    NodeID          int64           `db:"node_id"` // Foreign key to node(id)
}
```

---

### 6. HELPER FUNCTIONS CALLED

1. **decode(r.Body, &req)** - Parses JSON request body with strict validation
2. **repository.GetNodeRepository()** - Returns singleton NodeRepository instance
3. **archive.GetSubClusterByNode(cluster, hostname)** - Determines subcluster from topology
4. **archive.GetMetricConfigSubCluster(cluster, sc)** - Retrieves metric configurations
5. **metricListToNames(metricList)** - Converts metric map to name list
6. **metricdispatch.GetHealthCheckRepo(cluster)** - Gets health check repository for metric store
7. **healthRepo.HealthCheck(cluster, nodes, metrics)** - Performs health check on metric store
8. **determineState(states []string)** - Converts state string array to SchedulerState enum
9. **repo.BatchUpdateNodeStates(updates)** - Executes transactional batch database insert
10. **handleError()** - Returns standardized JSON error response

---

### 7. ERROR RESPONSES

**ErrorResponse Structure**:
```go
type ErrorResponse struct {
    Status string `json:"status"` // HTTP status text
    Error  string `json:"error"`  // Error message
}
```

**HTTP Status Codes:**
- **400 Bad Request** - JSON parsing failed
- **401 Unauthorized** - No authentication token or invalid token
- **403 Forbidden** - User lacks required permissions
- **500 Internal Server Error** - Database operations fail

**Note:** The current implementation does NOT return an error response on successful completion. It logs errors but doesn't fail the HTTP request when individual node updates fail during batch operations.

---

### 8. TIMING AND LOGGING

The endpoint records performance metrics:
- `Timer updateNodeStates, MemStore HealthCheck: <duration>` - Health check query time
- `Timer updateNodeStates, SQLite Inserts: <duration>` - Database transaction time
- Error logging at WARN level for missing metric stores
- Error logging at ERROR level for individual node failures during batch operations
- Debug logging for state update completion

---

This comprehensive implementation provides efficient batch processing of node state updates with transaction support, health check integration, and graceful error handling.

agentId: ab03c542dd3b3a6c0 (for resuming to continue this agent's work if needed)
<usage>total_tokens: 53155
tool_uses: 25
duration_ms: 103232</usage>

Prompt:
Find and thoroughly analyze the REST implementation for updating node states in the cc-backend project. Look in internal/api/ for the REST handler. I need:
1. The full implementation of the UpdateNodeStates REST endpoint
2. The request format/struct it expects
3. All validation and business logic
4. Database operations performed
5. Any helper functions called

Search in internal/api/rest.go and related files. Also check internal/repository/ for any node-related repository methods used.


Now let me read the actual current files to verify the details.