     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→package repository
     7→
     8→import (
     9→	"context"
    10→	"database/sql"
    11→	"encoding/json"
    12→	"fmt"
    13→	"slices"
    14→	"sort"
    15→	"strings"
    16→	"sync"
    17→	"time"
    18→
    19→	"github.com/ClusterCockpit/cc-backend/internal/graph/model"
    20→	"github.com/ClusterCockpit/cc-backend/pkg/archive"
    21→	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
    22→	"github.com/ClusterCockpit/cc-lib/v2/lrucache"
    23→	"github.com/ClusterCockpit/cc-lib/v2/schema"
    24→	sq "github.com/Masterminds/squirrel"
    25→	"github.com/jmoiron/sqlx"
    26→)
    27→
    28→var (
    29→	nodeRepoOnce     sync.Once
    30→	nodeRepoInstance *NodeRepository
    31→)
    32→
    33→type NodeRepository struct {
    34→	DB        *sqlx.DB
    35→	stmtCache *sq.StmtCache
    36→	cache     *lrucache.Cache
    37→	driver    string
    38→}
    39→
    40→func GetNodeRepository() *NodeRepository {
    41→	nodeRepoOnce.Do(func() {
    42→		db := GetConnection()
    43→
    44→		nodeRepoInstance = &NodeRepository{
    45→			DB:     db.DB,
    46→			driver: db.Driver,
    47→
    48→			stmtCache: sq.NewStmtCache(db.DB),
    49→			cache:     lrucache.New(repoConfig.CacheSize),
    50→		}
    51→	})
    52→	return nodeRepoInstance
    53→}
    54→
    55→// latestStateCondition returns a squirrel expression that restricts node_state
    56→// rows to the latest per node_id using a correlated subquery.
    57→// Requires the query to join node and node_state tables.
    58→func latestStateCondition() sq.Sqlizer {
    59→	return sq.Expr(
    60→		"node_state.id = (SELECT ns2.id FROM node_state ns2 WHERE ns2.node_id = node.id ORDER BY ns2.time_stamp DESC LIMIT 1)",
    61→	)
    62→}
    63→
    64→// applyNodeFilters applies common NodeFilter conditions to a query that joins
    65→// the node and node_state tables with latestStateCondition.
    66→func applyNodeFilters(query sq.SelectBuilder, filters []*model.NodeFilter) sq.SelectBuilder {
    67→	for _, f := range filters {
    68→		if f.Cluster != nil {
    69→			query = buildStringCondition("node.cluster", f.Cluster, query)
    70→		}
    71→		if f.SubCluster != nil {
    72→			query = buildStringCondition("node.subcluster", f.SubCluster, query)
    73→		}
    74→		if f.Hostname != nil {
    75→			query = buildStringCondition("node.hostname", f.Hostname, query)
    76→		}
    77→		if f.SchedulerState != nil {
    78→			query = query.Where("node_state.node_state = ?", f.SchedulerState)
    79→		}
    80→		if f.HealthState != nil {
    81→			query = query.Where("node_state.health_state = ?", f.HealthState)
    82→		}
    83→	}
    84→	return query
    85→}
    86→
    87→func (r *NodeRepository) FetchMetadata(hostname string, cluster string) (map[string]string, error) {
    88→	start := time.Now()
    89→
    90→	RawMetaData := make([]byte, 0)
    91→
    92→	if err := sq.Select("node.meta_data").From("node").
    93→		Where("node.hostname = ?", hostname).
    94→		Where("node.cluster = ?", cluster).
    95→		RunWith(r.stmtCache).QueryRow().Scan(&RawMetaData); err != nil {
    96→		cclog.Warn("Error while scanning for node metadata")
    97→		return nil, err
    98→	}
    99→
   100→	if len(RawMetaData) == 0 {
   101→		return nil, nil
   102→	}
   103→
   104→	MetaData := make(map[string]string)
   105→
   106→	if err := json.Unmarshal(RawMetaData, &MetaData); err != nil {
   107→		cclog.Warn("Error while unmarshaling raw metadata json")
   108→		return nil, err
   109→	}
   110→
   111→	cclog.Debugf("Timer FetchMetadata %s", time.Since(start))
   112→	return MetaData, nil
   113→}
   114→
   115→func (r *NodeRepository) GetNode(hostname string, cluster string, withMeta bool) (*schema.Node, error) {
   116→	node := &schema.Node{}
   117→	if err := sq.Select("node.hostname", "node.cluster", "node.subcluster",
   118→		"node_state.node_state", "node_state.health_state").
   119→		From("node").
   120→		Join("node_state ON node_state.node_id = node.id").
   121→		Where(latestStateCondition()).
   122→		Where("node.hostname = ?", hostname).
   123→		Where("node.cluster = ?", cluster).
   124→		RunWith(r.DB).
   125→		QueryRow().Scan(&node.Hostname, &node.Cluster, &node.SubCluster, &node.NodeState, &node.HealthState); err != nil {
   126→		cclog.Warnf("Error while querying node '%s' from database: %v", hostname, err)
   127→		return nil, err
   128→	}
   129→
   130→	if withMeta {
   131→		var err error
   132→		var meta map[string]string
   133→		if meta, err = r.FetchMetadata(hostname, cluster); err != nil {
   134→			cclog.Warnf("Error while fetching metadata for node '%s'", hostname)
   135→			return nil, err
   136→		}
   137→		node.MetaData = meta
   138→	}
   139→
   140→	return node, nil
   141→}
   142→
   143→func (r *NodeRepository) GetNodeByID(id int64, withMeta bool) (*schema.Node, error) {
   144→	node := &schema.Node{}
   145→	if err := sq.Select("node.hostname", "node.cluster", "node.subcluster",
   146→		"node_state.node_state", "node_state.health_state").
   147→		From("node").
   148→		Join("node_state ON node_state.node_id = node.id").
   149→		Where(latestStateCondition()).
   150→		Where("node.id = ?", id).
   151→		RunWith(r.DB).
   152→		QueryRow().Scan(&node.Hostname, &node.Cluster, &node.SubCluster, &node.NodeState, &node.HealthState); err != nil {
   153→		cclog.Warnf("Error while querying node ID '%d' from database: %v", id, err)
   154→		return nil, err
   155→	}
   156→
   157→	if withMeta {
   158→		meta, metaErr := r.FetchMetadata(node.Hostname, node.Cluster)
   159→		if metaErr != nil {
   160→			cclog.Warnf("Error while fetching metadata for node ID '%d': %v", id, metaErr)
   161→			return nil, metaErr
   162→		}
   163→		node.MetaData = meta
   164→	}
   165→
   166→	return node, nil
   167→}
   168→
   169→// const NamedNodeInsert string = `
   170→// INSERT INTO node (time_stamp, hostname, cluster, subcluster, node_state, health_state,
   171→//
   172→//	cpus_allocated, cpus_total, memory_allocated, memory_total, gpus_allocated, gpus_total)
   173→//	VALUES (:time_stamp, :hostname, :cluster, :subcluster, :node_state, :health_state,
   174→//	:cpus_allocated, :cpus_total, :memory_allocated, :memory_total, :gpus_allocated, :gpus_total);`
   175→
   176→const NamedNodeInsert string = `
   177→INSERT INTO node (hostname, cluster, subcluster)
   178→	VALUES (:hostname, :cluster, :subcluster);`
   179→
   180→// AddNode adds a Node to the node table. This can be triggered by a node collector registration or
   181→// from a nodestate update from the job scheduler.
   182→func (r *NodeRepository) AddNode(node *schema.NodeDB) (int64, error) {
   183→	var err error
   184→
   185→	res, err := r.DB.NamedExec(NamedNodeInsert, node)
   186→	if err != nil {
   187→		cclog.Errorf("Error while adding node '%v' to database", node.Hostname)
   188→		return 0, err
   189→	}
   190→	node.ID, err = res.LastInsertId()
   191→	if err != nil {
   192→		cclog.Errorf("Error while getting last insert id for node '%v' from database", node.Hostname)
   193→		return 0, err
   194→	}
   195→
   196→	return node.ID, nil
   197→}
   198→
   199→const NamedNodeStateInsert string = `
   200→INSERT INTO node_state (time_stamp, node_state, health_state, health_metrics,
   201→	cpus_allocated, memory_allocated, gpus_allocated, jobs_running, node_id)
   202→	VALUES (:time_stamp, :node_state, :health_state, :health_metrics,
   203→	:cpus_allocated, :memory_allocated, :gpus_allocated, :jobs_running, :node_id);`
   204→
   205→// TODO: Add real Monitoring Health State
   206→
   207→// UpdateNodeState is called from the Node REST API to add a row in the node state table
   208→func (r *NodeRepository) UpdateNodeState(hostname string, cluster string, nodeState *schema.NodeStateDB) error {
   209→	var id int64
   210→
   211→	if err := sq.Select("id").From("node").
   212→		Where("node.hostname = ?", hostname).Where("node.cluster = ?", cluster).RunWith(r.DB).
   213→		QueryRow().Scan(&id); err != nil {
   214→		if err == sql.ErrNoRows {
   215→			subcluster, err := archive.GetSubClusterByNode(cluster, hostname)
   216→			if err != nil {
   217→				cclog.Errorf("Error while getting subcluster for node '%s' in cluster '%s': %v", hostname, cluster, err)
   218→				return err
   219→			}
   220→			node := schema.NodeDB{
   221→				Hostname: hostname, Cluster: cluster, SubCluster: subcluster,
   222→			}
   223→			id, err = r.AddNode(&node)
   224→			if err != nil {
   225→				cclog.Errorf("Error while adding node '%s' to database: %v", hostname, err)
   226→				return err
   227→			}
   228→
   229→			cclog.Debugf("Added node '%s' to database", hostname)
   230→		} else {
   231→			cclog.Warnf("Error while querying node '%v' from database", id)
   232→			return err
   233→		}
   234→	}
   235→
   236→	nodeState.NodeID = id
   237→
   238→	_, err := r.DB.NamedExec(NamedNodeStateInsert, nodeState)
   239→	if err != nil {
   240→		cclog.Errorf("Error while adding node state for '%v' to database", hostname)
   241→		return err
   242→	}
   243→	cclog.Debugf("Updated node state for '%s' in database", hostname)
   244→	return nil
   245→}
   246→
   247→// func (r *NodeRepository) UpdateHealthState(hostname string, healthState *schema.MonitoringState) error {
   248→// 	if _, err := sq.Update("node").Set("health_state", healthState).Where("node.id = ?", id).RunWith(r.DB).Exec(); err != nil {
   249→// 		cclog.Errorf("error while updating node '%d'", id)
   250→// 		return err
   251→// 	}
   252→//
   253→// 	return nil
   254→// }
   255→
   256→// NodeStateWithNode combines a node state row with denormalized node info.
   257→type NodeStateWithNode struct {
   258→	ID              int64  `db:"id"`
   259→	TimeStamp       int64  `db:"time_stamp"`
   260→	NodeState       string `db:"node_state"`
   261→	HealthState     string `db:"health_state"`
   262→	HealthMetrics   string `db:"health_metrics"`
   263→	CpusAllocated   int    `db:"cpus_allocated"`
   264→	MemoryAllocated int64  `db:"memory_allocated"`
   265→	GpusAllocated   int    `db:"gpus_allocated"`
   266→	JobsRunning     int    `db:"jobs_running"`
   267→	Hostname        string `db:"hostname"`
   268→	Cluster         string `db:"cluster"`
   269→	SubCluster      string `db:"subcluster"`
   270→}
   271→
   272→// FindNodeStatesBefore returns all node_state rows with time_stamp < cutoff,
   273→// joined with node info for denormalized archiving.
   274→func (r *NodeRepository) FindNodeStatesBefore(cutoff int64) ([]NodeStateWithNode, error) {
   275→	rows, err := sq.Select(
   276→		"node_state.id", "node_state.time_stamp", "node_state.node_state",
   277→		"node_state.health_state", "node_state.health_metrics",
   278→		"node_state.cpus_allocated", "node_state.memory_allocated",
   279→		"node_state.gpus_allocated", "node_state.jobs_running",
   280→		"node.hostname", "node.cluster", "node.subcluster",
   281→	).
   282→		From("node_state").
   283→		Join("node ON node_state.node_id = node.id").
   284→		Where(sq.Lt{"node_state.time_stamp": cutoff}).
   285→		Where("node_state.id NOT IN (SELECT ns2.id FROM node_state ns2 WHERE ns2.time_stamp = (SELECT MAX(ns3.time_stamp) FROM node_state ns3 WHERE ns3.node_id = ns2.node_id))").
   286→		OrderBy("node.cluster ASC", "node.subcluster ASC", "node.hostname ASC", "node_state.time_stamp ASC").
   287→		RunWith(r.DB).Query()
   288→	if err != nil {
   289→		return nil, err
   290→	}
   291→	defer rows.Close()
   292→
   293→	var result []NodeStateWithNode
   294→	for rows.Next() {
   295→		var ns NodeStateWithNode
   296→		var healthMetrics sql.NullString
   297→		if err := rows.Scan(&ns.ID, &ns.TimeStamp, &ns.NodeState,
   298→			&ns.HealthState, &healthMetrics,
   299→			&ns.CpusAllocated, &ns.MemoryAllocated,
   300→			&ns.GpusAllocated, &ns.JobsRunning,
   301→			&ns.Hostname, &ns.Cluster, &ns.SubCluster); err != nil {
   302→			return nil, err
   303→		}
   304→		ns.HealthMetrics = healthMetrics.String
   305→		result = append(result, ns)
   306→	}
   307→	return result, nil
   308→}
   309→
   310→// DeleteNodeStatesBefore removes node_state rows with time_stamp < cutoff,
   311→// but always preserves the row with the latest timestamp per node_id.
   312→func (r *NodeRepository) DeleteNodeStatesBefore(cutoff int64) (int64, error) {
   313→	res, err := r.DB.Exec(
   314→		`DELETE FROM node_state WHERE time_stamp < ?
   315→		 AND id NOT IN (
   316→		   SELECT id FROM node_state ns2
   317→		   WHERE ns2.time_stamp = (SELECT MAX(ns3.time_stamp) FROM node_state ns3 WHERE ns3.node_id = ns2.node_id)
   318→		 )`,
   319→		cutoff,
   320→	)
   321→	if err != nil {
   322→		return 0, err
   323→	}
   324→	return res.RowsAffected()
   325→}
   326→
   327→func (r *NodeRepository) DeleteNode(id int64) error {
   328→	_, err := r.DB.Exec(`DELETE FROM node WHERE node.id = ?`, id)
   329→	if err != nil {
   330→		cclog.Errorf("Error while deleting node '%d' from DB", id)
   331→		return err
   332→	}
   333→	cclog.Infof("deleted node '%d' from DB", id)
   334→	return nil
   335→}
   336→
   337→// QueryNodes returns a list of nodes based on a node filter. It always operates
   338→// on the last state (largest timestamp).
   339→func (r *NodeRepository) QueryNodes(
   340→	ctx context.Context,
   341→	filters []*model.NodeFilter,
   342→	page *model.PageRequest,
   343→	order *model.OrderByInput, // Currently unused!
   344→) ([]*schema.Node, error) {
   345→	query, qerr := AccessCheck(ctx,
   346→		sq.Select("node.hostname", "node.cluster", "node.subcluster",
   347→			"node_state.node_state", "node_state.health_state").
   348→			From("node").
   349→			Join("node_state ON node_state.node_id = node.id").
   350→			Where(latestStateCondition()))
   351→	if qerr != nil {
   352→		return nil, qerr
   353→	}
   354→
   355→	query = applyNodeFilters(query, filters)
   356→	query = query.OrderBy("node.hostname ASC")
   357→
   358→	if page != nil && page.ItemsPerPage != -1 {
   359→		limit := uint64(page.ItemsPerPage)
   360→		query = query.Offset((uint64(page.Page) - 1) * limit).Limit(limit)
   361→	}
   362→
   363→	rows, err := query.RunWith(r.stmtCache).Query()
   364→	if err != nil {
   365→		queryString, queryVars, _ := query.ToSql()
   366→		cclog.Errorf("Error while running query '%s' %v: %v", queryString, queryVars, err)
   367→		return nil, err
   368→	}
   369→	defer rows.Close()
   370→
   371→	nodes := make([]*schema.Node, 0)
   372→	for rows.Next() {
   373→		node := schema.Node{}
   374→		if err := rows.Scan(&node.Hostname, &node.Cluster, &node.SubCluster,
   375→			&node.NodeState, &node.HealthState); err != nil {
   376→			cclog.Warn("Error while scanning rows (QueryNodes)")
   377→			return nil, err
   378→		}
   379→		nodes = append(nodes, &node)
   380→	}
   381→
   382→	if err := rows.Err(); err != nil {
   383→		return nil, err
   384→	}
   385→
   386→	return nodes, nil
   387→}
   388→
   389→// QueryNodesWithMeta returns a list of nodes based on a node filter. It always operates
   390→// on the last state (largest timestamp). It includes both (!) optional JSON column data
   391→func (r *NodeRepository) QueryNodesWithMeta(
   392→	ctx context.Context,
   393→	filters []*model.NodeFilter,
   394→	page *model.PageRequest,
   395→	order *model.OrderByInput, // Currently unused!
   396→) ([]*schema.Node, error) {
   397→	query, qerr := AccessCheck(ctx,
   398→		sq.Select("node.hostname", "node.cluster", "node.subcluster",
   399→			"node_state.node_state", "node_state.health_state",
   400→			"node.meta_data", "node_state.health_metrics").
   401→			From("node").
   402→			Join("node_state ON node_state.node_id = node.id").
   403→			Where(latestStateCondition()))
   404→	if qerr != nil {
   405→		return nil, qerr
   406→	}
   407→
   408→	query = applyNodeFilters(query, filters)
   409→	query = query.OrderBy("node.hostname ASC")
   410→
   411→	if page != nil && page.ItemsPerPage != -1 {
   412→		limit := uint64(page.ItemsPerPage)
   413→		query = query.Offset((uint64(page.Page) - 1) * limit).Limit(limit)
   414→	}
   415→
   416→	rows, err := query.RunWith(r.stmtCache).Query()
   417→	if err != nil {
   418→		queryString, queryVars, _ := query.ToSql()
   419→		cclog.Errorf("Error while running query '%s' %v: %v", queryString, queryVars, err)
   420→		return nil, err
   421→	}
   422→	defer rows.Close()
   423→
   424→	nodes := make([]*schema.Node, 0)
   425→	for rows.Next() {
   426→		node := schema.Node{}
   427→		RawMetaData := make([]byte, 0)
   428→		RawMetricHealth := make([]byte, 0)
   429→
   430→		if err := rows.Scan(&node.Hostname, &node.Cluster, &node.SubCluster,
   431→			&node.NodeState, &node.HealthState, &RawMetaData, &RawMetricHealth); err != nil {
   432→			cclog.Warn("Error while scanning rows (QueryNodes)")
   433→			return nil, err
   434→		}
   435→
   436→		if len(RawMetaData) == 0 {
   437→			node.MetaData = nil
   438→		} else {
   439→			metaData := make(map[string]string)
   440→			if err := json.Unmarshal(RawMetaData, &metaData); err != nil {
   441→				cclog.Warn("Error while unmarshaling raw metadata json")
   442→				return nil, err
   443→			}
   444→			node.MetaData = metaData
   445→		}
   446→
   447→		if len(RawMetricHealth) == 0 {
   448→			node.HealthData = nil
   449→		} else {
   450→			healthData := make(map[string][]string)
   451→			if err := json.Unmarshal(RawMetricHealth, &healthData); err != nil {
   452→				cclog.Warn("Error while unmarshaling raw healthdata json")
   453→				return nil, err
   454→			}
   455→			node.HealthData = healthData
   456→		}
   457→
   458→		nodes = append(nodes, &node)
   459→	}
   460→
   461→	if err := rows.Err(); err != nil {
   462→		return nil, err
   463→	}
   464→
   465→	return nodes, nil
   466→}
   467→
   468→// CountNodes returns the total matched nodes based on a node filter. It always operates
   469→// on the last state (largest timestamp) per node.
   470→func (r *NodeRepository) CountNodes(
   471→	ctx context.Context,
   472→	filters []*model.NodeFilter,
   473→) (int, error) {
   474→	query, qerr := AccessCheck(ctx,
   475→		sq.Select("COUNT(*)").
   476→			From("node").
   477→			Join("node_state ON node_state.node_id = node.id").
   478→			Where(latestStateCondition()))
   479→	if qerr != nil {
   480→		return 0, qerr
   481→	}
   482→
   483→	query = applyNodeFilters(query, filters)
   484→
   485→	var count int
   486→	if err := query.RunWith(r.stmtCache).QueryRow().Scan(&count); err != nil {
   487→		queryString, queryVars, _ := query.ToSql()
   488→		cclog.Errorf("Error while running query '%s' %v: %v", queryString, queryVars, err)
   489→		return 0, err
   490→	}
   491→
   492→	return count, nil
   493→}
   494→
   495→func (r *NodeRepository) ListNodes(cluster string) ([]*schema.Node, error) {
   496→	q := sq.Select("node.hostname", "node.cluster", "node.subcluster",
   497→		"node_state.node_state", "node_state.health_state").
   498→		From("node").
   499→		Join("node_state ON node_state.node_id = node.id").
   500→		Where(latestStateCondition()).
   501→		Where("node.cluster = ?", cluster).
   502→		OrderBy("node.hostname ASC")
   503→
   504→	rows, err := q.RunWith(r.DB).Query()
   505→	if err != nil {
   506→		cclog.Warn("Error while querying node list")
   507→		return nil, err
   508→	}
   509→	nodeList := make([]*schema.Node, 0, 100)
   510→	defer rows.Close()
   511→	for rows.Next() {
   512→		node := &schema.Node{}
   513→		if err := rows.Scan(&node.Hostname, &node.Cluster,
   514→			&node.SubCluster, &node.NodeState, &node.HealthState); err != nil {
   515→			cclog.Warn("Error while scanning node list (ListNodes)")
   516→			return nil, err
   517→		}
   518→
   519→		nodeList = append(nodeList, node)
   520→	}
   521→
   522→	return nodeList, nil
   523→}
   524→
   525→func (r *NodeRepository) MapNodes(cluster string) (map[string]string, error) {
   526→	q := sq.Select("node.hostname", "node_state.node_state").
   527→		From("node").
   528→		Join("node_state ON node_state.node_id = node.id").
   529→		Where(latestStateCondition()).
   530→		Where("node.cluster = ?", cluster).
   531→		OrderBy("node.hostname ASC")
   532→
   533→	rows, err := q.RunWith(r.DB).Query()
   534→	if err != nil {
   535→		cclog.Warn("Error while querying node list")
   536→		return nil, err
   537→	}
   538→
   539→	stateMap := make(map[string]string)
   540→	defer rows.Close()
   541→	for rows.Next() {
   542→		var hostname, nodestate string
   543→		if err := rows.Scan(&hostname, &nodestate); err != nil {
   544→			cclog.Warn("Error while scanning node list (MapNodes)")
   545→			return nil, err
   546→		}
   547→
   548→		stateMap[hostname] = nodestate
   549→	}
   550→
   551→	return stateMap, nil
   552→}
   553→
   554→func (r *NodeRepository) CountStates(ctx context.Context, filters []*model.NodeFilter, column string) ([]*model.NodeStates, error) {
   555→	query, qerr := AccessCheck(ctx,
   556→		sq.Select(column, "COUNT(*) as count").
   557→			From("node").
   558→			Join("node_state ON node_state.node_id = node.id").
   559→			Where(latestStateCondition()).
   560→			GroupBy(column))
   561→	if qerr != nil {
   562→		return nil, qerr
   563→	}
   564→
   565→	query = applyNodeFilters(query, filters)
   566→
   567→	rows, err := query.RunWith(r.stmtCache).Query()
   568→	if err != nil {
   569→		queryString, queryVars, _ := query.ToSql()
   570→		cclog.Errorf("Error while running query '%s' %v: %v", queryString, queryVars, err)
   571→		return nil, err
   572→	}
   573→	defer rows.Close()
   574→
   575→	nodes := make([]*model.NodeStates, 0)
   576→	for rows.Next() {
   577→		var state string
   578→		var count int
   579→		if err := rows.Scan(&state, &count); err != nil {
   580→			cclog.Warn("Error while scanning rows (CountStates)")
   581→			return nil, err
   582→		}
   583→		nodes = append(nodes, &model.NodeStates{State: state, Count: count})
   584→	}
   585→
   586→	if err := rows.Err(); err != nil {
   587→		return nil, err
   588→	}
   589→
   590→	return nodes, nil
   591→}
   592→
   593→func (r *NodeRepository) CountStatesTimed(ctx context.Context, filters []*model.NodeFilter, column string) ([]*model.NodeStatesTimed, error) {
   594→	query, qerr := AccessCheck(ctx, sq.Select(column, "time_stamp", "count(*) as count").From("node")) // "cluster"?
   595→	if qerr != nil {
   596→		return nil, qerr
   597→	}
   598→
   599→	query = query.Join("node_state ON node_state.node_id = node.id")
   600→
   601→	for _, f := range filters {
   602→		// Required
   603→		if f.TimeStart != nil {
   604→			query = query.Where("time_stamp > ?", f.TimeStart)
   605→		}
   606→		// Optional
   607→		if f.Hostname != nil {
   608→			query = buildStringCondition("hostname", f.Hostname, query)
   609→		}
   610→		if f.Cluster != nil {
   611→			query = buildStringCondition("cluster", f.Cluster, query)
   612→		}
   613→		if f.SubCluster != nil {
   614→			query = buildStringCondition("subcluster", f.SubCluster, query)
   615→		}
   616→		if f.SchedulerState != nil {
   617→			query = query.Where("node_state = ?", f.SchedulerState)
   618→		}
   619→		if f.HealthState != nil {
   620→			query = query.Where("health_state = ?", f.HealthState)
   621→		}
   622→	}
   623→
   624→	// Add Group and Order
   625→	query = query.GroupBy(column + ", time_stamp").OrderBy("time_stamp ASC")
   626→
   627→	rows, err := query.RunWith(r.stmtCache).Query()
   628→	if err != nil {
   629→		queryString, queryVars, _ := query.ToSql()
   630→		cclog.Errorf("Error while running query '%s' %v: %v", queryString, queryVars, err)
   631→		return nil, err
   632→	}
   633→	defer rows.Close()
   634→
   635→	rawData := make(map[string][][]int)
   636→	for rows.Next() {
   637→		var state string
   638→		var timestamp, count int
   639→
   640→		if err := rows.Scan(&state, &timestamp, &count); err != nil {
   641→			cclog.Warnf("Error while scanning rows (CountStatesTimed) at time '%d'", timestamp)
   642→			return nil, err
   643→		}
   644→
   645→		if rawData[state] == nil {
   646→			rawData[state] = [][]int{make([]int, 0), make([]int, 0)}
   647→		}
   648→
   649→		rawData[state][0] = append(rawData[state][0], timestamp)
   650→		rawData[state][1] = append(rawData[state][1], count)
   651→	}
   652→
   653→	if err := rows.Err(); err != nil {
   654→		return nil, err
   655→	}
   656→
   657→	timedStates := make([]*model.NodeStatesTimed, 0)
   658→	for state, data := range rawData {
   659→		entry := model.NodeStatesTimed{State: state, Times: data[0], Counts: data[1]}
   660→		timedStates = append(timedStates, &entry)
   661→	}
   662→
   663→	return timedStates, nil
   664→}
   665→
   666→func (r *NodeRepository) GetNodesForList(
   667→	ctx context.Context,
   668→	cluster string,
   669→	subCluster string,
   670→	stateFilter string,
   671→	nodeFilter string,
   672→	page *model.PageRequest,
   673→) ([]string, map[string]string, int, bool, error) {
   674→	// Init Return Vars
   675→	nodes := make([]string, 0)
   676→	stateMap := make(map[string]string)
   677→	countNodes := 0
   678→	hasNextPage := false
   679→
   680→	// Build Filters
   681→	queryFilters := make([]*model.NodeFilter, 0)
   682→	if cluster != "" {
   683→		queryFilters = append(queryFilters, &model.NodeFilter{Cluster: &model.StringInput{Eq: &cluster}})
   684→	}
   685→	if subCluster != "" {
   686→		queryFilters = append(queryFilters, &model.NodeFilter{SubCluster: &model.StringInput{Eq: &subCluster}})
   687→	}
   688→	if nodeFilter != "" && stateFilter != "notindb" {
   689→		queryFilters = append(queryFilters, &model.NodeFilter{Hostname: &model.StringInput{Contains: &nodeFilter}})
   690→	}
   691→	if stateFilter != "all" && stateFilter != "notindb" {
   692→		queryState := schema.SchedulerState(stateFilter)
   693→		queryFilters = append(queryFilters, &model.NodeFilter{SchedulerState: &queryState})
   694→	}
   695→	// if healthFilter != "all" {
   696→	// 	filters = append(filters, &model.NodeFilter{HealthState: &healthFilter})
   697→	// }
   698→
   699→	// Special Case: Disable Paging for missing nodes filter, save IPP for later
   700→	var backupItems int
   701→	if stateFilter == "notindb" {
   702→		backupItems = page.ItemsPerPage
   703→		page.ItemsPerPage = -1
   704→	}
   705→
   706→	// Query Nodes From DB
   707→	rawNodes, serr := r.QueryNodes(ctx, queryFilters, page, nil) // Order not Used
   708→	if serr != nil {
   709→		cclog.Warn("error while loading node database data (Resolver.NodeMetricsList)")
   710→		return nil, nil, 0, false, serr
   711→	}
   712→
   713→	// Intermediate Node Result Info
   714→	for _, node := range rawNodes {
   715→		if node == nil {
   716→			continue
   717→		}
   718→		nodes = append(nodes, node.Hostname)
   719→		stateMap[node.Hostname] = string(node.NodeState)
   720→	}
   721→
   722→	// Special Case: Find Nodes not in DB node table but in metricStore only
   723→	if stateFilter == "notindb" {
   724→		// Reapply Original Paging
   725→		page.ItemsPerPage = backupItems
   726→		// Get Nodes From Topology
   727→		var topoNodes []string
   728→		if subCluster != "" {
   729→			scNodes := archive.NodeLists[cluster][subCluster]
   730→			topoNodes = scNodes.PrintList()
   731→		} else {
   732→			subClusterNodeLists := archive.NodeLists[cluster]
   733→			for _, nodeList := range subClusterNodeLists {
   734→				topoNodes = append(topoNodes, nodeList.PrintList()...)
   735→			}
   736→		}
   737→		// Compare to all nodes from cluster/subcluster in DB
   738→		var missingNodes []string
   739→		for _, scanNode := range topoNodes {
   740→			if !slices.Contains(nodes, scanNode) {
   741→				missingNodes = append(missingNodes, scanNode)
   742→			}
   743→		}
   744→		// Filter nodes by name
   745→		if nodeFilter != "" {
   746→			filteredNodesByName := []string{}
   747→			for _, missingNode := range missingNodes {
   748→				if strings.Contains(missingNode, nodeFilter) {
   749→					filteredNodesByName = append(filteredNodesByName, missingNode)
   750→				}
   751→			}
   752→			missingNodes = filteredNodesByName
   753→		}
   754→		// Sort Missing Nodes Alphanumerically
   755→		slices.Sort(missingNodes)
   756→		// Total Missing
   757→		countNodes = len(missingNodes)
   758→		// Apply paging
   759→		if countNodes > page.ItemsPerPage {
   760→			start := (page.Page - 1) * page.ItemsPerPage
   761→			end := start + page.ItemsPerPage
   762→			if end > countNodes {
   763→				end = countNodes
   764→				hasNextPage = false
   765→			} else {
   766→				hasNextPage = true
   767→			}
   768→			nodes = missingNodes[start:end]
   769→		} else {
   770→			nodes = missingNodes
   771→		}
   772→
   773→	} else {
   774→		// DB Nodes: Count and derive hasNextPage from count
   775→		var cerr error
   776→		countNodes, cerr = r.CountNodes(ctx, queryFilters)
   777→		if cerr != nil {
   778→			cclog.Warn("error while counting node database data (Resolver.NodeMetricsList)")
   779→			return nil, nil, 0, false, cerr
   780→		}
   781→		hasNextPage = page.Page*page.ItemsPerPage < countNodes
   782→	}
   783→
   784→	// Fallback for non-init'd node table in DB; Ignores stateFilter
   785→	if stateFilter == "all" && countNodes == 0 {
   786→		nodes, countNodes, hasNextPage = getNodesFromTopol(cluster, subCluster, nodeFilter, page)
   787→	}
   788→
   789→	return nodes, stateMap, countNodes, hasNextPage, nil
   790→}
   791→
   792→func AccessCheck(ctx context.Context, query sq.SelectBuilder) (sq.SelectBuilder, error) {
   793→	user := GetUserFromContext(ctx)
   794→	return AccessCheckWithUser(user, query)
   795→}
   796→
   797→func AccessCheckWithUser(user *schema.User, query sq.SelectBuilder) (sq.SelectBuilder, error) {
   798→	if user == nil {
   799→		var qnil sq.SelectBuilder
   800→		return qnil, fmt.Errorf("user context is nil")
   801→	}
   802→
   803→	switch {
   804→	// case len(user.Roles) == 1 && user.HasRole(schema.RoleApi): // API-User : Access NodeInfos
   805→	// 	return query, nil
   806→	case user.HasAnyRole([]schema.Role{schema.RoleAdmin, schema.RoleSupport}): // Admin & Support : Access NodeInfos
   807→		return query, nil
   808→	default: // No known Role: No Access, return error
   809→		var qnil sq.SelectBuilder
   810→		return qnil, fmt.Errorf("user has no or unknown roles")
   811→	}
   812→}
   813→
   814→func getNodesFromTopol(cluster string, subCluster string, nodeFilter string, page *model.PageRequest) ([]string, int, bool) {
   815→	// 0) Init additional vars
   816→	hasNextPage := false
   817→	totalNodes := 0
   818→
   819→	// 1) Get list of all nodes
   820→	var topolNodes []string
   821→	if subCluster != "" {
   822→		scNodes := archive.NodeLists[cluster][subCluster]
   823→		topolNodes = scNodes.PrintList()
   824→	} else {
   825→		subClusterNodeLists := archive.NodeLists[cluster]
   826→		for _, nodeList := range subClusterNodeLists {
   827→			topolNodes = append(topolNodes, nodeList.PrintList()...)
   828→		}
   829→	}
   830→
   831→	// 2) Filter nodes
   832→	if nodeFilter != "" {
   833→		filteredNodes := []string{}
   834→		for _, node := range topolNodes {
   835→			if strings.Contains(node, nodeFilter) {
   836→				filteredNodes = append(filteredNodes, node)
   837→			}
   838→		}
   839→		topolNodes = filteredNodes
   840→	}
   841→
   842→	// 2.1) Count total nodes && Sort nodes -> Sorting invalidated after ccms return ...
   843→	totalNodes = len(topolNodes)
   844→	sort.Strings(topolNodes)
   845→
   846→	// 3) Apply paging
   847→	if len(topolNodes) > page.ItemsPerPage {
   848→		start := (page.Page - 1) * page.ItemsPerPage
   849→		end := start + page.ItemsPerPage
   850→		if end >= len(topolNodes) {
   851→			end = len(topolNodes)
   852→			hasNextPage = false
   853→		} else {
   854→			hasNextPage = true
   855→		}
   856→		topolNodes = topolNodes[start:end]
   857→	}
   858→
   859→	return topolNodes, totalNodes, hasNextPage
   860→}
   861→

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package repository

import (
	"context"
	"database/sql"
	"encoding/json"
	"fmt"
	"slices"
	"sort"
	"strings"
	"sync"
	"time"

	"github.com/ClusterCockpit/cc-backend/internal/graph/model"
	"github.com/ClusterCockpit/cc-backend/pkg/archive"
	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
	"github.com/ClusterCockpit/cc-lib/v2/lrucache"
	"github.com/ClusterCockpit/cc-lib/v2/schema"
	sq "github.com/Masterminds/squirrel"
	"github.com/jmoiron/sqlx"
)

var (
	nodeRepoOnce     sync.Once
	nodeRepoInstance *NodeRepository
)

type NodeRepository struct {
	DB        *sqlx.DB
	stmtCache *sq.StmtCache
	cache     *lrucache.Cache
	driver    string
}

func GetNodeRepository() *NodeRepository {
	nodeRepoOnce.Do(func() {
		db := GetConnection()

		nodeRepoInstance = &NodeRepository{
			DB:     db.DB,
			driver: db.Driver,

			stmtCache: sq.NewStmtCache(db.DB),
			cache:     lrucache.New(repoConfig.CacheSize),
		}
	})
	return nodeRepoInstance
}

// latestStateCondition returns a squirrel expression that restricts node_state
// rows to the latest per node_id using a correlated subquery.
// Requires the query to join node and node_state tables.
func latestStateCondition() sq.Sqlizer {
	return sq.Expr(
		"node_state.id = (SELECT ns2.id FROM node_state ns2 WHERE ns2.node_id = node.id ORDER BY ns2.time_stamp DESC LIMIT 1)",
	)
}

// applyNodeFilters applies common NodeFilter conditions to a query that joins
// the node and node_state tables with latestStateCondition.
func applyNodeFilters(query sq.SelectBuilder, filters []*model.NodeFilter) sq.SelectBuilder {
	for _, f := range filters {
		if f.Cluster != nil {
			query = buildStringCondition("node.cluster", f.Cluster, query)
		}
		if f.SubCluster != nil {
			query = buildStringCondition("node.subcluster", f.SubCluster, query)
		}
		if f.Hostname != nil {
			query = buildStringCondition("node.hostname", f.Hostname, query)
		}
		if f.SchedulerState != nil {
			query = query.Where("node_state.node_state = ?", f.SchedulerState)
		}
		if f.HealthState != nil {
			query = query.Where("node_state.health_state = ?", f.HealthState)
		}
	}
	return query
}

func (r *NodeRepository) FetchMetadata(hostname string, cluster string) (map[string]string, error) {
	start := time.Now()

	RawMetaData := make([]byte, 0)

	if err := sq.Select("node.meta_data").From("node").
		Where("node.hostname = ?", hostname).
		Where("node.cluster = ?", cluster).
		RunWith(r.stmtCache).QueryRow().Scan(&RawMetaData); err != nil {
		cclog.Warn("Error while scanning for node metadata")
		return nil, err
	}

	if len(RawMetaData) == 0 {
		return nil, nil
	}

	MetaData := make(map[string]string)

	if err := json.Unmarshal(RawMetaData, &MetaData); err != nil {
		cclog.Warn("Error while unmarshaling raw metadata json")
		return nil, err
	}

	cclog.Debugf("Timer FetchMetadata %s", time.Since(start))
	return MetaData, nil
}

func (r *NodeRepository) GetNode(hostname string, cluster string, withMeta bool) (*schema.Node, error) {
	node := &schema.Node{}
	if err := sq.Select("node.hostname", "node.cluster", "node.subcluster",
		"node_state.node_state", "node_state.health_state").
		From("node").
		Join("node_state ON node_state.node_id = node.id").
		Where(latestStateCondition()).
		Where("node.hostname = ?", hostname).
		Where("node.cluster = ?", cluster).
		RunWith(r.DB).
		QueryRow().Scan(&node.Hostname, &node.Cluster, &node.SubCluster, &node.NodeState, &node.HealthState); err != nil {
		cclog.Warnf("Error while querying node '%s' from database: %v", hostname, err)
		return nil, err
	}

	if withMeta {
		var err error
		var meta map[string]string
		if meta, err = r.FetchMetadata(hostname, cluster); err != nil {
			cclog.Warnf("Error while fetching metadata for node '%s'", hostname)
			return nil, err
		}
		node.MetaData = meta
	}

	return node, nil
}

func (r *NodeRepository) GetNodeByID(id int64, withMeta bool) (*schema.Node, error) {
	node := &schema.Node{}
	if err := sq.Select("node.hostname", "node.cluster", "node.subcluster",
		"node_state.node_state", "node_state.health_state").
		From("node").
		Join("node_state ON node_state.node_id = node.id").
		Where(latestStateCondition()).
		Where("node.id = ?", id).
		RunWith(r.DB).
		QueryRow().Scan(&node.Hostname, &node.Cluster, &node.SubCluster, &node.NodeState, &node.HealthState); err != nil {
		cclog.Warnf("Error while querying node ID '%d' from database: %v", id, err)
		return nil, err
	}

	if withMeta {
		meta, metaErr := r.FetchMetadata(node.Hostname, node.Cluster)
		if metaErr != nil {
			cclog.Warnf("Error while fetching metadata for node ID '%d': %v", id, metaErr)
			return nil, metaErr
		}
		node.MetaData = meta
	}

	return node, nil
}

// const NamedNodeInsert string = `
// INSERT INTO node (time_stamp, hostname, cluster, subcluster, node_state, health_state,
//
//	cpus_allocated, cpus_total, memory_allocated, memory_total, gpus_allocated, gpus_total)
//	VALUES (:time_stamp, :hostname, :cluster, :subcluster, :node_state, :health_state,
//	:cpus_allocated, :cpus_total, :memory_allocated, :memory_total, :gpus_allocated, :gpus_total);`

const NamedNodeInsert string = `
INSERT INTO node (hostname, cluster, subcluster)
	VALUES (:hostname, :cluster, :subcluster);`

// AddNode adds a Node to the node table. This can be triggered by a node collector registration or
// from a nodestate update from the job scheduler.
func (r *NodeRepository) AddNode(node *schema.NodeDB) (int64, error) {
	var err error

	res, err := r.DB.NamedExec(NamedNodeInsert, node)
	if err != nil {
		cclog.Errorf("Error while adding node '%v' to database", node.Hostname)
		return 0, err
	}
	node.ID, err = res.LastInsertId()
	if err != nil {
		cclog.Errorf("Error while getting last insert id for node '%v' from database", node.Hostname)
		return 0, err
	}

	return node.ID, nil
}

const NamedNodeStateInsert string = `
INSERT INTO node_state (time_stamp, node_state, health_state, health_metrics,
	cpus_allocated, memory_allocated, gpus_allocated, jobs_running, node_id)
	VALUES (:time_stamp, :node_state, :health_state, :health_metrics,
	:cpus_allocated, :memory_allocated, :gpus_allocated, :jobs_running, :node_id);`

// TODO: Add real Monitoring Health State

// UpdateNodeState is called from the Node REST API to add a row in the node state table
func (r *NodeRepository) UpdateNodeState(hostname string, cluster string, nodeState *schema.NodeStateDB) error {
	var id int64

	if err := sq.Select("id").From("node").
		Where("node.hostname = ?", hostname).Where("node.cluster = ?", cluster).RunWith(r.DB).
		QueryRow().Scan(&id); err != nil {
		if err == sql.ErrNoRows {
			subcluster, err := archive.GetSubClusterByNode(cluster, hostname)
			if err != nil {
				cclog.Errorf("Error while getting subcluster for node '%s' in cluster '%s': %v", hostname, cluster, err)
				return err
			}
			node := schema.NodeDB{
				Hostname: hostname, Cluster: cluster, SubCluster: subcluster,
			}
			id, err = r.AddNode(&node)
			if err != nil {
				cclog.Errorf("Error while adding node '%s' to database: %v", hostname, err)
				return err
			}

			cclog.Debugf("Added node '%s' to database", hostname)
		} else {
			cclog.Warnf("Error while querying node '%v' from database", id)
			return err
		}
	}

	nodeState.NodeID = id

	_, err := r.DB.NamedExec(NamedNodeStateInsert, nodeState)
	if err != nil {
		cclog.Errorf("Error while adding node state for '%v' to database", hostname)
		return err
	}
	cclog.Debugf("Updated node state for '%s' in database", hostname)
	return nil
}

// func (r *NodeRepository) UpdateHealthState(hostname string, healthState *schema.MonitoringState) error {
// 	if _, err := sq.Update("node").Set("health_state", healthState).Where("node.id = ?", id).RunWith(r.DB).Exec(); err != nil {
// 		cclog.Errorf("error while updating node '%d'", id)
// 		return err
// 	}
//
// 	return nil
// }

// NodeStateWithNode combines a node state row with denormalized node info.
type NodeStateWithNode struct {
	ID              int64  `db:"id"`
	TimeStamp       int64  `db:"time_stamp"`
	NodeState       string `db:"node_state"`
	HealthState     string `db:"health_state"`
	HealthMetrics   string `db:"health_metrics"`
	CpusAllocated   int    `db:"cpus_allocated"`
	MemoryAllocated int64  `db:"memory_allocated"`
	GpusAllocated   int    `db:"gpus_allocated"`
	JobsRunning     int    `db:"jobs_running"`
	Hostname        string `db:"hostname"`
	Cluster         string `db:"cluster"`
	SubCluster      string `db:"subcluster"`
}

// FindNodeStatesBefore returns all node_state rows with time_stamp < cutoff,
// joined with node info for denormalized archiving.
func (r *NodeRepository) FindNodeStatesBefore(cutoff int64) ([]NodeStateWithNode, error) {
	rows, err := sq.Select(
		"node_state.id", "node_state.time_stamp", "node_state.node_state",
		"node_state.health_state", "node_state.health_metrics",
		"node_state.cpus_allocated", "node_state.memory_allocated",
		"node_state.gpus_allocated", "node_state.jobs_running",
		"node.hostname", "node.cluster", "node.subcluster",
	).
		From("node_state").
		Join("node ON node_state.node_id = node.id").
		Where(sq.Lt{"node_state.time_stamp": cutoff}).
		Where("node_state.id NOT IN (SELECT ns2.id FROM node_state ns2 WHERE ns2.time_stamp = (SELECT MAX(ns3.time_stamp) FROM node_state ns3 WHERE ns3.node_id = ns2.node_id))").
		OrderBy("node.cluster ASC", "node.subcluster ASC", "node.hostname ASC", "node_state.time_stamp ASC").
		RunWith(r.DB).Query()
	if err != nil {
		return nil, err
	}
	defer rows.Close()

	var result []NodeStateWithNode
	for rows.Next() {
		var ns NodeStateWithNode
		var healthMetrics sql.NullString
		if err := rows.Scan(&ns.ID, &ns.TimeStamp, &ns.NodeState,
			&ns.HealthState, &healthMetrics,
			&ns.CpusAllocated, &ns.MemoryAllocated,
			&ns.GpusAllocated, &ns.JobsRunning,
			&ns.Hostname, &ns.Cluster, &ns.SubCluster); err != nil {
			return nil, err
		}
		ns.HealthMetrics = healthMetrics.String
		result = append(result, ns)
	}
	return result, nil
}

// DeleteNodeStatesBefore removes node_state rows with time_stamp < cutoff,
// but always preserves the row with the latest timestamp per node_id.
func (r *NodeRepository) DeleteNodeStatesBefore(cutoff int64) (int64, error) {
	res, err := r.DB.Exec(
		`DELETE FROM node_state WHERE time_stamp < ?
		 AND id NOT IN (
		   SELECT id FROM node_state ns2
		   WHERE ns2.time_stamp = (SELECT MAX(ns3.time_stamp) FROM node_state ns3 WHERE ns3.node_id = ns2.node_id)
		 )`,
		cutoff,
	)
	if err != nil {
		return 0, err
	}
	return res.RowsAffected()
}

func (r *NodeRepository) DeleteNode(id int64) error {
	_, err := r.DB.Exec(`DELETE FROM node WHERE node.id = ?`, id)
	if err != nil {
		cclog.Errorf("Error while deleting node '%d' from DB", id)
		return err
	}
	cclog.Infof("deleted node '%d' from DB", id)
	return nil
}

// QueryNodes returns a list of nodes based on a node filter. It always operates
// on the last state (largest timestamp).
func (r *NodeRepository) QueryNodes(
	ctx context.Context,
	filters []*model.NodeFilter,
	page *model.PageRequest,
	order *model.OrderByInput, // Currently unused!
) ([]*schema.Node, error) {
	query, qerr := AccessCheck(ctx,
		sq.Select("node.hostname", "node.cluster", "node.subcluster",
			"node_state.node_state", "node_state.health_state").
			From("node").
			Join("node_state ON node_state.node_id = node.id").
			Where(latestStateCondition()))
	if qerr != nil {
		return nil, qerr
	}

	query = applyNodeFilters(query, filters)
	query = query.OrderBy("node.hostname ASC")

	if page != nil && page.ItemsPerPage != -1 {
		limit := uint64(page.ItemsPerPage)
		query = query.Offset((uint64(page.Page) - 1) * limit).Limit(limit)
	}

	rows, err := query.RunWith(r.stmtCache).Query()
	if err != nil {
		queryString, queryVars, _ := query.ToSql()
		cclog.Errorf("Error while running query '%s' %v: %v", queryString, queryVars, err)
		return nil, err
	}
	defer rows.Close()

	nodes := make([]*schema.Node, 0)
	for rows.Next() {
		node := schema.Node{}
		if err := rows.Scan(&node.Hostname, &node.Cluster, &node.SubCluster,
			&node.NodeState, &node.HealthState); err != nil {
			cclog.Warn("Error while scanning rows (QueryNodes)")
			return nil, err
		}
		nodes = append(nodes, &node)
	}

	if err := rows.Err(); err != nil {
		return nil, err
	}

	return nodes, nil
}

// QueryNodesWithMeta returns a list of nodes based on a node filter. It always operates
// on the last state (largest timestamp). It includes both (!) optional JSON column data
func (r *NodeRepository) QueryNodesWithMeta(
	ctx context.Context,
	filters []*model.NodeFilter,
	page *model.PageRequest,
	order *model.OrderByInput, // Currently unused!
) ([]*schema.Node, error) {
	query, qerr := AccessCheck(ctx,
		sq.Select("node.hostname", "node.cluster", "node.subcluster",
			"node_state.node_state", "node_state.health_state",
			"node.meta_data", "node_state.health_metrics").
			From("node").
			Join("node_state ON node_state.node_id = node.id").
			Where(latestStateCondition()))
	if qerr != nil {
		return nil, qerr
	}

	query = applyNodeFilters(query, filters)
	query = query.OrderBy("node.hostname ASC")

	if page != nil && page.ItemsPerPage != -1 {
		limit := uint64(page.ItemsPerPage)
		query = query.Offset((uint64(page.Page) - 1) * limit).Limit(limit)
	}

	rows, err := query.RunWith(r.stmtCache).Query()
	if err != nil {
		queryString, queryVars, _ := query.ToSql()
		cclog.Errorf("Error while running query '%s' %v: %v", queryString, queryVars, err)
		return nil, err
	}
	defer rows.Close()

	nodes := make([]*schema.Node, 0)
	for rows.Next() {
		node := schema.Node{}
		RawMetaData := make([]byte, 0)
		RawMetricHealth := make([]byte, 0)

		if err := rows.Scan(&node.Hostname, &node.Cluster, &node.SubCluster,
			&node.NodeState, &node.HealthState, &RawMetaData, &RawMetricHealth); err != nil {
			cclog.Warn("Error while scanning rows (QueryNodes)")
			return nil, err
		}

		if len(RawMetaData) == 0 {
			node.MetaData = nil
		} else {
			metaData := make(map[string]string)
			if err := json.Unmarshal(RawMetaData, &metaData); err != nil {
				cclog.Warn("Error while unmarshaling raw metadata json")
				return nil, err
			}
			node.MetaData = metaData
		}

		if len(RawMetricHealth) == 0 {
			node.HealthData = nil
		} else {
			healthData := make(map[string][]string)
			if err := json.Unmarshal(RawMetricHealth, &healthData); err != nil {
				cclog.Warn("Error while unmarshaling raw healthdata json")
				return nil, err
			}
			node.HealthData = healthData
		}

		nodes = append(nodes, &node)
	}

	if err := rows.Err(); err != nil {
		return nil, err
	}

	return nodes, nil
}

// CountNodes returns the total matched nodes based on a node filter. It always operates
// on the last state (largest timestamp) per node.
func (r *NodeRepository) CountNodes(
	ctx context.Context,
	filters []*model.NodeFilter,
) (int, error) {
	query, qerr := AccessCheck(ctx,
		sq.Select("COUNT(*)").
			From("node").
			Join("node_state ON node_state.node_id = node.id").
			Where(latestStateCondition()))
	if qerr != nil {
		return 0, qerr
	}

	query = applyNodeFilters(query, filters)

	var count int
	if err := query.RunWith(r.stmtCache).QueryRow().Scan(&count); err != nil {
		queryString, queryVars, _ := query.ToSql()
		cclog.Errorf("Error while running query '%s' %v: %v", queryString, queryVars, err)
		return 0, err
	}

	return count, nil
}

func (r *NodeRepository) ListNodes(cluster string) ([]*schema.Node, error) {
	q := sq.Select("node.hostname", "node.cluster", "node.subcluster",
		"node_state.node_state", "node_state.health_state").
		From("node").
		Join("node_state ON node_state.node_id = node.id").
		Where(latestStateCondition()).
		Where("node.cluster = ?", cluster).
		OrderBy("node.hostname ASC")

	rows, err := q.RunWith(r.DB).Query()
	if err != nil {
		cclog.Warn("Error while querying node list")
		return nil, err
	}
	nodeList := make([]*schema.Node, 0, 100)
	defer rows.Close()
	for rows.Next() {
		node := &schema.Node{}
		if err := rows.Scan(&node.Hostname, &node.Cluster,
			&node.SubCluster, &node.NodeState, &node.HealthState); err != nil {
			cclog.Warn("Error while scanning node list (ListNodes)")
			return nil, err
		}

		nodeList = append(nodeList, node)
	}

	return nodeList, nil
}

func (r *NodeRepository) MapNodes(cluster string) (map[string]string, error) {
	q := sq.Select("node.hostname", "node_state.node_state").
		From("node").
		Join("node_state ON node_state.node_id = node.id").
		Where(latestStateCondition()).
		Where("node.cluster = ?", cluster).
		OrderBy("node.hostname ASC")

	rows, err := q.RunWith(r.DB).Query()
	if err != nil {
		cclog.Warn("Error while querying node list")
		return nil, err
	}

	stateMap := make(map[string]string)
	defer rows.Close()
	for rows.Next() {
		var hostname, nodestate string
		if err := rows.Scan(&hostname, &nodestate); err != nil {
			cclog.Warn("Error while scanning node list (MapNodes)")
			return nil, err
		}

		stateMap[hostname] = nodestate
	}

	return stateMap, nil
}

func (r *NodeRepository) CountStates(ctx context.Context, filters []*model.NodeFilter, column string) ([]*model.NodeStates, error) {
	query, qerr := AccessCheck(ctx,
		sq.Select(column, "COUNT(*) as count").
			From("node").
			Join("node_state ON node_state.node_id = node.id").
			Where(latestStateCondition()).
			GroupBy(column))
	if qerr != nil {
		return nil, qerr
	}

	query = applyNodeFilters(query, filters)

	rows, err := query.RunWith(r.stmtCache).Query()
	if err != nil {
		queryString, queryVars, _ := query.ToSql()
		cclog.Errorf("Error while running query '%s' %v: %v", queryString, queryVars, err)
		return nil, err
	}
	defer rows.Close()

	nodes := make([]*model.NodeStates, 0)
	for rows.Next() {
		var state string
		var count int
		if err := rows.Scan(&state, &count); err != nil {
			cclog.Warn("Error while scanning rows (CountStates)")
			return nil, err
		}
		nodes = append(nodes, &model.NodeStates{State: state, Count: count})
	}

	if err := rows.Err(); err != nil {
		return nil, err
	}

	return nodes, nil
}

func (r *NodeRepository) CountStatesTimed(ctx context.Context, filters []*model.NodeFilter, column string) ([]*model.NodeStatesTimed, error) {
	query, qerr := AccessCheck(ctx, sq.Select(column, "time_stamp", "count(*) as count").From("node")) // "cluster"?
	if qerr != nil {
		return nil, qerr
	}

	query = query.Join("node_state ON node_state.node_id = node.id")

	for _, f := range filters {
		// Required
		if f.TimeStart != nil {
			query = query.Where("time_stamp > ?", f.TimeStart)
		}
		// Optional
		if f.Hostname != nil {
			query = buildStringCondition("hostname", f.Hostname, query)
		}
		if f.Cluster != nil {
			query = buildStringCondition("cluster", f.Cluster, query)
		}
		if f.SubCluster != nil {
			query = buildStringCondition("subcluster", f.SubCluster, query)
		}
		if f.SchedulerState != nil {
			query = query.Where("node_state = ?", f.SchedulerState)
		}
		if f.HealthState != nil {
			query = query.Where("health_state = ?", f.HealthState)
		}
	}

	// Add Group and Order
	query = query.GroupBy(column + ", time_stamp").OrderBy("time_stamp ASC")

	rows, err := query.RunWith(r.stmtCache).Query()
	if err != nil {
		queryString, queryVars, _ := query.ToSql()
		cclog.Errorf("Error while running query '%s' %v: %v", queryString, queryVars, err)
		return nil, err
	}
	defer rows.Close()

	rawData := make(map[string][][]int)
	for rows.Next() {
		var state string
		var timestamp, count int

		if err := rows.Scan(&state, &timestamp, &count); err != nil {
			cclog.Warnf("Error while scanning rows (CountStatesTimed) at time '%d'", timestamp)
			return nil, err
		}

		if rawData[state] == nil {
			rawData[state] = [][]int{make([]int, 0), make([]int, 0)}
		}

		rawData[state][0] = append(rawData[state][0], timestamp)
		rawData[state][1] = append(rawData[state][1], count)
	}

	if err := rows.Err(); err != nil {
		return nil, err
	}

	timedStates := make([]*model.NodeStatesTimed, 0)
	for state, data := range rawData {
		entry := model.NodeStatesTimed{State: state, Times: data[0], Counts: data[1]}
		timedStates = append(timedStates, &entry)
	}

	return timedStates, nil
}

func (r *NodeRepository) GetNodesForList(
	ctx context.Context,
	cluster string,
	subCluster string,
	stateFilter string,
	nodeFilter string,
	page *model.PageRequest,
) ([]string, map[string]string, int, bool, error) {
	// Init Return Vars
	nodes := make([]string, 0)
	stateMap := make(map[string]string)
	countNodes := 0
	hasNextPage := false

	// Build Filters
	queryFilters := make([]*model.NodeFilter, 0)
	if cluster != "" {
		queryFilters = append(queryFilters, &model.NodeFilter{Cluster: &model.StringInput{Eq: &cluster}})
	}
	if subCluster != "" {
		queryFilters = append(queryFilters, &model.NodeFilter{SubCluster: &model.StringInput{Eq: &subCluster}})
	}
	if nodeFilter != "" && stateFilter != "notindb" {
		queryFilters = append(queryFilters, &model.NodeFilter{Hostname: &model.StringInput{Contains: &nodeFilter}})
	}
	if stateFilter != "all" && stateFilter != "notindb" {
		queryState := schema.SchedulerState(stateFilter)
		queryFilters = append(queryFilters, &model.NodeFilter{SchedulerState: &queryState})
	}
	// if healthFilter != "all" {
	// 	filters = append(filters, &model.NodeFilter{HealthState: &healthFilter})
	// }

	// Special Case: Disable Paging for missing nodes filter, save IPP for later
	var backupItems int
	if stateFilter == "notindb" {
		backupItems = page.ItemsPerPage
		page.ItemsPerPage = -1
	}

	// Query Nodes From DB
	rawNodes, serr := r.QueryNodes(ctx, queryFilters, page, nil) // Order not Used
	if serr != nil {
		cclog.Warn("error while loading node database data (Resolver.NodeMetricsList)")
		return nil, nil, 0, false, serr
	}

	// Intermediate Node Result Info
	for _, node := range rawNodes {
		if node == nil {
			continue
		}
		nodes = append(nodes, node.Hostname)
		stateMap[node.Hostname] = string(node.NodeState)
	}

	// Special Case: Find Nodes not in DB node table but in metricStore only
	if stateFilter == "notindb" {
		// Reapply Original Paging
		page.ItemsPerPage = backupItems
		// Get Nodes From Topology
		var topoNodes []string
		if subCluster != "" {
			scNodes := archive.NodeLists[cluster][subCluster]
			topoNodes = scNodes.PrintList()
		} else {
			subClusterNodeLists := archive.NodeLists[cluster]
			for _, nodeList := range subClusterNodeLists {
				topoNodes = append(topoNodes, nodeList.PrintList()...)
			}
		}
		// Compare to all nodes from cluster/subcluster in DB
		var missingNodes []string
		for _, scanNode := range topoNodes {
			if !slices.Contains(nodes, scanNode) {
				missingNodes = append(missingNodes, scanNode)
			}
		}
		// Filter nodes by name
		if nodeFilter != "" {
			filteredNodesByName := []string{}
			for _, missingNode := range missingNodes {
				if strings.Contains(missingNode, nodeFilter) {
					filteredNodesByName = append(filteredNodesByName, missingNode)
				}
			}
			missingNodes = filteredNodesByName
		}
		// Sort Missing Nodes Alphanumerically
		slices.Sort(missingNodes)
		// Total Missing
		countNodes = len(missingNodes)
		// Apply paging
		if countNodes > page.ItemsPerPage {
			start := (page.Page - 1) * page.ItemsPerPage
			end := start + page.ItemsPerPage
			if end > countNodes {
				end = countNodes
				hasNextPage = false
			} else {
				hasNextPage = true
			}
			nodes = missingNodes[start:end]
		} else {
			nodes = missingNodes
		}

	} else {
		// DB Nodes: Count and derive hasNextPage from count
		var cerr error
		countNodes, cerr = r.CountNodes(ctx, queryFilters)
		if cerr != nil {
			cclog.Warn("error while counting node database data (Resolver.NodeMetricsList)")
			return nil, nil, 0, false, cerr
		}
		hasNextPage = page.Page*page.ItemsPerPage < countNodes
	}

	// Fallback for non-init'd node table in DB; Ignores stateFilter
	if stateFilter == "all" && countNodes == 0 {
		nodes, countNodes, hasNextPage = getNodesFromTopol(cluster, subCluster, nodeFilter, page)
	}

	return nodes, stateMap, countNodes, hasNextPage, nil
}

func AccessCheck(ctx context.Context, query sq.SelectBuilder) (sq.SelectBuilder, error) {
	user := GetUserFromContext(ctx)
	return AccessCheckWithUser(user, query)
}

func AccessCheckWithUser(user *schema.User, query sq.SelectBuilder) (sq.SelectBuilder, error) {
	if user == nil {
		var qnil sq.SelectBuilder
		return qnil, fmt.Errorf("user context is nil")
	}

	switch {
	// case len(user.Roles) == 1 && user.HasRole(schema.RoleApi): // API-User : Access NodeInfos
	// 	return query, nil
	case user.HasAnyRole([]schema.Role{schema.RoleAdmin, schema.RoleSupport}): // Admin & Support : Access NodeInfos
		return query, nil
	default: // No known Role: No Access, return error
		var qnil sq.SelectBuilder
		return qnil, fmt.Errorf("user has no or unknown roles")
	}
}

func getNodesFromTopol(cluster string, subCluster string, nodeFilter string, page *model.PageRequest) ([]string, int, bool) {
	// 0) Init additional vars
	hasNextPage := false
	totalNodes := 0

	// 1) Get list of all nodes
	var topolNodes []string
	if subCluster != "" {
		scNodes := archive.NodeLists[cluster][subCluster]
		topolNodes = scNodes.PrintList()
	} else {
		subClusterNodeLists := archive.NodeLists[cluster]
		for _, nodeList := range subClusterNodeLists {
			topolNodes = append(topolNodes, nodeList.PrintList()...)
		}
	}

	// 2) Filter nodes
	if nodeFilter != "" {
		filteredNodes := []string{}
		for _, node := range topolNodes {
			if strings.Contains(node, nodeFilter) {
				filteredNodes = append(filteredNodes, node)
			}
		}
		topolNodes = filteredNodes
	}

	// 2.1) Count total nodes && Sort nodes -> Sorting invalidated after ccms return ...
	totalNodes = len(topolNodes)
	sort.Strings(topolNodes)

	// 3) Apply paging
	if len(topolNodes) > page.ItemsPerPage {
		start := (page.Page - 1) * page.ItemsPerPage
		end := start + page.ItemsPerPage
		if end >= len(topolNodes) {
			end = len(topolNodes)
			hasNextPage = false
		} else {
			hasNextPage = true
		}
		topolNodes = topolNodes[start:end]
	}

	return topolNodes, totalNodes, hasNextPage
}


All the `defer rows.Close()` calls are already present in the current state of all three files. Let me verify against what's actually changed via git diff.


Bash

git diff internal/repository/stats.go internal/repository/tags.go internal/repository/node.go

Show staged/unstaged changes in the three files


diff --git a/internal/repository/node.go b/internal/repository/node.go
index 09415bef..116b9868 100644
--- a/internal/repository/node.go
+++ b/internal/repository/node.go
@@ -366,19 +366,23 @@ func (r *NodeRepository) QueryNodes(
 		cclog.Errorf("Error while running query '%s' %v: %v", queryString, queryVars, err)
 		return nil, err
 	}
+	defer rows.Close()

 	nodes := make([]*schema.Node, 0)
 	for rows.Next() {
 		node := schema.Node{}
 		if err := rows.Scan(&node.Hostname, &node.Cluster, &node.SubCluster,
 			&node.NodeState, &node.HealthState); err != nil {
-			rows.Close()
 			cclog.Warn("Error while scanning rows (QueryNodes)")
 			return nil, err
 		}
 		nodes = append(nodes, &node)
 	}

+	if err := rows.Err(); err != nil {
+		return nil, err
+	}
+
 	return nodes, nil
 }

@@ -415,6 +419,7 @@ func (r *NodeRepository) QueryNodesWithMeta(
 		cclog.Errorf("Error while running query '%s' %v: %v", queryString, queryVars, err)
 		return nil, err
 	}
+	defer rows.Close()

 	nodes := make([]*schema.Node, 0)
 	for rows.Next() {
@@ -424,7 +429,6 @@ func (r *NodeRepository) QueryNodesWithMeta(

 		if err := rows.Scan(&node.Hostname, &node.Cluster, &node.SubCluster,
 			&node.NodeState, &node.HealthState, &RawMetaData, &RawMetricHealth); err != nil {
-			rows.Close()
 			cclog.Warn("Error while scanning rows (QueryNodes)")
 			return nil, err
 		}
@@ -454,6 +458,10 @@ func (r *NodeRepository) QueryNodesWithMeta(
 		nodes = append(nodes, &node)
 	}

+	if err := rows.Err(); err != nil {
+		return nil, err
+	}
+
 	return nodes, nil
 }

@@ -545,10 +553,11 @@ func (r *NodeRepository) MapNodes(cluster string) (map[string]string, error) {

 func (r *NodeRepository) CountStates(ctx context.Context, filters []*model.NodeFilter, column string) ([]*model.NodeStates, error) {
 	query, qerr := AccessCheck(ctx,
-		sq.Select(column).
+		sq.Select(column, "COUNT(*) as count").
 			From("node").
 			Join("node_state ON node_state.node_id = node.id").
-			Where(latestStateCondition()))
+			Where(latestStateCondition()).
+			GroupBy(column))
 	if qerr != nil {
 		return nil, qerr
 	}
@@ -561,23 +570,21 @@ func (r *NodeRepository) CountStates(ctx context.Context, filters []*model.NodeF
 		cclog.Errorf("Error while running query '%s' %v: %v", queryString, queryVars, err)
 		return nil, err
 	}
+	defer rows.Close()

-	stateMap := map[string]int{}
+	nodes := make([]*model.NodeStates, 0)
 	for rows.Next() {
 		var state string
-		if err := rows.Scan(&state); err != nil {
-			rows.Close()
+		var count int
+		if err := rows.Scan(&state, &count); err != nil {
 			cclog.Warn("Error while scanning rows (CountStates)")
 			return nil, err
 		}
-
-		stateMap[state] += 1
+		nodes = append(nodes, &model.NodeStates{State: state, Count: count})
 	}

-	nodes := make([]*model.NodeStates, 0)
-	for state, counts := range stateMap {
-		node := model.NodeStates{State: state, Count: counts}
-		nodes = append(nodes, &node)
+	if err := rows.Err(); err != nil {
+		return nil, err
 	}

 	return nodes, nil
@@ -623,6 +630,7 @@ func (r *NodeRepository) CountStatesTimed(ctx context.Context, filters []*model.
 		cclog.Errorf("Error while running query '%s' %v: %v", queryString, queryVars, err)
 		return nil, err
 	}
+	defer rows.Close()

 	rawData := make(map[string][][]int)
 	for rows.Next() {
@@ -630,7 +638,6 @@ func (r *NodeRepository) CountStatesTimed(ctx context.Context, filters []*model.
 		var timestamp, count int

 		if err := rows.Scan(&state, &timestamp, &count); err != nil {
-			rows.Close()
 			cclog.Warnf("Error while scanning rows (CountStatesTimed) at time '%d'", timestamp)
 			return nil, err
 		}
@@ -643,6 +650,10 @@ func (r *NodeRepository) CountStatesTimed(ctx context.Context, filters []*model.
 		rawData[state][1] = append(rawData[state][1], count)
 	}

+	if err := rows.Err(); err != nil {
+		return nil, err
+	}
+
 	timedStates := make([]*model.NodeStatesTimed, 0)
 	for state, data := range rawData {
 		entry := model.NodeStatesTimed{State: state, Times: data[0], Counts: data[1]}
diff --git a/internal/repository/stats.go b/internal/repository/stats.go
index 0ceb92f2..4a61f62d 100644
--- a/internal/repository/stats.go
+++ b/internal/repository/stats.go
@@ -235,6 +235,7 @@ func (r *JobRepository) JobsStatsGrouped(
 		cclog.Warn("Error while querying DB for job statistics")
 		return nil, err
 	}
+	defer rows.Close()

 	stats := make([]*model.JobsStatistics, 0, 100)

@@ -320,6 +321,10 @@ func (r *JobRepository) JobsStatsGrouped(
 		}
 	}

+	if err := rows.Err(); err != nil {
+		return nil, err
+	}
+
 	cclog.Debugf("Timer JobsStatsGrouped %s", time.Since(start))
 	return stats, nil
 }
@@ -440,6 +445,7 @@ func (r *JobRepository) JobCountGrouped(
 		cclog.Warn("Error while querying DB for job statistics")
 		return nil, err
 	}
+	defer rows.Close()

 	stats := make([]*model.JobsStatistics, 0, 100)

@@ -459,6 +465,10 @@ func (r *JobRepository) JobCountGrouped(
 		}
 	}

+	if err := rows.Err(); err != nil {
+		return nil, err
+	}
+
 	cclog.Debugf("Timer JobCountGrouped %s", time.Since(start))
 	return stats, nil
 }
@@ -496,6 +506,7 @@ func (r *JobRepository) AddJobCountGrouped(
 		cclog.Warn("Error while querying DB for job statistics")
 		return nil, err
 	}
+	defer rows.Close()

 	counts := make(map[string]int)

@@ -511,6 +522,10 @@ func (r *JobRepository) AddJobCountGrouped(
 		}
 	}

+	if err := rows.Err(); err != nil {
+		return nil, err
+	}
+
 	switch kind {
 	case "running":
 		for _, s := range stats {
@@ -550,23 +565,13 @@ func (r *JobRepository) AddJobCount(
 	if err != nil {
 		return nil, err
 	}
-	rows, err := query.RunWith(r.DB).Query()
-	if err != nil {
-		cclog.Warn("Error while querying DB for job statistics")
+	var cnt sql.NullInt64
+	if err := query.RunWith(r.DB).QueryRow().Scan(&cnt); err != nil {
+		cclog.Warn("Error while querying DB for job count")
 		return nil, err
 	}

-	var count int
-
-	for rows.Next() {
-		var cnt sql.NullInt64
-		if err := rows.Scan(&cnt); err != nil {
-			cclog.Warn("Error while scanning rows")
-			return nil, err
-		}
-
-		count = int(cnt.Int64)
-	}
+	count := int(cnt.Int64)

 	switch kind {
 	case "running":
@@ -755,6 +760,7 @@ func (r *JobRepository) jobsStatisticsHistogram(
 		cclog.Error("Error while running query")
 		return nil, err
 	}
+	defer rows.Close()

 	points := make([]*model.HistoPoint, 0)
 	// is it possible to introduce zero values here? requires info about bincount
@@ -767,6 +773,11 @@ func (r *JobRepository) jobsStatisticsHistogram(

 		points = append(points, &point)
 	}
+
+	if err := rows.Err(); err != nil {
+		return nil, err
+	}
+
 	cclog.Debugf("Timer jobsStatisticsHistogram %s", time.Since(start))
 	return points, nil
 }
@@ -823,6 +834,7 @@ func (r *JobRepository) jobsDurationStatisticsHistogram(
 		cclog.Error("Error while running query")
 		return nil, err
 	}
+	defer rows.Close()

 	// Match query results to pre-initialized bins.
 	// point.Value from query is the bin number; multiply by binSizeSeconds to match bin.Value.
@@ -841,6 +853,10 @@ func (r *JobRepository) jobsDurationStatisticsHistogram(
 		}
 	}

+	if err := rows.Err(); err != nil {
+		return nil, err
+	}
+
 	cclog.Debugf("Timer jobsStatisticsHistogram %s", time.Since(start))
 	return points, nil
 }
@@ -948,6 +964,7 @@ func (r *JobRepository) jobsMetricStatisticsHistogram(
 		cclog.Errorf("Error while running mainQuery: %s", err)
 		return nil, err
 	}
+	defer rows.Close()

 	// Pre-initialize bins with calculated min/max ranges.
 	// Example: peak=1000, bins=10 -> bin 1=[0,100), bin 2=[100,200), ..., bin 10=[900,1000]
@@ -976,6 +993,10 @@ func (r *JobRepository) jobsMetricStatisticsHistogram(
 		}
 	}

+	if err := rows.Err(); err != nil {
+		return nil, err
+	}
+
 	result := model.MetricHistoPoints{Metric: metric, Unit: unit, Stat: &footprintStat, Data: points}

 	cclog.Debugf("Timer jobsStatisticsHistogram %s", time.Since(start))
diff --git a/internal/repository/tags.go b/internal/repository/tags.go
index 943dda66..39ccd90d 100644
--- a/internal/repository/tags.go
+++ b/internal/repository/tags.go
@@ -283,6 +283,7 @@ func (r *JobRepository) CountTags(user *schema.User) (tags []schema.Tag, counts
 	if err != nil {
 		return nil, nil, err
 	}
+	defer xrows.Close()

 	for xrows.Next() {
 		var t schema.Tag
@@ -300,6 +301,10 @@ func (r *JobRepository) CountTags(user *schema.User) (tags []schema.Tag, counts
 		}
 	}

+	if err := xrows.Err(); err != nil {
+		return nil, nil, err
+	}
+
 	// Query and Count Jobs with attached Tags
 	q := sq.Select("t.tag_type, t.tag_name, t.id, count(jt.tag_id)").
 		From("tag t").
@@ -334,6 +339,7 @@ func (r *JobRepository) CountTags(user *schema.User) (tags []schema.Tag, counts
 	if err != nil {
 		return nil, nil, err
 	}
+	defer rows.Close()

 	counts = make(map[string]int)
 	for rows.Next() {
@@ -511,6 +517,7 @@ func (r *JobRepository) GetTags(user *schema.User, job *int64) ([]*schema.Tag, e
 		cclog.Errorf("Error get tags with %s: %v", s, err)
 		return nil, err
 	}
+	defer rows.Close()

 	tags := make([]*schema.Tag, 0)
 	for rows.Next() {
@@ -529,6 +536,10 @@ func (r *JobRepository) GetTags(user *schema.User, job *int64) ([]*schema.Tag, e
 		}
 	}

+	if err := rows.Err(); err != nil {
+		return nil, err
+	}
+
 	return tags, nil
 }

@@ -544,6 +555,7 @@ func (r *JobRepository) GetTagsDirect(job *int64) ([]*schema.Tag, error) {
 		cclog.Errorf("Error get tags with %s: %v", s, err)
 		return nil, err
 	}
+	defer rows.Close()

 	tags := make([]*schema.Tag, 0)
 	for rows.Next() {
@@ -555,6 +567,10 @@ func (r *JobRepository) GetTagsDirect(job *int64) ([]*schema.Tag, error) {
 		tags = append(tags, tag)
 	}

+	if err := rows.Err(); err != nil {
+		return nil, err
+	}
+
 	return tags, nil
 }

@@ -582,6 +598,7 @@ func (r *JobRepository) getArchiveTags(job *int64) ([]*schema.Tag, error) {
 		cclog.Errorf("Error get tags with %s: %v", s, err)
 		return nil, err
 	}
+	defer rows.Close()

 	tags := make([]*schema.Tag, 0)
 	for rows.Next() {
@@ -593,6 +610,10 @@ func (r *JobRepository) getArchiveTags(job *int64) ([]*schema.Tag, error) {
 		tags = append(tags, tag)
 	}

+	if err := rows.Err(); err != nil {
+		return nil, err
+	}
+
 	return tags, nil
 }


All the changes from the plan are already applied as unstaged modifications. The diff shows:

1. **stats.go**: Added `defer rows.Close()` + `rows.Err()` checks in 6 functions, plus refactored `AddJobCount` to use `QueryRow` instead of `Query`
2. **tags.go**: Added `defer xrows.Close()`/`defer rows.Close()` + `rows.Err()` checks in 5 locations
3. **node.go**: Added `defer rows.Close()` + `rows.Err()` checks in 4 functions, removed manual `rows.Close()` calls, plus refactored `CountStates` to use SQL `COUNT(*)`

Let me verify it builds and passes tests.


Build all packages


Run go vet on repository package


Run repository tests


ok  	github.com/ClusterCockpit/cc-backend/internal/repository	(cached)


All changes are already in place and verified:

- **Build**: passes
- **go vet**: passes
- **Tests**: pass

The changes are currently unstaged. All 16 locations from the plan have been fixed — `defer rows.Close()` added after each `Query()` error check, manual `rows.Close()` calls in error paths removed, and `rows.Err()` checks added where missing. The `AddJobCount` function was also simplified from `Query` + loop to `QueryRow().Scan()` since it returns a single value.