     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→package repository
     7→
     8→import (
     9→	"context"
    10→	"database/sql"
    11→	"encoding/json"
    12→	"fmt"
    13→	"slices"
    14→	"sort"
    15→	"strings"
    16→	"sync"
    17→	"time"
    18→
    19→	"github.com/ClusterCockpit/cc-backend/internal/graph/model"
    20→	"github.com/ClusterCockpit/cc-backend/pkg/archive"
    21→	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
    22→	"github.com/ClusterCockpit/cc-lib/v2/lrucache"
    23→	"github.com/ClusterCockpit/cc-lib/v2/schema"
    24→	sq "github.com/Masterminds/squirrel"
    25→	"github.com/jmoiron/sqlx"
    26→)
    27→
    28→var (
    29→	nodeRepoOnce     sync.Once
    30→	nodeRepoInstance *NodeRepository
    31→)
    32→
    33→type NodeRepository struct {
    34→	DB        *sqlx.DB
    35→	stmtCache *sq.StmtCache
    36→	cache     *lrucache.Cache
    37→	driver    string
    38→}
    39→
    40→func GetNodeRepository() *NodeRepository {
    41→	nodeRepoOnce.Do(func() {
    42→		db := GetConnection()
    43→
    44→		nodeRepoInstance = &NodeRepository{
    45→			DB:     db.DB,
    46→			driver: db.Driver,
    47→
    48→			stmtCache: sq.NewStmtCache(db.DB),
    49→			cache:     lrucache.New(repoConfig.CacheSize),
    50→		}
    51→	})
    52→	return nodeRepoInstance
    53→}
    54→
    55→// latestStateCondition returns a squirrel expression that restricts node_state
    56→// rows to the latest per node_id using a correlated subquery.
    57→// Requires the query to join node and node_state tables.
    58→func latestStateCondition() sq.Sqlizer {
    59→	return sq.Expr(
    60→		"node_state.id = (SELECT ns2.id FROM node_state ns2 WHERE ns2.node_id = node.id ORDER BY ns2.time_stamp DESC LIMIT 1)",
    61→	)
    62→}
    63→
    64→// applyNodeFilters applies common NodeFilter conditions to a query that joins
    65→// the node and node_state tables with latestStateCondition.
    66→func applyNodeFilters(query sq.SelectBuilder, filters []*model.NodeFilter) sq.SelectBuilder {
    67→	for _, f := range filters {
    68→		if f.Cluster != nil {
    69→			query = buildStringCondition("node.cluster", f.Cluster, query)
    70→		}
    71→		if f.SubCluster != nil {
    72→			query = buildStringCondition("node.subcluster", f.SubCluster, query)
    73→		}
    74→		if f.Hostname != nil {
    75→			query = buildStringCondition("node.hostname", f.Hostname, query)
    76→		}
    77→		if f.SchedulerState != nil {
    78→			query = query.Where("node_state.node_state = ?", f.SchedulerState)
    79→		}
    80→		if f.HealthState != nil {
    81→			query = query.Where("node_state.health_state = ?", f.HealthState)
    82→		}
    83→	}
    84→	return query
    85→}
    86→
    87→func (r *NodeRepository) FetchMetadata(hostname string, cluster string) (map[string]string, error) {
    88→	start := time.Now()
    89→
    90→	RawMetaData := make([]byte, 0)
    91→
    92→	if err := sq.Select("node.meta_data").From("node").
    93→		Where("node.hostname = ?", hostname).
    94→		Where("node.cluster = ?", cluster).
    95→		RunWith(r.stmtCache).QueryRow().Scan(&RawMetaData); err != nil {
    96→		cclog.Warn("Error while scanning for node metadata")
    97→		return nil, err
    98→	}
    99→
   100→	if len(RawMetaData) == 0 {
   101→		return nil, nil
   102→	}
   103→
   104→	MetaData := make(map[string]string)
   105→
   106→	if err := json.Unmarshal(RawMetaData, &MetaData); err != nil {
   107→		cclog.Warn("Error while unmarshaling raw metadata json")
   108→		return nil, err
   109→	}
   110→
   111→	cclog.Debugf("Timer FetchMetadata %s", time.Since(start))
   112→	return MetaData, nil
   113→}
   114→
   115→func (r *NodeRepository) GetNode(hostname string, cluster string, withMeta bool) (*schema.Node, error) {
   116→	node := &schema.Node{}
   117→	if err := sq.Select("node.hostname", "node.cluster", "node.subcluster",
   118→		"node_state.node_state", "node_state.health_state").
   119→		From("node").
   120→		Join("node_state ON node_state.node_id = node.id").
   121→		Where(latestStateCondition()).
   122→		Where("node.hostname = ?", hostname).
   123→		Where("node.cluster = ?", cluster).
   124→		RunWith(r.DB).
   125→		QueryRow().Scan(&node.Hostname, &node.Cluster, &node.SubCluster, &node.NodeState, &node.HealthState); err != nil {
   126→		cclog.Warnf("Error while querying node '%s' from database: %v", hostname, err)
   127→		return nil, err
   128→	}
   129→
   130→	if withMeta {
   131→		var err error
   132→		var meta map[string]string
   133→		if meta, err = r.FetchMetadata(hostname, cluster); err != nil {
   134→			cclog.Warnf("Error while fetching metadata for node '%s'", hostname)
   135→			return nil, err
   136→		}
   137→		node.MetaData = meta
   138→	}
   139→
   140→	return node, nil
   141→}
   142→
   143→func (r *NodeRepository) GetNodeByID(id int64, withMeta bool) (*schema.Node, error) {
   144→	node := &schema.Node{}
   145→	if err := sq.Select("node.hostname", "node.cluster", "node.subcluster",
   146→		"node_state.node_state", "node_state.health_state").
   147→		From("node").
   148→		Join("node_state ON node_state.node_id = node.id").
   149→		Where(latestStateCondition()).
   150→		Where("node.id = ?", id).
   151→		RunWith(r.DB).
   152→		QueryRow().Scan(&node.Hostname, &node.Cluster, &node.SubCluster, &node.NodeState, &node.HealthState); err != nil {
   153→		cclog.Warnf("Error while querying node ID '%d' from database: %v", id, err)
   154→		return nil, err
   155→	}
   156→
   157→	if withMeta {
   158→		meta, metaErr := r.FetchMetadata(node.Hostname, node.Cluster)
   159→		if metaErr != nil {
   160→			cclog.Warnf("Error while fetching metadata for node ID '%d': %v", id, metaErr)
   161→			return nil, metaErr
   162→		}
   163→		node.MetaData = meta
   164→	}
   165→
   166→	return node, nil
   167→}
   168→
   169→// const NamedNodeInsert string = `
   170→// INSERT INTO node (time_stamp, hostname, cluster, subcluster, node_state, health_state,
   171→//
   172→//	cpus_allocated, cpus_total, memory_allocated, memory_total, gpus_allocated, gpus_total)
   173→//	VALUES (:time_stamp, :hostname, :cluster, :subcluster, :node_state, :health_state,
   174→//	:cpus_allocated, :cpus_total, :memory_allocated, :memory_total, :gpus_allocated, :gpus_total);`
   175→
   176→const NamedNodeInsert string = `
   177→INSERT INTO node (hostname, cluster, subcluster)
   178→	VALUES (:hostname, :cluster, :subcluster);`
   179→
   180→// AddNode adds a Node to the node table. This can be triggered by a node collector registration or
   181→// from a nodestate update from the job scheduler.
   182→func (r *NodeRepository) AddNode(node *schema.NodeDB) (int64, error) {
   183→	var err error
   184→
   185→	res, err := r.DB.NamedExec(NamedNodeInsert, node)
   186→	if err != nil {
   187→		cclog.Errorf("Error while adding node '%v' to database", node.Hostname)
   188→		return 0, err
   189→	}
   190→	node.ID, err = res.LastInsertId()
   191→	if err != nil {
   192→		cclog.Errorf("Error while getting last insert id for node '%v' from database", node.Hostname)
   193→		return 0, err
   194→	}
   195→
   196→	return node.ID, nil
   197→}
   198→
   199→const NamedNodeStateInsert string = `
   200→INSERT INTO node_state (time_stamp, node_state, health_state, health_metrics,
   201→	cpus_allocated, memory_allocated, gpus_allocated, jobs_running, node_id)
   202→	VALUES (:time_stamp, :node_state, :health_state, :health_metrics,
   203→	:cpus_allocated, :memory_allocated, :gpus_allocated, :jobs_running, :node_id);`
   204→
   205→// TODO: Add real Monitoring Health State
   206→
   207→// UpdateNodeState is called from the Node REST API to add a row in the node state table
   208→func (r *NodeRepository) UpdateNodeState(hostname string, cluster string, nodeState *schema.NodeStateDB) error {
   209→	var id int64
   210→
   211→	if err := sq.Select("id").From("node").
   212→		Where("node.hostname = ?", hostname).Where("node.cluster = ?", cluster).RunWith(r.DB).
   213→		QueryRow().Scan(&id); err != nil {
   214→		if err == sql.ErrNoRows {
   215→			subcluster, err := archive.GetSubClusterByNode(cluster, hostname)
   216→			if err != nil {
   217→				cclog.Errorf("Error while getting subcluster for node '%s' in cluster '%s': %v", hostname, cluster, err)
   218→				return err
   219→			}
   220→			node := schema.NodeDB{
   221→				Hostname: hostname, Cluster: cluster, SubCluster: subcluster,
   222→			}
   223→			id, err = r.AddNode(&node)
   224→			if err != nil {
   225→				cclog.Errorf("Error while adding node '%s' to database: %v", hostname, err)
   226→				return err
   227→			}
   228→
   229→			cclog.Debugf("Added node '%s' to database", hostname)
   230→		} else {
   231→			cclog.Warnf("Error while querying node '%v' from database", id)
   232→			return err
   233→		}
   234→	}
   235→
   236→	nodeState.NodeID = id
   237→
   238→	_, err := r.DB.NamedExec(NamedNodeStateInsert, nodeState)
   239→	if err != nil {
   240→		cclog.Errorf("Error while adding node state for '%v' to database", hostname)
   241→		return err
   242→	}
   243→	cclog.Debugf("Updated node state for '%s' in database", hostname)
   244→	return nil
   245→}
   246→
   247→// func (r *NodeRepository) UpdateHealthState(hostname string, healthState *schema.MonitoringState) error {
   248→// 	if _, err := sq.Update("node").Set("health_state", healthState).Where("node.id = ?", id).RunWith(r.DB).Exec(); err != nil {
   249→// 		cclog.Errorf("error while updating node '%d'", id)
   250→// 		return err
   251→// 	}
   252→//
   253→// 	return nil
   254→// }
   255→
   256→// NodeStateWithNode combines a node state row with denormalized node info.
   257→type NodeStateWithNode struct {
   258→	ID              int64  `db:"id"`
   259→	TimeStamp       int64  `db:"time_stamp"`
   260→	NodeState       string `db:"node_state"`
   261→	HealthState     string `db:"health_state"`
   262→	HealthMetrics   string `db:"health_metrics"`
   263→	CpusAllocated   int    `db:"cpus_allocated"`
   264→	MemoryAllocated int64  `db:"memory_allocated"`
   265→	GpusAllocated   int    `db:"gpus_allocated"`
   266→	JobsRunning     int    `db:"jobs_running"`
   267→	Hostname        string `db:"hostname"`
   268→	Cluster         string `db:"cluster"`
   269→	SubCluster      string `db:"subcluster"`
   270→}
   271→
   272→// FindNodeStatesBefore returns all node_state rows with time_stamp < cutoff,
   273→// joined with node info for denormalized archiving.
   274→func (r *NodeRepository) FindNodeStatesBefore(cutoff int64) ([]NodeStateWithNode, error) {
   275→	rows, err := sq.Select(
   276→		"node_state.id", "node_state.time_stamp", "node_state.node_state",
   277→		"node_state.health_state", "node_state.health_metrics",
   278→		"node_state.cpus_allocated", "node_state.memory_allocated",
   279→		"node_state.gpus_allocated", "node_state.jobs_running",
   280→		"node.hostname", "node.cluster", "node.subcluster",
   281→	).
   282→		From("node_state").
   283→		Join("node ON node_state.node_id = node.id").
   284→		Where(sq.Lt{"node_state.time_stamp": cutoff}).
   285→		Where("node_state.id NOT IN (SELECT ns2.id FROM node_state ns2 WHERE ns2.time_stamp = (SELECT MAX(ns3.time_stamp) FROM node_state ns3 WHERE ns3.node_id = ns2.node_id))").
   286→		OrderBy("node.cluster ASC", "node.subcluster ASC", "node.hostname ASC", "node_state.time_stamp ASC").
   287→		RunWith(r.DB).Query()
   288→	if err != nil {
   289→		return nil, err
   290→	}
   291→	defer rows.Close()
   292→
   293→	var result []NodeStateWithNode
   294→	for rows.Next() {
   295→		var ns NodeStateWithNode
   296→		var healthMetrics sql.NullString
   297→		if err := rows.Scan(&ns.ID, &ns.TimeStamp, &ns.NodeState,
   298→			&ns.HealthState, &healthMetrics,
   299→			&ns.CpusAllocated, &ns.MemoryAllocated,
   300→			&ns.GpusAllocated, &ns.JobsRunning,
   301→			&ns.Hostname, &ns.Cluster, &ns.SubCluster); err != nil {
   302→			return nil, err
   303→		}
   304→		ns.HealthMetrics = healthMetrics.String
   305→		result = append(result, ns)
   306→	}
   307→	return result, nil
   308→}
   309→
   310→// DeleteNodeStatesBefore removes node_state rows with time_stamp < cutoff,
   311→// but always preserves the row with the latest timestamp per node_id.
   312→func (r *NodeRepository) DeleteNodeStatesBefore(cutoff int64) (int64, error) {
   313→	res, err := r.DB.Exec(
   314→		`DELETE FROM node_state WHERE time_stamp < ?
   315→		 AND id NOT IN (
   316→		   SELECT id FROM node_state ns2
   317→		   WHERE ns2.time_stamp = (SELECT MAX(ns3.time_stamp) FROM node_state ns3 WHERE ns3.node_id = ns2.node_id)
   318→		 )`,
   319→		cutoff,
   320→	)
   321→	if err != nil {
   322→		return 0, err
   323→	}
   324→	return res.RowsAffected()
   325→}
   326→
   327→func (r *NodeRepository) DeleteNode(id int64) error {
   328→	_, err := r.DB.Exec(`DELETE FROM node WHERE node.id = ?`, id)
   329→	if err != nil {
   330→		cclog.Errorf("Error while deleting node '%d' from DB", id)
   331→		return err
   332→	}
   333→	cclog.Infof("deleted node '%d' from DB", id)
   334→	return nil
   335→}
   336→
   337→// QueryNodes returns a list of nodes based on a node filter. It always operates
   338→// on the last state (largest timestamp).
   339→func (r *NodeRepository) QueryNodes(
   340→	ctx context.Context,
   341→	filters []*model.NodeFilter,
   342→	page *model.PageRequest,
   343→	order *model.OrderByInput, // Currently unused!
   344→) ([]*schema.Node, error) {
   345→	query, qerr := AccessCheck(ctx,
   346→		sq.Select("node.hostname", "node.cluster", "node.subcluster",
   347→			"node_state.node_state", "node_state.health_state").
   348→			From("node").
   349→			Join("node_state ON node_state.node_id = node.id").
   350→			Where(latestStateCondition()))
   351→	if qerr != nil {
   352→		return nil, qerr
   353→	}
   354→
   355→	query = applyNodeFilters(query, filters)
   356→	query = query.OrderBy("node.hostname ASC")
   357→
   358→	if page != nil && page.ItemsPerPage != -1 {
   359→		limit := uint64(page.ItemsPerPage)
   360→		query = query.Offset((uint64(page.Page) - 1) * limit).Limit(limit)
   361→	}
   362→
   363→	rows, err := query.RunWith(r.stmtCache).Query()
   364→	if err != nil {
   365→		queryString, queryVars, _ := query.ToSql()
   366→		cclog.Errorf("Error while running query '%s' %v: %v", queryString, queryVars, err)
   367→		return nil, err
   368→	}
   369→
   370→	nodes := make([]*schema.Node, 0)
   371→	for rows.Next() {
   372→		node := schema.Node{}
   373→		if err := rows.Scan(&node.Hostname, &node.Cluster, &node.SubCluster,
   374→			&node.NodeState, &node.HealthState); err != nil {
   375→			rows.Close()
   376→			cclog.Warn("Error while scanning rows (QueryNodes)")
   377→			return nil, err
   378→		}
   379→		nodes = append(nodes, &node)
   380→	}
   381→
   382→	return nodes, nil
   383→}
   384→
   385→// QueryNodesWithMeta returns a list of nodes based on a node filter. It always operates
   386→// on the last state (largest timestamp). It includes both (!) optional JSON column data
   387→func (r *NodeRepository) QueryNodesWithMeta(
   388→	ctx context.Context,
   389→	filters []*model.NodeFilter,
   390→	page *model.PageRequest,
   391→	order *model.OrderByInput, // Currently unused!
   392→) ([]*schema.Node, error) {
   393→	query, qerr := AccessCheck(ctx,
   394→		sq.Select("node.hostname", "node.cluster", "node.subcluster",
   395→			"node_state.node_state", "node_state.health_state",
   396→			"node.meta_data", "node_state.health_metrics").
   397→			From("node").
   398→			Join("node_state ON node_state.node_id = node.id").
   399→			Where(latestStateCondition()))
   400→	if qerr != nil {
   401→		return nil, qerr
   402→	}
   403→
   404→	query = applyNodeFilters(query, filters)
   405→	query = query.OrderBy("node.hostname ASC")
   406→
   407→	if page != nil && page.ItemsPerPage != -1 {
   408→		limit := uint64(page.ItemsPerPage)
   409→		query = query.Offset((uint64(page.Page) - 1) * limit).Limit(limit)
   410→	}
   411→
   412→	rows, err := query.RunWith(r.stmtCache).Query()
   413→	if err != nil {
   414→		queryString, queryVars, _ := query.ToSql()
   415→		cclog.Errorf("Error while running query '%s' %v: %v", queryString, queryVars, err)
   416→		return nil, err
   417→	}
   418→
   419→	nodes := make([]*schema.Node, 0)
   420→	for rows.Next() {
   421→		node := schema.Node{}
   422→		RawMetaData := make([]byte, 0)
   423→		RawMetricHealth := make([]byte, 0)
   424→
   425→		if err := rows.Scan(&node.Hostname, &node.Cluster, &node.SubCluster,
   426→			&node.NodeState, &node.HealthState, &RawMetaData, &RawMetricHealth); err != nil {
   427→			rows.Close()
   428→			cclog.Warn("Error while scanning rows (QueryNodes)")
   429→			return nil, err
   430→		}
   431→
   432→		if len(RawMetaData) == 0 {
   433→			node.MetaData = nil
   434→		} else {
   435→			metaData := make(map[string]string)
   436→			if err := json.Unmarshal(RawMetaData, &metaData); err != nil {
   437→				cclog.Warn("Error while unmarshaling raw metadata json")
   438→				return nil, err
   439→			}
   440→			node.MetaData = metaData
   441→		}
   442→
   443→		if len(RawMetricHealth) == 0 {
   444→			node.HealthData = nil
   445→		} else {
   446→			healthData := make(map[string][]string)
   447→			if err := json.Unmarshal(RawMetricHealth, &healthData); err != nil {
   448→				cclog.Warn("Error while unmarshaling raw healthdata json")
   449→				return nil, err
   450→			}
   451→			node.HealthData = healthData
   452→		}
   453→
   454→		nodes = append(nodes, &node)
   455→	}
   456→
   457→	return nodes, nil
   458→}
   459→
   460→// CountNodes returns the total matched nodes based on a node filter. It always operates
   461→// on the last state (largest timestamp) per node.
   462→func (r *NodeRepository) CountNodes(
   463→	ctx context.Context,
   464→	filters []*model.NodeFilter,
   465→) (int, error) {
   466→	query, qerr := AccessCheck(ctx,
   467→		sq.Select("COUNT(*)").
   468→			From("node").
   469→			Join("node_state ON node_state.node_id = node.id").
   470→			Where(latestStateCondition()))
   471→	if qerr != nil {
   472→		return 0, qerr
   473→	}
   474→
   475→	query = applyNodeFilters(query, filters)
   476→
   477→	var count int
   478→	if err := query.RunWith(r.stmtCache).QueryRow().Scan(&count); err != nil {
   479→		queryString, queryVars, _ := query.ToSql()
   480→		cclog.Errorf("Error while running query '%s' %v: %v", queryString, queryVars, err)
   481→		return 0, err
   482→	}
   483→
   484→	return count, nil
   485→}
   486→
   487→func (r *NodeRepository) ListNodes(cluster string) ([]*schema.Node, error) {
   488→	q := sq.Select("node.hostname", "node.cluster", "node.subcluster",
   489→		"node_state.node_state", "node_state.health_state").
   490→		From("node").
   491→		Join("node_state ON node_state.node_id = node.id").
   492→		Where(latestStateCondition()).
   493→		Where("node.cluster = ?", cluster).
   494→		OrderBy("node.hostname ASC")
   495→
   496→	rows, err := q.RunWith(r.DB).Query()
   497→	if err != nil {
   498→		cclog.Warn("Error while querying node list")
   499→		return nil, err
   500→	}
   501→	nodeList := make([]*schema.Node, 0, 100)
   502→	defer rows.Close()
   503→	for rows.Next() {
   504→		node := &schema.Node{}
   505→		if err := rows.Scan(&node.Hostname, &node.Cluster,
   506→			&node.SubCluster, &node.NodeState, &node.HealthState); err != nil {
   507→			cclog.Warn("Error while scanning node list (ListNodes)")
   508→			return nil, err
   509→		}
   510→
   511→		nodeList = append(nodeList, node)
   512→	}
   513→
   514→	return nodeList, nil
   515→}
   516→
   517→func (r *NodeRepository) MapNodes(cluster string) (map[string]string, error) {
   518→	q := sq.Select("node.hostname", "node_state.node_state").
   519→		From("node").
   520→		Join("node_state ON node_state.node_id = node.id").
   521→		Where(latestStateCondition()).
   522→		Where("node.cluster = ?", cluster).
   523→		OrderBy("node.hostname ASC")
   524→
   525→	rows, err := q.RunWith(r.DB).Query()
   526→	if err != nil {
   527→		cclog.Warn("Error while querying node list")
   528→		return nil, err
   529→	}
   530→
   531→	stateMap := make(map[string]string)
   532→	defer rows.Close()
   533→	for rows.Next() {
   534→		var hostname, nodestate string
   535→		if err := rows.Scan(&hostname, &nodestate); err != nil {
   536→			cclog.Warn("Error while scanning node list (MapNodes)")
   537→			return nil, err
   538→		}
   539→
   540→		stateMap[hostname] = nodestate
   541→	}
   542→
   543→	return stateMap, nil
   544→}
   545→
   546→func (r *NodeRepository) CountStates(ctx context.Context, filters []*model.NodeFilter, column string) ([]*model.NodeStates, error) {
   547→	query, qerr := AccessCheck(ctx,
   548→		sq.Select(column).
   549→			From("node").
   550→			Join("node_state ON node_state.node_id = node.id").
   551→			Where(latestStateCondition()))
   552→	if qerr != nil {
   553→		return nil, qerr
   554→	}
   555→
   556→	query = applyNodeFilters(query, filters)
   557→
   558→	rows, err := query.RunWith(r.stmtCache).Query()
   559→	if err != nil {
   560→		queryString, queryVars, _ := query.ToSql()
   561→		cclog.Errorf("Error while running query '%s' %v: %v", queryString, queryVars, err)
   562→		return nil, err
   563→	}
   564→
   565→	stateMap := map[string]int{}
   566→	for rows.Next() {
   567→		var state string
   568→		if err := rows.Scan(&state); err != nil {
   569→			rows.Close()
   570→			cclog.Warn("Error while scanning rows (CountStates)")
   571→			return nil, err
   572→		}
   573→
   574→		stateMap[state] += 1
   575→	}
   576→
   577→	nodes := make([]*model.NodeStates, 0)
   578→	for state, counts := range stateMap {
   579→		node := model.NodeStates{State: state, Count: counts}
   580→		nodes = append(nodes, &node)
   581→	}
   582→
   583→	return nodes, nil
   584→}
   585→
   586→func (r *NodeRepository) CountStatesTimed(ctx context.Context, filters []*model.NodeFilter, column string) ([]*model.NodeStatesTimed, error) {
   587→	query, qerr := AccessCheck(ctx, sq.Select(column, "time_stamp", "count(*) as count").From("node")) // "cluster"?
   588→	if qerr != nil {
   589→		return nil, qerr
   590→	}
   591→
   592→	query = query.Join("node_state ON node_state.node_id = node.id")
   593→
   594→	for _, f := range filters {
   595→		// Required
   596→		if f.TimeStart != nil {
   597→			query = query.Where("time_stamp > ?", f.TimeStart)
   598→		}
   599→		// Optional
   600→		if f.Hostname != nil {
   601→			query = buildStringCondition("hostname", f.Hostname, query)
   602→		}
   603→		if f.Cluster != nil {
   604→			query = buildStringCondition("cluster", f.Cluster, query)
   605→		}
   606→		if f.SubCluster != nil {
   607→			query = buildStringCondition("subcluster", f.SubCluster, query)
   608→		}
   609→		if f.SchedulerState != nil {
   610→			query = query.Where("node_state = ?", f.SchedulerState)
   611→		}
   612→		if f.HealthState != nil {
   613→			query = query.Where("health_state = ?", f.HealthState)
   614→		}
   615→	}
   616→
   617→	// Add Group and Order
   618→	query = query.GroupBy(column + ", time_stamp").OrderBy("time_stamp ASC")
   619→
   620→	rows, err := query.RunWith(r.stmtCache).Query()
   621→	if err != nil {
   622→		queryString, queryVars, _ := query.ToSql()
   623→		cclog.Errorf("Error while running query '%s' %v: %v", queryString, queryVars, err)
   624→		return nil, err
   625→	}
   626→
   627→	rawData := make(map[string][][]int)
   628→	for rows.Next() {
   629→		var state string
   630→		var timestamp, count int
   631→
   632→		if err := rows.Scan(&state, &timestamp, &count); err != nil {
   633→			rows.Close()
   634→			cclog.Warnf("Error while scanning rows (CountStatesTimed) at time '%d'", timestamp)
   635→			return nil, err
   636→		}
   637→
   638→		if rawData[state] == nil {
   639→			rawData[state] = [][]int{make([]int, 0), make([]int, 0)}
   640→		}
   641→
   642→		rawData[state][0] = append(rawData[state][0], timestamp)
   643→		rawData[state][1] = append(rawData[state][1], count)
   644→	}
   645→
   646→	timedStates := make([]*model.NodeStatesTimed, 0)
   647→	for state, data := range rawData {
   648→		entry := model.NodeStatesTimed{State: state, Times: data[0], Counts: data[1]}
   649→		timedStates = append(timedStates, &entry)
   650→	}
   651→
   652→	return timedStates, nil
   653→}
   654→
   655→func (r *NodeRepository) GetNodesForList(
   656→	ctx context.Context,
   657→	cluster string,
   658→	subCluster string,
   659→	stateFilter string,
   660→	nodeFilter string,
   661→	page *model.PageRequest,
   662→) ([]string, map[string]string, int, bool, error) {
   663→	// Init Return Vars
   664→	nodes := make([]string, 0)
   665→	stateMap := make(map[string]string)
   666→	countNodes := 0
   667→	hasNextPage := false
   668→
   669→	// Build Filters
   670→	queryFilters := make([]*model.NodeFilter, 0)
   671→	if cluster != "" {
   672→		queryFilters = append(queryFilters, &model.NodeFilter{Cluster: &model.StringInput{Eq: &cluster}})
   673→	}
   674→	if subCluster != "" {
   675→		queryFilters = append(queryFilters, &model.NodeFilter{SubCluster: &model.StringInput{Eq: &subCluster}})
   676→	}
   677→	if nodeFilter != "" && stateFilter != "notindb" {
   678→		queryFilters = append(queryFilters, &model.NodeFilter{Hostname: &model.StringInput{Contains: &nodeFilter}})
   679→	}
   680→	if stateFilter != "all" && stateFilter != "notindb" {
   681→		queryState := schema.SchedulerState(stateFilter)
   682→		queryFilters = append(queryFilters, &model.NodeFilter{SchedulerState: &queryState})
   683→	}
   684→	// if healthFilter != "all" {
   685→	// 	filters = append(filters, &model.NodeFilter{HealthState: &healthFilter})
   686→	// }
   687→
   688→	// Special Case: Disable Paging for missing nodes filter, save IPP for later
   689→	var backupItems int
   690→	if stateFilter == "notindb" {
   691→		backupItems = page.ItemsPerPage
   692→		page.ItemsPerPage = -1
   693→	}
   694→
   695→	// Query Nodes From DB
   696→	rawNodes, serr := r.QueryNodes(ctx, queryFilters, page, nil) // Order not Used
   697→	if serr != nil {
   698→		cclog.Warn("error while loading node database data (Resolver.NodeMetricsList)")
   699→		return nil, nil, 0, false, serr
   700→	}
   701→
   702→	// Intermediate Node Result Info
   703→	for _, node := range rawNodes {
   704→		if node == nil {
   705→			continue
   706→		}
   707→		nodes = append(nodes, node.Hostname)
   708→		stateMap[node.Hostname] = string(node.NodeState)
   709→	}
   710→
   711→	// Special Case: Find Nodes not in DB node table but in metricStore only
   712→	if stateFilter == "notindb" {
   713→		// Reapply Original Paging
   714→		page.ItemsPerPage = backupItems
   715→		// Get Nodes From Topology
   716→		var topoNodes []string
   717→		if subCluster != "" {
   718→			scNodes := archive.NodeLists[cluster][subCluster]
   719→			topoNodes = scNodes.PrintList()
   720→		} else {
   721→			subClusterNodeLists := archive.NodeLists[cluster]
   722→			for _, nodeList := range subClusterNodeLists {
   723→				topoNodes = append(topoNodes, nodeList.PrintList()...)
   724→			}
   725→		}
   726→		// Compare to all nodes from cluster/subcluster in DB
   727→		var missingNodes []string
   728→		for _, scanNode := range topoNodes {
   729→			if !slices.Contains(nodes, scanNode) {
   730→				missingNodes = append(missingNodes, scanNode)
   731→			}
   732→		}
   733→		// Filter nodes by name
   734→		if nodeFilter != "" {
   735→			filteredNodesByName := []string{}
   736→			for _, missingNode := range missingNodes {
   737→				if strings.Contains(missingNode, nodeFilter) {
   738→					filteredNodesByName = append(filteredNodesByName, missingNode)
   739→				}
   740→			}
   741→			missingNodes = filteredNodesByName
   742→		}
   743→		// Sort Missing Nodes Alphanumerically
   744→		slices.Sort(missingNodes)
   745→		// Total Missing
   746→		countNodes = len(missingNodes)
   747→		// Apply paging
   748→		if countNodes > page.ItemsPerPage {
   749→			start := (page.Page - 1) * page.ItemsPerPage
   750→			end := start + page.ItemsPerPage
   751→			if end > countNodes {
   752→				end = countNodes
   753→				hasNextPage = false
   754→			} else {
   755→				hasNextPage = true
   756→			}
   757→			nodes = missingNodes[start:end]
   758→		} else {
   759→			nodes = missingNodes
   760→		}
   761→
   762→	} else {
   763→		// DB Nodes: Count and derive hasNextPage from count
   764→		var cerr error
   765→		countNodes, cerr = r.CountNodes(ctx, queryFilters)
   766→		if cerr != nil {
   767→			cclog.Warn("error while counting node database data (Resolver.NodeMetricsList)")
   768→			return nil, nil, 0, false, cerr
   769→		}
   770→		hasNextPage = page.Page*page.ItemsPerPage < countNodes
   771→	}
   772→
   773→	// Fallback for non-init'd node table in DB; Ignores stateFilter
   774→	if stateFilter == "all" && countNodes == 0 {
   775→		nodes, countNodes, hasNextPage = getNodesFromTopol(cluster, subCluster, nodeFilter, page)
   776→	}
   777→
   778→	return nodes, stateMap, countNodes, hasNextPage, nil
   779→}
   780→
   781→func AccessCheck(ctx context.Context, query sq.SelectBuilder) (sq.SelectBuilder, error) {
   782→	user := GetUserFromContext(ctx)
   783→	return AccessCheckWithUser(user, query)
   784→}
   785→
   786→func AccessCheckWithUser(user *schema.User, query sq.SelectBuilder) (sq.SelectBuilder, error) {
   787→	if user == nil {
   788→		var qnil sq.SelectBuilder
   789→		return qnil, fmt.Errorf("user context is nil")
   790→	}
   791→
   792→	switch {
   793→	// case len(user.Roles) == 1 && user.HasRole(schema.RoleApi): // API-User : Access NodeInfos
   794→	// 	return query, nil
   795→	case user.HasAnyRole([]schema.Role{schema.RoleAdmin, schema.RoleSupport}): // Admin & Support : Access NodeInfos
   796→		return query, nil
   797→	default: // No known Role: No Access, return error
   798→		var qnil sq.SelectBuilder
   799→		return qnil, fmt.Errorf("user has no or unknown roles")
   800→	}
   801→}
   802→
   803→func getNodesFromTopol(cluster string, subCluster string, nodeFilter string, page *model.PageRequest) ([]string, int, bool) {
   804→	// 0) Init additional vars
   805→	hasNextPage := false
   806→	totalNodes := 0
   807→
   808→	// 1) Get list of all nodes
   809→	var topolNodes []string
   810→	if subCluster != "" {
   811→		scNodes := archive.NodeLists[cluster][subCluster]
   812→		topolNodes = scNodes.PrintList()
   813→	} else {
   814→		subClusterNodeLists := archive.NodeLists[cluster]
   815→		for _, nodeList := range subClusterNodeLists {
   816→			topolNodes = append(topolNodes, nodeList.PrintList()...)
   817→		}
   818→	}
   819→
   820→	// 2) Filter nodes
   821→	if nodeFilter != "" {
   822→		filteredNodes := []string{}
   823→		for _, node := range topolNodes {
   824→			if strings.Contains(node, nodeFilter) {
   825→				filteredNodes = append(filteredNodes, node)
   826→			}
   827→		}
   828→		topolNodes = filteredNodes
   829→	}
   830→
   831→	// 2.1) Count total nodes && Sort nodes -> Sorting invalidated after ccms return ...
   832→	totalNodes = len(topolNodes)
   833→	sort.Strings(topolNodes)
   834→
   835→	// 3) Apply paging
   836→	if len(topolNodes) > page.ItemsPerPage {
   837→		start := (page.Page - 1) * page.ItemsPerPage
   838→		end := start + page.ItemsPerPage
   839→		if end >= len(topolNodes) {
   840→			end = len(topolNodes)
   841→			hasNextPage = false
   842→		} else {
   843→			hasNextPage = true
   844→		}
   845→		topolNodes = topolNodes[start:end]
   846→	}
   847→
   848→	return topolNodes, totalNodes, hasNextPage
   849→}
   850→

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package repository

import (
	"context"
	"database/sql"
	"encoding/json"
	"fmt"
	"slices"
	"sort"
	"strings"
	"sync"
	"time"

	"github.com/ClusterCockpit/cc-backend/internal/graph/model"
	"github.com/ClusterCockpit/cc-backend/pkg/archive"
	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
	"github.com/ClusterCockpit/cc-lib/v2/lrucache"
	"github.com/ClusterCockpit/cc-lib/v2/schema"
	sq "github.com/Masterminds/squirrel"
	"github.com/jmoiron/sqlx"
)

var (
	nodeRepoOnce     sync.Once
	nodeRepoInstance *NodeRepository
)

type NodeRepository struct {
	DB        *sqlx.DB
	stmtCache *sq.StmtCache
	cache     *lrucache.Cache
	driver    string
}

func GetNodeRepository() *NodeRepository {
	nodeRepoOnce.Do(func() {
		db := GetConnection()

		nodeRepoInstance = &NodeRepository{
			DB:     db.DB,
			driver: db.Driver,

			stmtCache: sq.NewStmtCache(db.DB),
			cache:     lrucache.New(repoConfig.CacheSize),
		}
	})
	return nodeRepoInstance
}

// latestStateCondition returns a squirrel expression that restricts node_state
// rows to the latest per node_id using a correlated subquery.
// Requires the query to join node and node_state tables.
func latestStateCondition() sq.Sqlizer {
	return sq.Expr(
		"node_state.id = (SELECT ns2.id FROM node_state ns2 WHERE ns2.node_id = node.id ORDER BY ns2.time_stamp DESC LIMIT 1)",
	)
}

// applyNodeFilters applies common NodeFilter conditions to a query that joins
// the node and node_state tables with latestStateCondition.
func applyNodeFilters(query sq.SelectBuilder, filters []*model.NodeFilter) sq.SelectBuilder {
	for _, f := range filters {
		if f.Cluster != nil {
			query = buildStringCondition("node.cluster", f.Cluster, query)
		}
		if f.SubCluster != nil {
			query = buildStringCondition("node.subcluster", f.SubCluster, query)
		}
		if f.Hostname != nil {
			query = buildStringCondition("node.hostname", f.Hostname, query)
		}
		if f.SchedulerState != nil {
			query = query.Where("node_state.node_state = ?", f.SchedulerState)
		}
		if f.HealthState != nil {
			query = query.Where("node_state.health_state = ?", f.HealthState)
		}
	}
	return query
}

func (r *NodeRepository) FetchMetadata(hostname string, cluster string) (map[string]string, error) {
	start := time.Now()

	RawMetaData := make([]byte, 0)

	if err := sq.Select("node.meta_data").From("node").
		Where("node.hostname = ?", hostname).
		Where("node.cluster = ?", cluster).
		RunWith(r.stmtCache).QueryRow().Scan(&RawMetaData); err != nil {
		cclog.Warn("Error while scanning for node metadata")
		return nil, err
	}

	if len(RawMetaData) == 0 {
		return nil, nil
	}

	MetaData := make(map[string]string)

	if err := json.Unmarshal(RawMetaData, &MetaData); err != nil {
		cclog.Warn("Error while unmarshaling raw metadata json")
		return nil, err
	}

	cclog.Debugf("Timer FetchMetadata %s", time.Since(start))
	return MetaData, nil
}

func (r *NodeRepository) GetNode(hostname string, cluster string, withMeta bool) (*schema.Node, error) {
	node := &schema.Node{}
	if err := sq.Select("node.hostname", "node.cluster", "node.subcluster",
		"node_state.node_state", "node_state.health_state").
		From("node").
		Join("node_state ON node_state.node_id = node.id").
		Where(latestStateCondition()).
		Where("node.hostname = ?", hostname).
		Where("node.cluster = ?", cluster).
		RunWith(r.DB).
		QueryRow().Scan(&node.Hostname, &node.Cluster, &node.SubCluster, &node.NodeState, &node.HealthState); err != nil {
		cclog.Warnf("Error while querying node '%s' from database: %v", hostname, err)
		return nil, err
	}

	if withMeta {
		var err error
		var meta map[string]string
		if meta, err = r.FetchMetadata(hostname, cluster); err != nil {
			cclog.Warnf("Error while fetching metadata for node '%s'", hostname)
			return nil, err
		}
		node.MetaData = meta
	}

	return node, nil
}

func (r *NodeRepository) GetNodeByID(id int64, withMeta bool) (*schema.Node, error) {
	node := &schema.Node{}
	if err := sq.Select("node.hostname", "node.cluster", "node.subcluster",
		"node_state.node_state", "node_state.health_state").
		From("node").
		Join("node_state ON node_state.node_id = node.id").
		Where(latestStateCondition()).
		Where("node.id = ?", id).
		RunWith(r.DB).
		QueryRow().Scan(&node.Hostname, &node.Cluster, &node.SubCluster, &node.NodeState, &node.HealthState); err != nil {
		cclog.Warnf("Error while querying node ID '%d' from database: %v", id, err)
		return nil, err
	}

	if withMeta {
		meta, metaErr := r.FetchMetadata(node.Hostname, node.Cluster)
		if metaErr != nil {
			cclog.Warnf("Error while fetching metadata for node ID '%d': %v", id, metaErr)
			return nil, metaErr
		}
		node.MetaData = meta
	}

	return node, nil
}

// const NamedNodeInsert string = `
// INSERT INTO node (time_stamp, hostname, cluster, subcluster, node_state, health_state,
//
//	cpus_allocated, cpus_total, memory_allocated, memory_total, gpus_allocated, gpus_total)
//	VALUES (:time_stamp, :hostname, :cluster, :subcluster, :node_state, :health_state,
//	:cpus_allocated, :cpus_total, :memory_allocated, :memory_total, :gpus_allocated, :gpus_total);`

const NamedNodeInsert string = `
INSERT INTO node (hostname, cluster, subcluster)
	VALUES (:hostname, :cluster, :subcluster);`

// AddNode adds a Node to the node table. This can be triggered by a node collector registration or
// from a nodestate update from the job scheduler.
func (r *NodeRepository) AddNode(node *schema.NodeDB) (int64, error) {
	var err error

	res, err := r.DB.NamedExec(NamedNodeInsert, node)
	if err != nil {
		cclog.Errorf("Error while adding node '%v' to database", node.Hostname)
		return 0, err
	}
	node.ID, err = res.LastInsertId()
	if err != nil {
		cclog.Errorf("Error while getting last insert id for node '%v' from database", node.Hostname)
		return 0, err
	}

	return node.ID, nil
}

const NamedNodeStateInsert string = `
INSERT INTO node_state (time_stamp, node_state, health_state, health_metrics,
	cpus_allocated, memory_allocated, gpus_allocated, jobs_running, node_id)
	VALUES (:time_stamp, :node_state, :health_state, :health_metrics,
	:cpus_allocated, :memory_allocated, :gpus_allocated, :jobs_running, :node_id);`

// TODO: Add real Monitoring Health State

// UpdateNodeState is called from the Node REST API to add a row in the node state table
func (r *NodeRepository) UpdateNodeState(hostname string, cluster string, nodeState *schema.NodeStateDB) error {
	var id int64

	if err := sq.Select("id").From("node").
		Where("node.hostname = ?", hostname).Where("node.cluster = ?", cluster).RunWith(r.DB).
		QueryRow().Scan(&id); err != nil {
		if err == sql.ErrNoRows {
			subcluster, err := archive.GetSubClusterByNode(cluster, hostname)
			if err != nil {
				cclog.Errorf("Error while getting subcluster for node '%s' in cluster '%s': %v", hostname, cluster, err)
				return err
			}
			node := schema.NodeDB{
				Hostname: hostname, Cluster: cluster, SubCluster: subcluster,
			}
			id, err = r.AddNode(&node)
			if err != nil {
				cclog.Errorf("Error while adding node '%s' to database: %v", hostname, err)
				return err
			}

			cclog.Debugf("Added node '%s' to database", hostname)
		} else {
			cclog.Warnf("Error while querying node '%v' from database", id)
			return err
		}
	}

	nodeState.NodeID = id

	_, err := r.DB.NamedExec(NamedNodeStateInsert, nodeState)
	if err != nil {
		cclog.Errorf("Error while adding node state for '%v' to database", hostname)
		return err
	}
	cclog.Debugf("Updated node state for '%s' in database", hostname)
	return nil
}

// func (r *NodeRepository) UpdateHealthState(hostname string, healthState *schema.MonitoringState) error {
// 	if _, err := sq.Update("node").Set("health_state", healthState).Where("node.id = ?", id).RunWith(r.DB).Exec(); err != nil {
// 		cclog.Errorf("error while updating node '%d'", id)
// 		return err
// 	}
//
// 	return nil
// }

// NodeStateWithNode combines a node state row with denormalized node info.
type NodeStateWithNode struct {
	ID              int64  `db:"id"`
	TimeStamp       int64  `db:"time_stamp"`
	NodeState       string `db:"node_state"`
	HealthState     string `db:"health_state"`
	HealthMetrics   string `db:"health_metrics"`
	CpusAllocated   int    `db:"cpus_allocated"`
	MemoryAllocated int64  `db:"memory_allocated"`
	GpusAllocated   int    `db:"gpus_allocated"`
	JobsRunning     int    `db:"jobs_running"`
	Hostname        string `db:"hostname"`
	Cluster         string `db:"cluster"`
	SubCluster      string `db:"subcluster"`
}

// FindNodeStatesBefore returns all node_state rows with time_stamp < cutoff,
// joined with node info for denormalized archiving.
func (r *NodeRepository) FindNodeStatesBefore(cutoff int64) ([]NodeStateWithNode, error) {
	rows, err := sq.Select(
		"node_state.id", "node_state.time_stamp", "node_state.node_state",
		"node_state.health_state", "node_state.health_metrics",
		"node_state.cpus_allocated", "node_state.memory_allocated",
		"node_state.gpus_allocated", "node_state.jobs_running",
		"node.hostname", "node.cluster", "node.subcluster",
	).
		From("node_state").
		Join("node ON node_state.node_id = node.id").
		Where(sq.Lt{"node_state.time_stamp": cutoff}).
		Where("node_state.id NOT IN (SELECT ns2.id FROM node_state ns2 WHERE ns2.time_stamp = (SELECT MAX(ns3.time_stamp) FROM node_state ns3 WHERE ns3.node_id = ns2.node_id))").
		OrderBy("node.cluster ASC", "node.subcluster ASC", "node.hostname ASC", "node_state.time_stamp ASC").
		RunWith(r.DB).Query()
	if err != nil {
		return nil, err
	}
	defer rows.Close()

	var result []NodeStateWithNode
	for rows.Next() {
		var ns NodeStateWithNode
		var healthMetrics sql.NullString
		if err := rows.Scan(&ns.ID, &ns.TimeStamp, &ns.NodeState,
			&ns.HealthState, &healthMetrics,
			&ns.CpusAllocated, &ns.MemoryAllocated,
			&ns.GpusAllocated, &ns.JobsRunning,
			&ns.Hostname, &ns.Cluster, &ns.SubCluster); err != nil {
			return nil, err
		}
		ns.HealthMetrics = healthMetrics.String
		result = append(result, ns)
	}
	return result, nil
}

// DeleteNodeStatesBefore removes node_state rows with time_stamp < cutoff,
// but always preserves the row with the latest timestamp per node_id.
func (r *NodeRepository) DeleteNodeStatesBefore(cutoff int64) (int64, error) {
	res, err := r.DB.Exec(
		`DELETE FROM node_state WHERE time_stamp < ?
		 AND id NOT IN (
		   SELECT id FROM node_state ns2
		   WHERE ns2.time_stamp = (SELECT MAX(ns3.time_stamp) FROM node_state ns3 WHERE ns3.node_id = ns2.node_id)
		 )`,
		cutoff,
	)
	if err != nil {
		return 0, err
	}
	return res.RowsAffected()
}

func (r *NodeRepository) DeleteNode(id int64) error {
	_, err := r.DB.Exec(`DELETE FROM node WHERE node.id = ?`, id)
	if err != nil {
		cclog.Errorf("Error while deleting node '%d' from DB", id)
		return err
	}
	cclog.Infof("deleted node '%d' from DB", id)
	return nil
}

// QueryNodes returns a list of nodes based on a node filter. It always operates
// on the last state (largest timestamp).
func (r *NodeRepository) QueryNodes(
	ctx context.Context,
	filters []*model.NodeFilter,
	page *model.PageRequest,
	order *model.OrderByInput, // Currently unused!
) ([]*schema.Node, error) {
	query, qerr := AccessCheck(ctx,
		sq.Select("node.hostname", "node.cluster", "node.subcluster",
			"node_state.node_state", "node_state.health_state").
			From("node").
			Join("node_state ON node_state.node_id = node.id").
			Where(latestStateCondition()))
	if qerr != nil {
		return nil, qerr
	}

	query = applyNodeFilters(query, filters)
	query = query.OrderBy("node.hostname ASC")

	if page != nil && page.ItemsPerPage != -1 {
		limit := uint64(page.ItemsPerPage)
		query = query.Offset((uint64(page.Page) - 1) * limit).Limit(limit)
	}

	rows, err := query.RunWith(r.stmtCache).Query()
	if err != nil {
		queryString, queryVars, _ := query.ToSql()
		cclog.Errorf("Error while running query '%s' %v: %v", queryString, queryVars, err)
		return nil, err
	}

	nodes := make([]*schema.Node, 0)
	for rows.Next() {
		node := schema.Node{}
		if err := rows.Scan(&node.Hostname, &node.Cluster, &node.SubCluster,
			&node.NodeState, &node.HealthState); err != nil {
			rows.Close()
			cclog.Warn("Error while scanning rows (QueryNodes)")
			return nil, err
		}
		nodes = append(nodes, &node)
	}

	return nodes, nil
}

// QueryNodesWithMeta returns a list of nodes based on a node filter. It always operates
// on the last state (largest timestamp). It includes both (!) optional JSON column data
func (r *NodeRepository) QueryNodesWithMeta(
	ctx context.Context,
	filters []*model.NodeFilter,
	page *model.PageRequest,
	order *model.OrderByInput, // Currently unused!
) ([]*schema.Node, error) {
	query, qerr := AccessCheck(ctx,
		sq.Select("node.hostname", "node.cluster", "node.subcluster",
			"node_state.node_state", "node_state.health_state",
			"node.meta_data", "node_state.health_metrics").
			From("node").
			Join("node_state ON node_state.node_id = node.id").
			Where(latestStateCondition()))
	if qerr != nil {
		return nil, qerr
	}

	query = applyNodeFilters(query, filters)
	query = query.OrderBy("node.hostname ASC")

	if page != nil && page.ItemsPerPage != -1 {
		limit := uint64(page.ItemsPerPage)
		query = query.Offset((uint64(page.Page) - 1) * limit).Limit(limit)
	}

	rows, err := query.RunWith(r.stmtCache).Query()
	if err != nil {
		queryString, queryVars, _ := query.ToSql()
		cclog.Errorf("Error while running query '%s' %v: %v", queryString, queryVars, err)
		return nil, err
	}

	nodes := make([]*schema.Node, 0)
	for rows.Next() {
		node := schema.Node{}
		RawMetaData := make([]byte, 0)
		RawMetricHealth := make([]byte, 0)

		if err := rows.Scan(&node.Hostname, &node.Cluster, &node.SubCluster,
			&node.NodeState, &node.HealthState, &RawMetaData, &RawMetricHealth); err != nil {
			rows.Close()
			cclog.Warn("Error while scanning rows (QueryNodes)")
			return nil, err
		}

		if len(RawMetaData) == 0 {
			node.MetaData = nil
		} else {
			metaData := make(map[string]string)
			if err := json.Unmarshal(RawMetaData, &metaData); err != nil {
				cclog.Warn("Error while unmarshaling raw metadata json")
				return nil, err
			}
			node.MetaData = metaData
		}

		if len(RawMetricHealth) == 0 {
			node.HealthData = nil
		} else {
			healthData := make(map[string][]string)
			if err := json.Unmarshal(RawMetricHealth, &healthData); err != nil {
				cclog.Warn("Error while unmarshaling raw healthdata json")
				return nil, err
			}
			node.HealthData = healthData
		}

		nodes = append(nodes, &node)
	}

	return nodes, nil
}

// CountNodes returns the total matched nodes based on a node filter. It always operates
// on the last state (largest timestamp) per node.
func (r *NodeRepository) CountNodes(
	ctx context.Context,
	filters []*model.NodeFilter,
) (int, error) {
	query, qerr := AccessCheck(ctx,
		sq.Select("COUNT(*)").
			From("node").
			Join("node_state ON node_state.node_id = node.id").
			Where(latestStateCondition()))
	if qerr != nil {
		return 0, qerr
	}

	query = applyNodeFilters(query, filters)

	var count int
	if err := query.RunWith(r.stmtCache).QueryRow().Scan(&count); err != nil {
		queryString, queryVars, _ := query.ToSql()
		cclog.Errorf("Error while running query '%s' %v: %v", queryString, queryVars, err)
		return 0, err
	}

	return count, nil
}

func (r *NodeRepository) ListNodes(cluster string) ([]*schema.Node, error) {
	q := sq.Select("node.hostname", "node.cluster", "node.subcluster",
		"node_state.node_state", "node_state.health_state").
		From("node").
		Join("node_state ON node_state.node_id = node.id").
		Where(latestStateCondition()).
		Where("node.cluster = ?", cluster).
		OrderBy("node.hostname ASC")

	rows, err := q.RunWith(r.DB).Query()
	if err != nil {
		cclog.Warn("Error while querying node list")
		return nil, err
	}
	nodeList := make([]*schema.Node, 0, 100)
	defer rows.Close()
	for rows.Next() {
		node := &schema.Node{}
		if err := rows.Scan(&node.Hostname, &node.Cluster,
			&node.SubCluster, &node.NodeState, &node.HealthState); err != nil {
			cclog.Warn("Error while scanning node list (ListNodes)")
			return nil, err
		}

		nodeList = append(nodeList, node)
	}

	return nodeList, nil
}

func (r *NodeRepository) MapNodes(cluster string) (map[string]string, error) {
	q := sq.Select("node.hostname", "node_state.node_state").
		From("node").
		Join("node_state ON node_state.node_id = node.id").
		Where(latestStateCondition()).
		Where("node.cluster = ?", cluster).
		OrderBy("node.hostname ASC")

	rows, err := q.RunWith(r.DB).Query()
	if err != nil {
		cclog.Warn("Error while querying node list")
		return nil, err
	}

	stateMap := make(map[string]string)
	defer rows.Close()
	for rows.Next() {
		var hostname, nodestate string
		if err := rows.Scan(&hostname, &nodestate); err != nil {
			cclog.Warn("Error while scanning node list (MapNodes)")
			return nil, err
		}

		stateMap[hostname] = nodestate
	}

	return stateMap, nil
}

func (r *NodeRepository) CountStates(ctx context.Context, filters []*model.NodeFilter, column string) ([]*model.NodeStates, error) {
	query, qerr := AccessCheck(ctx,
		sq.Select(column).
			From("node").
			Join("node_state ON node_state.node_id = node.id").
			Where(latestStateCondition()))
	if qerr != nil {
		return nil, qerr
	}

	query = applyNodeFilters(query, filters)

	rows, err := query.RunWith(r.stmtCache).Query()
	if err != nil {
		queryString, queryVars, _ := query.ToSql()
		cclog.Errorf("Error while running query '%s' %v: %v", queryString, queryVars, err)
		return nil, err
	}

	stateMap := map[string]int{}
	for rows.Next() {
		var state string
		if err := rows.Scan(&state); err != nil {
			rows.Close()
			cclog.Warn("Error while scanning rows (CountStates)")
			return nil, err
		}

		stateMap[state] += 1
	}

	nodes := make([]*model.NodeStates, 0)
	for state, counts := range stateMap {
		node := model.NodeStates{State: state, Count: counts}
		nodes = append(nodes, &node)
	}

	return nodes, nil
}

func (r *NodeRepository) CountStatesTimed(ctx context.Context, filters []*model.NodeFilter, column string) ([]*model.NodeStatesTimed, error) {
	query, qerr := AccessCheck(ctx, sq.Select(column, "time_stamp", "count(*) as count").From("node")) // "cluster"?
	if qerr != nil {
		return nil, qerr
	}

	query = query.Join("node_state ON node_state.node_id = node.id")

	for _, f := range filters {
		// Required
		if f.TimeStart != nil {
			query = query.Where("time_stamp > ?", f.TimeStart)
		}
		// Optional
		if f.Hostname != nil {
			query = buildStringCondition("hostname", f.Hostname, query)
		}
		if f.Cluster != nil {
			query = buildStringCondition("cluster", f.Cluster, query)
		}
		if f.SubCluster != nil {
			query = buildStringCondition("subcluster", f.SubCluster, query)
		}
		if f.SchedulerState != nil {
			query = query.Where("node_state = ?", f.SchedulerState)
		}
		if f.HealthState != nil {
			query = query.Where("health_state = ?", f.HealthState)
		}
	}

	// Add Group and Order
	query = query.GroupBy(column + ", time_stamp").OrderBy("time_stamp ASC")

	rows, err := query.RunWith(r.stmtCache).Query()
	if err != nil {
		queryString, queryVars, _ := query.ToSql()
		cclog.Errorf("Error while running query '%s' %v: %v", queryString, queryVars, err)
		return nil, err
	}

	rawData := make(map[string][][]int)
	for rows.Next() {
		var state string
		var timestamp, count int

		if err := rows.Scan(&state, &timestamp, &count); err != nil {
			rows.Close()
			cclog.Warnf("Error while scanning rows (CountStatesTimed) at time '%d'", timestamp)
			return nil, err
		}

		if rawData[state] == nil {
			rawData[state] = [][]int{make([]int, 0), make([]int, 0)}
		}

		rawData[state][0] = append(rawData[state][0], timestamp)
		rawData[state][1] = append(rawData[state][1], count)
	}

	timedStates := make([]*model.NodeStatesTimed, 0)
	for state, data := range rawData {
		entry := model.NodeStatesTimed{State: state, Times: data[0], Counts: data[1]}
		timedStates = append(timedStates, &entry)
	}

	return timedStates, nil
}

func (r *NodeRepository) GetNodesForList(
	ctx context.Context,
	cluster string,
	subCluster string,
	stateFilter string,
	nodeFilter string,
	page *model.PageRequest,
) ([]string, map[string]string, int, bool, error) {
	// Init Return Vars
	nodes := make([]string, 0)
	stateMap := make(map[string]string)
	countNodes := 0
	hasNextPage := false

	// Build Filters
	queryFilters := make([]*model.NodeFilter, 0)
	if cluster != "" {
		queryFilters = append(queryFilters, &model.NodeFilter{Cluster: &model.StringInput{Eq: &cluster}})
	}
	if subCluster != "" {
		queryFilters = append(queryFilters, &model.NodeFilter{SubCluster: &model.StringInput{Eq: &subCluster}})
	}
	if nodeFilter != "" && stateFilter != "notindb" {
		queryFilters = append(queryFilters, &model.NodeFilter{Hostname: &model.StringInput{Contains: &nodeFilter}})
	}
	if stateFilter != "all" && stateFilter != "notindb" {
		queryState := schema.SchedulerState(stateFilter)
		queryFilters = append(queryFilters, &model.NodeFilter{SchedulerState: &queryState})
	}
	// if healthFilter != "all" {
	// 	filters = append(filters, &model.NodeFilter{HealthState: &healthFilter})
	// }

	// Special Case: Disable Paging for missing nodes filter, save IPP for later
	var backupItems int
	if stateFilter == "notindb" {
		backupItems = page.ItemsPerPage
		page.ItemsPerPage = -1
	}

	// Query Nodes From DB
	rawNodes, serr := r.QueryNodes(ctx, queryFilters, page, nil) // Order not Used
	if serr != nil {
		cclog.Warn("error while loading node database data (Resolver.NodeMetricsList)")
		return nil, nil, 0, false, serr
	}

	// Intermediate Node Result Info
	for _, node := range rawNodes {
		if node == nil {
			continue
		}
		nodes = append(nodes, node.Hostname)
		stateMap[node.Hostname] = string(node.NodeState)
	}

	// Special Case: Find Nodes not in DB node table but in metricStore only
	if stateFilter == "notindb" {
		// Reapply Original Paging
		page.ItemsPerPage = backupItems
		// Get Nodes From Topology
		var topoNodes []string
		if subCluster != "" {
			scNodes := archive.NodeLists[cluster][subCluster]
			topoNodes = scNodes.PrintList()
		} else {
			subClusterNodeLists := archive.NodeLists[cluster]
			for _, nodeList := range subClusterNodeLists {
				topoNodes = append(topoNodes, nodeList.PrintList()...)
			}
		}
		// Compare to all nodes from cluster/subcluster in DB
		var missingNodes []string
		for _, scanNode := range topoNodes {
			if !slices.Contains(nodes, scanNode) {
				missingNodes = append(missingNodes, scanNode)
			}
		}
		// Filter nodes by name
		if nodeFilter != "" {
			filteredNodesByName := []string{}
			for _, missingNode := range missingNodes {
				if strings.Contains(missingNode, nodeFilter) {
					filteredNodesByName = append(filteredNodesByName, missingNode)
				}
			}
			missingNodes = filteredNodesByName
		}
		// Sort Missing Nodes Alphanumerically
		slices.Sort(missingNodes)
		// Total Missing
		countNodes = len(missingNodes)
		// Apply paging
		if countNodes > page.ItemsPerPage {
			start := (page.Page - 1) * page.ItemsPerPage
			end := start + page.ItemsPerPage
			if end > countNodes {
				end = countNodes
				hasNextPage = false
			} else {
				hasNextPage = true
			}
			nodes = missingNodes[start:end]
		} else {
			nodes = missingNodes
		}

	} else {
		// DB Nodes: Count and derive hasNextPage from count
		var cerr error
		countNodes, cerr = r.CountNodes(ctx, queryFilters)
		if cerr != nil {
			cclog.Warn("error while counting node database data (Resolver.NodeMetricsList)")
			return nil, nil, 0, false, cerr
		}
		hasNextPage = page.Page*page.ItemsPerPage < countNodes
	}

	// Fallback for non-init'd node table in DB; Ignores stateFilter
	if stateFilter == "all" && countNodes == 0 {
		nodes, countNodes, hasNextPage = getNodesFromTopol(cluster, subCluster, nodeFilter, page)
	}

	return nodes, stateMap, countNodes, hasNextPage, nil
}

func AccessCheck(ctx context.Context, query sq.SelectBuilder) (sq.SelectBuilder, error) {
	user := GetUserFromContext(ctx)
	return AccessCheckWithUser(user, query)
}

func AccessCheckWithUser(user *schema.User, query sq.SelectBuilder) (sq.SelectBuilder, error) {
	if user == nil {
		var qnil sq.SelectBuilder
		return qnil, fmt.Errorf("user context is nil")
	}

	switch {
	// case len(user.Roles) == 1 && user.HasRole(schema.RoleApi): // API-User : Access NodeInfos
	// 	return query, nil
	case user.HasAnyRole([]schema.Role{schema.RoleAdmin, schema.RoleSupport}): // Admin & Support : Access NodeInfos
		return query, nil
	default: // No known Role: No Access, return error
		var qnil sq.SelectBuilder
		return qnil, fmt.Errorf("user has no or unknown roles")
	}
}

func getNodesFromTopol(cluster string, subCluster string, nodeFilter string, page *model.PageRequest) ([]string, int, bool) {
	// 0) Init additional vars
	hasNextPage := false
	totalNodes := 0

	// 1) Get list of all nodes
	var topolNodes []string
	if subCluster != "" {
		scNodes := archive.NodeLists[cluster][subCluster]
		topolNodes = scNodes.PrintList()
	} else {
		subClusterNodeLists := archive.NodeLists[cluster]
		for _, nodeList := range subClusterNodeLists {
			topolNodes = append(topolNodes, nodeList.PrintList()...)
		}
	}

	// 2) Filter nodes
	if nodeFilter != "" {
		filteredNodes := []string{}
		for _, node := range topolNodes {
			if strings.Contains(node, nodeFilter) {
				filteredNodes = append(filteredNodes, node)
			}
		}
		topolNodes = filteredNodes
	}

	// 2.1) Count total nodes && Sort nodes -> Sorting invalidated after ccms return ...
	totalNodes = len(topolNodes)
	sort.Strings(topolNodes)

	// 3) Apply paging
	if len(topolNodes) > page.ItemsPerPage {
		start := (page.Page - 1) * page.ItemsPerPage
		end := start + page.ItemsPerPage
		if end >= len(topolNodes) {
			end = len(topolNodes)
			hasNextPage = false
		} else {
			hasNextPage = true
		}
		topolNodes = topolNodes[start:end]
	}

	return topolNodes, totalNodes, hasNextPage
}


Let me also check the remaining query functions in the repository.


<persisted-output>
Output too large (20.6KB). Full output saved to: /Users/jan/.claude/projects/-Users-jan-prg-CC-cc-backend/28147033-ddc8-4056-b064-e0558fbc614e/tool-results/toolu_01RcRyYrnuwPStQV97fpLy9v.txt

Preview (first 2KB):
/Users/jan/prg/CC/cc-backend/internal/repository/node.go-285-		Where("node_state.id NOT IN (SELECT ns2.id FROM node_state ns2 WHERE ns2.time_stamp = (SELECT MAX(ns3.time_stamp) FROM node_state ns3 WHERE ns3.node_id = ns2.node_id))").
/Users/jan/prg/CC/cc-backend/internal/repository/node.go-286-		OrderBy("node.cluster ASC", "node.subcluster ASC", "node.hostname ASC", "node_state.time_stamp ASC").
internal/repository/node.go:287:		RunWith(r.DB).Query()
/Users/jan/prg/CC/cc-backend/internal/repository/node.go-288-	if err != nil {
/Users/jan/prg/CC/cc-backend/internal/repository/node.go-289-		return nil, err
/Users/jan/prg/CC/cc-backend/internal/repository/node.go-290-	}
/Users/jan/prg/CC/cc-backend/internal/repository/node.go-291-	defer rows.Close()
/Users/jan/prg/CC/cc-backend/internal/repository/node.go-292-
--
/Users/jan/prg/CC/cc-backend/internal/repository/node.go-361-	}
/Users/jan/prg/CC/cc-backend/internal/repository/node.go-362-
internal/repository/node.go:363:	rows, err := query.RunWith(r.stmtCache).Query()
/Users/jan/prg/CC/cc-backend/internal/repository/node.go-364-	if err != nil {
internal/repository/node.go-365-		queryString, queryVars, _ := query.ToSql()
internal/repository/node.go-366-		cclog.Errorf("Error while running query '%s' %v: %v", queryString, queryVars, err)
/Users/jan/prg/CC/cc-backend/internal/repository/node.go-367-		return nil, err
/Users/jan/prg/CC/cc-backend/internal/repository/node.go-368-	}
--
/Users/jan/prg/CC/cc-backend/internal/repository/node.go-410-	}
/Users/jan/prg/CC/cc-backend/internal/repository/node.go-411-
internal/repository/node.go:412:	rows, err := query.RunWith(r.stmtCache).Query()
/Users/jan/prg/CC/cc-backend/internal/repository/node.go-413-	if err != nil {
internal/repository/node.go-414-		queryString, queryVars, _ := query.ToSql()
internal/repository/node.go-415-		cclog.Errorf("Error while running query '%s' %v: %v", queryString, queryVars, err)
...
</persisted-output>

/Users/jan/prg/CC/cc-backend/internal/repository/node.go-285-		Where("node_state.id NOT IN (SELECT ns2.id FROM node_state ns2 WHERE ns2.time_stamp = (SELECT MAX(ns3.time_stamp) FROM node_state ns3 WHERE ns3.node_id = ns2.node_id))").
/Users/jan/prg/CC/cc-backend/internal/repository/node.go-286-		OrderBy("node.cluster ASC", "node.subcluster ASC", "node.hostname ASC", "node_state.time_stamp ASC").
internal/repository/node.go:287:		RunWith(r.DB).Query()
/Users/jan/prg/CC/cc-backend/internal/repository/node.go-288-	if err != nil {
/Users/jan/prg/CC/cc-backend/internal/repository/node.go-289-		return nil, err
/Users/jan/prg/CC/cc-backend/internal/repository/node.go-290-	}
/Users/jan/prg/CC/cc-backend/internal/repository/node.go-291-	defer rows.Close()
/Users/jan/prg/CC/cc-backend/internal/repository/node.go-292-
--
/Users/jan/prg/CC/cc-backend/internal/repository/node.go-361-	}
/Users/jan/prg/CC/cc-backend/internal/repository/node.go-362-
internal/repository/node.go:363:	rows, err := query.RunWith(r.stmtCache).Query()
/Users/jan/prg/CC/cc-backend/internal/repository/node.go-364-	if err != nil {
internal/repository/node.go-365-		queryString, queryVars, _ := query.ToSql()
internal/repository/node.go-366-		cclog.Errorf("Error while running query '%s' %v: %v", queryString, queryVars, err)
/Users/jan/prg/CC/cc-backend/internal/repository/node.go-367-		return nil, err
/Users/jan/prg/CC/cc-backend/internal/repository/node.go-368-	}
--
/Users/jan/prg/CC/cc-backend/internal/repository/node.go-410-	}
/Users/jan/prg/CC/cc-backend/internal/repository/node.go-411-
internal/repository/node.go:412:	rows, err := query.RunWith(r.stmtCache).Query()
/Users/jan/prg/CC/cc-backend/internal/repository/node.go-413-	if err != nil {
internal/repository/node.go-414-		queryString, queryVars, _ := query.ToSql()
internal/repository/node.go-415-		cclog.Errorf("Error while running query '%s' %v: %v", queryString, queryVars, err)
/Users/jan/prg/CC/cc-backend/internal/repository/node.go-416-		return nil, err
/Users/jan/prg/CC/cc-backend/internal/repository/node.go-417-	}
--
/Users/jan/prg/CC/cc-backend/internal/repository/node.go-494-		OrderBy("node.hostname ASC")
/Users/jan/prg/CC/cc-backend/internal/repository/node.go-495-
internal/repository/node.go:496:	rows, err := q.RunWith(r.DB).Query()
/Users/jan/prg/CC/cc-backend/internal/repository/node.go-497-	if err != nil {
/Users/jan/prg/CC/cc-backend/internal/repository/node.go-498-		cclog.Warn("Error while querying node list")
/Users/jan/prg/CC/cc-backend/internal/repository/node.go-499-		return nil, err
/Users/jan/prg/CC/cc-backend/internal/repository/node.go-500-	}
internal/repository/node.go-501-	nodeList := make([]*schema.Node, 0, 100)
--
/Users/jan/prg/CC/cc-backend/internal/repository/node.go-523-		OrderBy("node.hostname ASC")
/Users/jan/prg/CC/cc-backend/internal/repository/node.go-524-
internal/repository/node.go:525:	rows, err := q.RunWith(r.DB).Query()
/Users/jan/prg/CC/cc-backend/internal/repository/node.go-526-	if err != nil {
/Users/jan/prg/CC/cc-backend/internal/repository/node.go-527-		cclog.Warn("Error while querying node list")
/Users/jan/prg/CC/cc-backend/internal/repository/node.go-528-		return nil, err
/Users/jan/prg/CC/cc-backend/internal/repository/node.go-529-	}
/Users/jan/prg/CC/cc-backend/internal/repository/node.go-530-
--
/Users/jan/prg/CC/cc-backend/internal/repository/node.go-556-	query = applyNodeFilters(query, filters)
/Users/jan/prg/CC/cc-backend/internal/repository/node.go-557-
internal/repository/node.go:558:	rows, err := query.RunWith(r.stmtCache).Query()
/Users/jan/prg/CC/cc-backend/internal/repository/node.go-559-	if err != nil {
internal/repository/node.go-560-		queryString, queryVars, _ := query.ToSql()
internal/repository/node.go-561-		cclog.Errorf("Error while running query '%s' %v: %v", queryString, queryVars, err)
/Users/jan/prg/CC/cc-backend/internal/repository/node.go-562-		return nil, err
/Users/jan/prg/CC/cc-backend/internal/repository/node.go-563-	}
--
/Users/jan/prg/CC/cc-backend/internal/repository/node.go-618-	query = query.GroupBy(column + ", time_stamp").OrderBy("time_stamp ASC")
/Users/jan/prg/CC/cc-backend/internal/repository/node.go-619-
internal/repository/node.go:620:	rows, err := query.RunWith(r.stmtCache).Query()
/Users/jan/prg/CC/cc-backend/internal/repository/node.go-621-	if err != nil {
internal/repository/node.go-622-		queryString, queryVars, _ := query.ToSql()
internal/repository/node.go-623-		cclog.Errorf("Error while running query '%s' %v: %v", queryString, queryVars, err)
/Users/jan/prg/CC/cc-backend/internal/repository/node.go-624-		return nil, err
/Users/jan/prg/CC/cc-backend/internal/repository/node.go-625-	}
--
internal/repository/jobCreate.go-76-	query := sq.Select(jobCacheColumns...).From("job_cache")
/Users/jan/prg/CC/cc-backend/internal/repository/jobCreate.go-77-
internal/repository/jobCreate.go:78:	rows, err := query.RunWith(r.stmtCache).Query()
/Users/jan/prg/CC/cc-backend/internal/repository/jobCreate.go-79-	if err != nil {
/Users/jan/prg/CC/cc-backend/internal/repository/jobCreate.go-80-		cclog.Errorf("Error while running query %v", err)
/Users/jan/prg/CC/cc-backend/internal/repository/jobCreate.go-81-		return nil, err
/Users/jan/prg/CC/cc-backend/internal/repository/jobCreate.go-82-	}
/Users/jan/prg/CC/cc-backend/internal/repository/jobCreate.go-83-	defer rows.Close()
--
/Users/jan/prg/CC/cc-backend/internal/repository/tags.go-331-	}
/Users/jan/prg/CC/cc-backend/internal/repository/tags.go-332-
internal/repository/tags.go:333:	rows, err := q.RunWith(r.stmtCache).Query()
/Users/jan/prg/CC/cc-backend/internal/repository/tags.go-334-	if err != nil {
/Users/jan/prg/CC/cc-backend/internal/repository/tags.go-335-		return nil, nil, err
/Users/jan/prg/CC/cc-backend/internal/repository/tags.go-336-	}
/Users/jan/prg/CC/cc-backend/internal/repository/tags.go-337-
/Users/jan/prg/CC/cc-backend/internal/repository/tags.go-338-	counts = make(map[string]int)
--
/Users/jan/prg/CC/cc-backend/internal/repository/tags.go-506-	}
/Users/jan/prg/CC/cc-backend/internal/repository/tags.go-507-
internal/repository/tags.go:508:	rows, err := q.RunWith(r.stmtCache).Query()
/Users/jan/prg/CC/cc-backend/internal/repository/tags.go-509-	if err != nil {
internal/repository/tags.go-510-		s, _, _ := q.ToSql()
internal/repository/tags.go-511-		cclog.Errorf("Error get tags with %s: %v", s, err)
/Users/jan/prg/CC/cc-backend/internal/repository/tags.go-512-		return nil, err
/Users/jan/prg/CC/cc-backend/internal/repository/tags.go-513-	}
--
/Users/jan/prg/CC/cc-backend/internal/repository/tags.go-539-	}
/Users/jan/prg/CC/cc-backend/internal/repository/tags.go-540-
internal/repository/tags.go:541:	rows, err := q.RunWith(r.stmtCache).Query()
/Users/jan/prg/CC/cc-backend/internal/repository/tags.go-542-	if err != nil {
internal/repository/tags.go-543-		s, _, _ := q.ToSql()
internal/repository/tags.go-544-		cclog.Errorf("Error get tags with %s: %v", s, err)
/Users/jan/prg/CC/cc-backend/internal/repository/tags.go-545-		return nil, err
/Users/jan/prg/CC/cc-backend/internal/repository/tags.go-546-	}
--
/Users/jan/prg/CC/cc-backend/internal/repository/tags.go-577-	}
/Users/jan/prg/CC/cc-backend/internal/repository/tags.go-578-
internal/repository/tags.go:579:	rows, err := q.RunWith(r.stmtCache).Query()
/Users/jan/prg/CC/cc-backend/internal/repository/tags.go-580-	if err != nil {
internal/repository/tags.go-581-		s, _, _ := q.ToSql()
internal/repository/tags.go-582-		cclog.Errorf("Error get tags with %s: %v", s, err)
/Users/jan/prg/CC/cc-backend/internal/repository/tags.go-583-		return nil, err
/Users/jan/prg/CC/cc-backend/internal/repository/tags.go-584-	}
--
/Users/jan/prg/CC/cc-backend/internal/repository/jobFind.go-104-	}
/Users/jan/prg/CC/cc-backend/internal/repository/jobFind.go-105-
internal/repository/jobFind.go:106:	rows, err := q.RunWith(r.stmtCache).Query()
/Users/jan/prg/CC/cc-backend/internal/repository/jobFind.go-107-	if err != nil {
internal/repository/jobFind.go-108-		cclog.Errorf("Error while running FindAll query for jobID=%d: %v", *jobID, err)
internal/repository/jobFind.go-109-		return nil, fmt.Errorf("failed to execute FindAll query: %w", err)
/Users/jan/prg/CC/cc-backend/internal/repository/jobFind.go-110-	}
/Users/jan/prg/CC/cc-backend/internal/repository/jobFind.go-111-	defer rows.Close()
--
/Users/jan/prg/CC/cc-backend/internal/repository/jobFind.go-137-	}
/Users/jan/prg/CC/cc-backend/internal/repository/jobFind.go-138-
internal/repository/jobFind.go:139:	rows, err := query.RunWith(r.stmtCache).Query()
/Users/jan/prg/CC/cc-backend/internal/repository/jobFind.go-140-	if err != nil {
internal/repository/jobFind.go-141-		cclog.Errorf("Error while running GetJobList query (limit=%d, offset=%d): %v", limit, offset, err)
internal/repository/jobFind.go-142-		return nil, fmt.Errorf("failed to execute GetJobList query: %w", err)
/Users/jan/prg/CC/cc-backend/internal/repository/jobFind.go-143-	}
/Users/jan/prg/CC/cc-backend/internal/repository/jobFind.go-144-	defer rows.Close()
--
/Users/jan/prg/CC/cc-backend/internal/repository/jobFind.go-289-	query = query.Where("EXISTS (SELECT 1 FROM json_each(job.resources) WHERE json_extract(value, '$.hostname') = ?)", hostname)
/Users/jan/prg/CC/cc-backend/internal/repository/jobFind.go-290-
internal/repository/jobFind.go:291:	rows, err := query.RunWith(r.stmtCache).Query()
/Users/jan/prg/CC/cc-backend/internal/repository/jobFind.go-292-	if err != nil {
internal/repository/jobFind.go-293-		cclog.Errorf("Error while running concurrent jobs query: %v", err)
internal/repository/jobFind.go-294-		return nil, fmt.Errorf("failed to execute concurrent jobs query: %w", err)
/Users/jan/prg/CC/cc-backend/internal/repository/jobFind.go-295-	}
/Users/jan/prg/CC/cc-backend/internal/repository/jobFind.go-296-	defer rows.Close()
--
/Users/jan/prg/CC/cc-backend/internal/repository/jobFind.go-317-	}
/Users/jan/prg/CC/cc-backend/internal/repository/jobFind.go-318-
internal/repository/jobFind.go:319:	rows, err = queryRunning.RunWith(r.stmtCache).Query()
/Users/jan/prg/CC/cc-backend/internal/repository/jobFind.go-320-	if err != nil {
internal/repository/jobFind.go-321-		cclog.Errorf("Error while running concurrent running jobs query: %v", err)
internal/repository/jobFind.go-322-		return nil, fmt.Errorf("failed to execute concurrent running jobs query: %w", err)
/Users/jan/prg/CC/cc-backend/internal/repository/jobFind.go-323-	}
/Users/jan/prg/CC/cc-backend/internal/repository/jobFind.go-324-	defer rows.Close()
--
/Users/jan/prg/CC/cc-backend/internal/repository/user.go-276-	}
/Users/jan/prg/CC/cc-backend/internal/repository/user.go-277-
internal/repository/user.go:278:	rows, err := q.RunWith(r.DB).Query()
/Users/jan/prg/CC/cc-backend/internal/repository/user.go-279-	if err != nil {
/Users/jan/prg/CC/cc-backend/internal/repository/user.go-280-		cclog.Warn("Error while querying user list")
/Users/jan/prg/CC/cc-backend/internal/repository/user.go-281-		return nil, err
/Users/jan/prg/CC/cc-backend/internal/repository/user.go-282-	}
/Users/jan/prg/CC/cc-backend/internal/repository/user.go-283-
--
/Users/jan/prg/CC/cc-backend/internal/repository/jobQuery.go-85-	}
/Users/jan/prg/CC/cc-backend/internal/repository/jobQuery.go-86-
internal/repository/jobQuery.go:87:	rows, err := query.RunWith(r.stmtCache).Query()
/Users/jan/prg/CC/cc-backend/internal/repository/jobQuery.go-88-	if err != nil {
internal/repository/jobQuery.go-89-		queryString, queryVars, _ := query.ToSql()
internal/repository/jobQuery.go-90-		return nil, fmt.Errorf("query failed [%s] %v: %w", queryString, queryVars, err)
/Users/jan/prg/CC/cc-backend/internal/repository/jobQuery.go-91-	}
/Users/jan/prg/CC/cc-backend/internal/repository/jobQuery.go-92-	defer rows.Close()
--
/Users/jan/prg/CC/cc-backend/internal/repository/job.go-425-		}
/Users/jan/prg/CC/cc-backend/internal/repository/job.go-426-
internal/repository/job.go:427:		rows, err := selectQuery.RunWith(r.DB).Query()
/Users/jan/prg/CC/cc-backend/internal/repository/job.go-428-		if err == nil {
/Users/jan/prg/CC/cc-backend/internal/repository/job.go-429-			defer rows.Close()
/Users/jan/prg/CC/cc-backend/internal/repository/job.go-430-			for rows.Next() {
/Users/jan/prg/CC/cc-backend/internal/repository/job.go-431-				var id int64
internal/repository/job.go-432-				if err := rows.Scan(&id); err == nil {
--
internal/repository/job.go-603-		rows, err := sq.Select(table+"."+selectColumn).Distinct().From(table).
/Users/jan/prg/CC/cc-backend/internal/repository/job.go-604-			Where(table+"."+whereColumn+" LIKE ?", fmt.Sprint("%", query, "%")).
internal/repository/job.go:605:			RunWith(r.stmtCache).Query()
/Users/jan/prg/CC/cc-backend/internal/repository/job.go-606-		if err != nil && err != sql.ErrNoRows {
internal/repository/job.go-607-			cclog.Errorf("Error while querying FindColumnValues (table=%s, column=%s): %v", table, selectColumn, err)
internal/repository/job.go-608-			return emptyResult, fmt.Errorf("failed to find column values: %w", err)
/Users/jan/prg/CC/cc-backend/internal/repository/job.go-609-		} else if err == nil {
/Users/jan/prg/CC/cc-backend/internal/repository/job.go-610-			defer rows.Close()
--
/Users/jan/prg/CC/cc-backend/internal/repository/job.go-661-		Where("job.job_state = 'running'").
/Users/jan/prg/CC/cc-backend/internal/repository/job.go-662-		Where("job.cluster = ?", cluster).
internal/repository/job.go:663:		RunWith(r.stmtCache).Query()
/Users/jan/prg/CC/cc-backend/internal/repository/job.go-664-	if err != nil {
internal/repository/job.go-665-		cclog.Errorf("Error while running AllocatedNodes query for cluster=%s: %v", cluster, err)
internal/repository/job.go-666-		return nil, fmt.Errorf("failed to query allocated nodes for cluster %s: %w", cluster, err)
/Users/jan/prg/CC/cc-backend/internal/repository/job.go-667-	}
/Users/jan/prg/CC/cc-backend/internal/repository/job.go-668-
--
/Users/jan/prg/CC/cc-backend/internal/repository/job.go-749-		Join("jobtag ON jobtag.job_id = job.id").
internal/repository/job.go-750-		Where(sq.Eq{"jobtag.tag_id": tagID}).Distinct()
internal/repository/job.go:751:	rows, err := query.RunWith(r.stmtCache).Query()
/Users/jan/prg/CC/cc-backend/internal/repository/job.go-752-	if err != nil {
internal/repository/job.go-753-		cclog.Errorf("Error while running FindJobIdsByTag query for tagID=%d: %v", tagID, err)
internal/repository/job.go-754-		return nil, fmt.Errorf("failed to find job IDs by tag %d: %w", tagID, err)
/Users/jan/prg/CC/cc-backend/internal/repository/job.go-755-	}
/Users/jan/prg/CC/cc-backend/internal/repository/job.go-756-	defer rows.Close()
--
/Users/jan/prg/CC/cc-backend/internal/repository/job.go-785-		Where("job.duration > ?", repoConfig.MinRunningJobDuration)
/Users/jan/prg/CC/cc-backend/internal/repository/job.go-786-
internal/repository/job.go:787:	rows, err := query.RunWith(r.stmtCache).Query()
/Users/jan/prg/CC/cc-backend/internal/repository/job.go-788-	if err != nil {
internal/repository/job.go-789-		cclog.Errorf("Error while running FindRunningJobs query for cluster=%s: %v", cluster, err)
internal/repository/job.go-790-		return nil, fmt.Errorf("failed to find running jobs for cluster %s: %w", cluster, err)
/Users/jan/prg/CC/cc-backend/internal/repository/job.go-791-	}
/Users/jan/prg/CC/cc-backend/internal/repository/job.go-792-	defer rows.Close()
--
/Users/jan/prg/CC/cc-backend/internal/repository/job.go-861-	query = query.OrderBy("job.cluster ASC", "job.subcluster ASC", "job.project ASC", "job.start_time ASC")
/Users/jan/prg/CC/cc-backend/internal/repository/job.go-862-
internal/repository/job.go:863:	rows, err := query.RunWith(r.stmtCache).Query()
/Users/jan/prg/CC/cc-backend/internal/repository/job.go-864-	if err != nil {
internal/repository/job.go-865-		cclog.Errorf("Error while running FindJobsBetween query: %v", err)
internal/repository/job.go-866-		return nil, fmt.Errorf("failed to find jobs between %d and %d: %w", startTimeBegin, startTimeEnd, err)
/Users/jan/prg/CC/cc-backend/internal/repository/job.go-867-	}
/Users/jan/prg/CC/cc-backend/internal/repository/job.go-868-	defer rows.Close()
--
internal/repository/job.go-1069-		Where(sq.Eq{"job.job_state": "running"})
/Users/jan/prg/CC/cc-backend/internal/repository/job.go-1070-
internal/repository/job.go:1071:	rows, err := q.RunWith(r.stmtCache).Query()
/Users/jan/prg/CC/cc-backend/internal/repository/job.go-1072-	if err != nil {
internal/repository/job.go-1073-		queryString, queryVars, _ := q.ToSql()
internal/repository/job.go-1074-		return nil, fmt.Errorf("query failed [%s] %v: %w", queryString, queryVars, err)
/Users/jan/prg/CC/cc-backend/internal/repository/job.go-1075-	}
/Users/jan/prg/CC/cc-backend/internal/repository/job.go-1076-	defer rows.Close()
--
/Users/jan/prg/CC/cc-backend/internal/repository/stats.go-231-	}
/Users/jan/prg/CC/cc-backend/internal/repository/stats.go-232-
internal/repository/stats.go:233:	rows, err := query.RunWith(r.DB).Query()
/Users/jan/prg/CC/cc-backend/internal/repository/stats.go-234-	if err != nil {
/Users/jan/prg/CC/cc-backend/internal/repository/stats.go-235-		cclog.Warn("Error while querying DB for job statistics")
/Users/jan/prg/CC/cc-backend/internal/repository/stats.go-236-		return nil, err
/Users/jan/prg/CC/cc-backend/internal/repository/stats.go-237-	}
/Users/jan/prg/CC/cc-backend/internal/repository/stats.go-238-
--
/Users/jan/prg/CC/cc-backend/internal/repository/stats.go-436-		return nil, err
/Users/jan/prg/CC/cc-backend/internal/repository/stats.go-437-	}
internal/repository/stats.go:438:	rows, err := query.RunWith(r.DB).Query()
/Users/jan/prg/CC/cc-backend/internal/repository/stats.go-439-	if err != nil {
/Users/jan/prg/CC/cc-backend/internal/repository/stats.go-440-		cclog.Warn("Error while querying DB for job statistics")
/Users/jan/prg/CC/cc-backend/internal/repository/stats.go-441-		return nil, err
/Users/jan/prg/CC/cc-backend/internal/repository/stats.go-442-	}
/Users/jan/prg/CC/cc-backend/internal/repository/stats.go-443-
--
/Users/jan/prg/CC/cc-backend/internal/repository/stats.go-492-		return nil, err
/Users/jan/prg/CC/cc-backend/internal/repository/stats.go-493-	}
internal/repository/stats.go:494:	rows, err := query.RunWith(r.DB).Query()
/Users/jan/prg/CC/cc-backend/internal/repository/stats.go-495-	if err != nil {
/Users/jan/prg/CC/cc-backend/internal/repository/stats.go-496-		cclog.Warn("Error while querying DB for job statistics")
/Users/jan/prg/CC/cc-backend/internal/repository/stats.go-497-		return nil, err
/Users/jan/prg/CC/cc-backend/internal/repository/stats.go-498-	}
/Users/jan/prg/CC/cc-backend/internal/repository/stats.go-499-
--
/Users/jan/prg/CC/cc-backend/internal/repository/stats.go-551-		return nil, err
/Users/jan/prg/CC/cc-backend/internal/repository/stats.go-552-	}
internal/repository/stats.go:553:	rows, err := query.RunWith(r.DB).Query()
/Users/jan/prg/CC/cc-backend/internal/repository/stats.go-554-	if err != nil {
/Users/jan/prg/CC/cc-backend/internal/repository/stats.go-555-		cclog.Warn("Error while querying DB for job statistics")
/Users/jan/prg/CC/cc-backend/internal/repository/stats.go-556-		return nil, err
/Users/jan/prg/CC/cc-backend/internal/repository/stats.go-557-	}
/Users/jan/prg/CC/cc-backend/internal/repository/stats.go-558-
--
/Users/jan/prg/CC/cc-backend/internal/repository/stats.go-751-	}
/Users/jan/prg/CC/cc-backend/internal/repository/stats.go-752-
internal/repository/stats.go:753:	rows, err := query.GroupBy("value").RunWith(r.DB).Query()
/Users/jan/prg/CC/cc-backend/internal/repository/stats.go-754-	if err != nil {
/Users/jan/prg/CC/cc-backend/internal/repository/stats.go-755-		cclog.Error("Error while running query")
/Users/jan/prg/CC/cc-backend/internal/repository/stats.go-756-		return nil, err
/Users/jan/prg/CC/cc-backend/internal/repository/stats.go-757-	}
/Users/jan/prg/CC/cc-backend/internal/repository/stats.go-758-
--
/Users/jan/prg/CC/cc-backend/internal/repository/stats.go-819-	}
/Users/jan/prg/CC/cc-backend/internal/repository/stats.go-820-
internal/repository/stats.go:821:	rows, err := query.GroupBy("value").RunWith(r.DB).Query()
/Users/jan/prg/CC/cc-backend/internal/repository/stats.go-822-	if err != nil {
/Users/jan/prg/CC/cc-backend/internal/repository/stats.go-823-		cclog.Error("Error while running query")
/Users/jan/prg/CC/cc-backend/internal/repository/stats.go-824-		return nil, err
/Users/jan/prg/CC/cc-backend/internal/repository/stats.go-825-	}
/Users/jan/prg/CC/cc-backend/internal/repository/stats.go-826-
--
/Users/jan/prg/CC/cc-backend/internal/repository/stats.go-944-	mainQuery = mainQuery.GroupBy("bin").OrderBy("bin")
/Users/jan/prg/CC/cc-backend/internal/repository/stats.go-945-
internal/repository/stats.go:946:	rows, err := mainQuery.RunWith(r.DB).Query()
/Users/jan/prg/CC/cc-backend/internal/repository/stats.go-947-	if err != nil {
internal/repository/stats.go-948-		cclog.Errorf("Error while running mainQuery: %s", err)
/Users/jan/prg/CC/cc-backend/internal/repository/stats.go-949-		return nil, err
/Users/jan/prg/CC/cc-backend/internal/repository/stats.go-950-	}
/Users/jan/prg/CC/cc-backend/internal/repository/stats.go-951-


internal/repository/node.go:291:	defer rows.Close()
internal/repository/node.go:502:	defer rows.Close()
internal/repository/node.go:532:	defer rows.Close()
internal/repository/job.go:429:			defer rows.Close()
internal/repository/job.go:610:			defer rows.Close()
internal/repository/job.go:670:	defer rows.Close()
internal/repository/job.go:756:	defer rows.Close()
internal/repository/job.go:792:	defer rows.Close()
internal/repository/job.go:868:	defer rows.Close()
internal/repository/job.go:1076:	defer rows.Close()
internal/repository/user.go:285:	defer rows.Close()
internal/repository/jobQuery.go:92:	defer rows.Close()
internal/repository/jobCreate.go:83:	defer rows.Close()
internal/repository/jobFind.go:111:	defer rows.Close()
internal/repository/jobFind.go:144:	defer rows.Close()
internal/repository/jobFind.go:296:	defer rows.Close()
internal/repository/jobFind.go:324:	defer rows.Close()
internal/repository/userConfig.go:75:		defer rows.Close()


Now I have a comprehensive picture. Let me read the full grep output to catch any remaining files.