     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→package repository
     7→
     8→import (
     9→	"context"
    10→	"database/sql"
    11→	"fmt"
    12→	"time"
    13→
    14→	"github.com/ClusterCockpit/cc-backend/internal/graph/model"
    15→	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
    16→	"github.com/ClusterCockpit/cc-lib/v2/schema"
    17→	sq "github.com/Masterminds/squirrel"
    18→)
    19→
    20→// Find executes a SQL query to find a specific batch job.
    21→// The job is queried using the batch job id, the cluster name,
    22→// and the start time of the job in UNIX epoch time seconds.
    23→// It returns a pointer to a schema.Job data structure and an error variable.
    24→// To check if no job was found test err == sql.ErrNoRows
    25→func (r *JobRepository) Find(
    26→	jobID *int64,
    27→	cluster *string,
    28→	startTime *int64,
    29→) (*schema.Job, error) {
    30→	if jobID == nil {
    31→		return nil, fmt.Errorf("jobID cannot be nil")
    32→	}
    33→
    34→	start := time.Now()
    35→	q := sq.Select(jobColumns...).From("job").
    36→		Where("job.job_id = ?", *jobID)
    37→
    38→	if cluster != nil {
    39→		q = q.Where("job.cluster = ?", *cluster)
    40→	}
    41→	if startTime != nil {
    42→		q = q.Where("job.start_time = ?", *startTime)
    43→	}
    44→
    45→	q = q.OrderBy("job.id DESC").Limit(1) // always use newest matching job by db id if more than one match
    46→
    47→	cclog.Debugf("Timer Find %s", time.Since(start))
    48→	return scanJob(q.RunWith(r.stmtCache).QueryRow())
    49→}
    50→
    51→// FindCached executes a SQL query to find a specific batch job from the job_cache table.
    52→// The job is queried using the batch job id, and optionally filtered by cluster name
    53→// and start time (UNIX epoch time seconds). This method uses cached job data which
    54→// may be stale but provides faster access than Find().
    55→// It returns a pointer to a schema.Job data structure and an error variable.
    56→// To check if no job was found test err == sql.ErrNoRows
    57→func (r *JobRepository) FindCached(
    58→	jobID *int64,
    59→	cluster *string,
    60→	startTime *int64,
    61→) (*schema.Job, error) {
    62→	if jobID == nil {
    63→		return nil, fmt.Errorf("jobID cannot be nil")
    64→	}
    65→
    66→	q := sq.Select(jobCacheColumns...).From("job_cache").
    67→		Where("job_cache.job_id = ?", *jobID)
    68→
    69→	if cluster != nil {
    70→		q = q.Where("job_cache.cluster = ?", *cluster)
    71→	}
    72→	if startTime != nil {
    73→		q = q.Where("job_cache.start_time = ?", *startTime)
    74→	}
    75→
    76→	q = q.OrderBy("job_cache.id DESC").Limit(1) // always use newest matching job by db id if more than one match
    77→
    78→	return scanJob(q.RunWith(r.stmtCache).QueryRow())
    79→}
    80→
    81→// FindAll executes a SQL query to find all batch jobs matching the given criteria.
    82→// Jobs are queried using the batch job id, and optionally filtered by cluster name
    83→// and start time (UNIX epoch time seconds).
    84→// It returns a slice of pointers to schema.Job data structures and an error variable.
    85→// An empty slice is returned if no matching jobs are found.
    86→func (r *JobRepository) FindAll(
    87→	jobID *int64,
    88→	cluster *string,
    89→	startTime *int64,
    90→) ([]*schema.Job, error) {
    91→	if jobID == nil {
    92→		return nil, fmt.Errorf("jobID cannot be nil")
    93→	}
    94→
    95→	start := time.Now()
    96→	q := sq.Select(jobColumns...).From("job").
    97→		Where("job.job_id = ?", *jobID)
    98→
    99→	if cluster != nil {
   100→		q = q.Where("job.cluster = ?", *cluster)
   101→	}
   102→	if startTime != nil {
   103→		q = q.Where("job.start_time = ?", *startTime)
   104→	}
   105→
   106→	rows, err := q.RunWith(r.stmtCache).Query()
   107→	if err != nil {
   108→		cclog.Errorf("Error while running FindAll query for jobID=%d: %v", *jobID, err)
   109→		return nil, fmt.Errorf("failed to execute FindAll query: %w", err)
   110→	}
   111→	defer rows.Close()
   112→
   113→	jobs := make([]*schema.Job, 0, 10)
   114→	for rows.Next() {
   115→		job, err := scanJob(rows)
   116→		if err != nil {
   117→			cclog.Warnf("Error while scanning rows in FindAll: %v", err)
   118→			return nil, fmt.Errorf("failed to scan job row: %w", err)
   119→		}
   120→		jobs = append(jobs, job)
   121→	}
   122→	cclog.Debugf("Timer FindAll %s", time.Since(start))
   123→	return jobs, nil
   124→}
   125→
   126→// GetJobList returns job IDs for non-running jobs.
   127→// This is useful to process large job counts and intended to be used
   128→// together with FindById to process jobs one by one.
   129→// Use limit and offset for pagination. Use limit=0 to get all results (not recommended for large datasets).
   130→func (r *JobRepository) GetJobList(limit int, offset int) ([]int64, error) {
   131→	query := sq.Select("id").From("job").
   132→		Where("job.job_state != 'running'")
   133→
   134→	// Add pagination if limit is specified
   135→	if limit > 0 {
   136→		query = query.Limit(uint64(limit)).Offset(uint64(offset))
   137→	}
   138→
   139→	rows, err := query.RunWith(r.stmtCache).Query()
   140→	if err != nil {
   141→		cclog.Errorf("Error while running GetJobList query (limit=%d, offset=%d): %v", limit, offset, err)
   142→		return nil, fmt.Errorf("failed to execute GetJobList query: %w", err)
   143→	}
   144→	defer rows.Close()
   145→
   146→	jl := make([]int64, 0, 1000)
   147→	for rows.Next() {
   148→		var id int64
   149→		err := rows.Scan(&id)
   150→		if err != nil {
   151→			cclog.Warnf("Error while scanning rows in GetJobList: %v", err)
   152→			return nil, fmt.Errorf("failed to scan job ID: %w", err)
   153→		}
   154→		jl = append(jl, id)
   155→	}
   156→
   157→	cclog.Debugf("JobRepository.GetJobList(): Return job count %d", len(jl))
   158→	return jl, nil
   159→}
   160→
   161→// FindByID executes a SQL query to find a specific batch job.
   162→// The job is queried using the database id.
   163→// It returns a pointer to a schema.Job data structure and an error variable.
   164→// To check if no job was found test err == sql.ErrNoRows
   165→func (r *JobRepository) FindByID(ctx context.Context, jobID int64) (*schema.Job, error) {
   166→	q := sq.Select(jobColumns...).
   167→		From("job").Where("job.id = ?", jobID)
   168→
   169→	q, qerr := SecurityCheck(ctx, q)
   170→	if qerr != nil {
   171→		return nil, qerr
   172→	}
   173→
   174→	return scanJob(q.RunWith(r.stmtCache).QueryRow())
   175→}
   176→
   177→// FindByIDWithUser executes a SQL query to find a specific batch job.
   178→// The job is queried using the database id. The user is passed directly,
   179→// instead as part of the context.
   180→// It returns a pointer to a schema.Job data structure and an error variable.
   181→// To check if no job was found test err == sql.ErrNoRows
   182→func (r *JobRepository) FindByIDWithUser(user *schema.User, jobID int64) (*schema.Job, error) {
   183→	q := sq.Select(jobColumns...).
   184→		From("job").Where("job.id = ?", jobID)
   185→
   186→	q, qerr := SecurityCheckWithUser(user, q)
   187→	if qerr != nil {
   188→		return nil, qerr
   189→	}
   190→
   191→	return scanJob(q.RunWith(r.stmtCache).QueryRow())
   192→}
   193→
   194→// FindByIDDirect executes a SQL query to find a specific batch job.
   195→// The job is queried using the database id.
   196→// It returns a pointer to a schema.Job data structure and an error variable.
   197→// To check if no job was found test err == sql.ErrNoRows
   198→func (r *JobRepository) FindByIDDirect(jobID int64) (*schema.Job, error) {
   199→	q := sq.Select(jobColumns...).
   200→		From("job").Where("job.id = ?", jobID)
   201→	return scanJob(q.RunWith(r.stmtCache).QueryRow())
   202→}
   203→
   204→// FindByJobID executes a SQL query to find a specific batch job.
   205→// The job is queried using the slurm id and the clustername.
   206→// It returns a pointer to a schema.Job data structure and an error variable.
   207→// To check if no job was found test err == sql.ErrNoRows
   208→func (r *JobRepository) FindByJobID(ctx context.Context, jobID int64, startTime int64, cluster string) (*schema.Job, error) {
   209→	q := sq.Select(jobColumns...).
   210→		From("job").
   211→		Where("job.job_id = ?", jobID).
   212→		Where("job.cluster = ?", cluster).
   213→		Where("job.start_time = ?", startTime)
   214→
   215→	q, qerr := SecurityCheck(ctx, q)
   216→	if qerr != nil {
   217→		return nil, qerr
   218→	}
   219→
   220→	return scanJob(q.RunWith(r.stmtCache).QueryRow())
   221→}
   222→
   223→// IsJobOwner checks if the specified user owns the batch job identified by jobID,
   224→// startTime, and cluster. Returns true if the user is the owner, false otherwise.
   225→// This method does not return errors; it returns false for both non-existent jobs
   226→// and jobs owned by other users.
   227→func (r *JobRepository) IsJobOwner(jobID int64, startTime int64, user string, cluster string) bool {
   228→	q := sq.Select("id").
   229→		From("job").
   230→		Where("job.job_id = ?", jobID).
   231→		Where("job.hpc_user = ?", user).
   232→		Where("job.cluster = ?", cluster).
   233→		Where("job.start_time = ?", startTime)
   234→
   235→	_, err := scanJob(q.RunWith(r.stmtCache).QueryRow())
   236→	if err != nil && err != sql.ErrNoRows {
   237→		cclog.Warnf("IsJobOwner: unexpected error for jobID=%d, user=%s, cluster=%s: %v", jobID, user, cluster, err)
   238→	}
   239→	return err != sql.ErrNoRows
   240→}
   241→
   242→func (r *JobRepository) FindConcurrentJobs(
   243→	ctx context.Context,
   244→	job *schema.Job,
   245→) (*model.JobLinkResultList, error) {
   246→	if job == nil {
   247→		return nil, nil
   248→	}
   249→
   250→	query, qerr := SecurityCheck(ctx, sq.Select("job.id", "job.job_id", "job.start_time").From("job"))
   251→	if qerr != nil {
   252→		return nil, qerr
   253→	}
   254→
   255→	query = query.Where("cluster = ?", job.Cluster)
   256→
   257→	if len(job.Resources) == 0 {
   258→		return nil, fmt.Errorf("job has no resources defined")
   259→	}
   260→
   261→	var startTime int64
   262→	var stopTime int64
   263→
   264→	startTime = job.StartTime
   265→	hostname := job.Resources[0].Hostname
   266→
   267→	if job.State == schema.JobStateRunning {
   268→		stopTime = time.Now().Unix()
   269→	} else {
   270→		stopTime = startTime + int64(job.Duration)
   271→	}
   272→
   273→	// Time buffer constant for finding overlapping jobs
   274→	// overlapBufferEnd: 200s buffer at job end to account for scheduling/cleanup overlap
   275→	const overlapBufferEnd = 200
   276→
   277→	stopTimeTail := stopTime - overlapBufferEnd
   278→	startTimeFront := startTime + overlapBufferEnd
   279→
   280→	queryRunning := query.Where("job.job_state = ?", "running").
   281→		Where("job.start_time <= ?", stopTimeTail)
   282→	// Get At Least One Exact Hostname Match from JSON Resources Array in Database
   283→	queryRunning = queryRunning.Where("EXISTS (SELECT 1 FROM json_each(job.resources) WHERE json_extract(value, '$.hostname') = ?)", hostname)
   284→
   285→	query = query.Where("job.job_state != ?", "running").
   286→		Where("job.start_time < ?", stopTimeTail).
   287→		Where("(job.start_time + job.duration) > ?", startTimeFront)
   288→	// Get At Least One Exact Hostname Match from JSON Resources Array in Database
   289→	query = query.Where("EXISTS (SELECT 1 FROM json_each(job.resources) WHERE json_extract(value, '$.hostname') = ?)", hostname)
   290→
   291→	rows, err := query.RunWith(r.stmtCache).Query()
   292→	if err != nil {
   293→		cclog.Errorf("Error while running concurrent jobs query: %v", err)
   294→		return nil, fmt.Errorf("failed to execute concurrent jobs query: %w", err)
   295→	}
   296→	defer rows.Close()
   297→
   298→	items := make([]*model.JobLink, 0, 10)
   299→	queryString := fmt.Sprintf("cluster=%s", job.Cluster)
   300→
   301→	for rows.Next() {
   302→		var id, jobID, startTime sql.NullInt64
   303→
   304→		if err = rows.Scan(&id, &jobID, &startTime); err != nil {
   305→			cclog.Warnf("Error while scanning concurrent job rows: %v", err)
   306→			return nil, fmt.Errorf("failed to scan concurrent job row: %w", err)
   307→		}
   308→
   309→		if id.Valid {
   310→			queryString += fmt.Sprintf("&jobId=%d", int(jobID.Int64))
   311→			items = append(items,
   312→				&model.JobLink{
   313→					ID:    fmt.Sprint(id.Int64),
   314→					JobID: int(jobID.Int64),
   315→				})
   316→		}
   317→	}
   318→
   319→	rows, err = queryRunning.RunWith(r.stmtCache).Query()
   320→	if err != nil {
   321→		cclog.Errorf("Error while running concurrent running jobs query: %v", err)
   322→		return nil, fmt.Errorf("failed to execute concurrent running jobs query: %w", err)
   323→	}
   324→	defer rows.Close()
   325→
   326→	for rows.Next() {
   327→		var id, jobID, startTime sql.NullInt64
   328→
   329→		if err := rows.Scan(&id, &jobID, &startTime); err != nil {
   330→			cclog.Warnf("Error while scanning running concurrent job rows: %v", err)
   331→			return nil, fmt.Errorf("failed to scan running concurrent job row: %w", err)
   332→		}
   333→
   334→		if id.Valid {
   335→			queryString += fmt.Sprintf("&jobId=%d", int(jobID.Int64))
   336→			items = append(items,
   337→				&model.JobLink{
   338→					ID:    fmt.Sprint(id.Int64),
   339→					JobID: int(jobID.Int64),
   340→				})
   341→		}
   342→	}
   343→
   344→	cnt := len(items)
   345→
   346→	return &model.JobLinkResultList{
   347→		ListQuery: &queryString,
   348→		Items:     items,
   349→		Count:     &cnt,
   350→	}, nil
   351→}
   352→

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package repository

import (
	"context"
	"database/sql"
	"fmt"
	"time"

	"github.com/ClusterCockpit/cc-backend/internal/graph/model"
	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
	"github.com/ClusterCockpit/cc-lib/v2/schema"
	sq "github.com/Masterminds/squirrel"
)

// Find executes a SQL query to find a specific batch job.
// The job is queried using the batch job id, the cluster name,
// and the start time of the job in UNIX epoch time seconds.
// It returns a pointer to a schema.Job data structure and an error variable.
// To check if no job was found test err == sql.ErrNoRows
func (r *JobRepository) Find(
	jobID *int64,
	cluster *string,
	startTime *int64,
) (*schema.Job, error) {
	if jobID == nil {
		return nil, fmt.Errorf("jobID cannot be nil")
	}

	start := time.Now()
	q := sq.Select(jobColumns...).From("job").
		Where("job.job_id = ?", *jobID)

	if cluster != nil {
		q = q.Where("job.cluster = ?", *cluster)
	}
	if startTime != nil {
		q = q.Where("job.start_time = ?", *startTime)
	}

	q = q.OrderBy("job.id DESC").Limit(1) // always use newest matching job by db id if more than one match

	cclog.Debugf("Timer Find %s", time.Since(start))
	return scanJob(q.RunWith(r.stmtCache).QueryRow())
}

// FindCached executes a SQL query to find a specific batch job from the job_cache table.
// The job is queried using the batch job id, and optionally filtered by cluster name
// and start time (UNIX epoch time seconds). This method uses cached job data which
// may be stale but provides faster access than Find().
// It returns a pointer to a schema.Job data structure and an error variable.
// To check if no job was found test err == sql.ErrNoRows
func (r *JobRepository) FindCached(
	jobID *int64,
	cluster *string,
	startTime *int64,
) (*schema.Job, error) {
	if jobID == nil {
		return nil, fmt.Errorf("jobID cannot be nil")
	}

	q := sq.Select(jobCacheColumns...).From("job_cache").
		Where("job_cache.job_id = ?", *jobID)

	if cluster != nil {
		q = q.Where("job_cache.cluster = ?", *cluster)
	}
	if startTime != nil {
		q = q.Where("job_cache.start_time = ?", *startTime)
	}

	q = q.OrderBy("job_cache.id DESC").Limit(1) // always use newest matching job by db id if more than one match

	return scanJob(q.RunWith(r.stmtCache).QueryRow())
}

// FindAll executes a SQL query to find all batch jobs matching the given criteria.
// Jobs are queried using the batch job id, and optionally filtered by cluster name
// and start time (UNIX epoch time seconds).
// It returns a slice of pointers to schema.Job data structures and an error variable.
// An empty slice is returned if no matching jobs are found.
func (r *JobRepository) FindAll(
	jobID *int64,
	cluster *string,
	startTime *int64,
) ([]*schema.Job, error) {
	if jobID == nil {
		return nil, fmt.Errorf("jobID cannot be nil")
	}

	start := time.Now()
	q := sq.Select(jobColumns...).From("job").
		Where("job.job_id = ?", *jobID)

	if cluster != nil {
		q = q.Where("job.cluster = ?", *cluster)
	}
	if startTime != nil {
		q = q.Where("job.start_time = ?", *startTime)
	}

	rows, err := q.RunWith(r.stmtCache).Query()
	if err != nil {
		cclog.Errorf("Error while running FindAll query for jobID=%d: %v", *jobID, err)
		return nil, fmt.Errorf("failed to execute FindAll query: %w", err)
	}
	defer rows.Close()

	jobs := make([]*schema.Job, 0, 10)
	for rows.Next() {
		job, err := scanJob(rows)
		if err != nil {
			cclog.Warnf("Error while scanning rows in FindAll: %v", err)
			return nil, fmt.Errorf("failed to scan job row: %w", err)
		}
		jobs = append(jobs, job)
	}
	cclog.Debugf("Timer FindAll %s", time.Since(start))
	return jobs, nil
}

// GetJobList returns job IDs for non-running jobs.
// This is useful to process large job counts and intended to be used
// together with FindById to process jobs one by one.
// Use limit and offset for pagination. Use limit=0 to get all results (not recommended for large datasets).
func (r *JobRepository) GetJobList(limit int, offset int) ([]int64, error) {
	query := sq.Select("id").From("job").
		Where("job.job_state != 'running'")

	// Add pagination if limit is specified
	if limit > 0 {
		query = query.Limit(uint64(limit)).Offset(uint64(offset))
	}

	rows, err := query.RunWith(r.stmtCache).Query()
	if err != nil {
		cclog.Errorf("Error while running GetJobList query (limit=%d, offset=%d): %v", limit, offset, err)
		return nil, fmt.Errorf("failed to execute GetJobList query: %w", err)
	}
	defer rows.Close()

	jl := make([]int64, 0, 1000)
	for rows.Next() {
		var id int64
		err := rows.Scan(&id)
		if err != nil {
			cclog.Warnf("Error while scanning rows in GetJobList: %v", err)
			return nil, fmt.Errorf("failed to scan job ID: %w", err)
		}
		jl = append(jl, id)
	}

	cclog.Debugf("JobRepository.GetJobList(): Return job count %d", len(jl))
	return jl, nil
}

// FindByID executes a SQL query to find a specific batch job.
// The job is queried using the database id.
// It returns a pointer to a schema.Job data structure and an error variable.
// To check if no job was found test err == sql.ErrNoRows
func (r *JobRepository) FindByID(ctx context.Context, jobID int64) (*schema.Job, error) {
	q := sq.Select(jobColumns...).
		From("job").Where("job.id = ?", jobID)

	q, qerr := SecurityCheck(ctx, q)
	if qerr != nil {
		return nil, qerr
	}

	return scanJob(q.RunWith(r.stmtCache).QueryRow())
}

// FindByIDWithUser executes a SQL query to find a specific batch job.
// The job is queried using the database id. The user is passed directly,
// instead as part of the context.
// It returns a pointer to a schema.Job data structure and an error variable.
// To check if no job was found test err == sql.ErrNoRows
func (r *JobRepository) FindByIDWithUser(user *schema.User, jobID int64) (*schema.Job, error) {
	q := sq.Select(jobColumns...).
		From("job").Where("job.id = ?", jobID)

	q, qerr := SecurityCheckWithUser(user, q)
	if qerr != nil {
		return nil, qerr
	}

	return scanJob(q.RunWith(r.stmtCache).QueryRow())
}

// FindByIDDirect executes a SQL query to find a specific batch job.
// The job is queried using the database id.
// It returns a pointer to a schema.Job data structure and an error variable.
// To check if no job was found test err == sql.ErrNoRows
func (r *JobRepository) FindByIDDirect(jobID int64) (*schema.Job, error) {
	q := sq.Select(jobColumns...).
		From("job").Where("job.id = ?", jobID)
	return scanJob(q.RunWith(r.stmtCache).QueryRow())
}

// FindByJobID executes a SQL query to find a specific batch job.
// The job is queried using the slurm id and the clustername.
// It returns a pointer to a schema.Job data structure and an error variable.
// To check if no job was found test err == sql.ErrNoRows
func (r *JobRepository) FindByJobID(ctx context.Context, jobID int64, startTime int64, cluster string) (*schema.Job, error) {
	q := sq.Select(jobColumns...).
		From("job").
		Where("job.job_id = ?", jobID).
		Where("job.cluster = ?", cluster).
		Where("job.start_time = ?", startTime)

	q, qerr := SecurityCheck(ctx, q)
	if qerr != nil {
		return nil, qerr
	}

	return scanJob(q.RunWith(r.stmtCache).QueryRow())
}

// IsJobOwner checks if the specified user owns the batch job identified by jobID,
// startTime, and cluster. Returns true if the user is the owner, false otherwise.
// This method does not return errors; it returns false for both non-existent jobs
// and jobs owned by other users.
func (r *JobRepository) IsJobOwner(jobID int64, startTime int64, user string, cluster string) bool {
	q := sq.Select("id").
		From("job").
		Where("job.job_id = ?", jobID).
		Where("job.hpc_user = ?", user).
		Where("job.cluster = ?", cluster).
		Where("job.start_time = ?", startTime)

	_, err := scanJob(q.RunWith(r.stmtCache).QueryRow())
	if err != nil && err != sql.ErrNoRows {
		cclog.Warnf("IsJobOwner: unexpected error for jobID=%d, user=%s, cluster=%s: %v", jobID, user, cluster, err)
	}
	return err != sql.ErrNoRows
}

func (r *JobRepository) FindConcurrentJobs(
	ctx context.Context,
	job *schema.Job,
) (*model.JobLinkResultList, error) {
	if job == nil {
		return nil, nil
	}

	query, qerr := SecurityCheck(ctx, sq.Select("job.id", "job.job_id", "job.start_time").From("job"))
	if qerr != nil {
		return nil, qerr
	}

	query = query.Where("cluster = ?", job.Cluster)

	if len(job.Resources) == 0 {
		return nil, fmt.Errorf("job has no resources defined")
	}

	var startTime int64
	var stopTime int64

	startTime = job.StartTime
	hostname := job.Resources[0].Hostname

	if job.State == schema.JobStateRunning {
		stopTime = time.Now().Unix()
	} else {
		stopTime = startTime + int64(job.Duration)
	}

	// Time buffer constant for finding overlapping jobs
	// overlapBufferEnd: 200s buffer at job end to account for scheduling/cleanup overlap
	const overlapBufferEnd = 200

	stopTimeTail := stopTime - overlapBufferEnd
	startTimeFront := startTime + overlapBufferEnd

	queryRunning := query.Where("job.job_state = ?", "running").
		Where("job.start_time <= ?", stopTimeTail)
	// Get At Least One Exact Hostname Match from JSON Resources Array in Database
	queryRunning = queryRunning.Where("EXISTS (SELECT 1 FROM json_each(job.resources) WHERE json_extract(value, '$.hostname') = ?)", hostname)

	query = query.Where("job.job_state != ?", "running").
		Where("job.start_time < ?", stopTimeTail).
		Where("(job.start_time + job.duration) > ?", startTimeFront)
	// Get At Least One Exact Hostname Match from JSON Resources Array in Database
	query = query.Where("EXISTS (SELECT 1 FROM json_each(job.resources) WHERE json_extract(value, '$.hostname') = ?)", hostname)

	rows, err := query.RunWith(r.stmtCache).Query()
	if err != nil {
		cclog.Errorf("Error while running concurrent jobs query: %v", err)
		return nil, fmt.Errorf("failed to execute concurrent jobs query: %w", err)
	}
	defer rows.Close()

	items := make([]*model.JobLink, 0, 10)
	queryString := fmt.Sprintf("cluster=%s", job.Cluster)

	for rows.Next() {
		var id, jobID, startTime sql.NullInt64

		if err = rows.Scan(&id, &jobID, &startTime); err != nil {
			cclog.Warnf("Error while scanning concurrent job rows: %v", err)
			return nil, fmt.Errorf("failed to scan concurrent job row: %w", err)
		}

		if id.Valid {
			queryString += fmt.Sprintf("&jobId=%d", int(jobID.Int64))
			items = append(items,
				&model.JobLink{
					ID:    fmt.Sprint(id.Int64),
					JobID: int(jobID.Int64),
				})
		}
	}

	rows, err = queryRunning.RunWith(r.stmtCache).Query()
	if err != nil {
		cclog.Errorf("Error while running concurrent running jobs query: %v", err)
		return nil, fmt.Errorf("failed to execute concurrent running jobs query: %w", err)
	}
	defer rows.Close()

	for rows.Next() {
		var id, jobID, startTime sql.NullInt64

		if err := rows.Scan(&id, &jobID, &startTime); err != nil {
			cclog.Warnf("Error while scanning running concurrent job rows: %v", err)
			return nil, fmt.Errorf("failed to scan running concurrent job row: %w", err)
		}

		if id.Valid {
			queryString += fmt.Sprintf("&jobId=%d", int(jobID.Int64))
			items = append(items,
				&model.JobLink{
					ID:    fmt.Sprint(id.Int64),
					JobID: int(jobID.Int64),
				})
		}
	}

	cnt := len(items)

	return &model.JobLinkResultList{
		ListQuery: &queryString,
		Items:     items,
		Count:     &cnt,
	}, nil
}


<persisted-output>
Output too large (49.5KB). Full output saved to: /Users/jan/.claude/projects/-Users-jan-prg-CC-cc-backend/520afa6a-6a70-437b-96c1-35c40ed3ec48/tool-results/toolu_01141KDjZ4GvFWJSbM7oMSN4.txt

Preview (first 2KB):
     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→// Package repository provides the data access layer for cc-backend using the repository pattern.
     7→//
     8→// The repository pattern abstracts database operations and provides a clean interface for
     9→// data access. Each major entity (Job, User, Node, Tag) has its own repository with CRUD
    10→// operations and specialized queries.
    11→//
    12→// # Database Connection
    13→//
    14→// Initialize the database connection before using any repository:
    15→//
    16→//	repository.Connect("sqlite3", "./var/job.db")
    17→//
    18→// # Configuration
    19→//
    20→// Optional: Configure repository settings before initialization:
    21→//
    22→//	repository.SetConfig(&repository.RepositoryConfig{
    23→//	    CacheSize: 2 * 1024 * 1024,     // 2MB cache
    24→//	    MaxOpenConnections: 8,           // Connection pool size
    25→//	    MinRunningJobDuration: 300,      // Filter threshold
    26→//	})
    27→//
    28→// If not configured, sensible defaults are used automatically.
    29→//
    30→// # Repositories
    31→//
    32→//   - JobRepository: Job lifecycle management and querying
    33→//   - UserRepository: User management and authentication
    34→//   - NodeRepository: Cluster node state tracking
    35→//   - Tags: Job tagging and categorization
    36→//
    37→// # Caching
    38→//
    39→// Repositories use LRU caching to improve performance. Cache keys are constructed
    40→// as "type:id" (e.g., "metadata:123"). Cache is automatically invalidated on
    41→// mutations to maintain consistency.
    42→//
    43→// # Transaction Support
    44→//
    45→// For batch operations, use transactions:
    46→//
    47→//	t, err := jobRepo.TransactionInit()
...
</persisted-output>

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

// Package repository provides the data access layer for cc-backend using the repository pattern.
//
// The repository pattern abstracts database operations and provides a clean interface for
// data access. Each major entity (Job, User, Node, Tag) has its own repository with CRUD
// operations and specialized queries.
//
// # Database Connection
//
// Initialize the database connection before using any repository:
//
//	repository.Connect("sqlite3", "./var/job.db")
//
// # Configuration
//
// Optional: Configure repository settings before initialization:
//
//	repository.SetConfig(&repository.RepositoryConfig{
//	    CacheSize: 2 * 1024 * 1024,     // 2MB cache
//	    MaxOpenConnections: 8,           // Connection pool size
//	    MinRunningJobDuration: 300,      // Filter threshold
//	})
//
// If not configured, sensible defaults are used automatically.
//
// # Repositories
//
//   - JobRepository: Job lifecycle management and querying
//   - UserRepository: User management and authentication
//   - NodeRepository: Cluster node state tracking
//   - Tags: Job tagging and categorization
//
// # Caching
//
// Repositories use LRU caching to improve performance. Cache keys are constructed
// as "type:id" (e.g., "metadata:123"). Cache is automatically invalidated on
// mutations to maintain consistency.
//
// # Transaction Support
//
// For batch operations, use transactions:
//
//	t, err := jobRepo.TransactionInit()
//	if err != nil {
//	    return err
//	}
//	defer t.Rollback() // Rollback if not committed
//
//	// Perform operations...
//	jobRepo.TransactionAdd(t, query, args...)
//
//	// Commit when done
//	if err := t.Commit(); err != nil {
//	    return err
//	}
package repository

import (
	"database/sql"
	"encoding/json"
	"errors"
	"fmt"
	"maps"
	"math"
	"sort"
	"strconv"
	"sync"
	"time"

	"github.com/ClusterCockpit/cc-backend/pkg/archive"
	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
	"github.com/ClusterCockpit/cc-lib/v2/lrucache"
	"github.com/ClusterCockpit/cc-lib/v2/schema"
	sq "github.com/Masterminds/squirrel"
	"github.com/jmoiron/sqlx"
)

var (
	// jobRepoOnce ensures singleton initialization of the JobRepository
	jobRepoOnce sync.Once
	// jobRepoInstance holds the single instance of JobRepository
	jobRepoInstance *JobRepository
)

// JobRepository provides database access for job-related operations.
// It implements the repository pattern to abstract database interactions
// and provides caching for improved performance.
//
// The repository is a singleton initialized via GetJobRepository().
// All database queries use prepared statements via stmtCache for efficiency.
// Frequently accessed data (metadata, energy footprints) is cached in an LRU cache.
type JobRepository struct {
	DB        *sqlx.DB        // Database connection pool
	stmtCache *sq.StmtCache   // Prepared statement cache for query optimization
	cache     *lrucache.Cache // LRU cache for metadata and footprint data
	driver    string          // Database driver name (e.g., "sqlite3")
	Mutex     sync.Mutex      // Mutex for thread-safe operations
}

// GetJobRepository returns the singleton instance of JobRepository.
// The repository is initialized lazily on first access with database connection,
// prepared statement cache, and LRU cache configured from repoConfig.
//
// This function is thread-safe and ensures only one instance is created.
// It must be called after Connect() has established a database connection.
func GetJobRepository() *JobRepository {
	jobRepoOnce.Do(func() {
		db := GetConnection()

		jobRepoInstance = &JobRepository{
			DB:     db.DB,
			driver: db.Driver,

			stmtCache: sq.NewStmtCache(db.DB),
			cache:     lrucache.New(repoConfig.CacheSize),
		}
	})
	return jobRepoInstance
}

// jobColumns defines the standard set of columns selected from the job table.
// Used consistently across all job queries to ensure uniform data retrieval.
var jobColumns []string = []string{
	"job.id", "job.job_id", "job.hpc_user", "job.project", "job.cluster", "job.subcluster",
	"job.start_time", "job.cluster_partition", "job.array_job_id", "job.num_nodes",
	"job.num_hwthreads", "job.num_acc", "job.shared", "job.monitoring_status",
	"job.smt", "job.job_state", "job.duration", "job.walltime", "job.resources",
	"job.footprint", "job.energy",
}

// jobCacheColumns defines columns from the job_cache table, mirroring jobColumns.
// Used for queries against cached job data for performance optimization.
var jobCacheColumns []string = []string{
	"job_cache.id", "job_cache.job_id", "job_cache.hpc_user", "job_cache.project", "job_cache.cluster",
	"job_cache.subcluster", "job_cache.start_time", "job_cache.cluster_partition",
	"job_cache.array_job_id", "job_cache.num_nodes", "job_cache.num_hwthreads",
	"job_cache.num_acc", "job_cache.shared", "job_cache.monitoring_status", "job_cache.smt",
	"job_cache.job_state", "job_cache.duration", "job_cache.walltime", "job_cache.resources",
	"job_cache.footprint", "job_cache.energy",
}

// scanJob converts a database row into a schema.Job struct.
// It handles JSON unmarshaling of resources and footprint fields,
// and calculates accurate duration for running jobs.
//
// Parameters:
//   - row: Database row implementing Scan() interface (sql.Row or sql.Rows)
//
// Returns the populated Job struct or an error if scanning or unmarshaling fails.
func scanJob(row interface{ Scan(...any) error }) (*schema.Job, error) {
	job := &schema.Job{}

	if err := row.Scan(
		&job.ID, &job.JobID, &job.User, &job.Project, &job.Cluster, &job.SubCluster,
		&job.StartTime, &job.Partition, &job.ArrayJobID, &job.NumNodes, &job.NumHWThreads,
		&job.NumAcc, &job.Shared, &job.MonitoringStatus, &job.SMT, &job.State,
		&job.Duration, &job.Walltime, &job.RawResources, &job.RawFootprint, &job.Energy); err != nil {
		cclog.Warnf("Error while scanning rows (Job): %v", err)
		return nil, err
	}

	if err := json.Unmarshal(job.RawResources, &job.Resources); err != nil {
		cclog.Warn("Error while unmarshaling raw resources json")
		return nil, err
	}
	job.RawResources = nil

	if err := json.Unmarshal(job.RawFootprint, &job.Footprint); err != nil {
		cclog.Warnf("Error while unmarshaling raw footprint json: %v", err)
		return nil, err
	}
	job.RawFootprint = nil

	// Always ensure accurate duration for running jobs
	if job.State == schema.JobStateRunning {
		job.Duration = int32(time.Now().Unix() - job.StartTime)
	}

	return job, nil
}

// Optimize performs database optimization by running VACUUM command.
// This reclaims unused space and defragments the database file.
// Should be run periodically during maintenance windows.
func (r *JobRepository) Optimize() error {
	if _, err := r.DB.Exec(`VACUUM`); err != nil {
		cclog.Errorf("Error while executing VACUUM: %v", err)
		return fmt.Errorf("failed to optimize database: %w", err)
	}
	return nil
}

// Flush removes all data from job-related tables (jobtag, tag, job).
// WARNING: This is a destructive operation that deletes all job data.
// Use with extreme caution, typically only for testing or complete resets.
func (r *JobRepository) Flush() error {
	if _, err := r.DB.Exec(`DELETE FROM jobtag`); err != nil {
		cclog.Errorf("Error while deleting from jobtag table: %v", err)
		return fmt.Errorf("failed to flush jobtag table: %w", err)
	}
	if _, err := r.DB.Exec(`DELETE FROM tag`); err != nil {
		cclog.Errorf("Error while deleting from tag table: %v", err)
		return fmt.Errorf("failed to flush tag table: %w", err)
	}
	if _, err := r.DB.Exec(`DELETE FROM job`); err != nil {
		cclog.Errorf("Error while deleting from job table: %v", err)
		return fmt.Errorf("failed to flush job table: %w", err)
	}
	return nil
}

// FetchMetadata retrieves and unmarshals the metadata JSON for a job.
// Metadata is cached with a 24-hour TTL to improve performance.
//
// The metadata field stores arbitrary key-value pairs associated with a job,
// such as tags, labels, or custom attributes added by external systems.
//
// Parameters:
//   - job: Job struct with valid ID field, metadata will be populated in job.MetaData
//
// Returns the metadata map or an error if the job is nil or database query fails.
func (r *JobRepository) FetchMetadata(job *schema.Job) (map[string]string, error) {
	if job == nil {
		return nil, fmt.Errorf("job cannot be nil")
	}

	start := time.Now()
	cachekey := fmt.Sprintf("metadata:%d", *job.ID)
	if cached := r.cache.Get(cachekey, nil); cached != nil {
		job.MetaData = cached.(map[string]string)
		return job.MetaData, nil
	}

	if err := sq.Select("job.meta_data").From("job").Where("job.id = ?", job.ID).
		RunWith(r.stmtCache).QueryRow().Scan(&job.RawMetaData); err != nil {
		cclog.Warnf("Error while scanning for job metadata (ID=%d): %v", *job.ID, err)
		return nil, fmt.Errorf("failed to fetch metadata for job %d: %w", *job.ID, err)
	}

	if len(job.RawMetaData) == 0 {
		return nil, nil
	}

	if err := json.Unmarshal(job.RawMetaData, &job.MetaData); err != nil {
		cclog.Warnf("Error while unmarshaling raw metadata json (ID=%d): %v", *job.ID, err)
		return nil, fmt.Errorf("failed to unmarshal metadata for job %d: %w", *job.ID, err)
	}

	r.cache.Put(cachekey, job.MetaData, len(job.RawMetaData), 24*time.Hour)
	cclog.Debugf("Timer FetchMetadata %s", time.Since(start))
	return job.MetaData, nil
}

// UpdateMetadata adds or updates a single metadata key-value pair for a job.
// The entire metadata map is re-marshaled and stored, and the cache is invalidated.
// Also triggers archive metadata update via archive.UpdateMetadata.
//
// Parameters:
//   - job: Job struct with valid ID, existing metadata will be fetched if not present
//   - key: Metadata key to set
//   - val: Metadata value to set
//
// Returns an error if the job is nil, metadata fetch fails, or database update fails.
func (r *JobRepository) UpdateMetadata(job *schema.Job, key, val string) (err error) {
	if job == nil {
		return fmt.Errorf("job cannot be nil")
	}

	cachekey := fmt.Sprintf("metadata:%d", *job.ID)
	r.cache.Del(cachekey)
	if job.MetaData == nil {
		if _, err = r.FetchMetadata(job); err != nil {
			cclog.Warnf("Error while fetching metadata for job, DB ID '%v'", *job.ID)
			return fmt.Errorf("failed to fetch metadata for job %d: %w", *job.ID, err)
		}
	}

	if job.MetaData != nil {
		cpy := make(map[string]string, len(job.MetaData)+1)
		maps.Copy(cpy, job.MetaData)
		cpy[key] = val
		job.MetaData = cpy
	} else {
		job.MetaData = map[string]string{key: val}
	}

	if job.RawMetaData, err = json.Marshal(job.MetaData); err != nil {
		cclog.Warnf("Error while marshaling metadata for job, DB ID '%v'", *job.ID)
		return fmt.Errorf("failed to marshal metadata for job %d: %w", *job.ID, err)
	}

	if _, err = sq.Update("job").
		Set("meta_data", job.RawMetaData).
		Where("job.id = ?", job.ID).
		RunWith(r.stmtCache).Exec(); err != nil {
		cclog.Warnf("Error while updating metadata for job, DB ID '%v'", *job.ID)
		return fmt.Errorf("failed to update metadata in database for job %d: %w", *job.ID, err)
	}

	r.cache.Put(cachekey, job.MetaData, len(job.RawMetaData), 24*time.Hour)
	return archive.UpdateMetadata(job, job.MetaData)
}

// FetchFootprint retrieves and unmarshals the performance footprint JSON for a job.
// Unlike FetchMetadata, footprints are NOT cached as they can be large and change frequently.
//
// The footprint contains summary statistics (avg/min/max) for monitored metrics,
// stored as JSON with keys like "cpu_load_avg", "mem_used_max", etc.
//
// Parameters:
//   - job: Job struct with valid ID, footprint will be populated in job.Footprint
//
// Returns the footprint map or an error if the job is nil or database query fails.
func (r *JobRepository) FetchFootprint(job *schema.Job) (map[string]float64, error) {
	if job == nil {
		return nil, fmt.Errorf("job cannot be nil")
	}

	start := time.Now()

	if err := sq.Select("job.footprint").From("job").Where("job.id = ?", job.ID).
		RunWith(r.stmtCache).QueryRow().Scan(&job.RawFootprint); err != nil {
		cclog.Warnf("Error while scanning for job footprint (ID=%d): %v", *job.ID, err)
		return nil, fmt.Errorf("failed to fetch footprint for job %d: %w", *job.ID, err)
	}

	if len(job.RawFootprint) == 0 {
		return nil, nil
	}

	if err := json.Unmarshal(job.RawFootprint, &job.Footprint); err != nil {
		cclog.Warnf("Error while unmarshaling raw footprint json (ID=%d): %v", *job.ID, err)
		return nil, fmt.Errorf("failed to unmarshal footprint for job %d: %w", *job.ID, err)
	}

	cclog.Debugf("Timer FetchFootprint %s", time.Since(start))
	return job.Footprint, nil
}

// FetchEnergyFootprint retrieves and unmarshals the energy footprint JSON for a job.
// Energy footprints are cached with a 24-hour TTL as they are frequently accessed but rarely change.
//
// The energy footprint contains calculated energy consumption (in kWh) per metric,
// stored as JSON with keys like "power_avg", "acc_power_avg", etc.
//
// Parameters:
//   - job: Job struct with valid ID, energy footprint will be populated in job.EnergyFootprint
//
// Returns the energy footprint map or an error if the job is nil or database query fails.
func (r *JobRepository) FetchEnergyFootprint(job *schema.Job) (map[string]float64, error) {
	if job == nil {
		return nil, fmt.Errorf("job cannot be nil")
	}

	start := time.Now()
	cachekey := fmt.Sprintf("energyFootprint:%d", *job.ID)
	if cached := r.cache.Get(cachekey, nil); cached != nil {
		job.EnergyFootprint = cached.(map[string]float64)
		return job.EnergyFootprint, nil
	}

	if err := sq.Select("job.energy_footprint").From("job").Where("job.id = ?", job.ID).
		RunWith(r.stmtCache).QueryRow().Scan(&job.RawEnergyFootprint); err != nil {
		cclog.Warnf("Error while scanning for job energy_footprint (ID=%d): %v", *job.ID, err)
		return nil, fmt.Errorf("failed to fetch energy footprint for job %d: %w", *job.ID, err)
	}

	if len(job.RawEnergyFootprint) == 0 {
		return nil, nil
	}

	if err := json.Unmarshal(job.RawEnergyFootprint, &job.EnergyFootprint); err != nil {
		cclog.Warnf("Error while unmarshaling raw energy footprint json (ID=%d): %v", *job.ID, err)
		return nil, fmt.Errorf("failed to unmarshal energy footprint for job %d: %w", *job.ID, err)
	}

	r.cache.Put(cachekey, job.EnergyFootprint, len(job.EnergyFootprint), 24*time.Hour)
	cclog.Debugf("Timer FetchEnergyFootprint %s", time.Since(start))
	return job.EnergyFootprint, nil
}

// DeleteJobsBefore removes jobs older than the specified start time.
// Optionally preserves tagged jobs to protect important data from deletion.
// Cache entries for deleted jobs are automatically invalidated.
//
// This is typically used for data retention policies and cleanup operations.
// WARNING: This is a destructive operation that permanently deletes job records.
//
// Parameters:
//   - startTime: Unix timestamp, jobs with start_time < this value will be deleted
//   - omitTagged: "none" = delete all jobs, "all" = skip any tagged jobs,
//     "user" = skip jobs with user-created tags (not auto-tagger types "app"/"jobClass")
//
// Returns the count of deleted jobs or an error if the operation fails.
func (r *JobRepository) DeleteJobsBefore(startTime int64, omitTagged string) (int, error) {
	var cnt int
	q := sq.Select("count(*)").From("job").Where("job.start_time < ?", startTime)

	switch omitTagged {
	case "all":
		q = q.Where("NOT EXISTS (SELECT 1 FROM jobtag WHERE jobtag.job_id = job.id)")
	case "user":
		q = q.Where("NOT EXISTS (SELECT 1 FROM jobtag JOIN tag ON tag.id = jobtag.tag_id WHERE jobtag.job_id = job.id AND tag.tag_type NOT IN ('app', 'jobClass'))")
	}

	if err := q.RunWith(r.DB).QueryRow().Scan(&cnt); err != nil {
		cclog.Errorf("Error counting jobs before %d: %v", startTime, err)
		return 0, err
	}

	// Invalidate cache for jobs being deleted (get job IDs first)
	if cnt > 0 {
		var jobIds []int64
		selectQuery := sq.Select("id").From("job").Where("job.start_time < ?", startTime)

		switch omitTagged {
		case "all":
			selectQuery = selectQuery.Where("NOT EXISTS (SELECT 1 FROM jobtag WHERE jobtag.job_id = job.id)")
		case "user":
			selectQuery = selectQuery.Where("NOT EXISTS (SELECT 1 FROM jobtag JOIN tag ON tag.id = jobtag.tag_id WHERE jobtag.job_id = job.id AND tag.tag_type NOT IN ('app', 'jobClass'))")
		}

		rows, err := selectQuery.RunWith(r.DB).Query()
		if err == nil {
			defer rows.Close()
			for rows.Next() {
				var id int64
				if err := rows.Scan(&id); err == nil {
					jobIds = append(jobIds, id)
				}
			}
			// Invalidate cache entries
			for _, id := range jobIds {
				r.cache.Del(fmt.Sprintf("metadata:%d", id))
				r.cache.Del(fmt.Sprintf("energyFootprint:%d", id))
			}
		}
	}

	qd := sq.Delete("job").Where("job.start_time < ?", startTime)

	switch omitTagged {
	case "all":
		qd = qd.Where("NOT EXISTS (SELECT 1 FROM jobtag WHERE jobtag.job_id = job.id)")
	case "user":
		qd = qd.Where("NOT EXISTS (SELECT 1 FROM jobtag JOIN tag ON tag.id = jobtag.tag_id WHERE jobtag.job_id = job.id AND tag.tag_type NOT IN ('app', 'jobClass'))")
	}
	_, err := qd.RunWith(r.DB).Exec()

	if err != nil {
		s, _, _ := qd.ToSql()
		cclog.Errorf(" DeleteJobsBefore(%d) with %s: error %#v", startTime, s, err)
	} else {
		cclog.Debugf("DeleteJobsBefore(%d): Deleted %d jobs", startTime, cnt)
	}
	return cnt, err
}

// DeleteJobByID permanently removes a single job by its database ID.
// Cache entries for the deleted job are automatically invalidated.
//
// Parameters:
//   - id: Database ID (primary key) of the job to delete
//
// Returns an error if the deletion fails.
func (r *JobRepository) DeleteJobByID(id int64) error {
	// Invalidate cache entries before deletion
	r.cache.Del(fmt.Sprintf("metadata:%d", id))
	r.cache.Del(fmt.Sprintf("energyFootprint:%d", id))

	qd := sq.Delete("job").Where("job.id = ?", id)
	_, err := qd.RunWith(r.DB).Exec()

	if err != nil {
		s, _, _ := qd.ToSql()
		cclog.Errorf("DeleteJobById(%d) with %s : error %#v", id, s, err)
	} else {
		cclog.Debugf("DeleteJobById(%d): Success", id)
	}
	return err
}

// FindUserOrProjectOrJobname attempts to interpret a search term as a job ID,
// username, project ID, or job name by querying the database.
//
// Search logic (in priority order):
//  1. If searchterm is numeric, treat as job ID (returned immediately)
//  2. Try exact match in job.hpc_user column (username)
//  3. Try LIKE match in hpc_user.name column (real name)
//  4. Try exact match in job.project column (project ID)
//  5. If no matches, return searchterm as jobname for GraphQL query
//
// This powers the searchbar functionality for flexible job searching.
// Requires authenticated user for database lookups (returns empty if user is nil).
//
// Parameters:
//   - user: Authenticated user context, required for database access
//   - searchterm: Search string to interpret
//
// Returns up to one non-empty value among (jobid, username, project, jobname).
func (r *JobRepository) FindUserOrProjectOrJobname(user *schema.User, searchterm string) (jobid string, username string, project string, jobname string) {
	if searchterm == "" {
		return "", "", "", ""
	}

	if _, err := strconv.Atoi(searchterm); err == nil { // Return empty on successful conversion: parent method will redirect for integer jobId
		return searchterm, "", "", ""
	} else { // Has to have letters and logged-in user for other guesses
		if user != nil {
			// Find username by username in job table (match)
			uresult, _ := r.FindColumnValue(user, searchterm, "job", "hpc_user", "hpc_user", false)
			if uresult != "" {
				return "", uresult, "", ""
			}
			// Find username by real name in hpc_user table (like)
			nresult, _ := r.FindColumnValue(user, searchterm, "hpc_user", "username", "name", true)
			if nresult != "" {
				return "", nresult, "", ""
			}
			// Find projectId by projectId in job table (match)
			presult, _ := r.FindColumnValue(user, searchterm, "job", "project", "project", false)
			if presult != "" {
				return "", "", presult, ""
			}
		}
		// Return searchterm if no match before: Forward as jobname query to GQL in handleSearchbar function
		return "", "", "", searchterm
	}
}

var (
	ErrNotFound  = errors.New("no such jobname, project or user")
	ErrForbidden = errors.New("not authorized")
)

// FindColumnValue performs a generic column lookup in a database table with role-based access control.
// Only users with admin, support, or manager roles can execute this query.
//
// Parameters:
//   - user: User context for authorization check
//   - searchterm: Value to search for (exact match or LIKE pattern)
//   - table: Database table name to query
//   - selectColumn: Column name to return in results
//   - whereColumn: Column name to filter on
//   - isLike: If true, use LIKE with wildcards; if false, use exact equality
//
// Returns the first matching value, ErrForbidden if user lacks permission,
// or ErrNotFound if no matches are found.
func (r *JobRepository) FindColumnValue(user *schema.User, searchterm string, table string, selectColumn string, whereColumn string, isLike bool) (result string, err error) {
	if user == nil {
		return "", fmt.Errorf("user cannot be nil")
	}

	compareStr := " = ?"
	query := searchterm
	if isLike {
		compareStr = " LIKE ?"
		query = "%" + searchterm + "%"
	}
	if user.HasAnyRole([]schema.Role{schema.RoleAdmin, schema.RoleSupport, schema.RoleManager}) {
		theQuery := sq.Select(table+"."+selectColumn).Distinct().From(table).
			Where(table+"."+whereColumn+compareStr, query)

		err := theQuery.RunWith(r.stmtCache).QueryRow().Scan(&result)

		if err != nil && err != sql.ErrNoRows {
			cclog.Warnf("Error while querying FindColumnValue (table=%s, column=%s): %v", table, selectColumn, err)
			return "", fmt.Errorf("failed to find column value: %w", err)
		} else if err == nil {
			return result, nil
		}
		return "", ErrNotFound
	} else {
		cclog.Infof("Non-Admin User %s : Requested Query '%s' on table '%s' : Forbidden", user.Name, query, table)
		return "", ErrForbidden
	}
}

// FindColumnValues performs a generic column lookup returning multiple matches with role-based access control.
// Similar to FindColumnValue but returns all matching values instead of just the first.
// Only users with admin, support, or manager roles can execute this query.
//
// Parameters:
//   - user: User context for authorization check
//   - query: Search pattern (always uses LIKE with wildcards)
//   - table: Database table name to query
//   - selectColumn: Column name to return in results
//   - whereColumn: Column name to filter on
//
// Returns a slice of matching values, ErrForbidden if user lacks permission,
// or ErrNotFound if no matches are found.
func (r *JobRepository) FindColumnValues(user *schema.User, query string, table string, selectColumn string, whereColumn string) (results []string, err error) {
	if user == nil {
		return nil, fmt.Errorf("user cannot be nil")
	}

	emptyResult := make([]string, 0)
	if user.HasAnyRole([]schema.Role{schema.RoleAdmin, schema.RoleSupport, schema.RoleManager}) {
		rows, err := sq.Select(table+"."+selectColumn).Distinct().From(table).
			Where(table+"."+whereColumn+" LIKE ?", fmt.Sprint("%", query, "%")).
			RunWith(r.stmtCache).Query()
		if err != nil && err != sql.ErrNoRows {
			cclog.Errorf("Error while querying FindColumnValues (table=%s, column=%s): %v", table, selectColumn, err)
			return emptyResult, fmt.Errorf("failed to find column values: %w", err)
		} else if err == nil {
			defer rows.Close()
			for rows.Next() {
				var result string
				err := rows.Scan(&result)
				if err != nil {
					cclog.Warnf("Error while scanning rows in FindColumnValues: %v", err)
					return emptyResult, fmt.Errorf("failed to scan column value: %w", err)
				}
				results = append(results, result)
			}
			return results, nil
		}
		return emptyResult, ErrNotFound

	} else {
		cclog.Infof("Non-Admin User %s : Requested Query '%s' on table '%s' : Forbidden", user.Name, query, table)
		return emptyResult, ErrForbidden
	}
}

// Partitions returns a list of distinct cluster partitions for a given cluster.
// Results are cached with a 1-hour TTL to improve performance.
//
// Parameters:
//   - cluster: Cluster name to query partitions for
//
// Returns a slice of partition names or an error if the database query fails.
func (r *JobRepository) Partitions(cluster string) ([]string, error) {
	var err error
	start := time.Now()
	partitions := r.cache.Get("partitions:"+cluster, func() (any, time.Duration, int) {
		parts := []string{}
		if err = r.DB.Select(&parts, `SELECT DISTINCT job.cluster_partition FROM job WHERE job.cluster = ?;`, cluster); err != nil {
			return nil, 0, 1000
		}

		return parts, 1 * time.Hour, 1
	})
	if err != nil {
		return nil, err
	}
	cclog.Debugf("Timer Partitions %s", time.Since(start))
	return partitions.([]string), nil
}

// AllocatedNodes returns a map of all subclusters to a map of hostnames to the amount of jobs running on that host.
// Hosts with zero jobs running on them will not show up!
func (r *JobRepository) AllocatedNodes(cluster string) (map[string]map[string]int, error) {
	start := time.Now()
	subclusters := make(map[string]map[string]int)
	rows, err := sq.Select("resources", "subcluster").From("job").
		Where("job.job_state = 'running'").
		Where("job.cluster = ?", cluster).
		RunWith(r.stmtCache).Query()
	if err != nil {
		cclog.Errorf("Error while running AllocatedNodes query for cluster=%s: %v", cluster, err)
		return nil, fmt.Errorf("failed to query allocated nodes for cluster %s: %w", cluster, err)
	}

	var raw []byte
	defer rows.Close()
	for rows.Next() {
		raw = raw[0:0]
		var resources []*schema.Resource
		var subcluster string
		if err := rows.Scan(&raw, &subcluster); err != nil {
			cclog.Warnf("Error while scanning rows in AllocatedNodes: %v", err)
			return nil, fmt.Errorf("failed to scan allocated nodes row: %w", err)
		}
		if err := json.Unmarshal(raw, &resources); err != nil {
			cclog.Warnf("Error while unmarshaling raw resources json in AllocatedNodes: %v", err)
			return nil, fmt.Errorf("failed to unmarshal resources in AllocatedNodes: %w", err)
		}

		hosts, ok := subclusters[subcluster]
		if !ok {
			hosts = make(map[string]int)
			subclusters[subcluster] = hosts
		}

		for _, resource := range resources {
			hosts[resource.Hostname] += 1
		}
	}

	cclog.Debugf("Timer AllocatedNodes %s", time.Since(start))
	return subclusters, nil
}

// StopJobsExceedingWalltimeBy marks running jobs as failed if they exceed their walltime limit.
// This is typically called periodically to clean up stuck or orphaned jobs.
//
// Jobs are marked with:
//   - monitoring_status: MonitoringStatusArchivingFailed
//   - duration: 0
//   - job_state: JobStateFailed
//
// Parameters:
//   - seconds: Grace period beyond walltime before marking as failed
//
// Returns an error if the database update fails.
// Logs the number of jobs marked as failed if any were affected.
func (r *JobRepository) StopJobsExceedingWalltimeBy(seconds int) error {
	start := time.Now()
	currentTime := time.Now().Unix()
	res, err := sq.Update("job").
		Set("monitoring_status", schema.MonitoringStatusArchivingFailed).
		Set("duration", 0).
		Set("job_state", schema.JobStateFailed).
		Where("job.job_state = 'running'").
		Where("job.walltime > 0").
		Where("job.start_time < (? - job.walltime)", currentTime-int64(seconds)).
		RunWith(r.DB).Exec()
	if err != nil {
		cclog.Warnf("Error while stopping jobs exceeding walltime: %v", err)
		return fmt.Errorf("failed to stop jobs exceeding walltime: %w", err)
	}

	rowsAffected, err := res.RowsAffected()
	if err != nil {
		cclog.Warnf("Error while fetching affected rows after stopping due to exceeded walltime: %v", err)
		return fmt.Errorf("failed to get rows affected count: %w", err)
	}

	if rowsAffected > 0 {
		cclog.Infof("%d jobs have been marked as failed due to running too long", rowsAffected)
	}
	cclog.Debugf("Timer StopJobsExceedingWalltimeBy %s", time.Since(start))
	return nil
}

// FindJobIdsByTag returns all job database IDs associated with a specific tag.
//
// Parameters:
//   - tagID: Database ID of the tag to search for
//
// Returns a slice of job IDs or an error if the query fails.
func (r *JobRepository) FindJobIdsByTag(tagID int64) ([]int64, error) {
	query := sq.Select("job.id").From("job").
		Join("jobtag ON jobtag.job_id = job.id").
		Where(sq.Eq{"jobtag.tag_id": tagID}).Distinct()
	rows, err := query.RunWith(r.stmtCache).Query()
	if err != nil {
		cclog.Errorf("Error while running FindJobIdsByTag query for tagID=%d: %v", tagID, err)
		return nil, fmt.Errorf("failed to find job IDs by tag %d: %w", tagID, err)
	}
	defer rows.Close()

	jobIds := make([]int64, 0, 100)

	for rows.Next() {
		var jobID int64

		if err := rows.Scan(&jobID); err != nil {
			cclog.Warnf("Error while scanning rows in FindJobIdsByTag: %v", err)
			return nil, fmt.Errorf("failed to scan job ID in FindJobIdsByTag: %w", err)
		}

		jobIds = append(jobIds, jobID)
	}

	return jobIds, nil
}

// FindRunningJobs returns all currently running jobs for a specific cluster.
// Filters out short-running jobs based on repoConfig.MinRunningJobDuration threshold.
//
// Parameters:
//   - cluster: Cluster name to filter jobs
//
// Returns a slice of running job objects or an error if the query fails.
func (r *JobRepository) FindRunningJobs(cluster string) ([]*schema.Job, error) {
	query := sq.Select(jobColumns...).From("job").
		Where("job.cluster = ?", cluster).
		Where("job.job_state = 'running'").
		Where("job.duration > ?", repoConfig.MinRunningJobDuration)

	rows, err := query.RunWith(r.stmtCache).Query()
	if err != nil {
		cclog.Errorf("Error while running FindRunningJobs query for cluster=%s: %v", cluster, err)
		return nil, fmt.Errorf("failed to find running jobs for cluster %s: %w", cluster, err)
	}
	defer rows.Close()

	jobs := make([]*schema.Job, 0, 50)
	for rows.Next() {
		job, err := scanJob(rows)
		if err != nil {
			cclog.Warnf("Error while scanning rows in FindRunningJobs: %v", err)
			return nil, fmt.Errorf("failed to scan job in FindRunningJobs: %w", err)
		}
		jobs = append(jobs, job)
	}

	cclog.Debugf("JobRepository.FindRunningJobs(): Return job count %d (cluster: %s)", len(jobs), cluster)
	return jobs, nil
}

// UpdateDuration recalculates and updates the duration field for all running jobs.
// Called periodically to keep job durations current without querying individual jobs.
//
// Duration is calculated as: current_time - job.start_time
//
// Returns an error if the database update fails.
func (r *JobRepository) UpdateDuration() error {
	stmnt := sq.Update("job").
		Set("duration", sq.Expr("? - job.start_time", time.Now().Unix())).
		Where("job_state = 'running'")

	_, err := stmnt.RunWith(r.stmtCache).Exec()
	if err != nil {
		cclog.Errorf("Error while updating duration for running jobs: %v", err)
		return fmt.Errorf("failed to update duration for running jobs: %w", err)
	}

	return nil
}

// FindJobsBetween returns jobs within a specified time range.
// If startTimeBegin is 0, returns all jobs before startTimeEnd.
// Optionally excludes tagged jobs from results.
//
// Parameters:
//   - startTimeBegin: Unix timestamp for range start (use 0 for unbounded start)
//   - startTimeEnd: Unix timestamp for range end
//   - omitTagged: "none" = include all jobs, "all" = exclude any tagged jobs,
//     "user" = exclude jobs with user-created tags (not auto-tagger types "app"/"jobClass")
//
// Returns a slice of jobs or an error if the time range is invalid or query fails.
func (r *JobRepository) FindJobsBetween(startTimeBegin int64, startTimeEnd int64, omitTagged string) ([]*schema.Job, error) {
	var query sq.SelectBuilder

	if startTimeBegin == startTimeEnd || startTimeBegin > startTimeEnd {
		return nil, errors.New("startTimeBegin is equal or larger startTimeEnd")
	}

	if startTimeBegin == 0 {
		cclog.Infof("Find jobs before %d", startTimeEnd)
		query = sq.Select(jobColumns...).From("job").Where("job.start_time < ?", startTimeEnd)
	} else {
		cclog.Infof("Find jobs between %d and %d", startTimeBegin, startTimeEnd)
		query = sq.Select(jobColumns...).From("job").Where("job.start_time BETWEEN ? AND ?", startTimeBegin, startTimeEnd)
	}

	switch omitTagged {
	case "all":
		query = query.Where("NOT EXISTS (SELECT 1 FROM jobtag WHERE jobtag.job_id = job.id)")
	case "user":
		query = query.Where("NOT EXISTS (SELECT 1 FROM jobtag JOIN tag ON tag.id = jobtag.tag_id WHERE jobtag.job_id = job.id AND tag.tag_type NOT IN ('app', 'jobClass'))")
	}

	query = query.OrderBy("job.cluster ASC", "job.subcluster ASC", "job.project ASC", "job.start_time ASC")

	rows, err := query.RunWith(r.stmtCache).Query()
	if err != nil {
		cclog.Errorf("Error while running FindJobsBetween query: %v", err)
		return nil, fmt.Errorf("failed to find jobs between %d and %d: %w", startTimeBegin, startTimeEnd, err)
	}
	defer rows.Close()

	jobs := make([]*schema.Job, 0, 50)
	for rows.Next() {
		job, err := scanJob(rows)
		if err != nil {
			cclog.Warnf("Error while scanning rows in FindJobsBetween: %v", err)
			return nil, fmt.Errorf("failed to scan job in FindJobsBetween: %w", err)
		}
		jobs = append(jobs, job)
	}

	cclog.Debugf("JobRepository.FindJobsBetween(): Return job count %d (omitTagged: %v)", len(jobs), omitTagged)
	return jobs, nil
}

// UpdateMonitoringStatus updates the monitoring status for a job and invalidates its cache entries.
// Cache invalidation affects both metadata and energy footprint to ensure consistency.
//
// Parameters:
//   - job: Database ID of the job to update
//   - monitoringStatus: New monitoring status value (see schema.MonitoringStatus constants)
//
// Returns an error if the database update fails.
func (r *JobRepository) UpdateMonitoringStatus(job int64, monitoringStatus int32) (err error) {
	// Invalidate cache entries as monitoring status affects job state
	r.cache.Del(fmt.Sprintf("metadata:%d", job))
	r.cache.Del(fmt.Sprintf("energyFootprint:%d", job))

	stmt := sq.Update("job").
		Set("monitoring_status", monitoringStatus).
		Where("job.id = ?", job)

	if _, err = stmt.RunWith(r.stmtCache).Exec(); err != nil {
		cclog.Errorf("Error while updating monitoring status for job %d: %v", job, err)
		return fmt.Errorf("failed to update monitoring status for job %d: %w", job, err)
	}
	return nil
}

// Execute runs a Squirrel UpdateBuilder statement against the database.
// This is a generic helper for executing pre-built update queries.
//
// Parameters:
//   - stmt: Squirrel UpdateBuilder with prepared update query
//
// Returns an error if the execution fails.
func (r *JobRepository) Execute(stmt sq.UpdateBuilder) error {
	if _, err := stmt.RunWith(r.stmtCache).Exec(); err != nil {
		cclog.Errorf("Error while executing statement: %v", err)
		return fmt.Errorf("failed to execute update statement: %w", err)
	}

	return nil
}

// MarkArchived adds monitoring status update to an existing UpdateBuilder statement.
// This is a builder helper used when constructing multi-field update queries.
//
// Parameters:
//   - stmt: Existing UpdateBuilder to modify
//   - monitoringStatus: Monitoring status value to set
//
// Returns the modified UpdateBuilder for method chaining.
func (r *JobRepository) MarkArchived(
	stmt sq.UpdateBuilder,
	monitoringStatus int32,
) sq.UpdateBuilder {
	return stmt.Set("monitoring_status", monitoringStatus)
}

// UpdateEnergy calculates and updates the energy consumption for a job.
// This is called for running jobs during intermediate updates or when archiving.
//
// Energy calculation formula:
//   - For "power" metrics: Energy (kWh) = (Power_avg * NumNodes * Duration_hours) / 1000
//   - For "energy" metrics: Currently not implemented (would need sum statistics)
//
// The calculation accounts for:
//   - Multi-node jobs: Multiplies by NumNodes to get total cluster energy
//   - Shared jobs: Node average is already based on partial resources, so NumNodes=1
//   - Unit conversion: Watts * hours / 1000 = kilowatt-hours (kWh)
//   - Rounding: Results rounded to 2 decimal places
func (r *JobRepository) UpdateEnergy(
	stmt sq.UpdateBuilder,
	jobMeta *schema.Job,
) (sq.UpdateBuilder, error) {
	sc, err := archive.GetSubCluster(jobMeta.Cluster, jobMeta.SubCluster)
	if err != nil {
		cclog.Errorf("cannot get subcluster: %s", err.Error())
		return stmt, err
	}
	energyFootprint := make(map[string]float64)

	// Accumulate total energy across all energy-related metrics
	totalEnergy := 0.0
	for _, fp := range sc.EnergyFootprint {
		// Calculate energy for this specific metric
		metricEnergy := 0.0
		if i, err := archive.MetricIndex(sc.MetricConfig, fp); err == nil {
			switch sc.MetricConfig[i].Energy {
			case "energy": // Metric already in energy units (Joules or Wh)
				cclog.Warnf("Update EnergyFootprint for Job %d and Metric %s on cluster %s: Set to 'energy' in cluster.json: Not implemented, will return 0.0", jobMeta.JobID, jobMeta.Cluster, fp)
				// FIXME: Needs sum as stats type to accumulate energy values over time
			case "power": // Metric in power units (Watts)
				// Energy (kWh) = Power (W) × Time (h) / 1000
				// Formula: (avg_power_per_node * num_nodes) * (duration_sec / 3600) / 1000
				//
				// Breakdown:
				//   LoadJobStat(jobMeta, fp, "avg") = average power per node (W)
				//   jobMeta.NumNodes = number of nodes (1 for shared jobs)
				//   jobMeta.Duration / 3600.0 = duration in hours
				//   / 1000.0 = convert Wh to kWh
				rawEnergy := ((LoadJobStat(jobMeta, fp, "avg") * float64(jobMeta.NumNodes)) * (float64(jobMeta.Duration) / 3600.0)) / 1000.0
				metricEnergy = math.Round(rawEnergy*100.0) / 100.0 // Round to 2 decimal places
			}
		} else {
			cclog.Warnf("Error while collecting energy metric %s for job, DB ID '%v', return '0.0'", fp, jobMeta.ID)
		}

		energyFootprint[fp] = metricEnergy
		totalEnergy += metricEnergy
	}

	var rawFootprint []byte
	if rawFootprint, err = json.Marshal(energyFootprint); err != nil {
		cclog.Warnf("Error while marshaling energy footprint for job INTO BYTES, DB ID '%v'", jobMeta.ID)
		return stmt, err
	}

	return stmt.Set("energy_footprint", string(rawFootprint)).Set("energy", (math.Round(totalEnergy*100.0) / 100.0)), nil
}

// UpdateFootprint calculates and updates the performance footprint for a job.
// This is called for running jobs during intermediate updates or when archiving.
//
// A footprint is a summary statistic (avg/min/max) for each monitored metric.
// The specific statistic type is defined in the cluster config's Footprint field.
// Results are stored as JSON with keys like "metric_avg", "metric_max", etc.
//
// Example: For a "cpu_load" metric with Footprint="avg", this stores
// the average CPU load across all nodes as "cpu_load_avg": 85.3
func (r *JobRepository) UpdateFootprint(
	stmt sq.UpdateBuilder,
	jobMeta *schema.Job,
) (sq.UpdateBuilder, error) {
	sc, err := archive.GetSubCluster(jobMeta.Cluster, jobMeta.SubCluster)
	if err != nil {
		cclog.Errorf("cannot get subcluster: %s", err.Error())
		return stmt, err
	}
	footprint := make(map[string]float64)

	// Build footprint map with metric_stattype as keys
	for _, fp := range sc.Footprint {
		// Determine which statistic to use: avg, min, or max
		// First check global metric config, then cluster-specific config
		var statType string
		for _, gm := range archive.GlobalMetricList {
			if gm.Name == fp {
				statType = gm.Footprint
			}
		}

		// Validate statistic type
		if statType != "avg" && statType != "min" && statType != "max" {
			cclog.Warnf("unknown statType for footprint update: %s", statType)
			return stmt, fmt.Errorf("unknown statType for footprint update: %s", statType)
		}

		// Override with cluster-specific config if available
		if i, err := archive.MetricIndex(sc.MetricConfig, fp); err != nil {
			statType = sc.MetricConfig[i].Footprint
		}

		// Store as "metric_stattype": value (e.g., "cpu_load_avg": 85.3)
		name := fmt.Sprintf("%s_%s", fp, statType)
		footprint[name] = LoadJobStat(jobMeta, fp, statType)
	}

	var rawFootprint []byte
	if rawFootprint, err = json.Marshal(footprint); err != nil {
		cclog.Warnf("Error while marshaling footprint for job INTO BYTES, DB ID '%v'", jobMeta.ID)
		return stmt, err
	}

	return stmt.Set("footprint", string(rawFootprint)), nil
}

// GetUsedNodes returns a map of cluster names to sorted lists of unique hostnames
// that are currently in use by jobs that started before the given timestamp and
// are still in running state.
//
// The timestamp parameter (ts) is compared against job.start_time to find
// relevant jobs. Returns an error if the database query fails or row iteration
// encounters errors. Individual row parsing errors are logged but don't fail
// the entire operation.
func (r *JobRepository) GetUsedNodes(ts int64) (map[string][]string, error) {
	// Note: Query expects index on (job_state, start_time) for optimal performance
	q := sq.Select("job.cluster", "job.resources").From("job").
		Where("job.start_time < ?", ts).
		Where(sq.Eq{"job.job_state": "running"})

	rows, err := q.RunWith(r.stmtCache).Query()
	if err != nil {
		queryString, queryVars, _ := q.ToSql()
		return nil, fmt.Errorf("query failed [%s] %v: %w", queryString, queryVars, err)
	}
	defer rows.Close()

	// Use a map of sets for efficient deduplication
	nodeSet := make(map[string]map[string]struct{})

	var (
		cluster      string
		rawResources []byte
		resources    []*schema.Resource
		skippedRows  int
	)

	for rows.Next() {
		if err := rows.Scan(&cluster, &rawResources); err != nil {
			cclog.Warnf("Error scanning job row in GetUsedNodes: %v", err)
			skippedRows++
			continue
		}

		resources = resources[:0] // Clear slice, keep capacity
		if err := json.Unmarshal(rawResources, &resources); err != nil {
			cclog.Warnf("Error unmarshaling resources for cluster %s: %v", cluster, err)
			skippedRows++
			continue
		}

		if len(resources) == 0 {
			cclog.Debugf("Job in cluster %s has no resources", cluster)
			continue
		}

		if _, ok := nodeSet[cluster]; !ok {
			nodeSet[cluster] = make(map[string]struct{})
		}

		for _, res := range resources {
			nodeSet[cluster][res.Hostname] = struct{}{}
		}
	}

	if err := rows.Err(); err != nil {
		return nil, fmt.Errorf("error iterating rows: %w", err)
	}

	if skippedRows > 0 {
		cclog.Warnf("GetUsedNodes: Skipped %d rows due to parsing errors", skippedRows)
	}

	// Convert sets to sorted slices
	nodeList := make(map[string][]string, len(nodeSet))
	for cluster, nodes := range nodeSet {
		list := make([]string, 0, len(nodes))
		for node := range nodes {
			list = append(list, node)
		}
		sort.Strings(list)
		nodeList[cluster] = list
	}

	return nodeList, nil
}


Let me find the `CountGroupedJobs` method in job.go.


No matches found


The `CountGroupedJobs` method doesn't exist in the codebase. Now let me check how `scanJob` works with `QueryRow` — I need to verify whether squirrel's `RunWith(stmtCache).QueryRow()` returns something that supports `QueryRowContext`.


internal/repository/job.go:154:func scanJob(row interface{ Scan(...any) error }) (*schema.Job, error) {
internal/repository/job.go-155-	job := &schema.Job{}
/Users/jan/prg/CC/cc-backend/internal/repository/job.go-156-
internal/repository/job.go-157-	if err := row.Scan(
/Users/jan/prg/CC/cc-backend/internal/repository/job.go-158-		&job.ID, &job.JobID, &job.User, &job.Project, &job.Cluster, &job.SubCluster,
/Users/jan/prg/CC/cc-backend/internal/repository/job.go-159-		&job.StartTime, &job.Partition, &job.ArrayJobID, &job.NumNodes, &job.NumHWThreads,


Now I have all the context. Let me implement all four fixes.

**Fix 1: Cache size** and **Fix 2: soft_heap_limit** in `dbConnection.go`:


Edit

connectionURLParams.Add("_cache_size", "-200000") // 200MB per connection (negative = KiB)


The file /Users/jan/prg/CC/cc-backend/internal/repository/dbConnection.go has been updated successfully.

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package repository

import (
	"database/sql"
	"fmt"
	"net/url"
	"sync"
	"time"

	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
	"github.com/jmoiron/sqlx"
	"github.com/mattn/go-sqlite3"
	"github.com/qustavo/sqlhooks/v2"
)

var (
	dbConnOnce     sync.Once
	dbConnInstance *DBConnection
)

type DBConnection struct {
	DB     *sqlx.DB
	Driver string
}

type DatabaseOptions struct {
	URL                   string
	MaxOpenConnections    int
	MaxIdleConnections    int
	ConnectionMaxLifetime time.Duration
	ConnectionMaxIdleTime time.Duration
}

func setupSqlite(db *sql.DB) error {
	pragmas := []string{
		"temp_store = memory",
	}

	for _, pragma := range pragmas {
		_, err := db.Exec("PRAGMA " + pragma)
		if err != nil {
			return err
		}
	}

	// Update query planner statistics so SQLite picks optimal indexes.
	// Without this, SQLite guesses row distributions and often chooses wrong
	// indexes for queries with IN clauses + ORDER BY, causing full table sorts
	// in temp B-trees instead of using covering indexes.
	if _, err := db.Exec("ANALYZE"); err != nil {
		cclog.Warnf("Failed to run ANALYZE: %v", err)
	}

	return nil
}

func Connect(db string) {
	var err error
	var dbHandle *sqlx.DB

	dbConnOnce.Do(func() {
		opts := DatabaseOptions{
			URL:                   db,
			MaxOpenConnections:    repoConfig.MaxOpenConnections,
			MaxIdleConnections:    repoConfig.MaxIdleConnections,
			ConnectionMaxLifetime: repoConfig.ConnectionMaxLifetime,
			ConnectionMaxIdleTime: repoConfig.ConnectionMaxIdleTime,
		}

		// TODO: Have separate DB handles for Writes and Reads
		// Optimize SQLite connection: https://kerkour.com/sqlite-for-servers
		connectionURLParams := make(url.Values)
		connectionURLParams.Add("_txlock", "immediate")
		connectionURLParams.Add("_journal_mode", "WAL")
		connectionURLParams.Add("_busy_timeout", "5000")
		connectionURLParams.Add("_synchronous", "NORMAL")
		connectionURLParams.Add("_cache_size", "1000000000")
		connectionURLParams.Add("_foreign_keys", "true")
		opts.URL = fmt.Sprintf("file:%s?%s", opts.URL, connectionURLParams.Encode())

		if cclog.Loglevel() == "debug" {
			sql.Register("sqlite3WithHooks", sqlhooks.Wrap(&sqlite3.SQLiteDriver{}, &Hooks{}))
			dbHandle, err = sqlx.Open("sqlite3WithHooks", opts.URL)
		} else {
			dbHandle, err = sqlx.Open("sqlite3", opts.URL)
		}

		if err != nil {
			cclog.Abortf("DB Connection: Could not connect to SQLite database with sqlx.Open().\nError: %s\n", err.Error())
		}

		err = setupSqlite(dbHandle.DB)
		if err != nil {
			cclog.Abortf("Failed sqlite db setup.\nError: %s\n", err.Error())
		}

		dbHandle.SetMaxOpenConns(opts.MaxOpenConnections)
		dbHandle.SetMaxIdleConns(opts.MaxIdleConnections)
		dbHandle.SetConnMaxLifetime(opts.ConnectionMaxLifetime)
		dbHandle.SetConnMaxIdleTime(opts.ConnectionMaxIdleTime)

		dbConnInstance = &DBConnection{DB: dbHandle}
		err = checkDBVersion(dbHandle.DB)
		if err != nil {
			cclog.Abortf("DB Connection: Failed DB version check.\nError: %s\n", err.Error())
		}
	})
}

func GetConnection() *DBConnection {
	if dbConnInstance == nil {
		cclog.Fatalf("Database connection not initialized!")
	}

	return dbConnInstance
}

// ResetConnection closes the current database connection and resets the connection state.
// This function is intended for testing purposes only to allow test isolation.
func ResetConnection() error {
	if dbConnInstance != nil && dbConnInstance.DB != nil {
		if err := dbConnInstance.DB.Close(); err != nil {
			return fmt.Errorf("failed to close database connection: %w", err)
		}
	}

	dbConnInstance = nil
	dbConnOnce = sync.Once{}
	jobRepoInstance = nil
	jobRepoOnce = sync.Once{}
	nodeRepoInstance = nil
	nodeRepoOnce = sync.Once{}
	userRepoInstance = nil
	userRepoOnce = sync.Once{}
	userCfgRepoInstance = nil
	userCfgRepoOnce = sync.Once{}

	return nil
}

-    connectionURLParams.Add("_cache_size", "1000000000")

+    connectionURLParams.Add("_cache_size", "-200000") // 200MB per connection (negative = KiB)


	pragmas := []string{
		"temp_store = memory",
	}

	pragmas := []string{
		"temp_store = memory",
		"soft_heap_limit = 1073741824", // 1GB process-wide soft limit; SQLite evicts cache pages to stay under
	}


	pragmas := []string{
		"temp_store = memory",
		"soft_heap_limit = 1073741824", // 1GB process-wide soft limit; SQLite evicts cache pages to stay under
	}

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package repository

import (
	"database/sql"
	"fmt"
	"net/url"
	"sync"
	"time"

	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
	"github.com/jmoiron/sqlx"
	"github.com/mattn/go-sqlite3"
	"github.com/qustavo/sqlhooks/v2"
)

var (
	dbConnOnce     sync.Once
	dbConnInstance *DBConnection
)

type DBConnection struct {
	DB     *sqlx.DB
	Driver string
}

type DatabaseOptions struct {
	URL                   string
	MaxOpenConnections    int
	MaxIdleConnections    int
	ConnectionMaxLifetime time.Duration
	ConnectionMaxIdleTime time.Duration
}

func setupSqlite(db *sql.DB) error {
	pragmas := []string{
		"temp_store = memory",
	}

	for _, pragma := range pragmas {
		_, err := db.Exec("PRAGMA " + pragma)
		if err != nil {
			return err
		}
	}

	// Update query planner statistics so SQLite picks optimal indexes.
	// Without this, SQLite guesses row distributions and often chooses wrong
	// indexes for queries with IN clauses + ORDER BY, causing full table sorts
	// in temp B-trees instead of using covering indexes.
	if _, err := db.Exec("ANALYZE"); err != nil {
		cclog.Warnf("Failed to run ANALYZE: %v", err)
	}

	return nil
}

func Connect(db string) {
	var err error
	var dbHandle *sqlx.DB

	dbConnOnce.Do(func() {
		opts := DatabaseOptions{
			URL:                   db,
			MaxOpenConnections:    repoConfig.MaxOpenConnections,
			MaxIdleConnections:    repoConfig.MaxIdleConnections,
			ConnectionMaxLifetime: repoConfig.ConnectionMaxLifetime,
			ConnectionMaxIdleTime: repoConfig.ConnectionMaxIdleTime,
		}

		// TODO: Have separate DB handles for Writes and Reads
		// Optimize SQLite connection: https://kerkour.com/sqlite-for-servers
		connectionURLParams := make(url.Values)
		connectionURLParams.Add("_txlock", "immediate")
		connectionURLParams.Add("_journal_mode", "WAL")
		connectionURLParams.Add("_busy_timeout", "5000")
		connectionURLParams.Add("_synchronous", "NORMAL")
		connectionURLParams.Add("_cache_size", "-200000") // 200MB per connection (negative = KiB)
		connectionURLParams.Add("_foreign_keys", "true")
		opts.URL = fmt.Sprintf("file:%s?%s", opts.URL, connectionURLParams.Encode())

		if cclog.Loglevel() == "debug" {
			sql.Register("sqlite3WithHooks", sqlhooks.Wrap(&sqlite3.SQLiteDriver{}, &Hooks{}))
			dbHandle, err = sqlx.Open("sqlite3WithHooks", opts.URL)
		} else {
			dbHandle, err = sqlx.Open("sqlite3", opts.URL)
		}

		if err != nil {
			cclog.Abortf("DB Connection: Could not connect to SQLite database with sqlx.Open().\nError: %s\n", err.Error())
		}

		err = setupSqlite(dbHandle.DB)
		if err != nil {
			cclog.Abortf("Failed sqlite db setup.\nError: %s\n", err.Error())
		}

		dbHandle.SetMaxOpenConns(opts.MaxOpenConnections)
		dbHandle.SetMaxIdleConns(opts.MaxIdleConnections)
		dbHandle.SetConnMaxLifetime(opts.ConnectionMaxLifetime)
		dbHandle.SetConnMaxIdleTime(opts.ConnectionMaxIdleTime)

		dbConnInstance = &DBConnection{DB: dbHandle}
		err = checkDBVersion(dbHandle.DB)
		if err != nil {
			cclog.Abortf("DB Connection: Failed DB version check.\nError: %s\n", err.Error())
		}
	})
}

func GetConnection() *DBConnection {
	if dbConnInstance == nil {
		cclog.Fatalf("Database connection not initialized!")
	}

	return dbConnInstance
}

// ResetConnection closes the current database connection and resets the connection state.
// This function is intended for testing purposes only to allow test isolation.
func ResetConnection() error {
	if dbConnInstance != nil && dbConnInstance.DB != nil {
		if err := dbConnInstance.DB.Close(); err != nil {
			return fmt.Errorf("failed to close database connection: %w", err)
		}
	}

	dbConnInstance = nil
	dbConnOnce = sync.Once{}
	jobRepoInstance = nil
	jobRepoOnce = sync.Once{}
	nodeRepoInstance = nil
	nodeRepoOnce = sync.Once{}
	userRepoInstance = nil
	userRepoOnce = sync.Once{}
	userCfgRepoInstance = nil
	userCfgRepoOnce = sync.Once{}

	return nil
}

+    "soft_heap_limit = 1073741824", // 1GB process-wide soft limit; SQLite evicts cache pages to stay under


**Fix 3: Pass context to database queries.**

`jobQuery.go` — `QueryJobs` and `CountJobs`:


rows, err := query.RunWith(r.stmtCache).Query()

rows, err := query.RunWith(r.stmtCache).QueryContext(ctx)


The file /Users/jan/prg/CC/cc-backend/internal/repository/jobQuery.go has been updated successfully.

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

// Package repository provides job query functionality with filtering, pagination,
// and security controls. This file contains the main query builders and security
// checks for job retrieval operations.
package repository

import (
	"context"
	"errors"
	"fmt"
	"regexp"
	"strings"
	"time"

	"github.com/ClusterCockpit/cc-backend/internal/config"
	"github.com/ClusterCockpit/cc-backend/internal/graph/model"
	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
	"github.com/ClusterCockpit/cc-lib/v2/schema"
	sq "github.com/Masterminds/squirrel"
)

const (
	// Default initial capacity for job result slices
	defaultJobsCapacity = 50
)

// QueryJobs retrieves jobs from the database with optional filtering, pagination,
// and sorting. Security controls are automatically applied based on the user context.
//
// Parameters:
//   - ctx: Context containing user authentication information
//   - filters: Optional job filters (cluster, state, user, time ranges, etc.)
//   - page: Optional pagination parameters (page number and items per page)
//   - order: Optional sorting specification (column or footprint field)
//
// Returns a slice of jobs matching the criteria, or an error if the query fails.
// The function enforces role-based access control through SecurityCheck.
func (r *JobRepository) QueryJobs(
	ctx context.Context,
	filters []*model.JobFilter,
	page *model.PageRequest,
	order *model.OrderByInput,
) ([]*schema.Job, error) {
	query, qerr := SecurityCheck(ctx, sq.Select(jobColumns...).From("job"))
	if qerr != nil {
		return nil, qerr
	}

	if order != nil {
		field := toSnakeCase(order.Field)
		if order.Type == "col" {
			switch order.Order {
			case model.SortDirectionEnumAsc:
				query = query.OrderBy(fmt.Sprintf("job.%s ASC", field))
			case model.SortDirectionEnumDesc:
				query = query.OrderBy(fmt.Sprintf("job.%s DESC", field))
			default:
				return nil, errors.New("invalid sorting order for column")
			}
		} else {
			// Order by footprint JSON field values
			query = query.Where("JSON_VALID(footprint)")
			switch order.Order {
			case model.SortDirectionEnumAsc:
				query = query.OrderBy(fmt.Sprintf("JSON_EXTRACT(footprint, \"$.%s\") ASC", field))
			case model.SortDirectionEnumDesc:
				query = query.OrderBy(fmt.Sprintf("JSON_EXTRACT(footprint, \"$.%s\") DESC", field))
			default:
				return nil, errors.New("invalid sorting order for footprint")
			}
		}
	}

	if page != nil && page.ItemsPerPage != -1 {
		limit := uint64(page.ItemsPerPage)
		query = query.Offset((uint64(page.Page) - 1) * limit).Limit(limit)
	}

	for _, f := range filters {
		query = BuildWhereClause(f, query)
	}

	rows, err := query.RunWith(r.stmtCache).Query()
	if err != nil {
		queryString, queryVars, _ := query.ToSql()
		return nil, fmt.Errorf("query failed [%s] %v: %w", queryString, queryVars, err)
	}
	defer rows.Close()

	jobs := make([]*schema.Job, 0, defaultJobsCapacity)
	for rows.Next() {
		job, err := scanJob(rows)
		if err != nil {
			cclog.Warnf("Error scanning job row: %v", err)
			return nil, fmt.Errorf("failed to scan job row: %w", err)
		}
		jobs = append(jobs, job)
	}

	if err := rows.Err(); err != nil {
		return nil, fmt.Errorf("error iterating job rows: %w", err)
	}

	return jobs, nil
}

// CountJobs returns the total number of jobs matching the given filters.
// Security controls are automatically applied based on the user context.
// Uses DISTINCT count to handle tag filters correctly (jobs may appear multiple
// times when joined with the tag table).
func (r *JobRepository) CountJobs(
	ctx context.Context,
	filters []*model.JobFilter,
) (int, error) {
	query, qerr := SecurityCheck(ctx, sq.Select("count(DISTINCT job.id)").From("job"))
	if qerr != nil {
		return 0, qerr
	}

	for _, f := range filters {
		query = BuildWhereClause(f, query)
	}

	var count int
	if err := query.RunWith(r.DB).Scan(&count); err != nil {
		return 0, fmt.Errorf("failed to count jobs: %w", err)
	}

	return count, nil
}

// SecurityCheckWithUser applies role-based access control filters to a job query
// based on the provided user's roles and permissions.
//
// Access rules by role:
//   - API role (exclusive): Full access to all jobs
//   - Admin/Support roles: Full access to all jobs
//   - Manager role: Access to jobs in managed projects plus own jobs
//   - User role: Access only to own jobs
//
// Returns an error if the user is nil or has no recognized roles.
func SecurityCheckWithUser(user *schema.User, query sq.SelectBuilder) (sq.SelectBuilder, error) {
	if user == nil {
		var qnil sq.SelectBuilder
		return qnil, fmt.Errorf("user context is nil")
	}

	switch {
	case len(user.Roles) == 1 && user.HasRole(schema.RoleAPI):
		return query, nil
	case user.HasAnyRole([]schema.Role{schema.RoleAdmin, schema.RoleSupport}):
		return query, nil
	case user.HasRole(schema.RoleManager):
		if len(user.Projects) != 0 {
			return query.Where(sq.Or{sq.Eq{"job.project": user.Projects}, sq.Eq{"job.hpc_user": user.Username}}), nil
		}
		cclog.Debugf("Manager '%s' has no assigned projects, restricting to personal jobs", user.Username)
		return query.Where("job.hpc_user = ?", user.Username), nil
	case user.HasRole(schema.RoleUser):
		return query.Where("job.hpc_user = ?", user.Username), nil
	default:
		var qnil sq.SelectBuilder
		return qnil, fmt.Errorf("user has no or unknown roles")
	}
}

// SecurityCheck extracts the user from the context and applies role-based access
// control filters to the query. This is a convenience wrapper around SecurityCheckWithUser.
func SecurityCheck(ctx context.Context, query sq.SelectBuilder) (sq.SelectBuilder, error) {
	user := GetUserFromContext(ctx)
	return SecurityCheckWithUser(user, query)
}

// BuildWhereClause constructs SQL WHERE conditions from a JobFilter and applies
// them to the query. Supports filtering by job properties (cluster, state, user),
// time ranges, resource usage, tags, and JSON field searches in meta_data,
// footprint, and resources columns.
func BuildWhereClause(filter *model.JobFilter, query sq.SelectBuilder) sq.SelectBuilder {
	// Primary Key
	if filter.DbID != nil {
		dbIDs := make([]string, len(filter.DbID))
		copy(dbIDs, filter.DbID)
		query = query.Where(sq.Eq{"job.id": dbIDs})
	}
	// Explicit indices
	if filter.Cluster != nil {
		query = buildStringCondition("job.cluster", filter.Cluster, query)
	}
	if filter.SubCluster != nil {
		query = buildStringCondition("job.subcluster", filter.SubCluster, query)
	}
	if filter.Partition != nil {
		query = buildStringCondition("job.cluster_partition", filter.Partition, query)
	}
	if filter.State != nil {
		states := make([]string, len(filter.State))
		for i, val := range filter.State {
			states[i] = string(val)
		}
		query = query.Where(sq.Eq{"job.job_state": states})
	}
	if filter.Shared != nil {
		query = query.Where("job.shared = ?", *filter.Shared)
	}
	if filter.Project != nil {
		query = buildStringCondition("job.project", filter.Project, query)
	}
	if filter.User != nil {
		query = buildStringCondition("job.hpc_user", filter.User, query)
	}
	if filter.NumNodes != nil {
		query = buildIntCondition("job.num_nodes", filter.NumNodes, query)
	}
	if filter.NumAccelerators != nil {
		query = buildIntCondition("job.num_acc", filter.NumAccelerators, query)
	}
	if filter.NumHWThreads != nil {
		query = buildIntCondition("job.num_hwthreads", filter.NumHWThreads, query)
	}
	if filter.ArrayJobID != nil {
		query = query.Where("job.array_job_id = ?", *filter.ArrayJobID)
	}
	if filter.StartTime != nil {
		query = buildTimeCondition("job.start_time", filter.StartTime, query)
	}
	if filter.Duration != nil {
		query = buildIntCondition("job.duration", filter.Duration, query)
	}
	if filter.Energy != nil {
		query = buildFloatCondition("job.energy", filter.Energy, query)
	}
	// Indices on Tag Table
	if filter.Tags != nil {
		// This is an OR-Logic query: Returns all distinct jobs with at least one of the requested tags; TODO: AND-Logic query?
		query = query.Join("jobtag ON jobtag.job_id = job.id").Where(sq.Eq{"jobtag.tag_id": filter.Tags}).Distinct()
	}
	// No explicit Indices
	if filter.JobID != nil {
		query = buildStringCondition("job.job_id", filter.JobID, query)
	}
	// Queries Within JSONs
	if filter.MetricStats != nil {
		for _, ms := range filter.MetricStats {
			query = buildFloatJSONCondition(ms.MetricName, ms.Range, query)
		}
	}
	if filter.Node != nil {
		query = buildResourceJSONCondition("hostname", filter.Node, query)
	}
	if filter.JobName != nil {
		query = buildMetaJSONCondition("jobName", filter.JobName, query)
	}
	if filter.Schedule != nil {
		interactiveJobname := "interactive"
		switch *filter.Schedule {
		case "interactive":
			iFilter := model.StringInput{Eq: &interactiveJobname}
			query = buildMetaJSONCondition("jobName", &iFilter, query)
		case "batch":
			sFilter := model.StringInput{Neq: &interactiveJobname}
			query = buildMetaJSONCondition("jobName", &sFilter, query)
		}
	}

	// Configurable Filter to exclude recently started jobs, see config.go: ShortRunningJobsDuration
	if filter.MinRunningFor != nil {
		now := time.Now().Unix()
		// Only jobs whose start timestamp is more than MinRunningFor seconds in the past
		// If a job completed within the configured timeframe, it will still show up after the start_time matches the condition!
		query = query.Where(sq.Lt{"job.start_time": (now - int64(*filter.MinRunningFor))})
	}
	return query
}

// buildIntCondition creates clauses for integer range filters, using BETWEEN only if required.
func buildIntCondition(field string, cond *config.IntRange, query sq.SelectBuilder) sq.SelectBuilder {
	if cond.From != 1 && cond.To != 0 {
		return query.Where(field+" BETWEEN ? AND ?", cond.From, cond.To)
	} else if cond.From != 1 && cond.To == 0 {
		return query.Where(field+" >= ?", cond.From)
	} else if cond.From == 1 && cond.To != 0 {
		return query.Where(field+" <= ?", cond.To)
	} else {
		return query
	}
}

// buildFloatCondition creates a clauses for float range filters, using BETWEEN only if required.
func buildFloatCondition(field string, cond *model.FloatRange, query sq.SelectBuilder) sq.SelectBuilder {
	if cond.From != 1.0 && cond.To != 0.0 {
		return query.Where(field+" BETWEEN ? AND ?", cond.From, cond.To)
	} else if cond.From != 1.0 && cond.To == 0.0 {
		return query.Where(field+" >= ?", cond.From)
	} else if cond.From == 1.0 && cond.To != 0.0 {
		return query.Where(field+" <= ?", cond.To)
	} else {
		return query
	}
}

// buildTimeCondition creates time range filters supporting absolute timestamps,
// relative time ranges (last6h, last24h, last7d, last30d), or open-ended ranges.
// Reminder: BETWEEN Queries are slower and dont use indices as frequently: Only use if both conditions required
func buildTimeCondition(field string, cond *config.TimeRange, query sq.SelectBuilder) sq.SelectBuilder {
	if cond.From != nil && cond.To != nil {
		return query.Where(field+" BETWEEN ? AND ?", cond.From.Unix(), cond.To.Unix())
	} else if cond.From != nil {
		return query.Where(field+" >= ?", cond.From.Unix())
	} else if cond.To != nil {
		return query.Where(field+" <= ?", cond.To.Unix())
	} else if cond.Range != "" {
		now := time.Now().Unix()
		var then int64
		switch cond.Range {
		case "last6h":
			then = now - (60 * 60 * 6)
		case "last24h":
			then = now - (60 * 60 * 24)
		case "last7d":
			then = now - (60 * 60 * 24 * 7)
		case "last30d":
			then = now - (60 * 60 * 24 * 30)
		default:
			cclog.Debugf("No known named timeRange: startTime.range = %s", cond.Range)
			return query
		}
		return query.Where(field+" >= ?", then)
	} else {
		return query
	}
}

// buildFloatJSONCondition creates a filter on a numeric field within the footprint JSON column, using BETWEEN only if required.
func buildFloatJSONCondition(jsonField string, cond *model.FloatRange, query sq.SelectBuilder) sq.SelectBuilder {
	query = query.Where("JSON_VALID(footprint)")
	if cond.From != 1.0 && cond.To != 0.0 {
		return query.Where("JSON_EXTRACT(footprint, \"$."+jsonField+"\") BETWEEN ? AND ?", cond.From, cond.To)
	} else if cond.From != 1.0 && cond.To == 0.0 {
		return query.Where("JSON_EXTRACT(footprint, \"$."+jsonField+"\") >= ?", cond.From)
	} else if cond.From == 1.0 && cond.To != 0.0 {
		return query.Where("JSON_EXTRACT(footprint, \"$."+jsonField+"\") <= ?", cond.To)
	} else {
		return query
	}
}

// buildStringCondition creates filters for string fields supporting equality,
// inequality, prefix, suffix, substring, and IN list matching.
func buildStringCondition(field string, cond *model.StringInput, query sq.SelectBuilder) sq.SelectBuilder {
	if cond.Eq != nil {
		return query.Where(field+" = ?", *cond.Eq)
	}
	if cond.Neq != nil {
		return query.Where(field+" != ?", *cond.Neq)
	}
	if cond.StartsWith != nil {
		return query.Where(field+" LIKE ?", fmt.Sprint(*cond.StartsWith, "%"))
	}
	if cond.EndsWith != nil {
		return query.Where(field+" LIKE ?", fmt.Sprint("%", *cond.EndsWith))
	}
	if cond.Contains != nil {
		return query.Where(field+" LIKE ?", fmt.Sprint("%", *cond.Contains, "%"))
	}
	if cond.In != nil {
		queryElements := make([]string, len(cond.In))
		copy(queryElements, cond.In)
		return query.Where(sq.Or{sq.Eq{field: queryElements}})
	}
	return query
}

// buildMetaJSONCondition creates filters on fields within the meta_data JSON column.
func buildMetaJSONCondition(jsonField string, cond *model.StringInput, query sq.SelectBuilder) sq.SelectBuilder {
	query = query.Where("JSON_VALID(meta_data)")
	if cond.Eq != nil {
		return query.Where("JSON_EXTRACT(meta_data, \"$."+jsonField+"\") = ?", *cond.Eq)
	}
	if cond.Neq != nil {
		return query.Where("JSON_EXTRACT(meta_data, \"$."+jsonField+"\") != ?", *cond.Neq)
	}
	if cond.StartsWith != nil {
		return query.Where("JSON_EXTRACT(meta_data, \"$."+jsonField+"\") LIKE ?", fmt.Sprint(*cond.StartsWith, "%"))
	}
	if cond.EndsWith != nil {
		return query.Where("JSON_EXTRACT(meta_data, \"$."+jsonField+"\") LIKE ?", fmt.Sprint("%", *cond.EndsWith))
	}
	if cond.Contains != nil {
		return query.Where("JSON_EXTRACT(meta_data, \"$."+jsonField+"\") LIKE ?", fmt.Sprint("%", *cond.Contains, "%"))
	}
	return query
}

// buildResourceJSONCondition creates filters on fields within the resources JSON array column.
// Uses json_each to search within array elements.
func buildResourceJSONCondition(jsonField string, cond *model.StringInput, query sq.SelectBuilder) sq.SelectBuilder {
	query = query.Where("JSON_VALID(resources)")
	if cond.Eq != nil {
		return query.Where("EXISTS (SELECT 1 FROM json_each(job.resources) WHERE json_extract(value, \"$."+jsonField+"\") = ?)", *cond.Eq)
	}
	if cond.Neq != nil { // Currently Unused
		return query.Where("EXISTS (SELECT 1 FROM json_each(job.resources) WHERE json_extract(value, \"$."+jsonField+"\") != ?)", *cond.Neq)
	}
	if cond.StartsWith != nil { // Currently Unused
		return query.Where("EXISTS (SELECT 1 FROM json_each(job.resources) WHERE json_extract(value, \"$."+jsonField+"\")) LIKE ?)", fmt.Sprint(*cond.StartsWith, "%"))
	}
	if cond.EndsWith != nil { // Currently Unused
		return query.Where("EXISTS (SELECT 1 FROM json_each(job.resources) WHERE json_extract(value, \"$."+jsonField+"\") LIKE ?)", fmt.Sprint("%", *cond.EndsWith))
	}
	if cond.Contains != nil {
		return query.Where("EXISTS (SELECT 1 FROM json_each(job.resources) WHERE json_extract(value, \"$."+jsonField+"\") LIKE ?)", fmt.Sprint("%", *cond.Contains, "%"))
	}
	return query
}

var (
	matchFirstCap = regexp.MustCompile("(.)([A-Z][a-z]+)")
	matchAllCap   = regexp.MustCompile("([a-z0-9])([A-Z])")
)

// toSnakeCase converts camelCase strings to snake_case for SQL column names.
// Includes security checks to prevent SQL injection attempts.
// Panics if potentially dangerous characters are detected.
func toSnakeCase(str string) string {
	for _, c := range str {
		if c == '\'' || c == '\\' || c == '"' || c == ';' || c == '-' || c == ' ' {
			cclog.Panicf("toSnakeCase: potentially dangerous character detected in input: %q", str)
		}
	}

	snake := matchFirstCap.ReplaceAllString(str, "${1}_${2}")
	snake = matchAllCap.ReplaceAllString(snake, "${1}_${2}")
	return strings.ToLower(snake)
}

-  rows, err := query.RunWith(r.stmtCache).Query()

+  rows, err := query.RunWith(r.stmtCache).QueryContext(ctx)


if err := query.RunWith(r.DB).Scan(&count); err != nil {

if err := query.RunWith(r.DB).QueryRowContext(ctx).Scan(&count); err != nil {