     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→package repository
     7→
     8→import (
     9→	"context"
    10→	"database/sql"
    11→	"fmt"
    12→	"time"
    13→
    14→	"github.com/ClusterCockpit/cc-backend/internal/graph/model"
    15→	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
    16→	"github.com/ClusterCockpit/cc-lib/v2/schema"
    17→	sq "github.com/Masterminds/squirrel"
    18→)
    19→
    20→// Find executes a SQL query to find a specific batch job.
    21→// The job is queried using the batch job id, the cluster name,
    22→// and the start time of the job in UNIX epoch time seconds.
    23→// It returns a pointer to a schema.Job data structure and an error variable.
    24→// To check if no job was found test err == sql.ErrNoRows
    25→func (r *JobRepository) Find(
    26→	jobID *int64,
    27→	cluster *string,
    28→	startTime *int64,
    29→) (*schema.Job, error) {
    30→	if jobID == nil {
    31→		return nil, fmt.Errorf("jobID cannot be nil")
    32→	}
    33→
    34→	start := time.Now()
    35→	q := sq.Select(jobColumns...).From("job").
    36→		Where("job.job_id = ?", *jobID)
    37→
    38→	if cluster != nil {
    39→		q = q.Where("job.cluster = ?", *cluster)
    40→	}
    41→	if startTime != nil {
    42→		q = q.Where("job.start_time = ?", *startTime)
    43→	}
    44→
    45→	q = q.OrderBy("job.id DESC").Limit(1) // always use newest matching job by db id if more than one match
    46→
    47→	cclog.Debugf("Timer Find %s", time.Since(start))
    48→	return scanJob(q.RunWith(r.stmtCache).QueryRow())
    49→}
    50→
    51→// FindCached executes a SQL query to find a specific batch job from the job_cache table.
    52→// The job is queried using the batch job id, and optionally filtered by cluster name
    53→// and start time (UNIX epoch time seconds). This method uses cached job data which
    54→// may be stale but provides faster access than Find().
    55→// It returns a pointer to a schema.Job data structure and an error variable.
    56→// To check if no job was found test err == sql.ErrNoRows
    57→func (r *JobRepository) FindCached(
    58→	jobID *int64,
    59→	cluster *string,
    60→	startTime *int64,
    61→) (*schema.Job, error) {
    62→	if jobID == nil {
    63→		return nil, fmt.Errorf("jobID cannot be nil")
    64→	}
    65→
    66→	q := sq.Select(jobCacheColumns...).From("job_cache").
    67→		Where("job_cache.job_id = ?", *jobID)
    68→
    69→	if cluster != nil {
    70→		q = q.Where("job_cache.cluster = ?", *cluster)
    71→	}
    72→	if startTime != nil {
    73→		q = q.Where("job_cache.start_time = ?", *startTime)
    74→	}
    75→
    76→	q = q.OrderBy("job_cache.id DESC").Limit(1) // always use newest matching job by db id if more than one match
    77→
    78→	return scanJob(q.RunWith(r.stmtCache).QueryRow())
    79→}
    80→
    81→// FindAll executes a SQL query to find all batch jobs matching the given criteria.
    82→// Jobs are queried using the batch job id, and optionally filtered by cluster name
    83→// and start time (UNIX epoch time seconds).
    84→// It returns a slice of pointers to schema.Job data structures and an error variable.
    85→// An empty slice is returned if no matching jobs are found.
    86→func (r *JobRepository) FindAll(
    87→	jobID *int64,
    88→	cluster *string,
    89→	startTime *int64,
    90→) ([]*schema.Job, error) {
    91→	if jobID == nil {
    92→		return nil, fmt.Errorf("jobID cannot be nil")
    93→	}
    94→
    95→	start := time.Now()
    96→	q := sq.Select(jobColumns...).From("job").
    97→		Where("job.job_id = ?", *jobID)
    98→
    99→	if cluster != nil {
   100→		q = q.Where("job.cluster = ?", *cluster)
   101→	}
   102→	if startTime != nil {
   103→		q = q.Where("job.start_time = ?", *startTime)
   104→	}
   105→
   106→	rows, err := q.RunWith(r.stmtCache).Query()
   107→	if err != nil {
   108→		cclog.Errorf("Error while running FindAll query for jobID=%d: %v", *jobID, err)
   109→		return nil, fmt.Errorf("failed to execute FindAll query: %w", err)
   110→	}
   111→	defer rows.Close()
   112→
   113→	jobs := make([]*schema.Job, 0, 10)
   114→	for rows.Next() {
   115→		job, err := scanJob(rows)
   116→		if err != nil {
   117→			cclog.Warnf("Error while scanning rows in FindAll: %v", err)
   118→			return nil, fmt.Errorf("failed to scan job row: %w", err)
   119→		}
   120→		jobs = append(jobs, job)
   121→	}
   122→	cclog.Debugf("Timer FindAll %s", time.Since(start))
   123→	return jobs, nil
   124→}
   125→
   126→// GetJobList returns job IDs for non-running jobs.
   127→// This is useful to process large job counts and intended to be used
   128→// together with FindById to process jobs one by one.
   129→// Use limit and offset for pagination. Use limit=0 to get all results (not recommended for large datasets).
   130→func (r *JobRepository) GetJobList(limit int, offset int) ([]int64, error) {
   131→	query := sq.Select("id").From("job").
   132→		Where("job.job_state != 'running'")
   133→
   134→	// Add pagination if limit is specified
   135→	if limit > 0 {
   136→		query = query.Limit(uint64(limit)).Offset(uint64(offset))
   137→	}
   138→
   139→	rows, err := query.RunWith(r.stmtCache).Query()
   140→	if err != nil {
   141→		cclog.Errorf("Error while running GetJobList query (limit=%d, offset=%d): %v", limit, offset, err)
   142→		return nil, fmt.Errorf("failed to execute GetJobList query: %w", err)
   143→	}
   144→	defer rows.Close()
   145→
   146→	jl := make([]int64, 0, 1000)
   147→	for rows.Next() {
   148→		var id int64
   149→		err := rows.Scan(&id)
   150→		if err != nil {
   151→			cclog.Warnf("Error while scanning rows in GetJobList: %v", err)
   152→			return nil, fmt.Errorf("failed to scan job ID: %w", err)
   153→		}
   154→		jl = append(jl, id)
   155→	}
   156→
   157→	cclog.Debugf("JobRepository.GetJobList(): Return job count %d", len(jl))
   158→	return jl, nil
   159→}
   160→
   161→// FindByID executes a SQL query to find a specific batch job.
   162→// The job is queried using the database id.
   163→// It returns a pointer to a schema.Job data structure and an error variable.
   164→// To check if no job was found test err == sql.ErrNoRows
   165→func (r *JobRepository) FindByID(ctx context.Context, jobID int64) (*schema.Job, error) {
   166→	q := sq.Select(jobColumns...).
   167→		From("job").Where("job.id = ?", jobID)
   168→
   169→	q, qerr := SecurityCheck(ctx, q)
   170→	if qerr != nil {
   171→		return nil, qerr
   172→	}
   173→
   174→	return scanJob(q.RunWith(r.stmtCache).QueryRow())
   175→}
   176→
   177→// FindByIDWithUser executes a SQL query to find a specific batch job.
   178→// The job is queried using the database id. The user is passed directly,
   179→// instead as part of the context.
   180→// It returns a pointer to a schema.Job data structure and an error variable.
   181→// To check if no job was found test err == sql.ErrNoRows
   182→func (r *JobRepository) FindByIDWithUser(user *schema.User, jobID int64) (*schema.Job, error) {
   183→	q := sq.Select(jobColumns...).
   184→		From("job").Where("job.id = ?", jobID)
   185→
   186→	q, qerr := SecurityCheckWithUser(user, q)
   187→	if qerr != nil {
   188→		return nil, qerr
   189→	}
   190→
   191→	return scanJob(q.RunWith(r.stmtCache).QueryRow())
   192→}
   193→
   194→// FindByIDDirect executes a SQL query to find a specific batch job.
   195→// The job is queried using the database id.
   196→// It returns a pointer to a schema.Job data structure and an error variable.
   197→// To check if no job was found test err == sql.ErrNoRows
   198→func (r *JobRepository) FindByIDDirect(jobID int64) (*schema.Job, error) {
   199→	q := sq.Select(jobColumns...).
   200→		From("job").Where("job.id = ?", jobID)
   201→	return scanJob(q.RunWith(r.stmtCache).QueryRow())
   202→}
   203→
   204→// FindByJobID executes a SQL query to find a specific batch job.
   205→// The job is queried using the slurm id and the clustername.
   206→// It returns a pointer to a schema.Job data structure and an error variable.
   207→// To check if no job was found test err == sql.ErrNoRows
   208→func (r *JobRepository) FindByJobID(ctx context.Context, jobID int64, startTime int64, cluster string) (*schema.Job, error) {
   209→	q := sq.Select(jobColumns...).
   210→		From("job").
   211→		Where("job.job_id = ?", jobID).
   212→		Where("job.cluster = ?", cluster).
   213→		Where("job.start_time = ?", startTime)
   214→
   215→	q, qerr := SecurityCheck(ctx, q)
   216→	if qerr != nil {
   217→		return nil, qerr
   218→	}
   219→
   220→	return scanJob(q.RunWith(r.stmtCache).QueryRow())
   221→}
   222→
   223→// IsJobOwner checks if the specified user owns the batch job identified by jobID,
   224→// startTime, and cluster. Returns true if the user is the owner, false otherwise.
   225→// This method does not return errors; it returns false for both non-existent jobs
   226→// and jobs owned by other users.
   227→func (r *JobRepository) IsJobOwner(jobID int64, startTime int64, user string, cluster string) bool {
   228→	q := sq.Select("id").
   229→		From("job").
   230→		Where("job.job_id = ?", jobID).
   231→		Where("job.hpc_user = ?", user).
   232→		Where("job.cluster = ?", cluster).
   233→		Where("job.start_time = ?", startTime)
   234→
   235→	_, err := scanJob(q.RunWith(r.stmtCache).QueryRow())
   236→	if err != nil && err != sql.ErrNoRows {
   237→		cclog.Warnf("IsJobOwner: unexpected error for jobID=%d, user=%s, cluster=%s: %v", jobID, user, cluster, err)
   238→	}
   239→	return err != sql.ErrNoRows
   240→}
   241→
   242→func (r *JobRepository) FindConcurrentJobs(
   243→	ctx context.Context,
   244→	job *schema.Job,
   245→) (*model.JobLinkResultList, error) {
   246→	if job == nil {
   247→		return nil, nil
   248→	}
   249→
   250→	query, qerr := SecurityCheck(ctx, sq.Select("job.id", "job.job_id", "job.start_time").From("job"))
   251→	if qerr != nil {
   252→		return nil, qerr
   253→	}
   254→
   255→	query = query.Where("cluster = ?", job.Cluster)
   256→
   257→	if len(job.Resources) == 0 {
   258→		return nil, fmt.Errorf("job has no resources defined")
   259→	}
   260→
   261→	var startTime int64
   262→	var stopTime int64
   263→
   264→	startTime = job.StartTime
   265→	hostname := job.Resources[0].Hostname
   266→
   267→	if job.State == schema.JobStateRunning {
   268→		stopTime = time.Now().Unix()
   269→	} else {
   270→		stopTime = startTime + int64(job.Duration)
   271→	}
   272→
   273→	// Time buffer constant for finding overlapping jobs
   274→	// overlapBufferEnd: 200s buffer at job end to account for scheduling/cleanup overlap
   275→	const overlapBufferEnd = 200
   276→
   277→	stopTimeTail := stopTime - overlapBufferEnd
   278→	startTimeFront := startTime + overlapBufferEnd
   279→
   280→	queryRunning := query.Where("job.job_state = ?", "running").
   281→		Where("job.start_time <= ?", stopTimeTail)
   282→	// Get At Least One Exact Hostname Match from JSON Resources Array in Database
   283→	queryRunning = queryRunning.Where("EXISTS (SELECT 1 FROM json_each(job.resources) WHERE json_extract(value, '$.hostname') = ?)", hostname)
   284→
   285→	query = query.Where("job.job_state != ?", "running").
   286→		Where("job.start_time < ?", stopTimeTail).
   287→		Where("(job.start_time + job.duration) > ?", startTimeFront)
   288→	// Get At Least One Exact Hostname Match from JSON Resources Array in Database
   289→	query = query.Where("EXISTS (SELECT 1 FROM json_each(job.resources) WHERE json_extract(value, '$.hostname') = ?)", hostname)
   290→
   291→	rows, err := query.RunWith(r.stmtCache).Query()
   292→	if err != nil {
   293→		cclog.Errorf("Error while running concurrent jobs query: %v", err)
   294→		return nil, fmt.Errorf("failed to execute concurrent jobs query: %w", err)
   295→	}
   296→	defer rows.Close()
   297→
   298→	items := make([]*model.JobLink, 0, 10)
   299→	queryString := fmt.Sprintf("cluster=%s", job.Cluster)
   300→
   301→	for rows.Next() {
   302→		var id, jobID, startTime sql.NullInt64
   303→
   304→		if err = rows.Scan(&id, &jobID, &startTime); err != nil {
   305→			cclog.Warnf("Error while scanning concurrent job rows: %v", err)
   306→			return nil, fmt.Errorf("failed to scan concurrent job row: %w", err)
   307→		}
   308→
   309→		if id.Valid {
   310→			queryString += fmt.Sprintf("&jobId=%d", int(jobID.Int64))
   311→			items = append(items,
   312→				&model.JobLink{
   313→					ID:    fmt.Sprint(id.Int64),
   314→					JobID: int(jobID.Int64),
   315→				})
   316→		}
   317→	}
   318→
   319→	rows, err = queryRunning.RunWith(r.stmtCache).Query()
   320→	if err != nil {
   321→		cclog.Errorf("Error while running concurrent running jobs query: %v", err)
   322→		return nil, fmt.Errorf("failed to execute concurrent running jobs query: %w", err)
   323→	}
   324→	defer rows.Close()
   325→
   326→	for rows.Next() {
   327→		var id, jobID, startTime sql.NullInt64
   328→
   329→		if err := rows.Scan(&id, &jobID, &startTime); err != nil {
   330→			cclog.Warnf("Error while scanning running concurrent job rows: %v", err)
   331→			return nil, fmt.Errorf("failed to scan running concurrent job row: %w", err)
   332→		}
   333→
   334→		if id.Valid {
   335→			queryString += fmt.Sprintf("&jobId=%d", int(jobID.Int64))
   336→			items = append(items,
   337→				&model.JobLink{
   338→					ID:    fmt.Sprint(id.Int64),
   339→					JobID: int(jobID.Int64),
   340→				})
   341→		}
   342→	}
   343→
   344→	cnt := len(items)
   345→
   346→	return &model.JobLinkResultList{
   347→		ListQuery: &queryString,
   348→		Items:     items,
   349→		Count:     &cnt,
   350→	}, nil
   351→}
   352→


<persisted-output>
Output too large (49.5KB). Full output saved to: /Users/jan/.claude/projects/-Users-jan-prg-CC-cc-backend/42401d2e-7d1c-4c0e-abe6-356cb2d48747/tool-results/toolu_01QZarGmibdGxEkEQSJcTph6.txt

Preview (first 2KB):
     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→// Package repository provides the data access layer for cc-backend using the repository pattern.
     7→//
     8→// The repository pattern abstracts database operations and provides a clean interface for
     9→// data access. Each major entity (Job, User, Node, Tag) has its own repository with CRUD
    10→// operations and specialized queries.
    11→//
    12→// # Database Connection
    13→//
    14→// Initialize the database connection before using any repository:
    15→//
    16→//	repository.Connect("sqlite3", "./var/job.db")
    17→//
    18→// # Configuration
    19→//
    20→// Optional: Configure repository settings before initialization:
    21→//
    22→//	repository.SetConfig(&repository.RepositoryConfig{
    23→//	    CacheSize: 2 * 1024 * 1024,     // 2MB cache
    24→//	    MaxOpenConnections: 8,           // Connection pool size
    25→//	    MinRunningJobDuration: 300,      // Filter threshold
    26→//	})
    27→//
    28→// If not configured, sensible defaults are used automatically.
    29→//
    30→// # Repositories
    31→//
    32→//   - JobRepository: Job lifecycle management and querying
    33→//   - UserRepository: User management and authentication
    34→//   - NodeRepository: Cluster node state tracking
    35→//   - Tags: Job tagging and categorization
    36→//
    37→// # Caching
    38→//
    39→// Repositories use LRU caching to improve performance. Cache keys are constructed
    40→// as "type:id" (e.g., "metadata:123"). Cache is automatically invalidated on
    41→// mutations to maintain consistency.
    42→//
    43→// # Transaction Support
    44→//
    45→// For batch operations, use transactions:
    46→//
    47→//	t, err := jobRepo.TransactionInit()
...
</persisted-output>


     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→// Package repository provides data access and persistence layer for ClusterCockpit.
     7→//
     8→// This file implements tag management functionality for job categorization and classification.
     9→// Tags support both manual assignment (via REST/GraphQL APIs) and automatic detection
    10→// (via tagger plugins). The implementation includes role-based access control through
    11→// tag scopes and maintains bidirectional consistency between the SQL database and
    12→// the file-based job archive.
    13→//
    14→// Database Schema:
    15→//
    16→//	CREATE TABLE tag (
    17→//	    id INTEGER PRIMARY KEY AUTOINCREMENT,
    18→//	    tag_type VARCHAR(255) NOT NULL,
    19→//	    tag_name VARCHAR(255) NOT NULL,
    20→//	    tag_scope VARCHAR(255) NOT NULL DEFAULT "global",
    21→//	    CONSTRAINT tag_unique UNIQUE (tag_type, tag_name, tag_scope)
    22→//	);
    23→//
    24→//	CREATE TABLE jobtag (
    25→//	    job_id INTEGER,
    26→//	    tag_id INTEGER,
    27→//	    PRIMARY KEY (job_id, tag_id),
    28→//	    FOREIGN KEY (job_id) REFERENCES job(id) ON DELETE CASCADE,
    29→//	    FOREIGN KEY (tag_id) REFERENCES tag(id) ON DELETE CASCADE
    30→//	);
    31→//
    32→// The jobtag junction table enables many-to-many relationships between jobs and tags.
    33→// CASCADE deletion ensures referential integrity when jobs or tags are removed.
    34→package repository
    35→
    36→import (
    37→	"errors"
    38→	"fmt"
    39→	"strings"
    40→
    41→	"github.com/ClusterCockpit/cc-backend/pkg/archive"
    42→	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
    43→	"github.com/ClusterCockpit/cc-lib/v2/schema"
    44→	sq "github.com/Masterminds/squirrel"
    45→)
    46→
    47→// Tag Scope Rules:
    48→//
    49→// Tags in ClusterCockpit have three visibility scopes that control who can see and use them:
    50→//
    51→//  1. "global" - Visible to all users, can be used by anyone
    52→//     Example: System-generated tags like "energy-efficient", "failed", "short"
    53→//
    54→//  2. "private" - Only visible to the creating user
    55→//     Example: Personal notes like "needs-review", "interesting-case"
    56→//
    57→//  3. "admin" - Only visible to users with admin or support roles
    58→//     Example: Internal notes like "hardware-issue", "billing-problem"
    59→//
    60→// Authorization Rules:
    61→//   - Regular users can only create/see "global" and their own "private" tags
    62→//   - Admin/Support can create/see all scopes including "admin" tags
    63→//   - Users can only add tags to jobs they have permission to view
    64→//   - Tag scope is enforced at query time in GetTags() and CountTags()
    65→
    66→// AddTag adds the tag with id `tagId` to the job with the database id `jobId`.
    67→// Requires user authentication for security checks.
    68→//
    69→// The user must have permission to view the job. Tag visibility is determined by scope:
    70→//   - "global" tags: visible to all users
    71→//   - "private" tags: only visible to the tag creator
    72→//   - "admin" tags: only visible to admin/support users
    73→func (r *JobRepository) AddTag(user *schema.User, job int64, tag int64) ([]*schema.Tag, error) {
    74→	j, err := r.FindByIDWithUser(user, job)
    75→	if err != nil {
    76→		cclog.Warnf("Error finding job %d for user %s: %v", job, user.Username, err)
    77→		return nil, err
    78→	}
    79→
    80→	return r.addJobTag(job, tag, j, func() ([]*schema.Tag, error) {
    81→		return r.GetTags(user, &job)
    82→	})
    83→}
    84→
    85→// AddTagDirect adds a tag without user security checks.
    86→// Use only for internal/admin operations.
    87→func (r *JobRepository) AddTagDirect(job int64, tag int64) ([]*schema.Tag, error) {
    88→	j, err := r.FindByIDDirect(job)
    89→	if err != nil {
    90→		cclog.Warnf("Error finding job %d: %v", job, err)
    91→		return nil, err
    92→	}
    93→
    94→	return r.addJobTag(job, tag, j, func() ([]*schema.Tag, error) {
    95→		return r.GetTagsDirect(&job)
    96→	})
    97→}
    98→
    99→// RemoveTag removes the tag with the database id `tag` from the job with the database id `job`.
   100→// Requires user authentication for security checks. Used by GraphQL API.
   101→func (r *JobRepository) RemoveTag(user *schema.User, job, tag int64) ([]*schema.Tag, error) {
   102→	j, err := r.FindByIDWithUser(user, job)
   103→	if err != nil {
   104→		cclog.Warnf("Error while finding job %d for user %s during tag removal: %v", job, user.Username, err)
   105→		return nil, err
   106→	}
   107→
   108→	q := sq.Delete("jobtag").Where("jobtag.job_id = ?", job).Where("jobtag.tag_id = ?", tag)
   109→
   110→	if _, err := q.RunWith(r.stmtCache).Exec(); err != nil {
   111→		s, _, _ := q.ToSql()
   112→		cclog.Errorf("Error removing tag with %s: %v", s, err)
   113→		return nil, err
   114→	}
   115→
   116→	tags, err := r.GetTags(user, &job)
   117→	if err != nil {
   118→		cclog.Warn("Error while getting tags for job")
   119→		return nil, err
   120→	}
   121→
   122→	archiveTags, err := r.getArchiveTags(&job)
   123→	if err != nil {
   124→		cclog.Warnf("Error while getting archive tags for job %d in RemoveTag: %v", job, err)
   125→		return nil, err
   126→	}
   127→
   128→	return tags, archive.UpdateTags(j, archiveTags)
   129→}
   130→
   131→// RemoveJobTagByRequest removes a tag from the job with the database id `job` by tag type, name, and scope.
   132→// Requires user authentication for security checks. Used by REST API.
   133→func (r *JobRepository) RemoveJobTagByRequest(user *schema.User, job int64, tagType string, tagName string, tagScope string) ([]*schema.Tag, error) {
   134→	// Get Tag ID to delete
   135→	tagID, exists := r.TagID(tagType, tagName, tagScope)
   136→	if !exists {
   137→		cclog.Warnf("Tag does not exist (name, type, scope): %s, %s, %s", tagName, tagType, tagScope)
   138→		return nil, fmt.Errorf("tag does not exist (name, type, scope): %s, %s, %s", tagName, tagType, tagScope)
   139→	}
   140→
   141→	// Get Job
   142→	j, err := r.FindByIDWithUser(user, job)
   143→	if err != nil {
   144→		cclog.Warnf("Error while finding job %d for user %s during tag removal by request: %v", job, user.Username, err)
   145→		return nil, err
   146→	}
   147→
   148→	// Handle Delete
   149→	q := sq.Delete("jobtag").Where("jobtag.job_id = ?", job).Where("jobtag.tag_id = ?", tagID)
   150→
   151→	if _, err := q.RunWith(r.stmtCache).Exec(); err != nil {
   152→		s, _, _ := q.ToSql()
   153→		cclog.Errorf("Error removing tag from table 'jobTag' with %s: %v", s, err)
   154→		return nil, err
   155→	}
   156→
   157→	tags, err := r.GetTags(user, &job)
   158→	if err != nil {
   159→		cclog.Warnf("Error while getting tags for job %d in RemoveJobTagByRequest: %v", job, err)
   160→		return nil, err
   161→	}
   162→
   163→	archiveTags, err := r.getArchiveTags(&job)
   164→	if err != nil {
   165→		cclog.Warnf("Error while getting archive tags for job %d in RemoveJobTagByRequest: %v", job, err)
   166→		return nil, err
   167→	}
   168→
   169→	return tags, archive.UpdateTags(j, archiveTags)
   170→}
   171→
   172→// removeTagFromArchiveJobs updates the job archive for all affected jobs after a tag deletion.
   173→//
   174→// This function is called asynchronously (via goroutine) after removing a tag from the database
   175→// to synchronize the file-based job archive with the database state. Errors are logged but not
   176→// returned since this runs in the background.
   177→//
   178→// Parameters:
   179→//   - jobIds: Database IDs of all jobs that had the deleted tag
   180→//
   181→// Implementation note: Each job is processed individually to handle partial failures gracefully.
   182→// If one job fails to update, others will still be processed.
   183→func (r *JobRepository) removeTagFromArchiveJobs(jobIds []int64) {
   184→	for _, j := range jobIds {
   185→		tags, err := r.getArchiveTags(&j)
   186→		if err != nil {
   187→			cclog.Warnf("Error while getting tags for job %d", j)
   188→			continue
   189→		}
   190→
   191→		job, err := r.FindByIDDirect(j)
   192→		if err != nil {
   193→			cclog.Warnf("Error while getting job %d", j)
   194→			continue
   195→		}
   196→
   197→		archive.UpdateTags(job, tags)
   198→	}
   199→}
   200→
   201→// Removes a tag from db by tag info
   202→// Used by REST API. Does not update tagged jobs in Job archive.
   203→func (r *JobRepository) RemoveTagByRequest(tagType string, tagName string, tagScope string) error {
   204→	// Get Tag ID to delete
   205→	tagID, exists := r.TagID(tagType, tagName, tagScope)
   206→	if !exists {
   207→		cclog.Warnf("Tag does not exist (name, type, scope): %s, %s, %s", tagName, tagType, tagScope)
   208→		return fmt.Errorf("tag does not exist (name, type, scope): %s, %s, %s", tagName, tagType, tagScope)
   209→	}
   210→
   211→	return r.RemoveTagByID(tagID)
   212→}
   213→
   214→// Removes a tag from db by tag id
   215→// Used by GraphQL API.
   216→func (r *JobRepository) RemoveTagByID(tagID int64) error {
   217→	jobIds, err := r.FindJobIdsByTag(tagID)
   218→	if err != nil {
   219→		return err
   220→	}
   221→
   222→	// Handle Delete JobTagTable
   223→	qJobTag := sq.Delete("jobtag").Where("jobtag.tag_id = ?", tagID)
   224→
   225→	if _, err := qJobTag.RunWith(r.stmtCache).Exec(); err != nil {
   226→		s, _, _ := qJobTag.ToSql()
   227→		cclog.Errorf("Error removing tag from table 'jobTag' with %s: %v", s, err)
   228→		return err
   229→	}
   230→
   231→	// Handle Delete TagTable
   232→	qTag := sq.Delete("tag").Where("tag.id = ?", tagID)
   233→
   234→	if _, err := qTag.RunWith(r.stmtCache).Exec(); err != nil {
   235→		s, _, _ := qTag.ToSql()
   236→		cclog.Errorf("Error removing tag from table 'tag' with %s: %v", s, err)
   237→		return err
   238→	}
   239→
   240→	// asynchronously update archive jobs
   241→	go r.removeTagFromArchiveJobs(jobIds)
   242→
   243→	return nil
   244→}
   245→
   246→// CreateTag creates a new tag with the specified type, name, and scope.
   247→// Returns the database ID of the newly created tag.
   248→//
   249→// Scope defaults to "global" if empty string is provided.
   250→// Valid scopes: "global", "private", "admin"
   251→//
   252→// Example:
   253→//
   254→//	tagID, err := repo.CreateTag("performance", "high-memory", "global")
   255→func (r *JobRepository) CreateTag(tagType string, tagName string, tagScope string) (tagID int64, err error) {
   256→	// Default to "Global" scope if none defined
   257→	if tagScope == "" {
   258→		tagScope = "global"
   259→	}
   260→
   261→	q := sq.Insert("tag").Columns("tag_type", "tag_name", "tag_scope").Values(tagType, tagName, tagScope)
   262→
   263→	res, err := q.RunWith(r.stmtCache).Exec()
   264→	if err != nil {
   265→		s, _, _ := q.ToSql()
   266→		cclog.Errorf("Error inserting tag with %s: %v", s, err)
   267→		return 0, err
   268→	}
   269→
   270→	return res.LastInsertId()
   271→}
   272→
   273→// CountTags returns all tags visible to the user and the count of jobs for each tag.
   274→// Applies scope-based filtering to respect tag visibility rules.
   275→//
   276→// Returns:
   277→//   - tags: slice of tags the user can see
   278→//   - counts: map of tag name to job count
   279→//   - err: any error encountered
   280→func (r *JobRepository) CountTags(user *schema.User) (tags []schema.Tag, counts map[string]int, err error) {
   281→	tags = make([]schema.Tag, 0, 100)
   282→	xrows, err := r.DB.Queryx("SELECT id, tag_type, tag_name, tag_scope FROM tag")
   283→	if err != nil {
   284→		return nil, nil, err
   285→	}
   286→	defer xrows.Close()
   287→
   288→	for xrows.Next() {
   289→		var t schema.Tag
   290→		if err = xrows.StructScan(&t); err != nil {
   291→			return nil, nil, err
   292→		}
   293→
   294→		// Handle Scope Filtering: Tag Scope is Global, Private (== Username) or User is auth'd to view Admin Tags
   295→		readable, err := r.checkScopeAuth(user, "read", t.Scope)
   296→		if err != nil {
   297→			return nil, nil, err
   298→		}
   299→		if readable {
   300→			tags = append(tags, t)
   301→		}
   302→	}
   303→
   304→	if err := xrows.Err(); err != nil {
   305→		return nil, nil, err
   306→	}
   307→
   308→	// Query and Count Jobs with attached Tags
   309→	q := sq.Select("t.tag_type, t.tag_name, t.id, count(jt.tag_id)").
   310→		From("tag t").
   311→		LeftJoin("jobtag jt ON t.id = jt.tag_id").
   312→		GroupBy("t.tag_type, t.tag_name")
   313→
   314→	// Build scope list for filtering
   315→	var scopeBuilder strings.Builder
   316→	scopeBuilder.WriteString(`"global"`)
   317→	if user != nil {
   318→		scopeBuilder.WriteString(`,"`)
   319→		scopeBuilder.WriteString(user.Username)
   320→		scopeBuilder.WriteString(`"`)
   321→		if user.HasAnyRole([]schema.Role{schema.RoleAdmin, schema.RoleSupport}) {
   322→			scopeBuilder.WriteString(`,"admin"`)
   323→		}
   324→	}
   325→	q = q.Where("t.tag_scope IN (" + scopeBuilder.String() + ")")
   326→
   327→	// Handle Job Ownership
   328→	if user != nil && user.HasAnyRole([]schema.Role{schema.RoleAdmin, schema.RoleSupport}) { // ADMIN || SUPPORT: Count all jobs
   329→		// cclog.Debug("CountTags: User Admin or Support -> Count all Jobs for Tags")
   330→		// Unchanged: Needs to be own case still, due to UserRole/NoRole compatibility handling in else case
   331→	} else if user != nil && user.HasRole(schema.RoleManager) { // MANAGER: Count own jobs plus project's jobs
   332→		// Build ("project1", "project2", ...) list of variable length directly in SQL string
   333→		q = q.Where("jt.job_id IN (SELECT id FROM job WHERE job.hpc_user = ? OR job.project IN (\""+strings.Join(user.Projects, "\",\"")+"\"))", user.Username)
   334→	} else if user != nil { // USER OR NO ROLE (Compatibility): Only count own jobs
   335→		q = q.Where("jt.job_id IN (SELECT id FROM job WHERE job.hpc_user = ?)", user.Username)
   336→	}
   337→
   338→	rows, err := q.RunWith(r.stmtCache).Query()
   339→	if err != nil {
   340→		return nil, nil, err
   341→	}
   342→	defer rows.Close()
   343→
   344→	counts = make(map[string]int)
   345→	for rows.Next() {
   346→		var tagType string
   347→		var tagName string
   348→		var tagID int
   349→		var count int
   350→		if err = rows.Scan(&tagType, &tagName, &tagID, &count); err != nil {
   351→			return nil, nil, err
   352→		}
   353→		// Use tagId as second Map-Key component to differentiate tags with identical names
   354→		counts[fmt.Sprint(tagType, tagName, tagID)] = count
   355→	}
   356→	err = rows.Err()
   357→
   358→	return tags, counts, err
   359→}
   360→
   361→var (
   362→	// ErrTagNotFound is returned when a tag ID or tag identifier (type, name, scope) does not exist in the database.
   363→	ErrTagNotFound = errors.New("the tag does not exist")
   364→
   365→	// ErrJobNotOwned is returned when a user attempts to tag a job they do not have permission to access.
   366→	ErrJobNotOwned = errors.New("user is not owner of job")
   367→
   368→	// ErrTagNoAccess is returned when a user attempts to use a tag they cannot access due to scope restrictions.
   369→	ErrTagNoAccess = errors.New("user not permitted to use that tag")
   370→
   371→	// ErrTagPrivateScope is returned when a user attempts to access another user's private tag.
   372→	ErrTagPrivateScope = errors.New("tag is private to another user")
   373→
   374→	// ErrTagAdminScope is returned when a non-admin user attempts to use an admin-scoped tag.
   375→	ErrTagAdminScope = errors.New("tag requires admin privileges")
   376→
   377→	// ErrTagsIncompatScopes is returned when attempting to combine admin and non-admin scoped tags in a single operation.
   378→	ErrTagsIncompatScopes = errors.New("combining admin and non-admin scoped tags not allowed")
   379→)
   380→
   381→// addJobTag is a helper function that inserts a job-tag association and updates the archive.
   382→//
   383→// This function performs three operations atomically:
   384→//  1. Inserts the job-tag association into the jobtag junction table
   385→//  2. Retrieves the updated tag list for the job (using the provided getTags callback)
   386→//  3. Updates the job archive with the new tags to maintain database-archive consistency
   387→//
   388→// Parameters:
   389→//   - jobId: Database ID of the job
   390→//   - tagId: Database ID of the tag to associate
   391→//   - job: Full job object needed for archive update
   392→//   - getTags: Callback function to retrieve updated tags (allows different security contexts)
   393→//
   394→// Returns the complete updated tag list for the job or an error.
   395→//
   396→// Note: This function does NOT validate tag scope permissions - callers must perform
   397→// authorization checks before invoking this helper.
   398→func (r *JobRepository) addJobTag(jobID int64, tagID int64, job *schema.Job, getTags func() ([]*schema.Tag, error)) ([]*schema.Tag, error) {
   399→	q := sq.Insert("jobtag").Columns("job_id", "tag_id").Values(jobID, tagID)
   400→
   401→	if _, err := q.RunWith(r.stmtCache).Exec(); err != nil {
   402→		s, _, _ := q.ToSql()
   403→		cclog.Errorf("Error adding tag with %s: %v", s, err)
   404→		return nil, err
   405→	}
   406→
   407→	tags, err := getTags()
   408→	if err != nil {
   409→		cclog.Warnf("Error getting tags for job %d: %v", jobID, err)
   410→		return nil, err
   411→	}
   412→
   413→	archiveTags, err := r.getArchiveTags(&jobID)
   414→	if err != nil {
   415→		cclog.Warnf("Error getting archive tags for job %d: %v", jobID, err)
   416→		return nil, err
   417→	}
   418→
   419→	return tags, archive.UpdateTags(job, archiveTags)
   420→}
   421→
   422→// AddTagOrCreate adds the tag with the specified type and name to the job with the database id `jobId`.
   423→// If such a tag does not yet exist, it is created.
   424→func (r *JobRepository) AddTagOrCreate(user *schema.User, jobID int64, tagType string, tagName string, tagScope string) (tagID int64, err error) {
   425→	// Default to "Global" scope if none defined
   426→	if tagScope == "" {
   427→		tagScope = "global"
   428→	}
   429→
   430→	writable, err := r.checkScopeAuth(user, "write", tagScope)
   431→	if err != nil {
   432→		return 0, err
   433→	}
   434→	if !writable {
   435→		return 0, fmt.Errorf("cannot write tag scope with current authorization")
   436→	}
   437→
   438→	tagID, exists := r.TagID(tagType, tagName, tagScope)
   439→	if !exists {
   440→		tagID, err = r.CreateTag(tagType, tagName, tagScope)
   441→		if err != nil {
   442→			return 0, err
   443→		}
   444→	}
   445→
   446→	if _, err := r.AddTag(user, jobID, tagID); err != nil {
   447→		return 0, err
   448→	}
   449→
   450→	return tagID, nil
   451→}
   452→
   453→func (r *JobRepository) AddTagOrCreateDirect(jobID int64, tagType string, tagName string) (tagID int64, err error) {
   454→	tagScope := "global"
   455→
   456→	tagID, exists := r.TagID(tagType, tagName, tagScope)
   457→	if !exists {
   458→		tagID, err = r.CreateTag(tagType, tagName, tagScope)
   459→		if err != nil {
   460→			return 0, err
   461→		}
   462→	}
   463→
   464→	cclog.Infof("Adding tag %s:%s:%s (direct)", tagType, tagName, tagScope)
   465→
   466→	if _, err := r.AddTagDirect(jobID, tagID); err != nil {
   467→		return 0, err
   468→	}
   469→
   470→	return tagID, nil
   471→}
   472→
   473→func (r *JobRepository) HasTag(jobID int64, tagType string, tagName string) bool {
   474→	var id int64
   475→	q := sq.Select("id").From("tag").Join("jobtag ON jobtag.tag_id = tag.id").
   476→		Where("jobtag.job_id = ?", jobID).Where("tag.tag_type = ?", tagType).
   477→		Where("tag.tag_name = ?", tagName)
   478→	err := q.RunWith(r.stmtCache).QueryRow().Scan(&id)
   479→	if err != nil {
   480→		return false
   481→	} else {
   482→		return true
   483→	}
   484→}
   485→
   486→// TagID returns the database id of the tag with the specified type and name.
   487→func (r *JobRepository) TagID(tagType string, tagName string, tagScope string) (tagID int64, exists bool) {
   488→	exists = true
   489→	if err := sq.Select("id").From("tag").
   490→		Where("tag.tag_type = ?", tagType).Where("tag.tag_name = ?", tagName).Where("tag.tag_scope = ?", tagScope).
   491→		RunWith(r.stmtCache).QueryRow().Scan(&tagID); err != nil {
   492→		exists = false
   493→	}
   494→	return
   495→}
   496→
   497→// TagInfo returns the database infos of the tag with the specified id.
   498→func (r *JobRepository) TagInfo(tagID int64) (tagType string, tagName string, tagScope string, exists bool) {
   499→	exists = true
   500→	if err := sq.Select("tag.tag_type", "tag.tag_name", "tag.tag_scope").From("tag").Where("tag.id = ?", tagID).
   501→		RunWith(r.stmtCache).QueryRow().Scan(&tagType, &tagName, &tagScope); err != nil {
   502→		exists = false
   503→	}
   504→	return
   505→}
   506→
   507→// GetTags returns a list of all scoped tags if job is nil or of the tags that the job with that database ID has.
   508→func (r *JobRepository) GetTags(user *schema.User, job *int64) ([]*schema.Tag, error) {
   509→	q := sq.Select("id", "tag_type", "tag_name", "tag_scope").From("tag")
   510→	if job != nil {
   511→		q = q.Join("jobtag ON jobtag.tag_id = tag.id").Where("jobtag.job_id = ?", *job)
   512→	}
   513→
   514→	rows, err := q.RunWith(r.stmtCache).Query()
   515→	if err != nil {
   516→		s, _, _ := q.ToSql()
   517→		cclog.Errorf("Error get tags with %s: %v", s, err)
   518→		return nil, err
   519→	}
   520→	defer rows.Close()
   521→
   522→	tags := make([]*schema.Tag, 0)
   523→	for rows.Next() {
   524→		tag := &schema.Tag{}
   525→		if err := rows.Scan(&tag.ID, &tag.Type, &tag.Name, &tag.Scope); err != nil {
   526→			cclog.Warnf("Error while scanning tag rows in GetTags: %v", err)
   527→			return nil, err
   528→		}
   529→		// Handle Scope Filtering: Tag Scope is Global, Private (== Username) or User is auth'd to view Admin Tags
   530→		readable, err := r.checkScopeAuth(user, "read", tag.Scope)
   531→		if err != nil {
   532→			return nil, err
   533→		}
   534→		if readable {
   535→			tags = append(tags, tag)
   536→		}
   537→	}
   538→
   539→	if err := rows.Err(); err != nil {
   540→		return nil, err
   541→	}
   542→
   543→	return tags, nil
   544→}
   545→
   546→func (r *JobRepository) GetTagsDirect(job *int64) ([]*schema.Tag, error) {
   547→	q := sq.Select("id", "tag_type", "tag_name", "tag_scope").From("tag")
   548→	if job != nil {
   549→		q = q.Join("jobtag ON jobtag.tag_id = tag.id").Where("jobtag.job_id = ?", *job)
   550→	}
   551→
   552→	rows, err := q.RunWith(r.stmtCache).Query()
   553→	if err != nil {
   554→		s, _, _ := q.ToSql()
   555→		cclog.Errorf("Error get tags with %s: %v", s, err)
   556→		return nil, err
   557→	}
   558→	defer rows.Close()
   559→
   560→	tags := make([]*schema.Tag, 0)
   561→	for rows.Next() {
   562→		tag := &schema.Tag{}
   563→		if err := rows.Scan(&tag.ID, &tag.Type, &tag.Name, &tag.Scope); err != nil {
   564→			cclog.Warnf("Error while scanning tag rows in GetTagsDirect: %v", err)
   565→			return nil, err
   566→		}
   567→		tags = append(tags, tag)
   568→	}
   569→
   570→	if err := rows.Err(); err != nil {
   571→		return nil, err
   572→	}
   573→
   574→	return tags, nil
   575→}
   576→
   577→// getArchiveTags returns all tags for a job WITHOUT applying scope-based filtering.
   578→//
   579→// This internal function is used exclusively for job archive synchronization where we need
   580→// to store all tags regardless of the current user's permissions. Unlike GetTags() which
   581→// filters by scope, this returns the complete unfiltered tag list.
   582→//
   583→// Parameters:
   584→//   - job: Pointer to job database ID, or nil to return all tags in the system
   585→//
   586→// Returns all tags without scope filtering, used only for archive operations.
   587→//
   588→// WARNING: Do NOT expose this function to user-facing APIs as it bypasses authorization.
   589→func (r *JobRepository) getArchiveTags(job *int64) ([]*schema.Tag, error) {
   590→	q := sq.Select("id", "tag_type", "tag_name", "tag_scope").From("tag")
   591→	if job != nil {
   592→		q = q.Join("jobtag ON jobtag.tag_id = tag.id").Where("jobtag.job_id = ?", *job)
   593→	}
   594→
   595→	rows, err := q.RunWith(r.stmtCache).Query()
   596→	if err != nil {
   597→		s, _, _ := q.ToSql()
   598→		cclog.Errorf("Error get tags with %s: %v", s, err)
   599→		return nil, err
   600→	}
   601→	defer rows.Close()
   602→
   603→	tags := make([]*schema.Tag, 0)
   604→	for rows.Next() {
   605→		tag := &schema.Tag{}
   606→		if err := rows.Scan(&tag.ID, &tag.Type, &tag.Name, &tag.Scope); err != nil {
   607→			cclog.Warnf("Error while scanning tag rows in getArchiveTags: %v", err)
   608→			return nil, err
   609→		}
   610→		tags = append(tags, tag)
   611→	}
   612→
   613→	if err := rows.Err(); err != nil {
   614→		return nil, err
   615→	}
   616→
   617→	return tags, nil
   618→}
   619→
   620→func (r *JobRepository) ImportTag(jobID int64, tagType string, tagName string, tagScope string) (err error) {
   621→	// Import has no scope ctx, only import from metafile to DB (No recursive archive update required), only returns err
   622→
   623→	tagID, exists := r.TagID(tagType, tagName, tagScope)
   624→	if !exists {
   625→		tagID, err = r.CreateTag(tagType, tagName, tagScope)
   626→		if err != nil {
   627→			return err
   628→		}
   629→	}
   630→
   631→	q := sq.Insert("jobtag").Columns("job_id", "tag_id").Values(jobID, tagID)
   632→
   633→	if _, err := q.RunWith(r.stmtCache).Exec(); err != nil {
   634→		s, _, _ := q.ToSql()
   635→		cclog.Errorf("Error adding tag on import with %s: %v", s, err)
   636→		return err
   637→	}
   638→
   639→	return nil
   640→}
   641→
   642→// checkScopeAuth validates whether a user is authorized to perform an operation on a tag with the given scope.
   643→//
   644→// This function implements the tag scope authorization matrix:
   645→//
   646→//	Scope        | Read Access                      | Write Access
   647→//	-------------|----------------------------------|----------------------------------
   648→//	"global"     | All users                        | Admin, Support, API-only
   649→//	"admin"      | Admin, Support                   | Admin, API-only
   650→//	<username>   | Owner only                       | Owner only (private tags)
   651→//
   652→// Parameters:
   653→//   - user: User attempting the operation (must not be nil)
   654→//   - operation: Either "read" or "write"
   655→//   - scope: Tag scope value ("global", "admin", or username for private tags)
   656→//
   657→// Returns:
   658→//   - pass: true if authorized, false if denied
   659→//   - err: error only if operation is invalid or user is nil
   660→//
   661→// Special cases:
   662→//   - API-only users (single role: RoleApi) can write to admin and global scopes for automation
   663→//   - Private tags use the username as scope, granting exclusive access to that user
   664→func (r *JobRepository) checkScopeAuth(user *schema.User, operation string, scope string) (pass bool, err error) {
   665→	if user != nil {
   666→		switch {
   667→		case operation == "write" && scope == "admin":
   668→			if user.HasRole(schema.RoleAdmin) || (len(user.Roles) == 1 && user.HasRole(schema.RoleAPI)) {
   669→				return true, nil
   670→			}
   671→			return false, nil
   672→		case operation == "write" && scope == "global":
   673→			if user.HasAnyRole([]schema.Role{schema.RoleAdmin, schema.RoleSupport}) || (len(user.Roles) == 1 && user.HasRole(schema.RoleAPI)) {
   674→				return true, nil
   675→			}
   676→			return false, nil
   677→		case operation == "write" && scope == user.Username:
   678→			return true, nil
   679→		case operation == "read" && scope == "admin":
   680→			return user.HasAnyRole([]schema.Role{schema.RoleAdmin, schema.RoleSupport}), nil
   681→		case operation == "read" && scope == "global":
   682→			return true, nil
   683→		case operation == "read" && scope == user.Username:
   684→			return true, nil
   685→		default:
   686→			if operation == "read" || operation == "write" {
   687→				// No acceptable scope: deny tag
   688→				return false, nil
   689→			} else {
   690→				return false, fmt.Errorf("error while checking tag operation auth: unknown operation (%s)", operation)
   691→			}
   692→		}
   693→	} else {
   694→		return false, fmt.Errorf("error while checking tag operation auth: no user in context")
   695→	}
   696→}
   697→


     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→package repository
     7→
     8→import (
     9→	"database/sql"
    10→	"embed"
    11→	"fmt"
    12→
    13→	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
    14→	"github.com/golang-migrate/migrate/v4"
    15→	"github.com/golang-migrate/migrate/v4/database/sqlite3"
    16→	"github.com/golang-migrate/migrate/v4/source/iofs"
    17→)
    18→
    19→// Version is the current database schema version required by this version of cc-backend.
    20→// When the database schema changes, this version is incremented and a new migration file
    21→// is added to internal/repository/migrations/sqlite3/.
    22→//
    23→// Version history:
    24→//   - Version 10: Current version
    25→//
    26→// Migration files are embedded at build time from the migrations directory.
    27→const Version uint = 10
    28→
    29→//go:embed migrations/*
    30→var migrationFiles embed.FS
    31→
    32→// checkDBVersion verifies that the database schema version matches the expected version.
    33→// This is called automatically during Connect() to ensure schema compatibility.
    34→//
    35→// Returns an error if:
    36→//   - Database version is older than expected (needs migration)
    37→//   - Database version is newer than expected (needs app upgrade)
    38→//   - Database is in a dirty state (failed migration)
    39→//
    40→// A "dirty" database indicates a migration was started but not completed successfully.
    41→// This requires manual intervention to fix the database and force the version.
    42→func checkDBVersion(db *sql.DB) error {
    43→	driver, err := sqlite3.WithInstance(db, &sqlite3.Config{})
    44→	if err != nil {
    45→		return err
    46→	}
    47→	d, err := iofs.New(migrationFiles, "migrations/sqlite3")
    48→	if err != nil {
    49→		return err
    50→	}
    51→
    52→	m, err := migrate.NewWithInstance("iofs", d, "sqlite3", driver)
    53→	if err != nil {
    54→		return err
    55→	}
    56→
    57→	v, dirty, err := m.Version()
    58→	if err != nil {
    59→		if err == migrate.ErrNilVersion {
    60→			cclog.Warn("Legacy database without version or missing database file!")
    61→		} else {
    62→			return err
    63→		}
    64→	}
    65→
    66→	if v < Version {
    67→		return fmt.Errorf("unsupported database version %d, need %d.\nPlease backup your database file and run cc-backend -migrate-db", v, Version)
    68→	} else if v > Version {
    69→		return fmt.Errorf("unsupported database version %d, need %d.\nPlease refer to documentation how to downgrade db with external migrate tool", v, Version)
    70→	}
    71→
    72→	if dirty {
    73→		return fmt.Errorf("last migration to version %d has failed, please fix the db manually and force version with -force-db flag", Version)
    74→	}
    75→
    76→	return nil
    77→}
    78→
    79→// getMigrateInstance creates a new migration instance for the given database file.
    80→// This is used internally by MigrateDB, RevertDB, and ForceDB.
    81→func getMigrateInstance(db string) (m *migrate.Migrate, err error) {
    82→	d, err := iofs.New(migrationFiles, "migrations/sqlite3")
    83→	if err != nil {
    84→		return nil, err
    85→	}
    86→
    87→	m, err = migrate.NewWithSourceInstance("iofs", d, fmt.Sprintf("sqlite3://%s?_foreign_keys=on", db))
    88→	if err != nil {
    89→		return nil, err
    90→	}
    91→
    92→	return m, nil
    93→}
    94→
    95→// MigrateDB applies all pending database migrations to bring the schema up to date.
    96→// This should be run with the -migrate-db flag before starting the application
    97→// after upgrading to a new version that requires schema changes.
    98→//
    99→// Process:
   100→//  1. Checks current database version
   101→//  2. Applies all migrations from current version to target Version
   102→//  3. Updates schema_migrations table to track applied migrations
   103→//
   104→// Important:
   105→//   - Always backup your database before running migrations
   106→//   - Migrations are irreversible without manual intervention
   107→//   - If a migration fails, the database is marked "dirty" and requires manual fix
   108→//
   109→// Usage:
   110→//
   111→//	cc-backend -migrate-db
   112→func MigrateDB(db string) error {
   113→	m, err := getMigrateInstance(db)
   114→	if err != nil {
   115→		return err
   116→	}
   117→
   118→	v, dirty, err := m.Version()
   119→	if err != nil {
   120→		if err == migrate.ErrNilVersion {
   121→			cclog.Info("Legacy database without version or missing database file!")
   122→		} else {
   123→			return err
   124→		}
   125→	}
   126→
   127→	if v < Version {
   128→		cclog.Infof("unsupported database version %d, need %d.\nPlease backup your database file and run cc-backend -migrate-db", v, Version)
   129→	}
   130→
   131→	if dirty {
   132→		return fmt.Errorf("last migration to version %d has failed, please fix the db manually and force version with -force-db flag", Version)
   133→	}
   134→
   135→	if err := m.Up(); err != nil {
   136→		if err == migrate.ErrNoChange {
   137→			cclog.Info("DB already up to date!")
   138→		} else {
   139→			return err
   140→		}
   141→	}
   142→
   143→	m.Close()
   144→	return nil
   145→}
   146→
   147→// RevertDB rolls back the database schema to the previous version (Version - 1).
   148→// This is primarily used for testing or emergency rollback scenarios.
   149→//
   150→// Warning:
   151→//   - This may cause data loss if newer schema added columns/tables
   152→//   - Always backup before reverting
   153→//   - Not all migrations are safely reversible
   154→//
   155→// Usage:
   156→//
   157→//	cc-backend -revert-db
   158→func RevertDB(db string) error {
   159→	m, err := getMigrateInstance(db)
   160→	if err != nil {
   161→		return err
   162→	}
   163→
   164→	if err := m.Migrate(Version - 1); err != nil {
   165→		if err == migrate.ErrNoChange {
   166→			cclog.Info("DB already up to date!")
   167→		} else {
   168→			return err
   169→		}
   170→	}
   171→
   172→	m.Close()
   173→	return nil
   174→}
   175→
   176→// ForceDB forces the database schema version to the current Version without running migrations.
   177→// This is only used to recover from failed migrations that left the database in a "dirty" state.
   178→//
   179→// When to use:
   180→//   - After manually fixing a failed migration
   181→//   - When you've manually applied schema changes and need to update the version marker
   182→//
   183→// Warning:
   184→//   - This does NOT apply any schema changes
   185→//   - Only use after manually verifying the schema is correct
   186→//   - Improper use can cause schema/version mismatch
   187→//
   188→// Usage:
   189→//
   190→//	cc-backend -force-db
   191→func ForceDB(db string) error {
   192→	m, err := getMigrateInstance(db)
   193→	if err != nil {
   194→		return err
   195→	}
   196→
   197→	if err := m.Force(int(Version)); err != nil {
   198→		return err
   199→	}
   200→
   201→	m.Close()
   202→	return nil
   203→}
   204→


     1→CREATE TABLE "job_cache" (
     2→    id INTEGER PRIMARY KEY,
     3→    job_id BIGINT NOT NULL,
     4→    cluster VARCHAR(255) NOT NULL,
     5→    subcluster VARCHAR(255) NOT NULL,
     6→    submit_time BIGINT NOT NULL DEFAULT 0, -- Unix timestamp
     7→    start_time BIGINT NOT NULL DEFAULT 0, -- Unix timestamp
     8→    hpc_user VARCHAR(255) NOT NULL,
     9→    project VARCHAR(255) NOT NULL,
    10→    cluster_partition VARCHAR(255),
    11→    array_job_id BIGINT,
    12→    duration INT NOT NULL,
    13→    walltime INT NOT NULL,
    14→    job_state VARCHAR(255) NOT NULL
    15→    CHECK (job_state IN (
    16→        'boot_fail', 'cancelled', 'completed', 'deadline',
    17→        'failed', 'node_fail', 'out_of_memory', 'pending',
    18→        'preempted', 'running', 'suspended', 'timeout'
    19→    )),
    20→    meta_data TEXT,          -- JSON
    21→    resources TEXT NOT NULL, -- JSON
    22→    num_nodes INT NOT NULL,
    23→    num_hwthreads INT,
    24→    num_acc INT,
    25→    smt TINYINT NOT NULL DEFAULT 1 CHECK (smt IN (0, 1)),
    26→    shared TEXT NOT NULL
    27→    CHECK (shared IN ("none", "single_user", "multi_user")),
    28→    monitoring_status TINYINT NOT NULL DEFAULT 1
    29→    CHECK (monitoring_status IN (0, 1, 2, 3)),
    30→    energy REAL NOT NULL DEFAULT 0.0,
    31→    energy_footprint TEXT DEFAULT NULL,
    32→    footprint TEXT DEFAULT NULL,
    33→    UNIQUE (job_id, cluster, start_time)
    34→);
    35→
    36→CREATE TABLE "job_new" (
    37→    id INTEGER PRIMARY KEY,
    38→    job_id BIGINT NOT NULL,
    39→    cluster TEXT NOT NULL,
    40→    subcluster TEXT NOT NULL,
    41→    submit_time BIGINT NOT NULL DEFAULT 0, -- Unix timestamp
    42→    start_time BIGINT NOT NULL DEFAULT 0, -- Unix timestamp
    43→    hpc_user TEXT NOT NULL,
    44→    project TEXT NOT NULL,
    45→    cluster_partition TEXT,
    46→    array_job_id BIGINT,
    47→    duration INT NOT NULL,
    48→    walltime INT NOT NULL,
    49→    job_state TEXT NOT NULL
    50→    CHECK (job_state IN (
    51→        'boot_fail', 'cancelled', 'completed', 'deadline',
    52→        'failed', 'node_fail', 'out_of_memory', 'pending',
    53→        'preempted', 'running', 'suspended', 'timeout'
    54→    )),
    55→    meta_data TEXT,          -- JSON
    56→    resources TEXT NOT NULL, -- JSON
    57→    num_nodes INT NOT NULL,
    58→    num_hwthreads INT,
    59→    num_acc INT,
    60→    smt INT NOT NULL DEFAULT 1,
    61→    shared TEXT NOT NULL
    62→    CHECK (shared IN ("none", "single_user", "multi_user")),
    63→    monitoring_status TINYINT NOT NULL DEFAULT 1
    64→    CHECK (monitoring_status IN (0, 1, 2, 3)),
    65→    energy REAL NOT NULL DEFAULT 0.0,
    66→    energy_footprint TEXT DEFAULT NULL,
    67→    footprint TEXT DEFAULT NULL,
    68→    UNIQUE (job_id, cluster, start_time)
    69→);
    70→
    71→
    72→CREATE TABLE IF NOT EXISTS lookup_exclusive (
    73→    id INTEGER PRIMARY KEY,
    74→    name TEXT NOT NULL UNIQUE
    75→);
    76→
    77→INSERT INTO lookup_exclusive (id, name) VALUES
    78→(0, 'multi_user'),
    79→(1, 'none'),
    80→(2, 'single_user');
    81→
    82→INSERT INTO job_new (
    83→    id, job_id, cluster, subcluster, submit_time, start_time, hpc_user, project,
    84→    cluster_partition, array_job_id, duration, walltime, job_state, meta_data, resources,
    85→    num_nodes, num_hwthreads, num_acc, smt, shared, monitoring_status, energy,
    86→    energy_footprint, footprint
    87→) SELECT
    88→    id,
    89→    job_id,
    90→    cluster,
    91→    subcluster,
    92→    0,
    93→    start_time,
    94→    hpc_user,
    95→    project,
    96→    cluster_partition,
    97→    array_job_id,
    98→    duration,
    99→    walltime,
   100→    job_state,
   101→    meta_data,
   102→    resources,
   103→    num_nodes,
   104→    num_hwthreads,
   105→    num_acc,
   106→    smt,
   107→    (
   108→        SELECT name FROM lookup_exclusive
   109→        WHERE id = job.exclusive
   110→    ),
   111→    monitoring_status,
   112→    energy,
   113→    energy_footprint,
   114→    footprint
   115→FROM job;
   116→
   117→DROP TABLE lookup_exclusive;
   118→DROP TABLE job; -- Deletes All Existing 'job' Indices; Recreate after Renaming
   119→ALTER TABLE job_new RENAME TO job;
   120→
   121→-- Recreate Indices from 08_add-footprint; include new 'shared' column
   122→-- Cluster Filter
   123→CREATE INDEX IF NOT EXISTS jobs_cluster_user ON job (cluster, hpc_user);
   124→CREATE INDEX IF NOT EXISTS jobs_cluster_project ON job (cluster, project);
   125→CREATE INDEX IF NOT EXISTS jobs_cluster_subcluster ON job (cluster, subcluster);
   126→-- Cluster Filter Sorting
   127→CREATE INDEX IF NOT EXISTS jobs_cluster_numnodes ON job (cluster, num_nodes);
   128→CREATE INDEX IF NOT EXISTS jobs_cluster_numhwthreads ON job (cluster, num_hwthreads);
   129→CREATE INDEX IF NOT EXISTS jobs_cluster_numacc ON job (cluster, num_acc);
   130→CREATE INDEX IF NOT EXISTS jobs_cluster_energy ON job (cluster, energy);
   131→
   132→-- Cluster Time Filter Sorting
   133→CREATE INDEX IF NOT EXISTS jobs_cluster_duration_starttime ON job (cluster, duration, start_time);
   134→CREATE INDEX IF NOT EXISTS jobs_cluster_starttime_duration ON job (cluster, start_time, duration);
   135→
   136→-- Cluster+Partition Filter
   137→CREATE INDEX IF NOT EXISTS jobs_cluster_partition_user ON job (cluster, cluster_partition, hpc_user);
   138→CREATE INDEX IF NOT EXISTS jobs_cluster_partition_project ON job (cluster, cluster_partition, project);
   139→CREATE INDEX IF NOT EXISTS jobs_cluster_partition_jobstate ON job (cluster, cluster_partition, job_state);
   140→CREATE INDEX IF NOT EXISTS jobs_cluster_partition_shared ON job (cluster, cluster_partition, shared);
   141→
   142→-- Cluster+Partition Filter Sorting
   143→CREATE INDEX IF NOT EXISTS jobs_cluster_partition_numnodes ON job (cluster, cluster_partition, num_nodes);
   144→CREATE INDEX IF NOT EXISTS jobs_cluster_partition_numhwthreads ON job (cluster, cluster_partition, num_hwthreads);
   145→CREATE INDEX IF NOT EXISTS jobs_cluster_partition_numacc ON job (cluster, cluster_partition, num_acc);
   146→CREATE INDEX IF NOT EXISTS jobs_cluster_partition_energy ON job (cluster, cluster_partition, energy);
   147→
   148→-- Cluster+Partition Time Filter Sorting
   149→CREATE INDEX IF NOT EXISTS jobs_cluster_partition_duration_starttime ON job (cluster, cluster_partition, duration, start_time);
   150→CREATE INDEX IF NOT EXISTS jobs_cluster_partition_starttime_duration ON job (cluster, cluster_partition, start_time, duration);
   151→
   152→-- Cluster+JobState Filter
   153→CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_user ON job (cluster, job_state, hpc_user);
   154→CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_project ON job (cluster, job_state, project);
   155→-- Cluster+JobState Filter Sorting
   156→CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_numnodes ON job (cluster, job_state, num_nodes);
   157→CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_numhwthreads ON job (cluster, job_state, num_hwthreads);
   158→CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_numacc ON job (cluster, job_state, num_acc);
   159→CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_energy ON job (cluster, job_state, energy);
   160→
   161→-- Cluster+JobState Time Filter Sorting
   162→CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_starttime_duration ON job (cluster, job_state, start_time, duration);
   163→CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_duration_starttime ON job (cluster, job_state, duration, start_time);
   164→
   165→-- Cluster+Shared Filter
   166→CREATE INDEX IF NOT EXISTS jobs_cluster_shared_user ON job (cluster, shared, hpc_user);
   167→CREATE INDEX IF NOT EXISTS jobs_cluster_shared_project ON job (cluster, shared, project);
   168→-- Cluster+Shared Filter Sorting
   169→CREATE INDEX IF NOT EXISTS jobs_cluster_shared_numnodes ON job (cluster, shared, num_nodes);
   170→CREATE INDEX IF NOT EXISTS jobs_cluster_shared_numhwthreads ON job (cluster, shared, num_hwthreads);
   171→CREATE INDEX IF NOT EXISTS jobs_cluster_shared_numacc ON job (cluster, shared, num_acc);
   172→CREATE INDEX IF NOT EXISTS jobs_cluster_shared_energy ON job (cluster, shared, energy);
   173→
   174→-- Cluster+Shared Time Filter Sorting
   175→CREATE INDEX IF NOT EXISTS jobs_cluster_shared_starttime_duration ON job (cluster, shared, start_time, duration);
   176→CREATE INDEX IF NOT EXISTS jobs_cluster_shared_duration_starttime ON job (cluster, shared, duration, start_time);
   177→
   178→-- User Filter
   179→-- User Filter Sorting
   180→CREATE INDEX IF NOT EXISTS jobs_user_numnodes ON job (hpc_user, num_nodes);
   181→CREATE INDEX IF NOT EXISTS jobs_user_numhwthreads ON job (hpc_user, num_hwthreads);
   182→CREATE INDEX IF NOT EXISTS jobs_user_numacc ON job (hpc_user, num_acc);
   183→CREATE INDEX IF NOT EXISTS jobs_user_energy ON job (hpc_user, energy);
   184→
   185→-- Cluster+Shared Time Filter Sorting
   186→CREATE INDEX IF NOT EXISTS jobs_user_starttime_duration ON job (hpc_user, start_time, duration);
   187→CREATE INDEX IF NOT EXISTS jobs_user_duration_starttime ON job (hpc_user, duration, start_time);
   188→
   189→-- Project Filter
   190→CREATE INDEX IF NOT EXISTS jobs_project_user ON job (project, hpc_user);
   191→-- Project Filter Sorting
   192→CREATE INDEX IF NOT EXISTS jobs_project_numnodes ON job (project, num_nodes);
   193→CREATE INDEX IF NOT EXISTS jobs_project_numhwthreads ON job (project, num_hwthreads);
   194→CREATE INDEX IF NOT EXISTS jobs_project_numacc ON job (project, num_acc);
   195→CREATE INDEX IF NOT EXISTS jobs_project_energy ON job (project, energy);
   196→
   197→-- Cluster+Shared Time Filter Sorting
   198→CREATE INDEX IF NOT EXISTS jobs_project_starttime_duration ON job (project, start_time, duration);
   199→CREATE INDEX IF NOT EXISTS jobs_project_duration_starttime ON job (project, duration, start_time);
   200→
   201→-- JobState Filter
   202→CREATE INDEX IF NOT EXISTS jobs_jobstate_user ON job (job_state, hpc_user);
   203→CREATE INDEX IF NOT EXISTS jobs_jobstate_project ON job (job_state, project);
   204→-- JobState Filter Sorting
   205→CREATE INDEX IF NOT EXISTS jobs_jobstate_numnodes ON job (job_state, num_nodes);
   206→CREATE INDEX IF NOT EXISTS jobs_jobstate_numhwthreads ON job (job_state, num_hwthreads);
   207→CREATE INDEX IF NOT EXISTS jobs_jobstate_numacc ON job (job_state, num_acc);
   208→CREATE INDEX IF NOT EXISTS jobs_jobstate_energy ON job (job_state, energy);
   209→
   210→-- Cluster+Shared Time Filter Sorting
   211→CREATE INDEX IF NOT EXISTS jobs_jobstate_starttime_duration ON job (job_state, start_time, duration);
   212→CREATE INDEX IF NOT EXISTS jobs_jobstate_duration_starttime ON job (job_state, duration, start_time);
   213→
   214→-- Shared Filter
   215→CREATE INDEX IF NOT EXISTS jobs_shared_user ON job (shared, hpc_user);
   216→CREATE INDEX IF NOT EXISTS jobs_shared_project ON job (shared, project);
   217→-- Shared Filter Sorting
   218→CREATE INDEX IF NOT EXISTS jobs_shared_numnodes ON job (shared, num_nodes);
   219→CREATE INDEX IF NOT EXISTS jobs_shared_numhwthreads ON job (shared, num_hwthreads);
   220→CREATE INDEX IF NOT EXISTS jobs_shared_numacc ON job (shared, num_acc);
   221→CREATE INDEX IF NOT EXISTS jobs_shared_energy ON job (shared, energy);
   222→
   223→-- Cluster+Shared Time Filter Sorting
   224→CREATE INDEX IF NOT EXISTS jobs_shared_starttime_duration ON job (shared, start_time, duration);
   225→CREATE INDEX IF NOT EXISTS jobs_shared_duration_starttime ON job (shared, duration, start_time);
   226→
   227→-- ArrayJob Filter
   228→CREATE INDEX IF NOT EXISTS jobs_arrayjobid_starttime ON job (array_job_id, start_time);
   229→CREATE INDEX IF NOT EXISTS jobs_cluster_arrayjobid_starttime ON job (cluster, array_job_id, start_time);
   230→
   231→-- Single filters with default starttime sorting
   232→CREATE INDEX IF NOT EXISTS jobs_duration_starttime ON job (duration, start_time);
   233→CREATE INDEX IF NOT EXISTS jobs_numnodes_starttime ON job (num_nodes, start_time);
   234→CREATE INDEX IF NOT EXISTS jobs_numhwthreads_starttime ON job (num_hwthreads, start_time);
   235→CREATE INDEX IF NOT EXISTS jobs_numacc_starttime ON job (num_acc, start_time);
   236→CREATE INDEX IF NOT EXISTS jobs_energy_starttime ON job (energy, start_time);
   237→
   238→-- Single filters with duration sorting
   239→CREATE INDEX IF NOT EXISTS jobs_starttime_duration ON job (start_time, duration);
   240→CREATE INDEX IF NOT EXISTS jobs_numnodes_duration ON job (num_nodes, duration);
   241→CREATE INDEX IF NOT EXISTS jobs_numhwthreads_duration ON job (num_hwthreads, duration);
   242→CREATE INDEX IF NOT EXISTS jobs_numacc_duration ON job (num_acc, duration);
   243→CREATE INDEX IF NOT EXISTS jobs_energy_duration ON job (energy, duration);
   244→
   245→-- Backup Indices For High Variety Columns
   246→CREATE INDEX IF NOT EXISTS jobs_starttime ON job (start_time);
   247→CREATE INDEX IF NOT EXISTS jobs_duration ON job (duration);
   248→
   249→-- Notes:
   250→-- Cluster+Partition+Jobstate Filter: Tested -> Full Array Of Combinations non-required
   251→-- Cluster+JobState+Shared Filter: Tested -> No further timing improvement
   252→-- JobState+Shared Filter: Tested -> No further timing improvement
   253→
   254→-- Optimize DB index usage
   255→PRAGMA optimize;
   256→
   257→-- Optimize DB size: https://sqlite.org/lang_vacuum.html
   258→-- Not allowed within a migration transaction; Keep command here for documentation and recommendation
   259→-- Command: 'VACUUM;'


     1→CREATE TABLE "node" (
     2→    id INTEGER PRIMARY KEY,
     3→    hostname VARCHAR(255) NOT NULL,
     4→    cluster VARCHAR(255) NOT NULL,
     5→    subcluster VARCHAR(255) NOT NULL,
     6→    meta_data TEXT,          -- JSON
     7→    UNIQUE (hostname, cluster)
     8→);
     9→
    10→CREATE TABLE "node_state" (
    11→    id INTEGER PRIMARY KEY,
    12→    time_stamp INTEGER NOT NULL,
    13→    jobs_running INTEGER DEFAULT 0 NOT NULL,
    14→    cpus_allocated INTEGER DEFAULT 0 NOT NULL,
    15→    memory_allocated INTEGER DEFAULT 0 NOT NULL,
    16→    gpus_allocated INTEGER DEFAULT 0 NOT NULL,
    17→    node_state VARCHAR(255) NOT NULL
    18→    CHECK (node_state IN (
    19→        'allocated', 'reserved', 'idle', 'mixed',
    20→        'down', 'unknown'
    21→    )),
    22→    health_state VARCHAR(255) NOT NULL
    23→    CHECK (health_state IN (
    24→        'full', 'partial', 'failed'
    25→    )),
    26→    health_metrics TEXT,          -- JSON array of strings
    27→    node_id INTEGER,
    28→    FOREIGN KEY (node_id) REFERENCES node (id)
    29→);
    30→
    31→-- Add NEW Indices For New Node Table VARCHAR Fields
    32→CREATE INDEX IF NOT EXISTS nodes_cluster ON node (cluster);
    33→CREATE INDEX IF NOT EXISTS nodes_cluster_subcluster ON node (cluster, subcluster);
    34→
    35→-- Add NEW Indices For New Node_State Table Fields
    36→CREATE INDEX IF NOT EXISTS nodestates_timestamp ON node_state (time_stamp);
    37→CREATE INDEX IF NOT EXISTS nodestates_state_timestamp ON node_state (node_state, time_stamp);
    38→CREATE INDEX IF NOT EXISTS nodestates_health_timestamp ON node_state (health_state, time_stamp);
    39→CREATE INDEX IF NOT EXISTS nodestates_nodeid_state ON node_state (node_id, node_state);
    40→CREATE INDEX IF NOT EXISTS nodestates_nodeid_health ON node_state (node_id, health_state);
    41→CREATE INDEX IF NOT EXISTS nodestates_nodeid_timestamp ON node_state (node_id, time_stamp DESC);
    42→
    43→-- Add NEW Indices For Increased Amounts of Tags
    44→CREATE INDEX IF NOT EXISTS tags_jobid ON jobtag (job_id);
    45→CREATE INDEX IF NOT EXISTS tags_tagid ON jobtag (tag_id);
    46→
    47→-- Optimize DB index usage
    48→PRAGMA optimize;


No matches found


internal/repository/jobFind.go:281:		Where("job.start_time <= ?", stopTimeTail)
internal/repository/jobFind.go:286:		Where("job.start_time < ?", stopTimeTail).
internal/repository/jobFind.go:287:		Where("(job.start_time + job.duration) > ?", startTimeFront)
internal/repository/tags.go:108:	q := sq.Delete("jobtag").Where("jobtag.job_id = ?", job).Where("jobtag.tag_id = ?", tag)
internal/repository/tags.go:149:	q := sq.Delete("jobtag").Where("jobtag.job_id = ?", job).Where("jobtag.tag_id = ?", tagID)
internal/repository/tags.go:217:	jobIds, err := r.FindJobIdsByTag(tagID)
internal/repository/tags.go:223:	qJobTag := sq.Delete("jobtag").Where("jobtag.tag_id = ?", tagID)
internal/repository/tags.go:232:	qTag := sq.Delete("tag").Where("tag.id = ?", tagID)
internal/repository/jobCreate_test.go:82:		_, err = r.DB.Exec("DELETE FROM job_cache WHERE job_id = ? AND cluster = ?", job.JobID, job.Cluster)
internal/repository/jobCreate_test.go:113:		_, err = r.DB.Exec("DELETE FROM job_cache WHERE id = ?", id)
internal/repository/jobCreate_test.go:155:		_, err = r.DB.Exec("DELETE FROM job_cache WHERE id = ?", id)
internal/repository/jobCreate_test.go:174:		_, err = r.DB.Exec("DELETE FROM job_cache WHERE id = ?", id)
internal/repository/jobCreate_test.go:187:		_, err = r.DB.Exec("DELETE FROM job_cache WHERE id = ?", id)
internal/repository/jobCreate_test.go:210:		_, err = r.DB.Exec("DELETE FROM job_cache WHERE id = ?", id)
internal/repository/jobCreate_test.go:238:		_, err = r.DB.Exec("DELETE FROM job WHERE id = ?", id)
internal/repository/jobCreate_test.go:257:		_, err = r.DB.Exec("DELETE FROM job_cache WHERE id = ?", id)
internal/repository/jobCreate_test.go:276:		_, err = r.DB.Exec("DELETE FROM job WHERE id = ?", id)
internal/repository/jobCreate_test.go:308:				_, err = r.DB.Exec("DELETE FROM job_cache WHERE id = ?", id)
internal/repository/jobCreate_test.go:327:				_, err = r.DB.Exec("DELETE FROM job WHERE id = ?", id)
internal/repository/jobCreate_test.go:362:		_, err = r.DB.Exec("DELETE FROM job WHERE id = ?", newID)
internal/repository/jobCreate_test.go:387:		_, err = r.DB.Exec("DELETE FROM job WHERE id = ?", newID)
internal/repository/jobCreate_test.go:397:		_, err := r.DB.Exec("DELETE FROM job_cache")
internal/repository/jobCreate_test.go:438:		_, err = r.DB.Exec("DELETE FROM job WHERE job_id IN (?, ?, ?)", job1.JobID, job2.JobID, job3.JobID)
internal/repository/jobCreate_test.go:444:		_, err := r.DB.Exec("DELETE FROM job_cache")
internal/repository/jobCreate_test.go:488:		_, err = r.DB.Exec("DELETE FROM job WHERE job_id = ? AND cluster = ?", job.JobID, job.Cluster)
internal/repository/jobCreate_test.go:494:		_, err := r.DB.Exec("DELETE FROM job_cache")
internal/repository/jobCreate_test.go:516:		_, err = r.DB.Exec("DELETE FROM job WHERE job_id = ? AND cluster = ?", job.JobID, job.Cluster)
internal/repository/jobCreate_test.go:522:		_, err := r.DB.Exec("DELETE FROM job_cache")
internal/repository/jobCreate_test.go:558:		_, err = r.DB.Exec("DELETE FROM job WHERE id = ?", id)
internal/repository/jobCreate_test.go:576:		_, err = r.DB.Exec("DELETE FROM jobtag WHERE job_id = ?", id)
internal/repository/jobCreate_test.go:578:		_, err = r.DB.Exec("DELETE FROM job WHERE id = ?", id)
internal/repository/jobCreate_test.go:604:		_, err = r.DB.Exec("DELETE FROM job WHERE id = ?", id)
internal/repository/job.go:201:	if _, err := r.DB.Exec(`DELETE FROM jobtag`); err != nil {
internal/repository/job.go:209:	if _, err := r.DB.Exec(`DELETE FROM job`); err != nil {
internal/repository/job.go:394://   - startTime: Unix timestamp, jobs with start_time < this value will be deleted
internal/repository/job.go:401:	q := sq.Select("count(*)").From("job").Where("job.start_time < ?", startTime)
internal/repository/job.go:418:		selectQuery := sq.Select("id").From("job").Where("job.start_time < ?", startTime)
internal/repository/job.go:444:	qd := sq.Delete("job").Where("job.start_time < ?", startTime)
internal/repository/job.go:475:	qd := sq.Delete("job").Where("job.id = ?", id)
internal/repository/job.go:721:		Where("job.start_time < (? - job.walltime)", currentTime-int64(seconds)).
internal/repository/job.go:741:// FindJobIdsByTag returns all job database IDs associated with a specific tag.
internal/repository/job.go:747:func (r *JobRepository) FindJobIdsByTag(tagID int64) ([]int64, error) {
internal/repository/job.go:753:		cclog.Errorf("Error while running FindJobIdsByTag query for tagID=%d: %v", tagID, err)
internal/repository/job.go:764:			cclog.Warnf("Error while scanning rows in FindJobIdsByTag: %v", err)
internal/repository/job.go:765:			return nil, fmt.Errorf("failed to scan job ID in FindJobIdsByTag: %w", err)
internal/repository/job.go:848:		query = sq.Select(jobColumns...).From("job").Where("job.start_time < ?", startTimeEnd)
internal/repository/job.go:1068:		Where("job.start_time < ?", ts).
internal/repository/jobCreate.go:103:	_, err = r.DB.Exec("DELETE FROM job_cache")
internal/repository/jobCreate.go:143:	_, err = r.DB.Exec("DELETE FROM job_cache WHERE id = ?", cacheID)


     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→package repository
     7→
     8→import (
     9→	"sync"
    10→
    11→	"github.com/ClusterCockpit/cc-lib/v2/schema"
    12→)
    13→
    14→// JobHook interface allows external components to hook into job lifecycle events.
    15→// Implementations can perform actions when jobs start or stop, such as tagging,
    16→// logging, notifications, or triggering external workflows.
    17→//
    18→// Example implementation:
    19→//
    20→//	type MyJobTagger struct{}
    21→//
    22→//	func (t *MyJobTagger) JobStartCallback(job *schema.Job) {
    23→//	    if job.NumNodes > 100 {
    24→//	        // Tag large jobs automatically
    25→//	    }
    26→//	}
    27→//
    28→//	func (t *MyJobTagger) JobStopCallback(job *schema.Job) {
    29→//	    if job.State == schema.JobStateFailed {
    30→//	        // Log or alert on failed jobs
    31→//	    }
    32→//	}
    33→//
    34→// Register hooks during application initialization:
    35→//
    36→//	repository.RegisterJobHook(&MyJobTagger{})
    37→type JobHook interface {
    38→	// JobStartCallback is invoked when one or more jobs start.
    39→	// This is called synchronously, so implementations should be fast.
    40→	JobStartCallback(job *schema.Job)
    41→
    42→	// JobStopCallback is invoked when a job completes.
    43→	// This is called synchronously, so implementations should be fast.
    44→	JobStopCallback(job *schema.Job)
    45→}
    46→
    47→var (
    48→	initOnce sync.Once
    49→	hooks    []JobHook
    50→)
    51→
    52→// RegisterJobHook registers a JobHook to receive job lifecycle callbacks.
    53→// Multiple hooks can be registered and will be called in registration order.
    54→// This function is safe to call multiple times and is typically called during
    55→// application initialization.
    56→//
    57→// Nil hooks are silently ignored to simplify conditional registration.
    58→func RegisterJobHook(hook JobHook) {
    59→	initOnce.Do(func() {
    60→		hooks = make([]JobHook, 0)
    61→	})
    62→
    63→	if hook != nil {
    64→		hooks = append(hooks, hook)
    65→	}
    66→}
    67→
    68→// CallJobStartHooks invokes all registered JobHook.JobStartCallback methods
    69→// for each job in the provided slice. This is called internally by the repository
    70→// when jobs are started (e.g., via StartJob or batch job imports).
    71→//
    72→// Hooks are called synchronously in registration order. If a hook panics,
    73→// the panic will propagate to the caller.
    74→func CallJobStartHooks(jobs []*schema.Job) {
    75→	if hooks == nil {
    76→		return
    77→	}
    78→
    79→	for _, hook := range hooks {
    80→		if hook != nil {
    81→			for _, job := range jobs {
    82→				hook.JobStartCallback(job)
    83→			}
    84→		}
    85→	}
    86→}
    87→
    88→// CallJobStopHooks invokes all registered JobHook.JobStopCallback methods
    89→// for the provided job. This is called internally by the repository when a
    90→// job completes (e.g., via StopJob or job state updates).
    91→//
    92→// Hooks are called synchronously in registration order. If a hook panics,
    93→// the panic will propagate to the caller.
    94→func CallJobStopHooks(job *schema.Job) {
    95→	if hooks == nil {
    96→		return
    97→	}
    98→
    99→	for _, hook := range hooks {
   100→		if hook != nil {
   101→			hook.JobStopCallback(job)
   102→		}
   103→	}
   104→}
   105→


   700→// This is typically called periodically to clean up stuck or orphaned jobs.
   701→//
   702→// Jobs are marked with:
   703→//   - monitoring_status: MonitoringStatusArchivingFailed
   704→//   - duration: 0
   705→//   - job_state: JobStateFailed
   706→//
   707→// Parameters:
   708→//   - seconds: Grace period beyond walltime before marking as failed
   709→//
   710→// Returns an error if the database update fails.
   711→// Logs the number of jobs marked as failed if any were affected.
   712→func (r *JobRepository) StopJobsExceedingWalltimeBy(seconds int) error {
   713→	start := time.Now()
   714→	currentTime := time.Now().Unix()
   715→	res, err := sq.Update("job").
   716→		Set("monitoring_status", schema.MonitoringStatusArchivingFailed).
   717→		Set("duration", 0).
   718→		Set("job_state", schema.JobStateFailed).
   719→		Where("job.job_state = 'running'").
   720→		Where("job.walltime > 0").
   721→		Where("job.start_time < (? - job.walltime)", currentTime-int64(seconds)).
   722→		RunWith(r.DB).Exec()
   723→	if err != nil {
   724→		cclog.Warnf("Error while stopping jobs exceeding walltime: %v", err)
   725→		return fmt.Errorf("failed to stop jobs exceeding walltime: %w", err)
   726→	}
   727→
   728→	rowsAffected, err := res.RowsAffected()
   729→	if err != nil {
   730→		cclog.Warnf("Error while fetching affected rows after stopping due to exceeded walltime: %v", err)
   731→		return fmt.Errorf("failed to get rows affected count: %w", err)
   732→	}
   733→
   734→	if rowsAffected > 0 {
   735→		cclog.Infof("%d jobs have been marked as failed due to running too long", rowsAffected)
   736→	}
   737→	cclog.Debugf("Timer StopJobsExceedingWalltimeBy %s", time.Since(start))
   738→	return nil
   739→}
   740→
   741→// FindJobIdsByTag returns all job database IDs associated with a specific tag.
   742→//
   743→// Parameters:
   744→//   - tagID: Database ID of the tag to search for
   745→//
   746→// Returns a slice of job IDs or an error if the query fails.
   747→func (r *JobRepository) FindJobIdsByTag(tagID int64) ([]int64, error) {
   748→	query := sq.Select("job.id").From("job").
   749→		Join("jobtag ON jobtag.job_id = job.id").
   750→		Where(sq.Eq{"jobtag.tag_id": tagID}).Distinct()
   751→	rows, err := query.RunWith(r.stmtCache).Query()
   752→	if err != nil {
   753→		cclog.Errorf("Error while running FindJobIdsByTag query for tagID=%d: %v", tagID, err)
   754→		return nil, fmt.Errorf("failed to find job IDs by tag %d: %w", tagID, err)
   755→	}
   756→	defer rows.Close()
   757→
   758→	jobIds := make([]int64, 0, 100)
   759→
   760→	for rows.Next() {
   761→		var jobID int64
   762→
   763→		if err := rows.Scan(&jobID); err != nil {
   764→			cclog.Warnf("Error while scanning rows in FindJobIdsByTag: %v", err)
   765→			return nil, fmt.Errorf("failed to scan job ID in FindJobIdsByTag: %w", err)
   766→		}
   767→
   768→		jobIds = append(jobIds, jobID)
   769→	}
   770→
   771→	return jobIds, nil
   772→}
   773→
   774→// FindRunningJobs returns all currently running jobs for a specific cluster.
   775→// Filters out short-running jobs based on repoConfig.MinRunningJobDuration threshold.
   776→//
   777→// Parameters:
   778→//   - cluster: Cluster name to filter jobs
   779→//
   780→// Returns a slice of running job objects or an error if the query fails.
   781→func (r *JobRepository) FindRunningJobs(cluster string) ([]*schema.Job, error) {
   782→	query := sq.Select(jobColumns...).From("job").
   783→		Where("job.cluster = ?", cluster).
   784→		Where("job.job_state = 'running'").
   785→		Where("job.duration > ?", repoConfig.MinRunningJobDuration)
   786→
   787→	rows, err := query.RunWith(r.stmtCache).Query()
   788→	if err != nil {
   789→		cclog.Errorf("Error while running FindRunningJobs query for cluster=%s: %v", cluster, err)
   790→		return nil, fmt.Errorf("failed to find running jobs for cluster %s: %w", cluster, err)
   791→	}
   792→	defer rows.Close()
   793→
   794→	jobs := make([]*schema.Job, 0, 50)
   795→	for rows.Next() {
   796→		job, err := scanJob(rows)
   797→		if err != nil {
   798→			cclog.Warnf("Error while scanning rows in FindRunningJobs: %v", err)
   799→			return nil, fmt.Errorf("failed to scan job in FindRunningJobs: %w", err)
   800→		}
   801→		jobs = append(jobs, job)
   802→	}
   803→
   804→	cclog.Debugf("JobRepository.FindRunningJobs(): Return job count %d (cluster: %s)", len(jobs), cluster)
   805→	return jobs, nil
   806→}
   807→
   808→// UpdateDuration recalculates and updates the duration field for all running jobs.
   809→// Called periodically to keep job durations current without querying individual jobs.
   810→//
   811→// Duration is calculated as: current_time - job.start_time
   812→//
   813→// Returns an error if the database update fails.
   814→func (r *JobRepository) UpdateDuration() error {
   815→	stmnt := sq.Update("job").
   816→		Set("duration", sq.Expr("? - job.start_time", time.Now().Unix())).
   817→		Where("job_state = 'running'")
   818→
   819→	_, err := stmnt.RunWith(r.stmtCache).Exec()
   820→	if err != nil {
   821→		cclog.Errorf("Error while updating duration for running jobs: %v", err)
   822→		return fmt.Errorf("failed to update duration for running jobs: %w", err)
   823→	}
   824→
   825→	return nil
   826→}
   827→
   828→// FindJobsBetween returns jobs within a specified time range.
   829→// If startTimeBegin is 0, returns all jobs before startTimeEnd.
   830→// Optionally excludes tagged jobs from results.
   831→//
   832→// Parameters:
   833→//   - startTimeBegin: Unix timestamp for range start (use 0 for unbounded start)
   834→//   - startTimeEnd: Unix timestamp for range end
   835→//   - omitTagged: "none" = include all jobs, "all" = exclude any tagged jobs,
   836→//     "user" = exclude jobs with user-created tags (not auto-tagger types "app"/"jobClass")
   837→//
   838→// Returns a slice of jobs or an error if the time range is invalid or query fails.
   839→func (r *JobRepository) FindJobsBetween(startTimeBegin int64, startTimeEnd int64, omitTagged string) ([]*schema.Job, error) {
   840→	var query sq.SelectBuilder
   841→
   842→	if startTimeBegin == startTimeEnd || startTimeBegin > startTimeEnd {
   843→		return nil, errors.New("startTimeBegin is equal or larger startTimeEnd")
   844→	}
   845→
   846→	if startTimeBegin == 0 {
   847→		cclog.Infof("Find jobs before %d", startTimeEnd)
   848→		query = sq.Select(jobColumns...).From("job").Where("job.start_time < ?", startTimeEnd)
   849→	} else {
   850→		cclog.Infof("Find jobs between %d and %d", startTimeBegin, startTimeEnd)
   851→		query = sq.Select(jobColumns...).From("job").Where("job.start_time BETWEEN ? AND ?", startTimeBegin, startTimeEnd)
   852→	}
   853→
   854→	switch omitTagged {
   855→	case "all":
   856→		query = query.Where("NOT EXISTS (SELECT 1 FROM jobtag WHERE jobtag.job_id = job.id)")
   857→	case "user":
   858→		query = query.Where("NOT EXISTS (SELECT 1 FROM jobtag JOIN tag ON tag.id = jobtag.tag_id WHERE jobtag.job_id = job.id AND tag.tag_type NOT IN ('app', 'jobClass'))")
   859→	}
   860→
   861→	query = query.OrderBy("job.cluster ASC", "job.subcluster ASC", "job.project ASC", "job.start_time ASC")
   862→
   863→	rows, err := query.RunWith(r.stmtCache).Query()
   864→	if err != nil {
   865→		cclog.Errorf("Error while running FindJobsBetween query: %v", err)
   866→		return nil, fmt.Errorf("failed to find jobs between %d and %d: %w", startTimeBegin, startTimeEnd, err)
   867→	}
   868→	defer rows.Close()
   869→
   870→	jobs := make([]*schema.Job, 0, 50)
   871→	for rows.Next() {
   872→		job, err := scanJob(rows)
   873→		if err != nil {
   874→			cclog.Warnf("Error while scanning rows in FindJobsBetween: %v", err)
   875→			return nil, fmt.Errorf("failed to scan job in FindJobsBetween: %w", err)
   876→		}
   877→		jobs = append(jobs, job)
   878→	}
   879→