   226→func (r *JobRepository) FetchMetadata(job *schema.Job) (map[string]string, error) {
   227→	if job == nil {
   228→		return nil, fmt.Errorf("job cannot be nil")
   229→	}
   230→
   231→	start := time.Now()
   232→	cachekey := fmt.Sprintf("metadata:%d", *job.ID)
   233→	if cached := r.cache.Get(cachekey, nil); cached != nil {
   234→		job.MetaData = cached.(map[string]string)
   235→		return job.MetaData, nil
   236→	}
   237→
   238→	if err := sq.Select("job.meta_data").From("job").Where("job.id = ?", job.ID).
   239→		RunWith(r.stmtCache).QueryRow().Scan(&job.RawMetaData); err != nil {
   240→		cclog.Warnf("Error while scanning for job metadata (ID=%d): %v", *job.ID, err)
   241→		return nil, fmt.Errorf("failed to fetch metadata for job %d: %w", *job.ID, err)
   242→	}
   243→
   244→	if len(job.RawMetaData) == 0 {
   245→		return nil, nil
   246→	}
   247→
   248→	if err := json.Unmarshal(job.RawMetaData, &job.MetaData); err != nil {
   249→		cclog.Warnf("Error while unmarshaling raw metadata json (ID=%d): %v", *job.ID, err)
   250→		return nil, fmt.Errorf("failed to unmarshal metadata for job %d: %w", *job.ID, err)
   251→	}
   252→
   253→	r.cache.Put(cachekey, job.MetaData, len(job.RawMetaData), 24*time.Hour)
   254→	cclog.Debugf("Timer FetchMetadata %s", time.Since(start))
   255→	return job.MetaData, nil
   256→}
   257→
   258→// UpdateMetadata adds or updates a single metadata key-value pair for a job.
   259→// The entire metadata map is re-marshaled and stored, and the cache is invalidated.
   260→// Also triggers archive metadata update via archive.UpdateMetadata.
   261→//
   262→// Parameters:
   263→//   - job: Job struct with valid ID, existing metadata will be fetched if not present
   264→//   - key: Metadata key to set
   265→//   - val: Metadata value to set
   266→//
   267→// Returns an error if the job is nil, metadata fetch fails, or database update fails.
   268→func (r *JobRepository) UpdateMetadata(job *schema.Job, key, val string) (err error) {
   269→	if job == nil {
   270→		return fmt.Errorf("job cannot be nil")
   271→	}
   272→
   273→	cachekey := fmt.Sprintf("metadata:%d", *job.ID)
   274→	r.cache.Del(cachekey)
   275→	if job.MetaData == nil {
   276→		if _, err = r.FetchMetadata(job); err != nil {
   277→			cclog.Warnf("Error while fetching metadata for job, DB ID '%v'", *job.ID)
   278→			return fmt.Errorf("failed to fetch metadata for job %d: %w", *job.ID, err)
   279→		}
   280→	}
   281→
   282→	if job.MetaData != nil {
   283→		cpy := make(map[string]string, len(job.MetaData)+1)
   284→		maps.Copy(cpy, job.MetaData)
   285→		cpy[key] = val
   286→		job.MetaData = cpy
   287→	} else {
   288→		job.MetaData = map[string]string{key: val}
   289→	}
   290→
   291→	if job.RawMetaData, err = json.Marshal(job.MetaData); err != nil {
   292→		cclog.Warnf("Error while marshaling metadata for job, DB ID '%v'", *job.ID)
   293→		return fmt.Errorf("failed to marshal metadata for job %d: %w", *job.ID, err)
   294→	}
   295→
   296→	if _, err = sq.Update("job").
   297→		Set("meta_data", job.RawMetaData).
   298→		Where("job.id = ?", job.ID).
   299→		RunWith(r.stmtCache).Exec(); err != nil {
   300→		cclog.Warnf("Error while updating metadata for job, DB ID '%v'", *job.ID)
   301→		return fmt.Errorf("failed to update metadata in database for job %d: %w", *job.ID, err)
   302→	}
   303→
   304→	r.cache.Put(cachekey, job.MetaData, len(job.RawMetaData), 24*time.Hour)
   305→	return archive.UpdateMetadata(job, job.MetaData)
   306→}
   307→
   308→// FetchFootprint retrieves and unmarshals the performance footprint JSON for a job.
   309→// Unlike FetchMetadata, footprints are NOT cached as they can be large and change frequently.
   310→//
   311→// The footprint contains summary statistics (avg/min/max) for monitored metrics,
   312→// stored as JSON with keys like "cpu_load_avg", "mem_used_max", etc.
   313→//
   314→// Parameters:
   315→//   - job: Job struct with valid ID, footprint will be populated in job.Footprint
   316→//
   317→// Returns the footprint map or an error if the job is nil or database query fails.
   318→func (r *JobRepository) FetchFootprint(job *schema.Job) (map[string]float64, error) {
   319→	if job == nil {
   320→		return nil, fmt.Errorf("job cannot be nil")
   321→	}
   322→
   323→	start := time.Now()
   324→
   325→	if err := sq.Select("job.footprint").From("job").Where("job.id = ?", job.ID).
   326→		RunWith(r.stmtCache).QueryRow().Scan(&job.RawFootprint); err != nil {
   327→		cclog.Warnf("Error while scanning for job footprint (ID=%d): %v", *job.ID, err)
   328→		return nil, fmt.Errorf("failed to fetch footprint for job %d: %w", *job.ID, err)
   329→	}
   330→
   331→	if len(job.RawFootprint) == 0 {
   332→		return nil, nil
   333→	}
   334→
   335→	if err := json.Unmarshal(job.RawFootprint, &job.Footprint); err != nil {
   336→		cclog.Warnf("Error while unmarshaling raw footprint json (ID=%d): %v", *job.ID, err)
   337→		return nil, fmt.Errorf("failed to unmarshal footprint for job %d: %w", *job.ID, err)
   338→	}
   339→
   340→	cclog.Debugf("Timer FetchFootprint %s", time.Since(start))
   341→	return job.Footprint, nil
   342→}
   343→
   344→// FetchEnergyFootprint retrieves and unmarshals the energy footprint JSON for a job.
   345→// Energy footprints are cached with a 24-hour TTL as they are frequently accessed but rarely change.
   346→//
   347→// The energy footprint contains calculated energy consumption (in kWh) per metric,
   348→// stored as JSON with keys like "power_avg", "acc_power_avg", etc.
   349→//
   350→// Parameters:
   351→//   - job: Job struct with valid ID, energy footprint will be populated in job.EnergyFootprint
   352→//
   353→// Returns the energy footprint map or an error if the job is nil or database query fails.
   354→func (r *JobRepository) FetchEnergyFootprint(job *schema.Job) (map[string]float64, error) {
   355→	if job == nil {
   356→		return nil, fmt.Errorf("job cannot be nil")
   357→	}
   358→
   359→	start := time.Now()
   360→	cachekey := fmt.Sprintf("energyFootprint:%d", *job.ID)
   361→	if cached := r.cache.Get(cachekey, nil); cached != nil {
   362→		job.EnergyFootprint = cached.(map[string]float64)
   363→		return job.EnergyFootprint, nil
   364→	}
   365→
   366→	if err := sq.Select("job.energy_footprint").From("job").Where("job.id = ?", job.ID).
   367→		RunWith(r.stmtCache).QueryRow().Scan(&job.RawEnergyFootprint); err != nil {
   368→		cclog.Warnf("Error while scanning for job energy_footprint (ID=%d): %v", *job.ID, err)
   369→		return nil, fmt.Errorf("failed to fetch energy footprint for job %d: %w", *job.ID, err)
   370→	}
   371→
   372→	if len(job.RawEnergyFootprint) == 0 {
   373→		return nil, nil
   374→	}
   375→
   376→	if err := json.Unmarshal(job.RawEnergyFootprint, &job.EnergyFootprint); err != nil {
   377→		cclog.Warnf("Error while unmarshaling raw energy footprint json (ID=%d): %v", *job.ID, err)
   378→		return nil, fmt.Errorf("failed to unmarshal energy footprint for job %d: %w", *job.ID, err)
   379→	}
   380→
   381→	r.cache.Put(cachekey, job.EnergyFootprint, len(job.EnergyFootprint), 24*time.Hour)
   382→	cclog.Debugf("Timer FetchEnergyFootprint %s", time.Since(start))
   383→	return job.EnergyFootprint, nil
   384→}
   385→
   386→// DeleteJobsBefore removes jobs older than the specified start time.
   387→// Optionally preserves tagged jobs to protect important data from deletion.
   388→// Cache entries for deleted jobs are automatically invalidated.
   389→//
   390→// This is typically used for data retention policies and cleanup operations.
   391→// WARNING: This is a destructive operation that permanently deletes job records.
   392→//
   393→// Parameters:
   394→//   - startTime: Unix timestamp, jobs with start_time < this value will be deleted
   395→//   - omitTagged: "none" = delete all jobs, "all" = skip any tagged jobs,
   396→//     "user" = skip jobs with user-created tags (not auto-tagger types "app"/"jobClass")
   397→//
   398→// Returns the count of deleted jobs or an error if the operation fails.
   399→func (r *JobRepository) DeleteJobsBefore(startTime int64, omitTagged string) (int, error) {
   400→	var cnt int
   401→	q := sq.Select("count(*)").From("job").Where("job.start_time < ?", startTime)
   402→
   403→	switch omitTagged {
   404→	case "all":
   405→		q = q.Where("NOT EXISTS (SELECT 1 FROM jobtag WHERE jobtag.job_id = job.id)")
   406→	case "user":
   407→		q = q.Where("NOT EXISTS (SELECT 1 FROM jobtag JOIN tag ON tag.id = jobtag.tag_id WHERE jobtag.job_id = job.id AND tag.tag_type NOT IN ('app', 'jobClass'))")
   408→	}
   409→
   410→	if err := q.RunWith(r.DB).QueryRow().Scan(&cnt); err != nil {
   411→		cclog.Errorf("Error counting jobs before %d: %v", startTime, err)
   412→		return 0, err
   413→	}
   414→
   415→	// Invalidate cache for jobs being deleted (get job IDs first)
   416→	if cnt > 0 {
   417→		var jobIds []int64
   418→		selectQuery := sq.Select("id").From("job").Where("job.start_time < ?", startTime)
   419→
   420→		switch omitTagged {
   421→		case "all":
   422→			selectQuery = selectQuery.Where("NOT EXISTS (SELECT 1 FROM jobtag WHERE jobtag.job_id = job.id)")
   423→		case "user":
   424→			selectQuery = selectQuery.Where("NOT EXISTS (SELECT 1 FROM jobtag JOIN tag ON tag.id = jobtag.tag_id WHERE jobtag.job_id = job.id AND tag.tag_type NOT IN ('app', 'jobClass'))")
   425→		}
   426→
   427→		rows, err := selectQuery.RunWith(r.DB).Query()
   428→		if err == nil {
   429→			defer rows.Close()
   430→			for rows.Next() {
   431→				var id int64
   432→				if err := rows.Scan(&id); err == nil {
   433→					jobIds = append(jobIds, id)
   434→				}
   435→			}
   436→			// Invalidate cache entries
   437→			for _, id := range jobIds {
   438→				r.cache.Del(fmt.Sprintf("metadata:%d", id))
   439→				r.cache.Del(fmt.Sprintf("energyFootprint:%d", id))
   440→			}
   441→		}
   442→	}
   443→
   444→	qd := sq.Delete("job").Where("job.start_time < ?", startTime)
   445→
   446→	switch omitTagged {
   447→	case "all":
   448→		qd = qd.Where("NOT EXISTS (SELECT 1 FROM jobtag WHERE jobtag.job_id = job.id)")
   449→	case "user":
   450→		qd = qd.Where("NOT EXISTS (SELECT 1 FROM jobtag JOIN tag ON tag.id = jobtag.tag_id WHERE jobtag.job_id = job.id AND tag.tag_type NOT IN ('app', 'jobClass'))")
   451→	}
   452→	_, err := qd.RunWith(r.DB).Exec()
   453→
   454→	if err != nil {
   455→		s, _, _ := qd.ToSql()
   456→		cclog.Errorf(" DeleteJobsBefore(%d) with %s: error %#v", startTime, s, err)
   457→	} else {
   458→		cclog.Debugf("DeleteJobsBefore(%d): Deleted %d jobs", startTime, cnt)
   459→	}
   460→	return cnt, err
   461→}
   462→
   463→// DeleteJobByID permanently removes a single job by its database ID.
   464→// Cache entries for the deleted job are automatically invalidated.
   465→//
   466→// Parameters:
   467→//   - id: Database ID (primary key) of the job to delete
   468→//
   469→// Returns an error if the deletion fails.
   470→func (r *JobRepository) DeleteJobByID(id int64) error {
   471→	// Invalidate cache entries before deletion
   472→	r.cache.Del(fmt.Sprintf("metadata:%d", id))
   473→	r.cache.Del(fmt.Sprintf("energyFootprint:%d", id))
   474→
   475→	qd := sq.Delete("job").Where("job.id = ?", id)
   476→	_, err := qd.RunWith(r.DB).Exec()
   477→
   478→	if err != nil {
   479→		s, _, _ := qd.ToSql()
   480→		cclog.Errorf("DeleteJobById(%d) with %s : error %#v", id, s, err)
   481→	} else {
   482→		cclog.Debugf("DeleteJobById(%d): Success", id)
   483→	}
   484→	return err
   485→}
   486→
   487→// FindUserOrProjectOrJobname attempts to interpret a search term as a job ID,
   488→// username, project ID, or job name by querying the database.
   489→//
   490→// Search logic (in priority order):
   491→//  1. If searchterm is numeric, treat as job ID (returned immediately)
   492→//  2. Try exact match in job.hpc_user column (username)
   493→//  3. Try LIKE match in hpc_user.name column (real name)
   494→//  4. Try exact match in job.project column (project ID)
   495→//  5. If no matches, return searchterm as jobname for GraphQL query
   496→//
   497→// This powers the searchbar functionality for flexible job searching.
   498→// Requires authenticated user for database lookups (returns empty if user is nil).
   499→//
   500→// Parameters:
   501→//   - user: Authenticated user context, required for database access
   502→//   - searchterm: Search string to interpret
   503→//
   504→// Returns up to one non-empty value among (jobid, username, project, jobname).
   505→func (r *JobRepository) FindUserOrProjectOrJobname(user *schema.User, searchterm string) (jobid string, username string, project string, jobname string) {
   506→	if searchterm == "" {
   507→		return "", "", "", ""
   508→	}
   509→
   510→	if _, err := strconv.Atoi(searchterm); err == nil { // Return empty on successful conversion: parent method will redirect for integer jobId
   511→		return searchterm, "", "", ""
   512→	} else { // Has to have letters and logged-in user for other guesses
   513→		if user != nil {
   514→			// Find username by username in job table (match)
   515→			uresult, _ := r.FindColumnValue(user, searchterm, "job", "hpc_user", "hpc_user", false)
   516→			if uresult != "" {
   517→				return "", uresult, "", ""
   518→			}
   519→			// Find username by real name in hpc_user table (like)
   520→			nresult, _ := r.FindColumnValue(user, searchterm, "hpc_user", "username", "name", true)
   521→			if nresult != "" {
   522→				return "", nresult, "", ""
   523→			}
   524→			// Find projectId by projectId in job table (match)
   525→			presult, _ := r.FindColumnValue(user, searchterm, "job", "project", "project", false)
   526→			if presult != "" {
   527→				return "", "", presult, ""
   528→			}
   529→		}
   530→		// Return searchterm if no match before: Forward as jobname query to GQL in handleSearchbar function
   531→		return "", "", "", searchterm
   532→	}
   533→}
   534→
   535→var (
   536→	ErrNotFound  = errors.New("no such jobname, project or user")
   537→	ErrForbidden = errors.New("not authorized")
   538→)
   539→
   540→// FindColumnValue performs a generic column lookup in a database table with role-based access control.
   541→// Only users with admin, support, or manager roles can execute this query.
   542→//
   543→// Parameters:
   544→//   - user: User context for authorization check
   545→//   - searchterm: Value to search for (exact match or LIKE pattern)
   546→//   - table: Database table name to query
   547→//   - selectColumn: Column name to return in results
   548→//   - whereColumn: Column name to filter on
   549→//   - isLike: If true, use LIKE with wildcards; if false, use exact equality
   550→//
   551→// Returns the first matching value, ErrForbidden if user lacks permission,
   552→// or ErrNotFound if no matches are found.
   553→func (r *JobRepository) FindColumnValue(user *schema.User, searchterm string, table string, selectColumn string, whereColumn string, isLike bool) (result string, err error) {
   554→	if user == nil {
   555→		return "", fmt.Errorf("user cannot be nil")
   556→	}
   557→
   558→	compareStr := " = ?"
   559→	query := searchterm
   560→	if isLike {
   561→		compareStr = " LIKE ?"
   562→		query = "%" + searchterm + "%"
   563→	}
   564→	if user.HasAnyRole([]schema.Role{schema.RoleAdmin, schema.RoleSupport, schema.RoleManager}) {
   565→		theQuery := sq.Select(table+"."+selectColumn).Distinct().From(table).
   566→			Where(table+"."+whereColumn+compareStr, query)
   567→
   568→		err := theQuery.RunWith(r.stmtCache).QueryRow().Scan(&result)
   569→
   570→		if err != nil && err != sql.ErrNoRows {
   571→			cclog.Warnf("Error while querying FindColumnValue (table=%s, column=%s): %v", table, selectColumn, err)
   572→			return "", fmt.Errorf("failed to find column value: %w", err)
   573→		} else if err == nil {
   574→			return result, nil
   575→		}
   576→		return "", ErrNotFound
   577→	} else {
   578→		cclog.Infof("Non-Admin User %s : Requested Query '%s' on table '%s' : Forbidden", user.Name, query, table)
   579→		return "", ErrForbidden
   580→	}
   581→}
   582→
   583→// FindColumnValues performs a generic column lookup returning multiple matches with role-based access control.
   584→// Similar to FindColumnValue but returns all matching values instead of just the first.
   585→// Only users with admin, support, or manager roles can execute this query.
   586→//
   587→// Parameters:
   588→//   - user: User context for authorization check
   589→//   - query: Search pattern (always uses LIKE with wildcards)
   590→//   - table: Database table name to query
   591→//   - selectColumn: Column name to return in results
   592→//   - whereColumn: Column name to filter on
   593→//
   594→// Returns a slice of matching values, ErrForbidden if user lacks permission,
   595→// or ErrNotFound if no matches are found.
   596→func (r *JobRepository) FindColumnValues(user *schema.User, query string, table string, selectColumn string, whereColumn string) (results []string, err error) {
   597→	if user == nil {
   598→		return nil, fmt.Errorf("user cannot be nil")
   599→	}
   600→
   601→	emptyResult := make([]string, 0)
   602→	if user.HasAnyRole([]schema.Role{schema.RoleAdmin, schema.RoleSupport, schema.RoleManager}) {
   603→		rows, err := sq.Select(table+"."+selectColumn).Distinct().From(table).
   604→			Where(table+"."+whereColumn+" LIKE ?", fmt.Sprint("%", query, "%")).
   605→			RunWith(r.stmtCache).Query()
   606→		if err != nil && err != sql.ErrNoRows {
   607→			cclog.Errorf("Error while querying FindColumnValues (table=%s, column=%s): %v", table, selectColumn, err)
   608→			return emptyResult, fmt.Errorf("failed to find column values: %w", err)
   609→		} else if err == nil {
   610→			defer rows.Close()
   611→			for rows.Next() {
   612→				var result string
   613→				err := rows.Scan(&result)
   614→				if err != nil {
   615→					cclog.Warnf("Error while scanning rows in FindColumnValues: %v", err)
   616→					return emptyResult, fmt.Errorf("failed to scan column value: %w", err)
   617→				}
   618→				results = append(results, result)
   619→			}
   620→			return results, nil
   621→		}
   622→		return emptyResult, ErrNotFound
   623→
   624→	} else {
   625→		cclog.Infof("Non-Admin User %s : Requested Query '%s' on table '%s' : Forbidden", user.Name, query, table)


   637→func (r *JobRepository) Partitions(cluster string) ([]string, error) {
   638→	var err error
   639→	start := time.Now()
   640→	partitions := r.cache.Get("partitions:"+cluster, func() (any, time.Duration, int) {
   641→		parts := []string{}
   642→		if err = r.DB.Select(&parts, `SELECT DISTINCT job.cluster_partition FROM job WHERE job.cluster = ?;`, cluster); err != nil {
   643→			return nil, 0, 1000
   644→		}
   645→
   646→		return parts, 1 * time.Hour, 1
   647→	})
   648→	if err != nil {
   649→		return nil, err
   650→	}
   651→	cclog.Debugf("Timer Partitions %s", time.Since(start))
   652→	return partitions.([]string), nil
   653→}
   654→
   655→// AllocatedNodes returns a map of all subclusters to a map of hostnames to the amount of jobs running on that host.
   656→// Hosts with zero jobs running on them will not show up!
   657→func (r *JobRepository) AllocatedNodes(cluster string) (map[string]map[string]int, error) {
   658→	start := time.Now()
   659→	subclusters := make(map[string]map[string]int)
   660→	rows, err := sq.Select("resources", "subcluster").From("job").
   661→		Where("job.job_state = 'running'").
   662→		Where("job.cluster = ?", cluster).
   663→		RunWith(r.stmtCache).Query()
   664→	if err != nil {
   665→		cclog.Errorf("Error while running AllocatedNodes query for cluster=%s: %v", cluster, err)
   666→		return nil, fmt.Errorf("failed to query allocated nodes for cluster %s: %w", cluster, err)
   667→	}
   668→
   669→	var raw []byte
   670→	defer rows.Close()
   671→	for rows.Next() {
   672→		raw = raw[0:0]
   673→		var resources []*schema.Resource
   674→		var subcluster string
   675→		if err := rows.Scan(&raw, &subcluster); err != nil {
   676→			cclog.Warnf("Error while scanning rows in AllocatedNodes: %v", err)
   677→			return nil, fmt.Errorf("failed to scan allocated nodes row: %w", err)
   678→		}
   679→		if err := json.Unmarshal(raw, &resources); err != nil {
   680→			cclog.Warnf("Error while unmarshaling raw resources json in AllocatedNodes: %v", err)
   681→			return nil, fmt.Errorf("failed to unmarshal resources in AllocatedNodes: %w", err)
   682→		}
   683→
   684→		hosts, ok := subclusters[subcluster]
   685→		if !ok {
   686→			hosts = make(map[string]int)
   687→			subclusters[subcluster] = hosts
   688→		}
   689→
   690→		for _, resource := range resources {
   691→			hosts[resource.Hostname] += 1
   692→		}
   693→	}
   694→
   695→	cclog.Debugf("Timer AllocatedNodes %s", time.Since(start))
   696→	return subclusters, nil
   697→}
   698→
   699→// StopJobsExceedingWalltimeBy marks running jobs as failed if they exceed their walltime limit.
   700→// This is typically called periodically to clean up stuck or orphaned jobs.
   701→//
   702→// Jobs are marked with:
   703→//   - monitoring_status: MonitoringStatusArchivingFailed
   704→//   - duration: 0
   705→//   - job_state: JobStateFailed
   706→//
   707→// Parameters:
   708→//   - seconds: Grace period beyond walltime before marking as failed
   709→//
   710→// Returns an error if the database update fails.
   711→// Logs the number of jobs marked as failed if any were affected.
   712→func (r *JobRepository) StopJobsExceedingWalltimeBy(seconds int) error {
   713→	start := time.Now()
   714→	currentTime := time.Now().Unix()
   715→	res, err := sq.Update("job").
   716→		Set("monitoring_status", schema.MonitoringStatusArchivingFailed).
   717→		Set("duration", 0).
   718→		Set("job_state", schema.JobStateFailed).
   719→		Where("job.job_state = 'running'").
   720→		Where("job.walltime > 0").
   721→		Where("job.start_time < (? - job.walltime)", currentTime-int64(seconds)).
   722→		RunWith(r.DB).Exec()
   723→	if err != nil {
   724→		cclog.Warnf("Error while stopping jobs exceeding walltime: %v", err)
   725→		return fmt.Errorf("failed to stop jobs exceeding walltime: %w", err)
   726→	}
   727→
   728→	rowsAffected, err := res.RowsAffected()
   729→	if err != nil {
   730→		cclog.Warnf("Error while fetching affected rows after stopping due to exceeded walltime: %v", err)
   731→		return fmt.Errorf("failed to get rows affected count: %w", err)
   732→	}
   733→
   734→	if rowsAffected > 0 {
   735→		cclog.Infof("%d jobs have been marked as failed due to running too long", rowsAffected)
   736→	}
   737→	cclog.Debugf("Timer StopJobsExceedingWalltimeBy %s", time.Since(start))
   738→	return nil
   739→}
   740→
   741→// FindJobIdsByTag returns all job database IDs associated with a specific tag.
   742→//
   743→// Parameters:
   744→//   - tagID: Database ID of the tag to search for
   745→//
   746→// Returns a slice of job IDs or an error if the query fails.
   747→func (r *JobRepository) FindJobIdsByTag(tagID int64) ([]int64, error) {
   748→	query := sq.Select("job.id").From("job").
   749→		Join("jobtag ON jobtag.job_id = job.id").
   750→		Where(sq.Eq{"jobtag.tag_id": tagID}).Distinct()
   751→	rows, err := query.RunWith(r.stmtCache).Query()
   752→	if err != nil {
   753→		cclog.Errorf("Error while running FindJobIdsByTag query for tagID=%d: %v", tagID, err)
   754→		return nil, fmt.Errorf("failed to find job IDs by tag %d: %w", tagID, err)
   755→	}
   756→	defer rows.Close()
   757→
   758→	jobIds := make([]int64, 0, 100)
   759→
   760→	for rows.Next() {
   761→		var jobID int64
   762→
   763→		if err := rows.Scan(&jobID); err != nil {
   764→			cclog.Warnf("Error while scanning rows in FindJobIdsByTag: %v", err)
   765→			return nil, fmt.Errorf("failed to scan job ID in FindJobIdsByTag: %w", err)
   766→		}
   767→
   768→		jobIds = append(jobIds, jobID)
   769→	}
   770→
   771→	return jobIds, nil
   772→}
   773→
   774→// FindRunningJobs returns all currently running jobs for a specific cluster.
   775→// Filters out short-running jobs based on repoConfig.MinRunningJobDuration threshold.
   776→//
   777→// Parameters:
   778→//   - cluster: Cluster name to filter jobs
   779→//
   780→// Returns a slice of running job objects or an error if the query fails.
   781→func (r *JobRepository) FindRunningJobs(cluster string) ([]*schema.Job, error) {
   782→	query := sq.Select(jobColumns...).From("job").
   783→		Where("job.cluster = ?", cluster).
   784→		Where("job.job_state = 'running'").
   785→		Where("job.duration > ?", repoConfig.MinRunningJobDuration)
   786→
   787→	rows, err := query.RunWith(r.stmtCache).Query()
   788→	if err != nil {
   789→		cclog.Errorf("Error while running FindRunningJobs query for cluster=%s: %v", cluster, err)
   790→		return nil, fmt.Errorf("failed to find running jobs for cluster %s: %w", cluster, err)
   791→	}
   792→	defer rows.Close()
   793→
   794→	jobs := make([]*schema.Job, 0, 50)
   795→	for rows.Next() {
   796→		job, err := scanJob(rows)
   797→		if err != nil {
   798→			cclog.Warnf("Error while scanning rows in FindRunningJobs: %v", err)
   799→			return nil, fmt.Errorf("failed to scan job in FindRunningJobs: %w", err)
   800→		}
   801→		jobs = append(jobs, job)
   802→	}
   803→
   804→	cclog.Debugf("JobRepository.FindRunningJobs(): Return job count %d (cluster: %s)", len(jobs), cluster)
   805→	return jobs, nil
   806→}
   807→
   808→// UpdateDuration recalculates and updates the duration field for all running jobs.
   809→// Called periodically to keep job durations current without querying individual jobs.
   810→//
   811→// Duration is calculated as: current_time - job.start_time
   812→//
   813→// Returns an error if the database update fails.
   814→func (r *JobRepository) UpdateDuration() error {
   815→	stmnt := sq.Update("job").
   816→		Set("duration", sq.Expr("? - job.start_time", time.Now().Unix())).
   817→		Where("job_state = 'running'")
   818→
   819→	_, err := stmnt.RunWith(r.stmtCache).Exec()
   820→	if err != nil {
   821→		cclog.Errorf("Error while updating duration for running jobs: %v", err)
   822→		return fmt.Errorf("failed to update duration for running jobs: %w", err)
   823→	}
   824→
   825→	return nil
   826→}
   827→
   828→// FindJobsBetween returns jobs within a specified time range.
   829→// If startTimeBegin is 0, returns all jobs before startTimeEnd.
   830→// Optionally excludes tagged jobs from results.
   831→//
   832→// Parameters:
   833→//   - startTimeBegin: Unix timestamp for range start (use 0 for unbounded start)
   834→//   - startTimeEnd: Unix timestamp for range end
   835→//   - omitTagged: "none" = include all jobs, "all" = exclude any tagged jobs,
   836→//     "user" = exclude jobs with user-created tags (not auto-tagger types "app"/"jobClass")
   837→//
   838→// Returns a slice of jobs or an error if the time range is invalid or query fails.
   839→func (r *JobRepository) FindJobsBetween(startTimeBegin int64, startTimeEnd int64, omitTagged string) ([]*schema.Job, error) {
   840→	var query sq.SelectBuilder
   841→
   842→	if startTimeBegin == startTimeEnd || startTimeBegin > startTimeEnd {
   843→		return nil, errors.New("startTimeBegin is equal or larger startTimeEnd")
   844→	}
   845→
   846→	if startTimeBegin == 0 {
   847→		cclog.Infof("Find jobs before %d", startTimeEnd)
   848→		query = sq.Select(jobColumns...).From("job").Where("job.start_time < ?", startTimeEnd)
   849→	} else {
   850→		cclog.Infof("Find jobs between %d and %d", startTimeBegin, startTimeEnd)
   851→		query = sq.Select(jobColumns...).From("job").Where("job.start_time BETWEEN ? AND ?", startTimeBegin, startTimeEnd)
   852→	}
   853→
   854→	switch omitTagged {
   855→	case "all":
   856→		query = query.Where("NOT EXISTS (SELECT 1 FROM jobtag WHERE jobtag.job_id = job.id)")
   857→	case "user":
   858→		query = query.Where("NOT EXISTS (SELECT 1 FROM jobtag JOIN tag ON tag.id = jobtag.tag_id WHERE jobtag.job_id = job.id AND tag.tag_type NOT IN ('app', 'jobClass'))")
   859→	}
   860→
   861→	query = query.OrderBy("job.cluster ASC", "job.subcluster ASC", "job.project ASC", "job.start_time ASC")
   862→
   863→	rows, err := query.RunWith(r.stmtCache).Query()
   864→	if err != nil {
   865→		cclog.Errorf("Error while running FindJobsBetween query: %v", err)
   866→		return nil, fmt.Errorf("failed to find jobs between %d and %d: %w", startTimeBegin, startTimeEnd, err)
   867→	}
   868→	defer rows.Close()
   869→
   870→	jobs := make([]*schema.Job, 0, 50)
   871→	for rows.Next() {
   872→		job, err := scanJob(rows)
   873→		if err != nil {
   874→			cclog.Warnf("Error while scanning rows in FindJobsBetween: %v", err)
   875→			return nil, fmt.Errorf("failed to scan job in FindJobsBetween: %w", err)
   876→		}
   877→		jobs = append(jobs, job)
   878→	}
   879→
   880→	cclog.Debugf("JobRepository.FindJobsBetween(): Return job count %d (omitTagged: %v)", len(jobs), omitTagged)
   881→	return jobs, nil
   882→}
   883→
   884→// UpdateMonitoringStatus updates the monitoring status for a job and invalidates its cache entries.
   885→// Cache invalidation affects both metadata and energy footprint to ensure consistency.
   886→//
   887→// Parameters:
   888→//   - job: Database ID of the job to update
   889→//   - monitoringStatus: New monitoring status value (see schema.MonitoringStatus constants)
   890→//
   891→// Returns an error if the database update fails.
   892→func (r *JobRepository) UpdateMonitoringStatus(job int64, monitoringStatus int32) (err error) {
   893→	// Invalidate cache entries as monitoring status affects job state
   894→	r.cache.Del(fmt.Sprintf("metadata:%d", job))
   895→	r.cache.Del(fmt.Sprintf("energyFootprint:%d", job))
   896→
   897→	stmt := sq.Update("job").
   898→		Set("monitoring_status", monitoringStatus).
   899→		Where("job.id = ?", job)
   900→
   901→	if _, err = stmt.RunWith(r.stmtCache).Exec(); err != nil {
   902→		cclog.Errorf("Error while updating monitoring status for job %d: %v", job, err)
   903→		return fmt.Errorf("failed to update monitoring status for job %d: %w", job, err)
   904→	}
   905→	return nil
   906→}
   907→
   908→// Execute runs a Squirrel UpdateBuilder statement against the database.
   909→// This is a generic helper for executing pre-built update queries.
   910→//
   911→// Parameters:
   912→//   - stmt: Squirrel UpdateBuilder with prepared update query
   913→//
   914→// Returns an error if the execution fails.
   915→func (r *JobRepository) Execute(stmt sq.UpdateBuilder) error {
   916→	if _, err := stmt.RunWith(r.stmtCache).Exec(); err != nil {
   917→		cclog.Errorf("Error while executing statement: %v", err)
   918→		return fmt.Errorf("failed to execute update statement: %w", err)
   919→	}
   920→
   921→	return nil
   922→}
   923→
   924→// MarkArchived adds monitoring status update to an existing UpdateBuilder statement.
   925→// This is a builder helper used when constructing multi-field update queries.
   926→//
   927→// Parameters:
   928→//   - stmt: Existing UpdateBuilder to modify
   929→//   - monitoringStatus: Monitoring status value to set
   930→//
   931→// Returns the modified UpdateBuilder for method chaining.
   932→func (r *JobRepository) MarkArchived(
   933→	stmt sq.UpdateBuilder,
   934→	monitoringStatus int32,
   935→) sq.UpdateBuilder {
   936→	return stmt.Set("monitoring_status", monitoringStatus)
   937→}
   938→
   939→// UpdateEnergy calculates and updates the energy consumption for a job.
   940→// This is called for running jobs during intermediate updates or when archiving.
   941→//
   942→// Energy calculation formula:
   943→//   - For "power" metrics: Energy (kWh) = (Power_avg * NumNodes * Duration_hours) / 1000
   944→//   - For "energy" metrics: Currently not implemented (would need sum statistics)
   945→//
   946→// The calculation accounts for:
   947→//   - Multi-node jobs: Multiplies by NumNodes to get total cluster energy
   948→//   - Shared jobs: Node average is already based on partial resources, so NumNodes=1
   949→//   - Unit conversion: Watts * hours / 1000 = kilowatt-hours (kWh)
   950→//   - Rounding: Results rounded to 2 decimal places
   951→func (r *JobRepository) UpdateEnergy(
   952→	stmt sq.UpdateBuilder,
   953→	jobMeta *schema.Job,
   954→) (sq.UpdateBuilder, error) {
   955→	sc, err := archive.GetSubCluster(jobMeta.Cluster, jobMeta.SubCluster)
   956→	if err != nil {
   957→		cclog.Errorf("cannot get subcluster: %s", err.Error())
   958→		return stmt, err
   959→	}
   960→	energyFootprint := make(map[string]float64)
   961→
   962→	// Accumulate total energy across all energy-related metrics
   963→	totalEnergy := 0.0
   964→	for _, fp := range sc.EnergyFootprint {
   965→		// Calculate energy for this specific metric
   966→		metricEnergy := 0.0
   967→		if i, err := archive.MetricIndex(sc.MetricConfig, fp); err == nil {
   968→			switch sc.MetricConfig[i].Energy {
   969→			case "energy": // Metric already in energy units (Joules or Wh)
   970→				cclog.Warnf("Update EnergyFootprint for Job %d and Metric %s on cluster %s: Set to 'energy' in cluster.json: Not implemented, will return 0.0", jobMeta.JobID, jobMeta.Cluster, fp)
   971→				// FIXME: Needs sum as stats type to accumulate energy values over time
   972→			case "power": // Metric in power units (Watts)
   973→				// Energy (kWh) = Power (W) × Time (h) / 1000
   974→				// Formula: (avg_power_per_node * num_nodes) * (duration_sec / 3600) / 1000
   975→				//
   976→				// Breakdown:
   977→				//   LoadJobStat(jobMeta, fp, "avg") = average power per node (W)
   978→				//   jobMeta.NumNodes = number of nodes (1 for shared jobs)
   979→				//   jobMeta.Duration / 3600.0 = duration in hours
   980→				//   / 1000.0 = convert Wh to kWh
   981→				rawEnergy := ((LoadJobStat(jobMeta, fp, "avg") * float64(jobMeta.NumNodes)) * (float64(jobMeta.Duration) / 3600.0)) / 1000.0
   982→				metricEnergy = math.Round(rawEnergy*100.0) / 100.0 // Round to 2 decimal places
   983→			}
   984→		} else {
   985→			cclog.Warnf("Error while collecting energy metric %s for job, DB ID '%v', return '0.0'", fp, jobMeta.ID)
   986→		}
   987→
   988→		energyFootprint[fp] = metricEnergy
   989→		totalEnergy += metricEnergy
   990→	}
   991→
   992→	var rawFootprint []byte
   993→	if rawFootprint, err = json.Marshal(energyFootprint); err != nil {
   994→		cclog.Warnf("Error while marshaling energy footprint for job INTO BYTES, DB ID '%v'", jobMeta.ID)
   995→		return stmt, err
   996→	}
   997→
   998→	return stmt.Set("energy_footprint", string(rawFootprint)).Set("energy", (math.Round(totalEnergy*100.0) / 100.0)), nil
   999→}
  1000→
  1001→// UpdateFootprint calculates and updates the performance footprint for a job.
  1002→// This is called for running jobs during intermediate updates or when archiving.
  1003→//
  1004→// A footprint is a summary statistic (avg/min/max) for each monitored metric.
  1005→// The specific statistic type is defined in the cluster config's Footprint field.
  1006→// Results are stored as JSON with keys like "metric_avg", "metric_max", etc.
  1007→//
  1008→// Example: For a "cpu_load" metric with Footprint="avg", this stores
  1009→// the average CPU load across all nodes as "cpu_load_avg": 85.3
  1010→func (r *JobRepository) UpdateFootprint(
  1011→	stmt sq.UpdateBuilder,
  1012→	jobMeta *schema.Job,
  1013→) (sq.UpdateBuilder, error) {
  1014→	sc, err := archive.GetSubCluster(jobMeta.Cluster, jobMeta.SubCluster)
  1015→	if err != nil {
  1016→		cclog.Errorf("cannot get subcluster: %s", err.Error())
  1017→		return stmt, err
  1018→	}
  1019→	footprint := make(map[string]float64)
  1020→
  1021→	// Build footprint map with metric_stattype as keys
  1022→	for _, fp := range sc.Footprint {
  1023→		// Determine which statistic to use: avg, min, or max
  1024→		// First check global metric config, then cluster-specific config
  1025→		var statType string
  1026→		for _, gm := range archive.GlobalMetricList {
  1027→			if gm.Name == fp {
  1028→				statType = gm.Footprint
  1029→			}
  1030→		}
  1031→
  1032→		// Validate statistic type
  1033→		if statType != "avg" && statType != "min" && statType != "max" {
  1034→			cclog.Warnf("unknown statType for footprint update: %s", statType)
  1035→			return stmt, fmt.Errorf("unknown statType for footprint update: %s", statType)
  1036→		}
  1037→
  1038→		// Override with cluster-specific config if available
  1039→		if i, err := archive.MetricIndex(sc.MetricConfig, fp); err != nil {
  1040→			statType = sc.MetricConfig[i].Footprint
  1041→		}
  1042→
  1043→		// Store as "metric_stattype": value (e.g., "cpu_load_avg": 85.3)
  1044→		name := fmt.Sprintf("%s_%s", fp, statType)
  1045→		footprint[name] = LoadJobStat(jobMeta, fp, statType)
  1046→	}
  1047→
  1048→	var rawFootprint []byte
  1049→	if rawFootprint, err = json.Marshal(footprint); err != nil {
  1050→		cclog.Warnf("Error while marshaling footprint for job INTO BYTES, DB ID '%v'", jobMeta.ID)
  1051→		return stmt, err
  1052→	}
  1053→
  1054→	return stmt.Set("footprint", string(rawFootprint)), nil
  1055→}
  1056→
  1057→// GetUsedNodes returns a map of cluster names to sorted lists of unique hostnames
  1058→// that are currently in use by jobs that started before the given timestamp and
  1059→// are still in running state.
  1060→//
  1061→// The timestamp parameter (ts) is compared against job.start_time to find
  1062→// relevant jobs. Returns an error if the database query fails or row iteration
  1063→// encounters errors. Individual row parsing errors are logged but don't fail
  1064→// the entire operation.
  1065→func (r *JobRepository) GetUsedNodes(ts int64) (map[string][]string, error) {
  1066→	// Note: Query expects index on (job_state, start_time) for optimal performance
  1067→	q := sq.Select("job.cluster", "job.resources").From("job").
  1068→		Where("job.start_time < ?", ts).
  1069→		Where(sq.Eq{"job.job_state": "running"})
  1070→
  1071→	rows, err := q.RunWith(r.stmtCache).Query()
  1072→	if err != nil {
  1073→		queryString, queryVars, _ := q.ToSql()
  1074→		return nil, fmt.Errorf("query failed [%s] %v: %w", queryString, queryVars, err)
  1075→	}
  1076→	defer rows.Close()
  1077→
  1078→	// Use a map of sets for efficient deduplication
  1079→	nodeSet := make(map[string]map[string]struct{})
  1080→
  1081→	var (
  1082→		cluster      string
  1083→		rawResources []byte
  1084→		resources    []*schema.Resource
  1085→		skippedRows  int
  1086→	)


internal/repository/node.go:286:		OrderBy("node.cluster ASC", "node.subcluster ASC", "node.hostname ASC", "node_state.time_stamp ASC").
internal/repository/node.go:343:	order *model.OrderByInput, // Currently unused!
internal/repository/node.go:356:	query = query.OrderBy("node.hostname ASC")
internal/repository/node.go:395:	order *model.OrderByInput, // Currently unused!
internal/repository/node.go:409:	query = query.OrderBy("node.hostname ASC")
internal/repository/node.go:502:		OrderBy("node.hostname ASC")
internal/repository/node.go:531:		OrderBy("node.hostname ASC")
internal/repository/node.go:625:	query = query.GroupBy(column + ", time_stamp").OrderBy("time_stamp ASC")
internal/repository/jobQuery.go:46:	order *model.OrderByInput,
internal/repository/jobQuery.go:58:				query = query.OrderBy(fmt.Sprintf("job.%s ASC", field))
internal/repository/jobQuery.go:60:				query = query.OrderBy(fmt.Sprintf("job.%s DESC", field))
internal/repository/jobQuery.go:69:				query = query.OrderBy(fmt.Sprintf("JSON_EXTRACT(footprint, \"$.%s\") ASC", field))
internal/repository/jobQuery.go:71:				query = query.OrderBy(fmt.Sprintf("JSON_EXTRACT(footprint, \"$.%s\") DESC", field))
internal/repository/repository_test.go:116:	order := &model.OrderByInput{Field: "startTime", Type: "col", Order: model.SortDirectionEnumDesc}
internal/repository/jobFind.go:45:	q = q.OrderBy("job.id DESC").Limit(1) // always use newest matching job by db id if more than one match
internal/repository/jobFind.go:76:	q = q.OrderBy("job_cache.id DESC").Limit(1) // always use newest matching job by db id if more than one match
internal/repository/job.go:861:	query = query.OrderBy("job.cluster ASC", "job.subcluster ASC", "job.project ASC", "job.start_time ASC")
internal/repository/stats.go:62:// into SQL GROUP BY clauses. GraphQL validation ensures only valid enum values are accepted.
internal/repository/stats.go:90://   - col: Column name to GROUP BY; empty string for total count without grouping
internal/repository/stats.go:131://   - col: Column name to GROUP BY; empty string for overall statistics without grouping
internal/repository/stats.go:226:		query = query.OrderBy(fmt.Sprintf("%s DESC", sortBy))
internal/repository/stats.go:960:	mainQuery = mainQuery.GroupBy("bin").OrderBy("bin")

[Showing results with pagination = limit: 100, offset: 0]


Perfect! I now have the complete picture. Let me compile the comprehensive analysis of the job table indexes.

## Complete Job Table Index Situation

### JOB TABLE SCHEMA (Final)

After migration 09_add-job-cache.up.sql, the job table has these columns:

```
id                INTEGER PRIMARY KEY
job_id            BIGINT NOT NULL
cluster           TEXT NOT NULL
subcluster        TEXT NOT NULL
submit_time       BIGINT NOT NULL DEFAULT 0
start_time        BIGINT NOT NULL DEFAULT 0
hpc_user          TEXT NOT NULL (renamed from 'user' in migration 08)
project           TEXT NOT NULL
cluster_partition TEXT (renamed from 'partition' in migration 08)
array_job_id      BIGINT
duration          INT NOT NULL
walltime          INT NOT NULL
job_state         TEXT NOT NULL
meta_data         TEXT (JSON)
resources         TEXT NOT NULL (JSON)
num_nodes         INT NOT NULL
num_hwthreads     INT
num_acc           INT
smt               INT NOT NULL DEFAULT 1
shared            TEXT NOT NULL (added in migration 09)
monitoring_status TINYINT NOT NULL DEFAULT 1
energy            REAL NOT NULL DEFAULT 0.0 (added in migration 08)
energy_footprint  TEXT DEFAULT NULL (added in migration 08)
footprint         TEXT DEFAULT NULL (added in migration 08)
UNIQUE (job_id, cluster, start_time)
```

### DROPPED INDEXES

Migration 08 dropped 8 indexes before creating a larger set:
1. job_stats
2. job_by_user
3. job_by_starttime
4. job_by_job_id
5. job_list
6. job_list_user
7. job_list_users
8. job_list_users_start

Migration 09 dropped all job indexes when the job table was recreated (line 118: "DROP TABLE job; -- Deletes All Existing 'job' Indices"). These were re-created with optimizations.

### CURRENT INDEXES (76 Total)

#### **Organized by Leading Column:**

**CLUSTER** (14 indexes):
1. jobs_cluster_user - (cluster, hpc_user)
2. jobs_cluster_project - (cluster, project)
3. jobs_cluster_subcluster - (cluster, subcluster)
4. jobs_cluster_numnodes - (cluster, num_nodes)
5. jobs_cluster_numhwthreads - (cluster, num_hwthreads)
6. jobs_cluster_numacc - (cluster, num_acc)
7. jobs_cluster_energy - (cluster, energy)
8. jobs_cluster_duration_starttime - (cluster, duration, start_time)
9. jobs_cluster_starttime_duration - (cluster, start_time, duration)
10. jobs_cluster_partition_user - (cluster, cluster_partition, hpc_user)
11. jobs_cluster_partition_project - (cluster, cluster_partition, project)
12. jobs_cluster_partition_jobstate - (cluster, cluster_partition, job_state)
13. jobs_cluster_partition_shared - (cluster, cluster_partition, shared)
14. jobs_cluster_partition_numnodes - (cluster, cluster_partition, num_nodes)

**CLUSTER+PARTITION** (10 indexes):
15. jobs_cluster_partition_numhwthreads - (cluster, cluster_partition, num_hwthreads)
16. jobs_cluster_partition_numacc - (cluster, cluster_partition, num_acc)
17. jobs_cluster_partition_energy - (cluster, cluster_partition, energy)
18. jobs_cluster_partition_duration_starttime - (cluster, cluster_partition, duration, start_time)
19. jobs_cluster_partition_starttime_duration - (cluster, cluster_partition, start_time, duration)
20. jobs_cluster_jobstate_user - (cluster, job_state, hpc_user)
21. jobs_cluster_jobstate_project - (cluster, job_state, project)
22. jobs_cluster_jobstate_numnodes - (cluster, job_state, num_nodes)
23. jobs_cluster_jobstate_numhwthreads - (cluster, job_state, num_hwthreads)
24. jobs_cluster_jobstate_numacc - (cluster, job_state, num_acc)

**CLUSTER+JOBSTATE** (8 indexes):
25. jobs_cluster_jobstate_energy - (cluster, job_state, energy)
26. jobs_cluster_jobstate_starttime_duration - (cluster, job_state, start_time, duration)
27. jobs_cluster_jobstate_duration_starttime - (cluster, job_state, duration, start_time)
28. jobs_cluster_shared_user - (cluster, shared, hpc_user)
29. jobs_cluster_shared_project - (cluster, shared, project)
30. jobs_cluster_shared_numnodes - (cluster, shared, num_nodes)
31. jobs_cluster_shared_numhwthreads - (cluster, shared, num_hwthreads)
32. jobs_cluster_shared_numacc - (cluster, shared, num_acc)

**CLUSTER+SHARED** (8 indexes):
33. jobs_cluster_shared_energy - (cluster, shared, energy)
34. jobs_cluster_shared_starttime_duration - (cluster, shared, start_time, duration)
35. jobs_cluster_shared_duration_starttime - (cluster, shared, duration, start_time)
36. jobs_user_numnodes - (hpc_user, num_nodes)
37. jobs_user_numhwthreads - (hpc_user, num_hwthreads)
38. jobs_user_numacc - (hpc_user, num_acc)
39. jobs_user_energy - (hpc_user, energy)
40. jobs_user_starttime_duration - (hpc_user, start_time, duration)

**HPC_USER** (8 indexes):
41. jobs_user_duration_starttime - (hpc_user, duration, start_time)
42. jobs_project_user - (project, hpc_user)
43. jobs_project_numnodes - (project, num_nodes)
44. jobs_project_numhwthreads - (project, num_hwthreads)
45. jobs_project_numacc - (project, num_acc)
46. jobs_project_energy - (project, energy)
47. jobs_project_starttime_duration - (project, start_time, duration)
48. jobs_project_duration_starttime - (project, duration, start_time)

**PROJECT** (8 indexes):
49. jobs_jobstate_user - (job_state, hpc_user)
50. jobs_jobstate_project - (job_state, project)
51. jobs_jobstate_numnodes - (job_state, num_nodes)
52. jobs_jobstate_numhwthreads - (job_state, num_hwthreads)
53. jobs_jobstate_numacc - (job_state, num_acc)
54. jobs_jobstate_energy - (job_state, energy)
55. jobs_jobstate_starttime_duration - (job_state, start_time, duration)
56. jobs_jobstate_duration_starttime - (job_state, duration, start_time)

**JOB_STATE** (8 indexes):
57. jobs_shared_user - (shared, hpc_user)
58. jobs_shared_project - (shared, project)
59. jobs_shared_numnodes - (shared, num_nodes)
60. jobs_shared_numhwthreads - (shared, num_hwthreads)
61. jobs_shared_numacc - (shared, num_acc)
62. jobs_shared_energy - (shared, energy)
63. jobs_shared_starttime_duration - (shared, start_time, duration)
64. jobs_shared_duration_starttime - (shared, duration, start_time)

**SHARED** (8 indexes):
65. jobs_arrayjobid_starttime - (array_job_id, start_time)
66. jobs_cluster_arrayjobid_starttime - (cluster, array_job_id, start_time)
67. jobs_duration_starttime - (duration, start_time)
68. jobs_numnodes_starttime - (num_nodes, start_time)
69. jobs_numhwthreads_starttime - (num_hwthreads, start_time)
70. jobs_numacc_starttime - (num_acc, start_time)
71. jobs_energy_starttime - (energy, start_time)
72. jobs_starttime_duration - (start_time, duration)

**ARRAY_JOB_ID / SINGLE COLUMN SORTS** (8 indexes):
73. jobs_numnodes_duration - (num_nodes, duration)
74. jobs_numhwthreads_duration - (num_hwthreads, duration)
75. jobs_numacc_duration - (num_acc, duration)
76. jobs_energy_duration - (energy, duration)
77. jobs_starttime - (start_time) — backup index
78. jobs_duration - (duration) — backup index

### KEY OBSERVATIONS

1. **76 CREATE INDEX statements** across migrations 02, 04, 08, and 09
2. **Composite indexes dominate** - most are 2-4 column indexes to support filtering + sorting combinations
3. **Covering pattern**: Many indexes include sorting columns (start_time, duration) as covered columns
4. **Leading columns**: cluster (heavily used), hpc_user, project, job_state, shared, array_job_id, duration, num_nodes, num_hwthreads, num_acc, energy
5. **Sorting columns**: start_time and duration appear in 30+ indexes as included columns
6. **Migration 08** introduced the comprehensive index strategy with 98 CREATE INDEX statements
7. **Migration 09** refined the indexes when renaming columns and adding the `shared` column, optimizing away some redundant indexes and restructuring the index set to 76 final indexes

The current schema follows a **query-driven indexing strategy** where each common filter combination with its default sort gets its own index to optimize query plans.

agentId: a1ede6e4db46fa8db (for resuming to continue this agent's work if needed)
<usage>total_tokens: 31547
tool_uses: 11
duration_ms: 21471</usage>

Prompt:
In /Users/jan/prg/CC/cc-backend, I need to understand the current index situation on the job table.

1. Read ALL migration files in `internal/repository/migrations/sqlite3/` - especially 08_add-footprint.up.sql and 09_add-job-cache.up.sql which define the current indexes
2. List EVERY CREATE INDEX statement for the job table across all migrations
3. Note which indexes were DROPped in later migrations
4. Read the job table schema (CREATE TABLE) from 01_init-schema.up.sql and any modifications in later migrations

I need the complete picture of:
- All 79+ indexes currently defined
- The job table column list with types
- Which columns have individual indexes vs composite indexes
- Any covering index patterns

Report the full list organized by leading column.


Perfect! Now I have a comprehensive understanding of all query patterns. Let me compile the complete analysis:

## COMPREHENSIVE JOB TABLE QUERY ANALYSIS FOR INDEX OPTIMIZATION

### Overview
The `job` table in this 20M row database is queried through multiple access patterns across 9 repository files. I've identified **28 distinct query sites** with varying WHERE, ORDER BY, GROUP BY, JOIN, and LIMIT patterns.

---

## 1. MAIN QUERYING FUNCTIONS (jobQuery.go)

### QueryJobs - Security-controlled list with filtering, pagination, sorting
**File:** `/Users/jan/prg/CC/cc-backend/internal/repository/jobQuery.go` (line 42)
- **Query Type:** SELECT with filters, pagination, sorting
- **FROM:** `FROM job`
- **WHERE Columns:** (from BuildWhereClause - see section below)
- **ORDER BY:** Dynamic columns via `order.Field` (user, project, duration, etc.) OR JSON_EXTRACT on footprint
- **LIMIT/PAGINATION:** Yes - `LIMIT` and `OFFSET`
- **JOINs:** `JOIN jobtag ON jobtag.job_id = job.id` (conditional, when tag filtering)
- **GROUP BY:** `DISTINCT` when joined with jobtag
- **Critical:** Security checks apply, respects user roles

### CountJobs - Count matching jobs (for pagination)
**File:** `/Users/jan/prg/CC/cc-backend/internal/repository/jobQuery.go` (line 115)
- **Query Type:** COUNT(DISTINCT job.id)
- **FROM:** `FROM job`
- **WHERE Columns:** Same as QueryJobs (applies BuildWhereClause)
- **JOINs:** `JOIN jobtag` (conditional, for tag filtering)
- **GROUP BY:** DISTINCT (when tags involved)
- **Purpose:** Pagination total count

---

## 2. BUILD WHERE CLAUSE - ALL POSSIBLE FILTER COLUMNS (jobQuery.go line 182)

This function builds WHERE conditions from filters. Here are ALL filter columns accessed:

| Column Name | Filter Type | Index? | Usage |
|------------|-------------|--------|-------|
| `job.id` | Primary Key lookup | **YES (PK)** | filter.DbID |
| `job.cluster` | String equality/LIKE | **CONSIDER** | Direct filter, JOIN target |
| `job.subcluster` | String equality/LIKE | **CONSIDER** | Geographic partitioning |
| `job.cluster_partition` | String equality/LIKE | **OPTIONAL** | Partition filtering |
| `job.job_state` | IN list | **CRITICAL** | Running/completed distinction |
| `job.shared` | Boolean | **OPTIONAL** | Shared job filtering |
| `job.project` | String equality/LIKE | **CONSIDER** | Project-scoped access |
| `job.hpc_user` | String equality/LIKE | **CRITICAL** | Security: user-scoped access |
| `job.num_nodes` | Range (BETWEEN/>=/<) | **OPTIONAL** | Resource filtering |
| `job.num_hwthreads` | Range (BETWEEN/>=/<) | **OPTIONAL** | Core count filtering |
| `job.num_acc` | Range (BETWEEN/>=/<) | **OPTIONAL** | Accelerator filtering |
| `job.array_job_id` | Direct equality | **OPTIONAL** | Array job grouping |
| `job.start_time` | Range/relative time | **CRITICAL** | Time filtering (last6h, last30d, etc.) |
| `job.duration` | Range (BETWEEN/>=/<) | **OPTIONAL** | Duration filtering |
| `job.energy` | Range (BETWEEN/>=/<) | **OPTIONAL** | Energy filtering |
| **JSON columns:** | | | |
| `job.footprint` | JSON_EXTRACT | **OPTIONAL** | Metric statistics in JSON |
| `job.meta_data` | JSON_EXTRACT | **OPTIONAL** | Job name, interactive flag |
| `job.resources` | json_each() EXISTS | **OPTIONAL** | Hostname lookup via JSON array |

**Additional Complex Filters:**
- `job.job_state` + `job.start_time` (MinRunningFor - recent job filtering)
- Tag-based filtering via JOIN jobtag table

---

## 3. FIND/LOOKUP QUERIES (jobFind.go)

### Find - Single job by job_id + cluster + start_time
**File:** `/Users/jan/prg/CC/cc-backend/internal/repository/jobFind.go` (line 25)
- **WHERE:** `job.job_id`, `job.cluster`, `job.start_time` (optional)
- **ORDER BY:** `job.id DESC`
- **LIMIT:** 1
- **Purpose:** Exact job lookup by SLURM credentials

### FindCached - Same but from job_cache table
**File:** `/Users/jan/prg/CC/cc-backend/internal/repository/jobFind.go` (line 57)
- **Same pattern as Find but on job_cache table**

### FindAll - All jobs matching job_id + cluster + start_time
**File:** `/Users/jan/prg/CC/cc-backend/internal/repository/jobFind.go` (line 86)
- **WHERE:** Same as Find
- **No LIMIT, returns all matches**

### GetJobList - IDs of non-running jobs (for archiving)
**File:** `/Users/jan/prg/CC/cc-backend/internal/repository/jobFind.go` (line 130)
- **WHERE:** `job.job_state != 'running'`
- **LIMIT/OFFSET:** Yes (pagination for batch processing)
- **Returns:** Job IDs only

### FindByID - Single job by database ID
**File:** `/Users/jan/prg/CC/cc-backend/internal/repository/jobFind.go` (line 165)
- **WHERE:** `job.id = ?` (PK)
- **Security:** Applies SecurityCheck

### FindByIDDirect - Same without security checks
**File:** `/Users/jan/prg/CC/cc-backend/internal/repository/jobFind.go` (line 198)
- **WHERE:** `job.id = ?`
- **No security checks**

### FindByJobID - Job by job_id + cluster + start_time with security
**File:** `/Users/jan/prg/CC/cc-backend/internal/repository/jobFind.go` (line 208)
- **WHERE:** All three: `job.job_id`, `job.cluster`, `job.start_time`
- **Security:** Applies SecurityCheck

### IsJobOwner - Check user ownership
**File:** `/Users/jan/prg/CC/cc-backend/internal/repository/jobFind.go` (line 227)
- **WHERE:** `job.job_id`, `job.hpc_user`, `job.cluster`, `job.start_time`
- **Returns:** Boolean

### FindConcurrentJobs - Complex overlapping job query
**File:** `/Users/jan/prg/CC/cc-backend/internal/repository/jobFind.go` (line 242)
- **WHERE Columns:** `job.cluster`, `job.job_state`, `job.start_time`, `job.duration`, plus JSON resource matching
- **WHERE Logic:**
  - For running jobs: `job.job_state = 'running'` AND `job.start_time <= stopTimeTail`
  - For completed: `job.job_state != 'running'` AND time overlap check + `(job.start_time + job.duration) > startTimeFront`
  - Plus: `EXISTS (json_each(job.resources)...)` for hostname matching
- **Security:** Applies SecurityCheck
- **Purpose:** Find jobs running on same node during overlapping time

---

## 4. STATISTICS & AGGREGATION QUERIES (stats.go)

### JobsStatsGrouped - Aggregated stats grouped by dimension
**File:** `/Users/jan/prg/CC/cc-backend/internal/repository/stats.go` (line 208)
- **GROUP BY:** `job.hpc_user` OR `job.project` OR `job.cluster` OR `job.subcluster`
- **SELECT:** COUNT, SUM, aggregations with complex CASE for running jobs
- **WHERE:** All filter columns via BuildWhereClause
- **ORDER BY:** Dynamic by sort dimension (totalJobs, totalWalltime, totalCoreHours, etc.)
- **LIMIT/OFFSET:** Yes (pagination)
- **JOINs:** `LEFT JOIN hpc_user ON hpc_user.username = job.hpc_user`
- **Purpose:** Dashboard statistics, user/project/cluster reports

### JobsStats - Overall aggregation (no grouping)
**File:** `/Users/jan/prg/CC/cc-backend/internal/repository/stats.go` (line 347)
- **SELECT:** COUNT, SUM across all jobs
- **WHERE:** All filter columns
- **No GROUP BY, no LIMIT**
- **Purpose:** Total system statistics

### JobCountGrouped - Count only (lightweight)
**File:** `/Users/jan/prg/CC/cc-backend/internal/repository/stats.go` (line 431)
- **GROUP BY:** Same as JobsStatsGrouped
- **WHERE:** Filter columns
- **Purpose:** Count breakdown by group

### AddJobCountGrouped - Augment stats with running/short job counts
**File:** `/Users/jan/prg/CC/cc-backend/internal/repository/stats.go` (line 490)
- **WHERE:** GROUP BY column + special `kind`: "running" or "short"
- **Additional Filter:** `job.job_state = 'running'` OR `job.duration < config.ShortRunningJobsDuration`

### AddJobCount - Augment overall stats with counts
**File:** `/Users/jan/prg/CC/cc-backend/internal/repository/stats.go` (line 556)
- **Same as AddJobCountGrouped but ungrouped**

### jobsStatisticsHistogram - Histogram generation
**File:** `/Users/jan/prg/CC/cc-backend/internal/repository/stats.go` (line 741)
- **GROUP BY:** value (computed column, e.g., `job.num_nodes`, `job.num_hwthreads`, `job.num_acc`)
- **WHERE:** Via BuildWhereClause + Security
- **Purpose:** Distribution visualization

### jobsDurationStatisticsHistogram - Duration histogram
**File:** `/Users/jan/prg/CC/cc-backend/internal/repository/stats.go` (line 805)
- **GROUP BY:** Computed bin from duration division
- **WHERE:** Via BuildWhereClause + Security
- **Purpose:** Duration distribution with zero-padded bins

### jobsMetricStatisticsHistogram - Metric footprint histogram
**File:** `/Users/jan/prg/CC/cc-backend/internal/repository/stats.go` (line 886)
- **WHERE:**
  - `job.footprint` JSON_VALID check
  - JSON_EXTRACT condition on `job.footprint`
  - Via BuildWhereClause + Security
- **GROUP BY:** Computed bin from JSON value
- **ORDER BY:** bin ASC
- **Purpose:** Metric distribution for completed jobs

---

## 5. SINGLE JOB METADATA QUERIES (job.go)

### FetchMetadata - Get meta_data JSON for one job
**File:** `/Users/jan/prg/CC/cc-backend/internal/repository/job.go` (line 238)
- **SELECT:** `job.meta_data`
- **WHERE:** `job.id = ?` (PK)
- **Cached:** 24 hours

### UpdateMetadata - Update one metadata field
**File:** `/Users/jan/prg/CC/cc-backend/internal/repository/job.go` (line 296)
- **WHERE:** `job.id = ?`

### FetchFootprint - Get footprint JSON
**File:** `/Users/jan/prg/CC/cc-backend/internal/repository/job.go` (line 325)
- **WHERE:** `job.id = ?`
- **Not cached**

### FetchEnergyFootprint - Get energy_footprint JSON
**File:** `/Users/jan/prg/CC/cc-backend/internal/repository/job.go` (line 366)
- **WHERE:** `job.id = ?`
- **Cached:** 24 hours

---

## 6. DELETION & ARCHIVE QUERIES (job.go)

### DeleteJobsBefore - Delete jobs older than timestamp
**File:** `/Users/jan/prg/CC/cc-backend/internal/repository/job.go` (line 399)
- **WHERE:**
  - `job.start_time < ?` (CRITICAL for deletion)
  - Optional: `NOT EXISTS (SELECT 1 FROM jobtag...)` depending on `omitTagged` mode
- **Two queries:** COUNT first, then SELECT for cache invalidation, then DELETE

### DeleteJobByID - Delete single job
**File:** `/Users/jan/prg/CC/cc-backend/internal/repository/job.go` (line 470)
- **WHERE:** `job.id = ?`

---

## 7. SEARCH & LOOKUP QUERIES (job.go)

### FindUserOrProjectOrJobname - Flexible search
**File:** `/Users/jan/prg/CC/cc-backend/internal/repository/job.go` (line 505)
- **Calls FindColumnValue on job/hpc_user tables**

### FindColumnValue - Generic column lookup
**File:** `/Users/jan/prg/CC/cc-backend/internal/repository/job.go` (line 553)
- **WHERE:** `table.whereColumn = ? OR table.whereColumn LIKE ?`
- **SELECT:** `table.selectColumn`
- **DISTINCT:** Yes
- **Authorization:** Admin/Support/Manager only

### FindColumnValues - Multi-result column lookup
**File:** `/Users/jan/prg/CC/cc-backend/internal/repository/job.go` (line 596)
- **WHERE:** `table.whereColumn LIKE ?`
- **Returns:** All matches

### Partitions - Get cluster partitions
**File:** `/Users/jan/prg/CC/cc-backend/internal/repository/job.go` (line 637)
- **WHERE:** `job.cluster = ?`
- **SELECT DISTINCT:** `job.cluster_partition`
- **Cached:** 1 hour

### AllocatedNodes - Node usage query
**File:** `/Users/jan/prg/CC/cc-backend/internal/repository/job.go` (line 657)
- **WHERE:** `job.job_state = 'running'` AND `job.cluster = ?`
- **SELECT:** `job.resources`, `job.subcluster`
- **Purpose:** Count jobs per node

---

## 8. RUNNING JOB QUERIES (job.go)

### FindRunningJobs - Get running jobs for a cluster
**File:** `/Users/jan/prg/CC/cc-backend/internal/repository/job.go` (line 781)
- **WHERE:** `job.cluster = ?` AND `job.job_state = 'running'` AND `job.duration > repoConfig.MinRunningJobDuration`

### StopJobsExceedingWalltimeBy - Mark stuck jobs as failed
**File:** `/Users/jan/prg/CC/cc-backend/internal/repository/job.go` (line 712)
- **UPDATE query with WHERE:** `job.job_state = 'running'` AND `job.walltime > 0` AND `job.start_time < (? - job.walltime)`

### UpdateDuration - Update duration for running jobs
**File:** `/Users/jan/prg/CC/cc-backend/internal/repository/job.go` (line 814)
- **UPDATE with WHERE:** `job_state = 'running'`

---

## 9. TIME RANGE & BATCH QUERIES (job.go)

### FindJobsBetween - Get jobs in time range
**File:** `/Users/jan/prg/CC/cc-backend/internal/repository/job.go` (line 839)
- **WHERE:** `job.start_time < endTime` OR `job.start_time BETWEEN beginTime AND endTime`
- **Optional:** NOT EXISTS for jobtag filtering
- **ORDER BY:** `job.cluster ASC, job.subcluster ASC, job.project ASC, job.start_time ASC`
- **Purpose:** Archiving, data export

### GetUsedNodes - Get running nodes before timestamp
**File:** `/Users/jan/prg/CC/cc-backend/internal/repository/job.go` (line 1065)
- **WHERE:** `job.start_time < ?` AND `job.job_state = 'running'`
- **SELECT:** `job.cluster, job.resources`

---

## 10. TAG-RELATED QUERIES (tags.go, job.go)

### FindJobIdsByTag - Get all jobs with a tag
**File:** `/Users/jan/prg/CC/cc-backend/internal/repository/job.go` (line 747)
- **WHERE:** Via `JOIN jobtag ON jobtag.job_id = job.id`
- **WHERE:** `jobtag.tag_id = ?`
- **SELECT DISTINCT:** `job.id`
- **Purpose:** Tag deletion cascade

### CountTags - Count jobs per tag
**File:** `/Users/jan/prg/CC/cc-backend/internal/repository/tags.go` (line 280)
- **FROM:** `tag t LEFT JOIN jobtag jt ON t.id = jt.tag_id`
- **WHERE:** `jt.job_id IN (SELECT id FROM job WHERE job.hpc_user = ? OR job.project IN (...))`
- **GROUP BY:** tag columns
- **Purpose:** Dashboard tag usage

---

## RECOMMENDED INDEX STRATEGY FOR 20M ROWS

Based on this comprehensive analysis, here's the recommended indexing strategy:

### CRITICAL INDEXES (Create First)
```sql
-- 1. job_state is queried in nearly every stats query
CREATE INDEX idx_job_job_state ON job(job_state);

-- 2. hpc_user is critical for security/filtering AND heavily used
CREATE INDEX idx_job_hpc_user ON job(hpc_user);

-- 3. start_time is used for ALL time-based filtering and range queries
CREATE INDEX idx_job_start_time ON job(start_time);

-- 4. cluster is frequently filtered and used for geographic partitioning
CREATE INDEX idx_job_cluster ON job(cluster);
```

### HIGH PRIORITY INDEXES
```sql
-- 5. Composite for FindConcurrentJobs overlap detection (very complex query)
CREATE INDEX idx_job_cluster_state_starttime ON job(cluster, job_state, start_time);

-- 6. Project filtering for access control
CREATE INDEX idx_job_project ON job(project);

-- 7. Partitions query (DISTINCT on cluster_partition by cluster)
CREATE INDEX idx_job_cluster_partition ON job(cluster, cluster_partition);

-- 8. Subcluster for AllocatedNodes query
CREATE INDEX idx_job_subcluster ON job(subcluster);

-- 9. Composite for GetUsedNodes query (start_time < ? AND job_state = 'running')
CREATE INDEX idx_job_running_nodes ON job(job_state, start_time);
```

### MEDIUM PRIORITY INDEXES
```sql
-- 10. Duration filtering (exists in multiple queries)
CREATE INDEX idx_job_duration ON job(duration);

-- 11. Array job grouping
CREATE INDEX idx_job_array_job_id ON job(array_job_id);

-- 12. Walltime for StopJobsExceedingWalltimeBy
CREATE INDEX idx_job_walltime ON job(walltime);

-- 13. Job ID lookup (for Find operations)
CREATE INDEX idx_job_job_id ON job(job_id);

-- 14. Energy for energy-based filtering
CREATE INDEX idx_job_energy ON job(energy);

-- 15. Composite for DeleteJobsBefore (very frequently called)
CREATE INDEX idx_job_start_time_state ON job(start_time, job_state);
```

### JSON INDEXES (Optional, for large deployments)
```sql
-- For meta_data JSON queries (jobName, interactive filtering)
CREATE INDEX idx_job_meta_data_jobname ON job(
    json_extract(meta_data, '$.jobName')
);

-- For footprint JSON histogram queries
CREATE INDEX idx_job_footprint_valid ON job(
    json_extract(footprint, '$.cpu_load_avg')
) WHERE json_valid(footprint);

-- For resources JSON array queries (hostname lookup)
CREATE INDEX idx_job_resources_hostname ON job(
    json_extract(resources, '$[0].hostname')
) WHERE json_valid(resources);
```

### COMPOSITE INDEXES FOR COMMON QUERY PATTERNS
```sql
-- SecurityCheck + common filter combo
CREATE INDEX idx_job_hpc_user_cluster_state ON job(hpc_user, cluster, job_state);

-- Stats queries: GROUP BY column + filter columns
CREATE INDEX idx_job_cluster_state_starttime_project ON job(
    cluster, job_state, start_time, project
);

CREATE INDEX idx_job_project_state_starttime ON job(
    project, job_state, start_time
);

-- Pagination: ORDER BY + LIMIT pattern
CREATE INDEX idx_job_start_time_desc ON job(start_time DESC);
```

---

## QUERY HOTSPOTS SUMMARY

**Hottest Query Patterns (prioritize these):**
1. **QueryJobs + CountJobs** - Core UI query, high concurrency, multiple WHERE conditions
2. **JobsStatsGrouped** - Dashboard, aggregation on millions of rows
3. **FindConcurrentJobs** - Complex overlap detection, multiple WHERE conditions + JSON
4. **DeleteJobsBefore** - Batch operation, scans large time ranges
5. **FindRunningJobs** - Periodic monitoring, filters on state + duration
6. **AllocatedNodes** - Node inventory, JSON parsing

**Security-critical columns (always indexed):**
- `job.hpc_user` (access control)
- `job.project` (access control)

**Time-critical columns (always indexed):**
- `job.start_time` (all historical queries)
- `job.job_state` (running vs. archived distinction)

**Resource-critical columns (indexed as secondary):**
- `job.cluster`, `job.subcluster`
- `job.num_nodes`, `job.num_acc`, `job.num_hwthreads`

---

This analysis covers 28 distinct query sites across 9 files with complete WHERE, ORDER BY, GROUP BY, JOIN, and pagination patterns for your 20M row job table.

agentId: a97d0003ebdc3eccd (for resuming to continue this agent's work if needed)
<usage>total_tokens: 84745
tool_uses: 14
duration_ms: 61574</usage>

Prompt:
In /Users/jan/prg/CC/cc-backend, I need to understand ALL query patterns that hit the `job` table. This is for index optimization on a 20M row table.

Search thoroughly for:

1. **All SQL queries on the job table** - search for `.From("job")`, `FROM job`, `from("job")` in all .go files
2. **All WHERE clause builders** - read `internal/repository/jobQuery.go` fully, especially `BuildWhereClause` to understand all possible filter columns
3. **All ORDER BY patterns** - search for `OrderBy`, `ORDER BY` related to jobs
4. **All GROUP BY patterns** - search for `GroupBy`, `GROUP BY` related to jobs
5. **All JOIN patterns** - search for joins on the job table (jobtag, etc.)

For each query site, report:
- File and function name
- WHERE columns used
- ORDER BY columns used
- Whether it has LIMIT/pagination
- Whether it uses GROUP BY

Be very thorough - check jobQuery.go, job.go, jobFind.go, jobCreate.go, stats.go, tags.go, and any other files that query the job table.


Excellent data. Now let me verify the actual query plans for the most critical patterns and check a few details.


Bash

sqlite3 ./var/job.db "
-- Pattern 1: Default job list (multi-state + ORDER BY start_time DESC)
EXPLAIN QUERY PLAN SELECT * FROM job WHERE job.job_state IN ('completed', 'running', 'failed') ORDER BY job.start_time DESC LIMIT 50 OFFSET 0;
" && echo '===P2===' && sqlite3 ./var/job.db "
-- Pattern 2: Cluster + state + ORDER BY start_time
EXPLAIN QUERY PLAN SELECT * FROM job WHERE job.cluster = 'fritz' AND job.job_state IN ('completed','running') ORDER BY job.start_time DESC LIMIT 50;
" && echo '===P3===' && sqlite3 ./var/job.db "
-- Pattern 3: User filter + ORDER BY start_time
EXPLAIN QUERY PLAN SELECT * FROM job WHERE job.hpc_user = 'alice' ORDER BY job.start_time DESC LIMIT 50;
" && echo '===P4===' && sqlite3 ./var/job.db "
-- Pattern 4: Stats aggregation with cluster + state
EXPLAIN QUERY PLAN SELECT COUNT(job.id), SUM(job.duration) FROM job WHERE job.cluster = 'fritz' AND job.job_state = 'completed';
" && echo '===P5===' && sqlite3 ./var/job.db "
-- Pattern 5: Find by job_id + cluster + start_time (unique lookup)
EXPLAIN QUERY PLAN SELECT * FROM job WHERE job.job_id = 123 AND job.cluster = 'fritz' AND job.start_time = 1000 ORDER BY job.id DESC LIMIT 1;
" && echo '===P6===' && sqlite3 ./var/job.db "
-- Pattern 6: Running jobs for cluster
EXPLAIN QUERY PLAN SELECT * FROM job WHERE job.cluster = 'fritz' AND job.job_state = 'running' AND job.duration > 300;
" && echo '===P7===' && sqlite3 ./var/job.db "
-- Pattern 7: Time range query
EXPLAIN QUERY PLAN SELECT * FROM job WHERE job.start_time BETWEEN 1700000000 AND 1710000000 ORDER BY job.cluster, job.subcluster, job.project, job.start_time;
" && echo '===P8===' && sqlite3 ./var/job.db "
-- Pattern 8: GroupBy user with cluster filter
EXPLAIN QUERY PLAN SELECT job.hpc_user, COUNT(job.id) FROM job WHERE job.cluster = 'fritz' GROUP BY job.hpc_user;
" && echo '===P9===' && sqlite3 ./var/job.db "
-- Pattern 9: Concurrent jobs (overlap detection)
EXPLAIN QUERY PLAN SELECT * FROM job WHERE job.cluster = 'fritz' AND job.job_state != 'running' AND job.start_time <= 1710000000 AND (job.start_time + job.duration) > 1700000000;
"

Check query plans for 9 critical query patterns


QUERY PLAN
`--SCAN job USING INDEX jobs_starttime
===P2===
QUERY PLAN
`--SEARCH job USING INDEX jobs_cluster_starttime_duration (cluster=?)
===P3===
QUERY PLAN
`--SEARCH job USING INDEX jobs_user_starttime_duration (hpc_user=?)
===P4===
QUERY PLAN
`--SEARCH job USING COVERING INDEX jobs_cluster_jobstate_duration_starttime (cluster=? AND job_state=?)
===P5===
QUERY PLAN
`--SEARCH job USING INDEX sqlite_autoindex_job_1 (job_id=? AND cluster=? AND start_time=?)
===P6===
QUERY PLAN
`--SEARCH job USING INDEX jobs_cluster_jobstate_duration_starttime (cluster=? AND job_state=? AND duration>?)
===P7===
QUERY PLAN
|--SEARCH job USING INDEX jobs_starttime (start_time>? AND start_time<?)
`--USE TEMP B-TREE FOR ORDER BY
===P8===
QUERY PLAN
`--SEARCH job USING COVERING INDEX jobs_cluster_user (cluster=?)
===P9===
QUERY PLAN
`--SEARCH job USING INDEX jobs_cluster_starttime_duration (cluster=? AND start_time<?)


sqlite3 ./var/job.db "
-- Pattern 10: Project filter + state + ORDER BY start_time (manager view)
EXPLAIN QUERY PLAN SELECT * FROM job WHERE job.project IN ('proj1','proj2','proj3') AND job.job_state IN ('completed','running') ORDER BY job.start_time DESC LIMIT 50;
" && echo '===P11===' && sqlite3 ./var/job.db "
-- Pattern 11: User security check (user role) + state + sort
EXPLAIN QUERY PLAN SELECT * FROM job WHERE job.hpc_user = 'alice' AND job.job_state IN ('completed','running','failed') ORDER BY job.start_time DESC LIMIT 50;
" && echo '===P12===' && sqlite3 ./var/job.db "
-- Pattern 12: Cluster + state + sort by duration
EXPLAIN QUERY PLAN SELECT * FROM job WHERE job.cluster = 'fritz' AND job.job_state = 'completed' ORDER BY job.duration DESC LIMIT 50;
" && echo '===P13===' && sqlite3 ./var/job.db "
-- Pattern 13: Cluster + state + sort by num_nodes
EXPLAIN QUERY PLAN SELECT * FROM job WHERE job.cluster = 'fritz' AND job.job_state = 'completed' ORDER BY job.num_nodes DESC LIMIT 50;
" && echo '===P14===' && sqlite3 ./var/job.db "
-- Pattern 14: Tag join query
EXPLAIN QUERY PLAN SELECT DISTINCT job.* FROM job JOIN jobtag ON jobtag.job_id = job.id WHERE jobtag.tag_id = 5 ORDER BY job.start_time DESC LIMIT 50;
" && echo '===P15===' && sqlite3 ./var/job.db "
-- Pattern 15: Delete jobs before timestamp
EXPLAIN QUERY PLAN SELECT job.id FROM job WHERE job.start_time < 1700000000;
" && echo '===P16===' && sqlite3 ./var/job.db "
-- Pattern 16: Non-running jobs list (GetJobList for archiving)
EXPLAIN QUERY PLAN SELECT job.id FROM job WHERE job.job_state != 'running' LIMIT 50 OFFSET 0;
"

Check query plans for remaining critical patterns


QUERY PLAN
|--SEARCH job USING INDEX jobs_jobstate_project (job_state=? AND project=?)
`--USE TEMP B-TREE FOR ORDER BY
===P11===
QUERY PLAN
`--SEARCH job USING INDEX jobs_user_starttime_duration (hpc_user=?)
===P12===
QUERY PLAN
`--SEARCH job USING INDEX jobs_cluster_jobstate_duration_starttime (cluster=? AND job_state=?)
===P13===
QUERY PLAN
`--SEARCH job USING INDEX jobs_cluster_jobstate_numnodes (cluster=? AND job_state=?)
===P14===
QUERY PLAN
|--SEARCH jobtag USING INDEX tags_tagid (tag_id=?)
|--SEARCH job USING INTEGER PRIMARY KEY (rowid=?)
|--USE TEMP B-TREE FOR DISTINCT
`--USE TEMP B-TREE FOR ORDER BY
===P15===
QUERY PLAN
`--SEARCH job USING COVERING INDEX jobs_starttime (start_time<?)
===P16===
QUERY PLAN
`--SCAN job USING COVERING INDEX jobs_jobstate_duration_starttime


# Check DB size and index overhead
sqlite3 ./var/job.db "
SELECT 'table_pages', (SELECT page_count FROM pragma_page_count()) as total_pages;
" && sqlite3 ./var/job.db "
SELECT 'page_size', (SELECT page_size FROM pragma_page_size());
" && echo "---" && sqlite3 ./var/job.db "
-- Check index sizes: compare with and without indexes
SELECT COUNT(*) as total_indexes FROM sqlite_master WHERE type='index' AND tbl_name='job';
"

Check DB size and index count


table_pages|561
page_size|4096
---
79


Now let me look at the migration 09 file to understand the index design rationale, and check which indexes are actually redundant.


   100→    job_state,
   101→    meta_data,
   102→    resources,
   103→    num_nodes,
   104→    num_hwthreads,
   105→    num_acc,
   106→    smt,
   107→    (
   108→        SELECT name FROM lookup_exclusive
   109→        WHERE id = job.exclusive
   110→    ),
   111→    monitoring_status,
   112→    energy,
   113→    energy_footprint,
   114→    footprint
   115→FROM job;
   116→
   117→DROP TABLE lookup_exclusive;
   118→DROP TABLE job; -- Deletes All Existing 'job' Indices; Recreate after Renaming
   119→ALTER TABLE job_new RENAME TO job;
   120→
   121→-- Recreate Indices from 08_add-footprint; include new 'shared' column
   122→-- Cluster Filter
   123→CREATE INDEX IF NOT EXISTS jobs_cluster_user ON job (cluster, hpc_user);
   124→CREATE INDEX IF NOT EXISTS jobs_cluster_project ON job (cluster, project);
   125→CREATE INDEX IF NOT EXISTS jobs_cluster_subcluster ON job (cluster, subcluster);
   126→-- Cluster Filter Sorting
   127→CREATE INDEX IF NOT EXISTS jobs_cluster_numnodes ON job (cluster, num_nodes);
   128→CREATE INDEX IF NOT EXISTS jobs_cluster_numhwthreads ON job (cluster, num_hwthreads);
   129→CREATE INDEX IF NOT EXISTS jobs_cluster_numacc ON job (cluster, num_acc);
   130→CREATE INDEX IF NOT EXISTS jobs_cluster_energy ON job (cluster, energy);
   131→
   132→-- Cluster Time Filter Sorting
   133→CREATE INDEX IF NOT EXISTS jobs_cluster_duration_starttime ON job (cluster, duration, start_time);
   134→CREATE INDEX IF NOT EXISTS jobs_cluster_starttime_duration ON job (cluster, start_time, duration);
   135→
   136→-- Cluster+Partition Filter
   137→CREATE INDEX IF NOT EXISTS jobs_cluster_partition_user ON job (cluster, cluster_partition, hpc_user);
   138→CREATE INDEX IF NOT EXISTS jobs_cluster_partition_project ON job (cluster, cluster_partition, project);
   139→CREATE INDEX IF NOT EXISTS jobs_cluster_partition_jobstate ON job (cluster, cluster_partition, job_state);
   140→CREATE INDEX IF NOT EXISTS jobs_cluster_partition_shared ON job (cluster, cluster_partition, shared);
   141→
   142→-- Cluster+Partition Filter Sorting
   143→CREATE INDEX IF NOT EXISTS jobs_cluster_partition_numnodes ON job (cluster, cluster_partition, num_nodes);
   144→CREATE INDEX IF NOT EXISTS jobs_cluster_partition_numhwthreads ON job (cluster, cluster_partition, num_hwthreads);
   145→CREATE INDEX IF NOT EXISTS jobs_cluster_partition_numacc ON job (cluster, cluster_partition, num_acc);
   146→CREATE INDEX IF NOT EXISTS jobs_cluster_partition_energy ON job (cluster, cluster_partition, energy);
   147→
   148→-- Cluster+Partition Time Filter Sorting
   149→CREATE INDEX IF NOT EXISTS jobs_cluster_partition_duration_starttime ON job (cluster, cluster_partition, duration, start_time);
   150→CREATE INDEX IF NOT EXISTS jobs_cluster_partition_starttime_duration ON job (cluster, cluster_partition, start_time, duration);
   151→
   152→-- Cluster+JobState Filter
   153→CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_user ON job (cluster, job_state, hpc_user);
   154→CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_project ON job (cluster, job_state, project);
   155→-- Cluster+JobState Filter Sorting
   156→CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_numnodes ON job (cluster, job_state, num_nodes);
   157→CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_numhwthreads ON job (cluster, job_state, num_hwthreads);
   158→CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_numacc ON job (cluster, job_state, num_acc);
   159→CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_energy ON job (cluster, job_state, energy);
   160→
   161→-- Cluster+JobState Time Filter Sorting
   162→CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_starttime_duration ON job (cluster, job_state, start_time, duration);
   163→CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_duration_starttime ON job (cluster, job_state, duration, start_time);
   164→
   165→-- Cluster+Shared Filter
   166→CREATE INDEX IF NOT EXISTS jobs_cluster_shared_user ON job (cluster, shared, hpc_user);
   167→CREATE INDEX IF NOT EXISTS jobs_cluster_shared_project ON job (cluster, shared, project);
   168→-- Cluster+Shared Filter Sorting
   169→CREATE INDEX IF NOT EXISTS jobs_cluster_shared_numnodes ON job (cluster, shared, num_nodes);
   170→CREATE INDEX IF NOT EXISTS jobs_cluster_shared_numhwthreads ON job (cluster, shared, num_hwthreads);
   171→CREATE INDEX IF NOT EXISTS jobs_cluster_shared_numacc ON job (cluster, shared, num_acc);
   172→CREATE INDEX IF NOT EXISTS jobs_cluster_shared_energy ON job (cluster, shared, energy);
   173→
   174→-- Cluster+Shared Time Filter Sorting
   175→CREATE INDEX IF NOT EXISTS jobs_cluster_shared_starttime_duration ON job (cluster, shared, start_time, duration);
   176→CREATE INDEX IF NOT EXISTS jobs_cluster_shared_duration_starttime ON job (cluster, shared, duration, start_time);
   177→
   178→-- User Filter
   179→-- User Filter Sorting
   180→CREATE INDEX IF NOT EXISTS jobs_user_numnodes ON job (hpc_user, num_nodes);
   181→CREATE INDEX IF NOT EXISTS jobs_user_numhwthreads ON job (hpc_user, num_hwthreads);
   182→CREATE INDEX IF NOT EXISTS jobs_user_numacc ON job (hpc_user, num_acc);
   183→CREATE INDEX IF NOT EXISTS jobs_user_energy ON job (hpc_user, energy);
   184→
   185→-- Cluster+Shared Time Filter Sorting
   186→CREATE INDEX IF NOT EXISTS jobs_user_starttime_duration ON job (hpc_user, start_time, duration);
   187→CREATE INDEX IF NOT EXISTS jobs_user_duration_starttime ON job (hpc_user, duration, start_time);
   188→
   189→-- Project Filter
   190→CREATE INDEX IF NOT EXISTS jobs_project_user ON job (project, hpc_user);
   191→-- Project Filter Sorting
   192→CREATE INDEX IF NOT EXISTS jobs_project_numnodes ON job (project, num_nodes);
   193→CREATE INDEX IF NOT EXISTS jobs_project_numhwthreads ON job (project, num_hwthreads);
   194→CREATE INDEX IF NOT EXISTS jobs_project_numacc ON job (project, num_acc);
   195→CREATE INDEX IF NOT EXISTS jobs_project_energy ON job (project, energy);
   196→
   197→-- Cluster+Shared Time Filter Sorting
   198→CREATE INDEX IF NOT EXISTS jobs_project_starttime_duration ON job (project, start_time, duration);
   199→CREATE INDEX IF NOT EXISTS jobs_project_duration_starttime ON job (project, duration, start_time);
   200→
   201→-- JobState Filter
   202→CREATE INDEX IF NOT EXISTS jobs_jobstate_user ON job (job_state, hpc_user);
   203→CREATE INDEX IF NOT EXISTS jobs_jobstate_project ON job (job_state, project);
   204→-- JobState Filter Sorting
   205→CREATE INDEX IF NOT EXISTS jobs_jobstate_numnodes ON job (job_state, num_nodes);
   206→CREATE INDEX IF NOT EXISTS jobs_jobstate_numhwthreads ON job (job_state, num_hwthreads);
   207→CREATE INDEX IF NOT EXISTS jobs_jobstate_numacc ON job (job_state, num_acc);
   208→CREATE INDEX IF NOT EXISTS jobs_jobstate_energy ON job (job_state, energy);
   209→
   210→-- Cluster+Shared Time Filter Sorting
   211→CREATE INDEX IF NOT EXISTS jobs_jobstate_starttime_duration ON job (job_state, start_time, duration);
   212→CREATE INDEX IF NOT EXISTS jobs_jobstate_duration_starttime ON job (job_state, duration, start_time);
   213→
   214→-- Shared Filter
   215→CREATE INDEX IF NOT EXISTS jobs_shared_user ON job (shared, hpc_user);
   216→CREATE INDEX IF NOT EXISTS jobs_shared_project ON job (shared, project);
   217→-- Shared Filter Sorting
   218→CREATE INDEX IF NOT EXISTS jobs_shared_numnodes ON job (shared, num_nodes);
   219→CREATE INDEX IF NOT EXISTS jobs_shared_numhwthreads ON job (shared, num_hwthreads);
   220→CREATE INDEX IF NOT EXISTS jobs_shared_numacc ON job (shared, num_acc);
   221→CREATE INDEX IF NOT EXISTS jobs_shared_energy ON job (shared, energy);
   222→
   223→-- Cluster+Shared Time Filter Sorting
   224→CREATE INDEX IF NOT EXISTS jobs_shared_starttime_duration ON job (shared, start_time, duration);
   225→CREATE INDEX IF NOT EXISTS jobs_shared_duration_starttime ON job (shared, duration, start_time);
   226→
   227→-- ArrayJob Filter
   228→CREATE INDEX IF NOT EXISTS jobs_arrayjobid_starttime ON job (array_job_id, start_time);
   229→CREATE INDEX IF NOT EXISTS jobs_cluster_arrayjobid_starttime ON job (cluster, array_job_id, start_time);
   230→
   231→-- Single filters with default starttime sorting
   232→CREATE INDEX IF NOT EXISTS jobs_duration_starttime ON job (duration, start_time);
   233→CREATE INDEX IF NOT EXISTS jobs_numnodes_starttime ON job (num_nodes, start_time);
   234→CREATE INDEX IF NOT EXISTS jobs_numhwthreads_starttime ON job (num_hwthreads, start_time);
   235→CREATE INDEX IF NOT EXISTS jobs_numacc_starttime ON job (num_acc, start_time);
   236→CREATE INDEX IF NOT EXISTS jobs_energy_starttime ON job (energy, start_time);
   237→
   238→-- Single filters with duration sorting
   239→CREATE INDEX IF NOT EXISTS jobs_starttime_duration ON job (start_time, duration);
   240→CREATE INDEX IF NOT EXISTS jobs_numnodes_duration ON job (num_nodes, duration);
   241→CREATE INDEX IF NOT EXISTS jobs_numhwthreads_duration ON job (num_hwthreads, duration);
   242→CREATE INDEX IF NOT EXISTS jobs_numacc_duration ON job (num_acc, duration);
   243→CREATE INDEX IF NOT EXISTS jobs_energy_duration ON job (energy, duration);
   244→
   245→-- Backup Indices For High Variety Columns
   246→CREATE INDEX IF NOT EXISTS jobs_starttime ON job (start_time);
   247→CREATE INDEX IF NOT EXISTS jobs_duration ON job (duration);
   248→
   249→-- Notes:

    job_state,
    meta_data,
    resources,
    num_nodes,
    num_hwthreads,
    num_acc,
    smt,
    (
        SELECT name FROM lookup_exclusive
        WHERE id = job.exclusive
    ),
    monitoring_status,
    energy,
    energy_footprint,
    footprint
FROM job;

DROP TABLE lookup_exclusive;
DROP TABLE job; -- Deletes All Existing 'job' Indices; Recreate after Renaming
ALTER TABLE job_new RENAME TO job;

-- Recreate Indices from 08_add-footprint; include new 'shared' column
-- Cluster Filter
CREATE INDEX IF NOT EXISTS jobs_cluster_user ON job (cluster, hpc_user);
CREATE INDEX IF NOT EXISTS jobs_cluster_project ON job (cluster, project);
CREATE INDEX IF NOT EXISTS jobs_cluster_subcluster ON job (cluster, subcluster);
-- Cluster Filter Sorting
CREATE INDEX IF NOT EXISTS jobs_cluster_numnodes ON job (cluster, num_nodes);
CREATE INDEX IF NOT EXISTS jobs_cluster_numhwthreads ON job (cluster, num_hwthreads);
CREATE INDEX IF NOT EXISTS jobs_cluster_numacc ON job (cluster, num_acc);
CREATE INDEX IF NOT EXISTS jobs_cluster_energy ON job (cluster, energy);

-- Cluster Time Filter Sorting
CREATE INDEX IF NOT EXISTS jobs_cluster_duration_starttime ON job (cluster, duration, start_time);
CREATE INDEX IF NOT EXISTS jobs_cluster_starttime_duration ON job (cluster, start_time, duration);

-- Cluster+Partition Filter
CREATE INDEX IF NOT EXISTS jobs_cluster_partition_user ON job (cluster, cluster_partition, hpc_user);
CREATE INDEX IF NOT EXISTS jobs_cluster_partition_project ON job (cluster, cluster_partition, project);
CREATE INDEX IF NOT EXISTS jobs_cluster_partition_jobstate ON job (cluster, cluster_partition, job_state);
CREATE INDEX IF NOT EXISTS jobs_cluster_partition_shared ON job (cluster, cluster_partition, shared);

-- Cluster+Partition Filter Sorting
CREATE INDEX IF NOT EXISTS jobs_cluster_partition_numnodes ON job (cluster, cluster_partition, num_nodes);
CREATE INDEX IF NOT EXISTS jobs_cluster_partition_numhwthreads ON job (cluster, cluster_partition, num_hwthreads);
CREATE INDEX IF NOT EXISTS jobs_cluster_partition_numacc ON job (cluster, cluster_partition, num_acc);
CREATE INDEX IF NOT EXISTS jobs_cluster_partition_energy ON job (cluster, cluster_partition, energy);

-- Cluster+Partition Time Filter Sorting
CREATE INDEX IF NOT EXISTS jobs_cluster_partition_duration_starttime ON job (cluster, cluster_partition, duration, start_time);
CREATE INDEX IF NOT EXISTS jobs_cluster_partition_starttime_duration ON job (cluster, cluster_partition, start_time, duration);

-- Cluster+JobState Filter
CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_user ON job (cluster, job_state, hpc_user);
CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_project ON job (cluster, job_state, project);
-- Cluster+JobState Filter Sorting
CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_numnodes ON job (cluster, job_state, num_nodes);
CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_numhwthreads ON job (cluster, job_state, num_hwthreads);
CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_numacc ON job (cluster, job_state, num_acc);
CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_energy ON job (cluster, job_state, energy);

-- Cluster+JobState Time Filter Sorting
CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_starttime_duration ON job (cluster, job_state, start_time, duration);
CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_duration_starttime ON job (cluster, job_state, duration, start_time);

-- Cluster+Shared Filter
CREATE INDEX IF NOT EXISTS jobs_cluster_shared_user ON job (cluster, shared, hpc_user);
CREATE INDEX IF NOT EXISTS jobs_cluster_shared_project ON job (cluster, shared, project);
-- Cluster+Shared Filter Sorting
CREATE INDEX IF NOT EXISTS jobs_cluster_shared_numnodes ON job (cluster, shared, num_nodes);
CREATE INDEX IF NOT EXISTS jobs_cluster_shared_numhwthreads ON job (cluster, shared, num_hwthreads);
CREATE INDEX IF NOT EXISTS jobs_cluster_shared_numacc ON job (cluster, shared, num_acc);
CREATE INDEX IF NOT EXISTS jobs_cluster_shared_energy ON job (cluster, shared, energy);

-- Cluster+Shared Time Filter Sorting
CREATE INDEX IF NOT EXISTS jobs_cluster_shared_starttime_duration ON job (cluster, shared, start_time, duration);
CREATE INDEX IF NOT EXISTS jobs_cluster_shared_duration_starttime ON job (cluster, shared, duration, start_time);

-- User Filter
-- User Filter Sorting
CREATE INDEX IF NOT EXISTS jobs_user_numnodes ON job (hpc_user, num_nodes);
CREATE INDEX IF NOT EXISTS jobs_user_numhwthreads ON job (hpc_user, num_hwthreads);
CREATE INDEX IF NOT EXISTS jobs_user_numacc ON job (hpc_user, num_acc);
CREATE INDEX IF NOT EXISTS jobs_user_energy ON job (hpc_user, energy);

-- Cluster+Shared Time Filter Sorting
CREATE INDEX IF NOT EXISTS jobs_user_starttime_duration ON job (hpc_user, start_time, duration);
CREATE INDEX IF NOT EXISTS jobs_user_duration_starttime ON job (hpc_user, duration, start_time);

-- Project Filter
CREATE INDEX IF NOT EXISTS jobs_project_user ON job (project, hpc_user);
-- Project Filter Sorting
CREATE INDEX IF NOT EXISTS jobs_project_numnodes ON job (project, num_nodes);
CREATE INDEX IF NOT EXISTS jobs_project_numhwthreads ON job (project, num_hwthreads);
CREATE INDEX IF NOT EXISTS jobs_project_numacc ON job (project, num_acc);
CREATE INDEX IF NOT EXISTS jobs_project_energy ON job (project, energy);

-- Cluster+Shared Time Filter Sorting
CREATE INDEX IF NOT EXISTS jobs_project_starttime_duration ON job (project, start_time, duration);
CREATE INDEX IF NOT EXISTS jobs_project_duration_starttime ON job (project, duration, start_time);

-- JobState Filter
CREATE INDEX IF NOT EXISTS jobs_jobstate_user ON job (job_state, hpc_user);
CREATE INDEX IF NOT EXISTS jobs_jobstate_project ON job (job_state, project);
-- JobState Filter Sorting
CREATE INDEX IF NOT EXISTS jobs_jobstate_numnodes ON job (job_state, num_nodes);
CREATE INDEX IF NOT EXISTS jobs_jobstate_numhwthreads ON job (job_state, num_hwthreads);
CREATE INDEX IF NOT EXISTS jobs_jobstate_numacc ON job (job_state, num_acc);
CREATE INDEX IF NOT EXISTS jobs_jobstate_energy ON job (job_state, energy);

-- Cluster+Shared Time Filter Sorting
CREATE INDEX IF NOT EXISTS jobs_jobstate_starttime_duration ON job (job_state, start_time, duration);
CREATE INDEX IF NOT EXISTS jobs_jobstate_duration_starttime ON job (job_state, duration, start_time);

-- Shared Filter
CREATE INDEX IF NOT EXISTS jobs_shared_user ON job (shared, hpc_user);
CREATE INDEX IF NOT EXISTS jobs_shared_project ON job (shared, project);
-- Shared Filter Sorting
CREATE INDEX IF NOT EXISTS jobs_shared_numnodes ON job (shared, num_nodes);
CREATE INDEX IF NOT EXISTS jobs_shared_numhwthreads ON job (shared, num_hwthreads);
CREATE INDEX IF NOT EXISTS jobs_shared_numacc ON job (shared, num_acc);
CREATE INDEX IF NOT EXISTS jobs_shared_energy ON job (shared, energy);

-- Cluster+Shared Time Filter Sorting
CREATE INDEX IF NOT EXISTS jobs_shared_starttime_duration ON job (shared, start_time, duration);
CREATE INDEX IF NOT EXISTS jobs_shared_duration_starttime ON job (shared, duration, start_time);

-- ArrayJob Filter
CREATE INDEX IF NOT EXISTS jobs_arrayjobid_starttime ON job (array_job_id, start_time);
CREATE INDEX IF NOT EXISTS jobs_cluster_arrayjobid_starttime ON job (cluster, array_job_id, start_time);

-- Single filters with default starttime sorting
CREATE INDEX IF NOT EXISTS jobs_duration_starttime ON job (duration, start_time);
CREATE INDEX IF NOT EXISTS jobs_numnodes_starttime ON job (num_nodes, start_time);
CREATE INDEX IF NOT EXISTS jobs_numhwthreads_starttime ON job (num_hwthreads, start_time);
CREATE INDEX IF NOT EXISTS jobs_numacc_starttime ON job (num_acc, start_time);
CREATE INDEX IF NOT EXISTS jobs_energy_starttime ON job (energy, start_time);

-- Single filters with duration sorting
CREATE INDEX IF NOT EXISTS jobs_starttime_duration ON job (start_time, duration);
CREATE INDEX IF NOT EXISTS jobs_numnodes_duration ON job (num_nodes, duration);
CREATE INDEX IF NOT EXISTS jobs_numhwthreads_duration ON job (num_hwthreads, duration);
CREATE INDEX IF NOT EXISTS jobs_numacc_duration ON job (num_acc, duration);
CREATE INDEX IF NOT EXISTS jobs_energy_duration ON job (energy, duration);

-- Backup Indices For High Variety Columns
CREATE INDEX IF NOT EXISTS jobs_starttime ON job (start_time);
CREATE INDEX IF NOT EXISTS jobs_duration ON job (duration);

-- Notes:


Now let me design the optimal index strategy with a Plan agent.


Agent

Design optimal index strategy

Prompt:
Design an optimal index strategy for the `job` table in a SQLite database with 20 million rows. The current setup has 79 indexes which is excessive — it confuses the query planner, wastes disk space (~2x overhead per index on 20M rows), and slows inserts/updates.

## Current Problem
- 79 indexes on one table, many redundant
- SQLite query planner picks wrong indexes without ANALYZE (e.g., `jobs_jobstate_energy` instead of `jobs_jobstate_starttime_duration` for ORDER BY start_time queries)
- `IN` clauses with multiple values + `ORDER BY` causes temp B-tree sorts
- Timeouts on filter queries at 20M rows

## Key Constraints
1. **SQLite only** — no partial indexes with expressions on columns (but expression indexes on JSON are supported)
2. Each additional index costs ~200-400MB at 20M rows
3. The `ANALYZE` fix (already implemented) helps the planner, but we still need fewer, better indexes
4. Indexes must be defined in a migration file

## Job Table Columns
```
id (PK), job_id, cluster, subcluster, submit_time, start_time, hpc_user, project,
cluster_partition, array_job_id, duration, walltime, job_state, meta_data (JSON),
resources (JSON), num_nodes, num_hwthreads, num_acc, smt, shared, monitoring_status,
energy, energy_footprint (JSON), footprint (JSON)
UNIQUE(job_id, cluster, start_time) — sqlite_autoindex_job_1
```

## Query Plans WITH ANALYZE (verified working)
These are the plans SQLite ALREADY chooses with ANALYZE and current indexes:

1. **Default job list** (multi-state IN + ORDER BY start_time DESC LIMIT 50): → `SCAN jobs_starttime` ✓
2. **Cluster + state + sort start_time**: → `SEARCH jobs_cluster_starttime_duration (cluster=?)` ✓
3. **User + sort start_time**: → `SEARCH jobs_user_starttime_duration (hpc_user=?)` ✓
4. **Cluster + state aggregation**: → `SEARCH COVERING jobs_cluster_jobstate_duration_starttime` ✓
5. **Unique lookup (job_id, cluster, start_time)**: → `SEARCH sqlite_autoindex_job_1` ✓
6. **Running jobs for cluster**: → `SEARCH jobs_cluster_jobstate_duration_starttime (cluster=? AND job_state=? AND duration>?)` ✓
7. **Time range**: → `SEARCH jobs_starttime (start_time BETWEEN)` ✓ (temp b-tree for multi-col ORDER BY is acceptable)
8. **Group by user with cluster**: → `SEARCH COVERING jobs_cluster_user (cluster=?)` ✓
9. **Concurrent jobs**: → `SEARCH jobs_cluster_starttime_duration (cluster=? AND start_time<?)` ✓
10. **Project IN + state IN + sort**: → `SEARCH jobs_jobstate_project (job_state=? AND project=?)` + TEMP B-TREE ⚠
11. **User + multi-state + sort**: → `SEARCH jobs_user_starttime_duration (hpc_user=?)` ✓
12. **Cluster + state + sort duration**: → `SEARCH jobs_cluster_jobstate_duration_starttime (cluster=? AND job_state=?)` ✓
13. **Cluster + state + sort num_nodes**: → `SEARCH jobs_cluster_jobstate_numnodes (cluster=? AND job_state=?)` ✓
14. **Tag join**: → `SEARCH tags_tagid` + PK lookup ✓
15. **Delete before timestamp**: → `SEARCH COVERING jobs_starttime (start_time<?)` ✓
16. **Non-running jobs list**: → `SCAN COVERING jobs_jobstate_duration_starttime` ✓

## All Query Patterns (28 sites across 9 files)

**High-frequency patterns (every page load):**
- `QueryJobs`: WHERE=(any filter combo from BuildWhereClause) ORDER BY=(start_time|duration|num_nodes|num_hwthreads|num_acc|energy|footprint_json) LIMIT+OFFSET
- `CountJobs`: WHERE=(same filters) COUNT(DISTINCT job.id)
- Security filter always adds: `hpc_user = ?` (user role) or `project IN (...)` (manager role)

**BuildWhereClause filter columns (in priority order):**
1. `cluster` — almost always present (equality)
2. `job_state` — almost always present (IN list, typically 1-3 values)
3. `start_time` — very common (range or relative: last6h/24h/7d/30d)
4. `hpc_user` — added by security for user role (equality)
5. `project` — added by security for manager role (IN list)
6. `duration` — common filter (range)
7. `subcluster` — occasional (equality)
8. `cluster_partition` — occasional (equality)
9. `num_nodes`, `num_hwthreads`, `num_acc` — occasional (range)
10. `energy` — occasional (range)
11. `shared` — occasional (equality)
12. `array_job_id` — rare (equality)
13. JSON columns (footprint, meta_data, resources) — rare, can't be indexed traditionally

**ORDER BY columns (one at a time, ASC or DESC):**
- `start_time` (default, ~80% of queries)
- `duration` (~10%)
- `num_nodes`, `num_hwthreads`, `num_acc`, `energy` (~10% combined)
- JSON footprint fields (rare, always requires temp sort)

**Statistics/aggregation patterns:**
- GROUP BY: `hpc_user`, `project`, `cluster`, `subcluster`
- Aggregates: COUNT, SUM(duration), SUM(num_nodes * duration), etc.

**Lookup patterns:**
- `(job_id, cluster, start_time)` — covered by UNIQUE constraint
- `job.id` (PK) — always indexed

## Design the optimal reduced index set
Requirements:
1. **Target: 20-25 indexes max** (down from 79)
2. Every query pattern above must use an index (no full table scans unless LIMIT is low enough)
3. For each remaining index, explain which query patterns it serves
4. List every dropped index and explain why it's redundant
5. Use composite index prefix properties (an index on (A, B, C) also covers queries on (A) and (A, B))
6. Prioritize covering indexes for aggregation queries where practical
7. Consider that `start_time` is the dominant sort column

Format the output as a concrete migration SQL file with comments explaining each index.