2024-04-11 23:04:30 +02:00
// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
2022-07-29 06:29:21 +02:00
// All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
2022-02-17 09:04:57 +01:00
package repository
import (
"context"
"errors"
"fmt"
"regexp"
"strings"
2022-03-01 16:00:44 +01:00
"time"
2022-02-17 09:04:57 +01:00
2022-06-21 17:52:36 +02:00
"github.com/ClusterCockpit/cc-backend/internal/graph/model"
"github.com/ClusterCockpit/cc-backend/pkg/log"
"github.com/ClusterCockpit/cc-backend/pkg/schema"
2022-02-17 09:04:57 +01:00
sq "github.com/Masterminds/squirrel"
)
2023-08-30 15:04:50 +02:00
func ( r * JobRepository ) QueryJobs (
ctx context . Context ,
2022-02-17 09:04:57 +01:00
filters [ ] * model . JobFilter ,
page * model . PageRequest ,
2024-06-28 16:48:10 +02:00
order * model . OrderByInput ,
) ( [ ] * schema . Job , error ) {
2023-08-30 15:04:50 +02:00
query , qerr := SecurityCheck ( ctx , sq . Select ( jobColumns ... ) . From ( "job" ) )
if qerr != nil {
return nil , qerr
}
2022-02-17 09:04:57 +01:00
if order != nil {
field := toSnakeCase ( order . Field )
2024-07-22 15:41:33 +02:00
if order . Type == "col" {
// "col": Fixed column name query
switch order . Order {
case model . SortDirectionEnumAsc :
query = query . OrderBy ( fmt . Sprintf ( "job.%s ASC" , field ) )
case model . SortDirectionEnumDesc :
query = query . OrderBy ( fmt . Sprintf ( "job.%s DESC" , field ) )
default :
return nil , errors . New ( "REPOSITORY/QUERY > invalid sorting order for column" )
}
} else {
// "foot": Order by footprint JSON field values
// Verify and Search Only in Valid Jsons
query = query . Where ( "JSON_VALID(meta_data)" )
switch order . Order {
case model . SortDirectionEnumAsc :
query = query . OrderBy ( fmt . Sprintf ( "JSON_EXTRACT(footprint, \"$.%s\") ASC" , field ) )
case model . SortDirectionEnumDesc :
query = query . OrderBy ( fmt . Sprintf ( "JSON_EXTRACT(footprint, \"$.%s\") DESC" , field ) )
default :
return nil , errors . New ( "REPOSITORY/QUERY > invalid sorting order for footprint" )
}
2022-02-17 09:04:57 +01:00
}
}
2022-02-22 14:07:49 +01:00
if page != nil && page . ItemsPerPage != - 1 {
2022-02-17 09:04:57 +01:00
limit := uint64 ( page . ItemsPerPage )
query = query . Offset ( ( uint64 ( page . Page ) - 1 ) * limit ) . Limit ( limit )
}
for _ , f := range filters {
query = BuildWhereClause ( f , query )
}
2022-02-22 09:25:41 +01:00
rows , err := query . RunWith ( r . stmtCache ) . Query ( )
2022-02-17 09:04:57 +01:00
if err != nil {
2023-06-20 15:52:16 +02:00
log . Errorf ( "Error while running query: %v" , err )
2022-02-17 09:04:57 +01:00
return nil , err
}
jobs := make ( [ ] * schema . Job , 0 , 50 )
for rows . Next ( ) {
2022-02-22 09:25:41 +01:00
job , err := scanJob ( rows )
2022-02-17 09:04:57 +01:00
if err != nil {
2022-03-03 14:54:37 +01:00
rows . Close ( )
2023-04-28 12:34:40 +02:00
log . Warn ( "Error while scanning rows (Jobs)" )
2022-02-17 09:04:57 +01:00
return nil , err
}
jobs = append ( jobs , job )
}
return jobs , nil
}
2023-08-30 15:04:50 +02:00
func ( r * JobRepository ) CountJobs (
2023-04-28 12:34:40 +02:00
ctx context . Context ,
2024-06-28 16:48:10 +02:00
filters [ ] * model . JobFilter ,
) ( int , error ) {
2024-08-02 16:42:55 +02:00
// DISTICT count for tags filters, does not affect other queries
2024-08-02 16:11:47 +02:00
query , qerr := SecurityCheck ( ctx , sq . Select ( "count(DISTINCT job.id)" ) . From ( "job" ) )
2023-04-28 12:34:40 +02:00
if qerr != nil {
2023-08-30 15:04:50 +02:00
return 0 , qerr
2023-04-28 12:34:40 +02:00
}
2022-02-17 09:04:57 +01:00
for _ , f := range filters {
query = BuildWhereClause ( f , query )
}
2023-06-01 13:57:35 +02:00
2022-02-17 09:04:57 +01:00
var count int
if err := query . RunWith ( r . DB ) . Scan ( & count ) ; err != nil {
return 0 , err
}
return count , nil
}
2023-06-20 12:54:26 +02:00
func SecurityCheck ( ctx context . Context , query sq . SelectBuilder ) ( sq . SelectBuilder , error ) {
2023-08-17 10:29:00 +02:00
user := GetUserFromContext ( ctx )
2023-06-12 11:35:16 +02:00
if user == nil {
var qnil sq . SelectBuilder
2023-12-08 12:03:04 +01:00
return qnil , fmt . Errorf ( "user context is nil" )
2024-07-03 17:24:26 +02:00
}
switch {
case len ( user . Roles ) == 1 && user . HasRole ( schema . RoleApi ) : // API-User : All jobs
return query , nil
case user . HasAnyRole ( [ ] schema . Role { schema . RoleAdmin , schema . RoleSupport } ) : // Admin & Support : All jobs
2023-01-27 18:36:58 +01:00
return query , nil
2024-07-03 17:24:26 +02:00
case user . HasRole ( schema . RoleManager ) : // Manager : Add filter for managed projects' jobs only + personal jobs
2023-02-20 11:24:18 +01:00
if len ( user . Projects ) != 0 {
2024-11-21 15:02:30 +01:00
return query . Where ( sq . Or { sq . Eq { "job.project" : user . Projects } , sq . Eq { "job.hpc_user" : user . Username } } ) , nil
2023-02-20 11:24:18 +01:00
} else {
2023-06-20 15:47:38 +02:00
log . Debugf ( "Manager-User '%s' has no defined projects to lookup! Query only personal jobs ..." , user . Username )
2024-11-21 15:02:30 +01:00
return query . Where ( "job.hpc_user = ?" , user . Username ) , nil
2023-02-20 11:24:18 +01:00
}
2024-07-03 17:24:26 +02:00
case user . HasRole ( schema . RoleUser ) : // User : Only personal jobs
2024-11-21 15:02:30 +01:00
return query . Where ( "job.hpc_user = ?" , user . Username ) , nil
2024-07-03 17:24:26 +02:00
default : // No known Role, return error
var qnil sq . SelectBuilder
return qnil , fmt . Errorf ( "user has no or unknown roles" )
2022-02-17 09:04:57 +01:00
}
}
// Build a sq.SelectBuilder out of a schema.JobFilter.
func BuildWhereClause ( filter * model . JobFilter , query sq . SelectBuilder ) sq . SelectBuilder {
if filter . Tags != nil {
2024-08-02 16:42:55 +02:00
// This is an OR-Logic query: Returns all distinct jobs with at least one of the requested tags; TODO: AND-Logic query?
2024-08-02 16:11:47 +02:00
query = query . Join ( "jobtag ON jobtag.job_id = job.id" ) . Where ( sq . Eq { "jobtag.tag_id" : filter . Tags } ) . Distinct ( )
2022-02-17 09:04:57 +01:00
}
if filter . JobID != nil {
query = buildStringCondition ( "job.job_id" , filter . JobID , query )
}
if filter . ArrayJobID != nil {
query = query . Where ( "job.array_job_id = ?" , * filter . ArrayJobID )
}
if filter . User != nil {
2024-11-21 15:02:30 +01:00
query = buildStringCondition ( "job.hpc_user" , filter . User , query )
2022-02-17 09:04:57 +01:00
}
if filter . Project != nil {
query = buildStringCondition ( "job.project" , filter . Project , query )
}
2023-01-11 16:25:02 +01:00
if filter . JobName != nil {
2024-04-22 12:14:40 +02:00
query = buildMetaJsonCondition ( "jobName" , filter . JobName , query )
2023-01-11 16:25:02 +01:00
}
2022-02-17 09:04:57 +01:00
if filter . Cluster != nil {
query = buildStringCondition ( "job.cluster" , filter . Cluster , query )
}
if filter . Partition != nil {
2024-11-21 15:02:30 +01:00
query = buildStringCondition ( "job.cluster_partition" , filter . Partition , query )
2022-02-17 09:04:57 +01:00
}
if filter . StartTime != nil {
query = buildTimeCondition ( "job.start_time" , filter . StartTime , query )
}
if filter . Duration != nil {
2022-03-01 16:00:44 +01:00
now := time . Now ( ) . Unix ( ) // There does not seam to be a portable way to get the current unix timestamp accross different DBs.
2022-03-01 16:04:27 +01:00
query = query . Where ( "(CASE WHEN job.job_state = 'running' THEN (? - job.start_time) ELSE job.duration END) BETWEEN ? AND ?" , now , filter . Duration . From , filter . Duration . To )
2022-02-17 09:04:57 +01:00
}
2022-03-02 10:48:52 +01:00
if filter . MinRunningFor != nil {
now := time . Now ( ) . Unix ( ) // There does not seam to be a portable way to get the current unix timestamp accross different DBs.
query = query . Where ( "(job.job_state != 'running' OR (? - job.start_time) > ?)" , now , * filter . MinRunningFor )
}
2022-02-17 09:04:57 +01:00
if filter . State != nil {
states := make ( [ ] string , len ( filter . State ) )
for i , val := range filter . State {
states [ i ] = string ( val )
}
query = query . Where ( sq . Eq { "job.job_state" : states } )
}
if filter . NumNodes != nil {
query = buildIntCondition ( "job.num_nodes" , filter . NumNodes , query )
}
if filter . NumAccelerators != nil {
query = buildIntCondition ( "job.num_acc" , filter . NumAccelerators , query )
}
if filter . NumHWThreads != nil {
query = buildIntCondition ( "job.num_hwthreads" , filter . NumHWThreads , query )
}
2023-06-28 13:35:41 +02:00
if filter . Node != nil {
query = buildStringCondition ( "job.resources" , filter . Node , query )
}
2024-10-01 16:25:09 +02:00
if filter . Energy != nil {
query = buildFloatCondition ( "job.energy" , filter . Energy , query )
}
2024-07-12 13:21:19 +02:00
if filter . MetricStats != nil {
2024-07-22 15:41:33 +02:00
for _ , ms := range filter . MetricStats {
query = buildFloatJsonCondition ( ms . MetricName , ms . Range , query )
2024-07-12 13:21:19 +02:00
}
2022-02-17 09:04:57 +01:00
}
return query
}
2022-09-07 12:24:45 +02:00
func buildIntCondition ( field string , cond * schema . IntRange , query sq . SelectBuilder ) sq . SelectBuilder {
2022-02-17 09:04:57 +01:00
return query . Where ( field + " BETWEEN ? AND ?" , cond . From , cond . To )
}
2024-10-01 16:25:09 +02:00
func buildFloatCondition ( field string , cond * model . FloatRange , query sq . SelectBuilder ) sq . SelectBuilder {
return query . Where ( field + " BETWEEN ? AND ?" , cond . From , cond . To )
}
2022-09-07 12:24:45 +02:00
func buildTimeCondition ( field string , cond * schema . TimeRange , query sq . SelectBuilder ) sq . SelectBuilder {
2022-02-17 09:04:57 +01:00
if cond . From != nil && cond . To != nil {
return query . Where ( field + " BETWEEN ? AND ?" , cond . From . Unix ( ) , cond . To . Unix ( ) )
} else if cond . From != nil {
return query . Where ( "? <= " + field , cond . From . Unix ( ) )
} else if cond . To != nil {
return query . Where ( field + " <= ?" , cond . To . Unix ( ) )
2024-10-10 18:35:53 +02:00
} else if cond . Range != "" {
now := time . Now ( ) . Unix ( )
var then int64
switch cond . Range {
case "last6h" :
then = now - ( 60 * 60 * 6 )
case "last24h" :
then = now - ( 60 * 60 * 24 )
case "last7d" :
then = now - ( 60 * 60 * 24 * 7 )
case "last30d" :
then = now - ( 60 * 60 * 24 * 30 )
default :
log . Debugf ( "No known named timeRange: startTime.range = %s" , cond . Range )
return query
}
return query . Where ( field + " BETWEEN ? AND ?" , then , now )
2022-02-17 09:04:57 +01:00
} else {
return query
}
}
2024-07-22 15:41:33 +02:00
func buildFloatJsonCondition ( condName string , condRange * model . FloatRange , query sq . SelectBuilder ) sq . SelectBuilder {
// Verify and Search Only in Valid Jsons
query = query . Where ( "JSON_VALID(footprint)" )
return query . Where ( "JSON_EXTRACT(footprint, \"$." + condName + "\") BETWEEN ? AND ?" , condRange . From , condRange . To )
2022-02-17 09:04:57 +01:00
}
func buildStringCondition ( field string , cond * model . StringInput , query sq . SelectBuilder ) sq . SelectBuilder {
if cond . Eq != nil {
return query . Where ( field + " = ?" , * cond . Eq )
}
2023-04-28 12:34:40 +02:00
if cond . Neq != nil {
return query . Where ( field + " != ?" , * cond . Neq )
}
2022-02-17 09:04:57 +01:00
if cond . StartsWith != nil {
return query . Where ( field + " LIKE ?" , fmt . Sprint ( * cond . StartsWith , "%" ) )
}
if cond . EndsWith != nil {
return query . Where ( field + " LIKE ?" , fmt . Sprint ( "%" , * cond . EndsWith ) )
}
if cond . Contains != nil {
return query . Where ( field + " LIKE ?" , fmt . Sprint ( "%" , * cond . Contains , "%" ) )
}
2023-02-20 10:20:08 +01:00
if cond . In != nil {
2023-06-30 16:55:34 +02:00
queryElements := make ( [ ] string , len ( cond . In ) )
2024-06-28 16:48:10 +02:00
copy ( queryElements , cond . In )
2023-06-30 16:55:34 +02:00
return query . Where ( sq . Or { sq . Eq { field : queryElements } } )
2023-02-20 10:20:08 +01:00
}
2022-02-17 09:04:57 +01:00
return query
}
2024-04-22 12:14:40 +02:00
func buildMetaJsonCondition ( jsonField string , cond * model . StringInput , query sq . SelectBuilder ) sq . SelectBuilder {
2024-05-06 13:15:15 +02:00
// Verify and Search Only in Valid Jsons
query = query . Where ( "JSON_VALID(meta_data)" )
// add "AND" Sql query Block for field match
2024-04-22 12:14:40 +02:00
if cond . Eq != nil {
return query . Where ( "JSON_EXTRACT(meta_data, \"$." + jsonField + "\") = ?" , * cond . Eq )
}
if cond . Neq != nil {
return query . Where ( "JSON_EXTRACT(meta_data, \"$." + jsonField + "\") != ?" , * cond . Neq )
}
if cond . StartsWith != nil {
return query . Where ( "JSON_EXTRACT(meta_data, \"$." + jsonField + "\") LIKE ?" , fmt . Sprint ( * cond . StartsWith , "%" ) )
}
if cond . EndsWith != nil {
return query . Where ( "JSON_EXTRACT(meta_data, \"$." + jsonField + "\") LIKE ?" , fmt . Sprint ( "%" , * cond . EndsWith ) )
}
if cond . Contains != nil {
return query . Where ( "JSON_EXTRACT(meta_data, \"$." + jsonField + "\") LIKE ?" , fmt . Sprint ( "%" , * cond . Contains , "%" ) )
}
return query
}
2024-06-28 16:48:10 +02:00
var (
matchFirstCap = regexp . MustCompile ( "(.)([A-Z][a-z]+)" )
matchAllCap = regexp . MustCompile ( "([a-z0-9])([A-Z])" )
)
2022-02-17 09:04:57 +01:00
func toSnakeCase ( str string ) string {
for _ , c := range str {
if c == '\'' || c == '\\' {
2023-01-23 18:48:06 +01:00
log . Panic ( "toSnakeCase() attack vector!" )
2022-02-17 09:04:57 +01:00
}
}
str = strings . ReplaceAll ( str , "'" , "" )
str = strings . ReplaceAll ( str , "\\" , "" )
snake := matchFirstCap . ReplaceAllString ( str , "${1}_${2}" )
snake = matchAllCap . ReplaceAllString ( snake , "${1}_${2}" )
return strings . ToLower ( snake )
}