Merge branch 'master' into 97_107_mark_and_show_shared

internal/repository/job.go

@@ -96,6 +96,50 @@ func scanJob(row interface{ Scan(...interface{}) error }) (*schema.Job, error) {
 	return job, nil
 }
 
+func (r *JobRepository) Optimize() error {
+	var err error
+
+	switch r.driver {
+	case "sqlite3":
+		if _, err = r.DB.Exec(`VACUUM`); err != nil {
+			return err
+		}
+	case "mysql":
+		log.Info("Optimize currently not supported for mysql driver")
+	}
+
+	return nil
+}
+
+func (r *JobRepository) Flush() error {
+	var err error
+
+	switch r.driver {
+	case "sqlite3":
+		if _, err = r.DB.Exec(`DELETE FROM jobtag`); err != nil {
+			return err
+		}
+		if _, err = r.DB.Exec(`DELETE FROM tag`); err != nil {
+			return err
+		}
+		if _, err = r.DB.Exec(`DELETE FROM job`); err != nil {
+			return err
+		}
+	case "mysql":
+		if _, err = r.DB.Exec(`TRUNCATE TABLE jobtag`); err != nil {
+			return err
+		}
+		if _, err = r.DB.Exec(`TRUNCATE TABLE tag`); err != nil {
+			return err
+		}
+		if _, err = r.DB.Exec(`TRUNCATE TABLE job`); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
 func scanJobLink(row interface{ Scan(...interface{}) error }) (*model.JobLink, error) {
 	jobLink := &model.JobLink{}
 	if err := row.Scan(
@@ -548,7 +592,7 @@ func (r *JobRepository) FindUserOrProjectOrJobname(ctx context.Context, searchte
 func (r *JobRepository) FindColumnValue(user *auth.User, searchterm string, table string, selectColumn string, whereColumn string, isLike bool) (result string, err error) {
 	compareStr := " = ?"
 	query := searchterm
-	if isLike == true {
+	if isLike {
 		compareStr = " LIKE ?"
 		query = "%" + searchterm + "%"
 	}
@@ -689,6 +733,38 @@ func (r *JobRepository) StopJobsExceedingWalltimeBy(seconds int) error {
 	return nil
 }
 
+func (r *JobRepository) FindJobsBefore(startTime int64) ([]*schema.Job, error) {
+
+	query := sq.Select(jobColumns...).From("job").Where(fmt.Sprintf(
+		"job.start_time < %d", startTime))
+
+	sql, args, err := query.ToSql()
+	if err != nil {
+		log.Warn("Error while converting query to sql")
+		return nil, err
+	}
+
+	log.Debugf("SQL query: `%s`, args: %#v", sql, args)
+	rows, err := query.RunWith(r.stmtCache).Query()
+	if err != nil {
+		log.Error("Error while running query")
+		return nil, err
+	}
+
+	jobs := make([]*schema.Job, 0, 50)
+	for rows.Next() {
+		job, err := scanJob(rows)
+		if err != nil {
+			rows.Close()
+			log.Warn("Error while scanning rows")
+			return nil, err
+		}
+		jobs = append(jobs, job)
+	}
+
+	return jobs, nil
+}
+
 // GraphQL validation should make sure that no unkown values can be specified.
 var groupBy2column = map[model.Aggregate]string{
 	model.AggregateUser:    "job.user",
@@ -706,9 +782,10 @@ func (r *JobRepository) JobsStatistics(ctx context.Context,
 	stats := map[string]*model.JobsStatistics{}
 	var castType string
 
-	if r.driver == "sqlite3" {
+	switch r.driver {
+	case "sqlite3":
 		castType = "int"
-	} else if r.driver == "mysql" {
+	case "mysql":
 		castType = "unsigned"
 	}
 
@@ -890,7 +967,6 @@ func (r *JobRepository) jobsStatisticsHistogram(ctx context.Context,
 	value string, filters []*model.JobFilter, id, col string) ([]*model.HistoPoint, error) {
 
 	start := time.Now()
-	query := sq.Select(value, "COUNT(job.id) AS count").From("job")
 	query, qerr := SecurityCheck(ctx, sq.Select(value, "COUNT(job.id) AS count").From("job"))
 
 	if qerr != nil {
@@ -924,3 +1000,121 @@ func (r *JobRepository) jobsStatisticsHistogram(ctx context.Context,
 	log.Infof("Timer jobsStatisticsHistogram %s", time.Since(start))
 	return points, nil
 }
+
+const NamedJobInsert string = `INSERT INTO job (
+	job_id, user, project, cluster, subcluster, ` + "`partition`" + `, array_job_id, num_nodes, num_hwthreads, num_acc,
+	exclusive, monitoring_status, smt, job_state, start_time, duration, walltime, resources, meta_data,
+	mem_used_max, flops_any_avg, mem_bw_avg, load_avg, net_bw_avg, net_data_vol_total, file_bw_avg, file_data_vol_total
+) VALUES (
+	:job_id, :user, :project, :cluster, :subcluster, :partition, :array_job_id, :num_nodes, :num_hwthreads, :num_acc,
+	:exclusive, :monitoring_status, :smt, :job_state, :start_time, :duration, :walltime, :resources, :meta_data,
+	:mem_used_max, :flops_any_avg, :mem_bw_avg, :load_avg, :net_bw_avg, :net_data_vol_total, :file_bw_avg, :file_data_vol_total
+);`
+
+func (r *JobRepository) InsertJob(job *schema.Job) (int64, error) {
+	res, err := r.DB.NamedExec(NamedJobInsert, job)
+	if err != nil {
+		log.Warn("Error while NamedJobInsert")
+		return 0, err
+	}
+	id, err := res.LastInsertId()
+	if err != nil {
+		log.Warn("Error while getting last insert ID")
+		return 0, err
+	}
+
+	return id, nil
+}
+
+type Transaction struct {
+	tx   *sqlx.Tx
+	stmt *sqlx.NamedStmt
+}
+
+func (r *JobRepository) TransactionInit() (*Transaction, error) {
+	var err error
+	t := new(Transaction)
+	// Inserts are bundled into transactions because in sqlite,
+	// that speeds up inserts A LOT.
+	t.tx, err = r.DB.Beginx()
+	if err != nil {
+		log.Warn("Error while bundling transactions")
+		return nil, err
+	}
+
+	t.stmt, err = t.tx.PrepareNamed(NamedJobInsert)
+	if err != nil {
+		log.Warn("Error while preparing namedJobInsert")
+		return nil, err
+	}
+
+	return t, nil
+}
+
+func (r *JobRepository) TransactionCommit(t *Transaction) error {
+	var err error
+	if t.tx != nil {
+		if err = t.tx.Commit(); err != nil {
+			log.Warn("Error while committing transactions")
+			return err
+		}
+	}
+
+	t.tx, err = r.DB.Beginx()
+	if err != nil {
+		log.Warn("Error while bundling transactions")
+		return err
+	}
+
+	t.stmt = t.tx.NamedStmt(t.stmt)
+	return nil
+}
+
+func (r *JobRepository) TransactionEnd(t *Transaction) error {
+	if err := t.tx.Commit(); err != nil {
+		log.Warn("Error while committing SQL transactions")
+		return err
+	}
+
+	return nil
+}
+
+func (r *JobRepository) TransactionAdd(t *Transaction, job schema.Job) (int64, error) {
+	res, err := t.stmt.Exec(job)
+	if err != nil {
+		log.Errorf("repository initDB(): %v", err)
+		return 0, err
+	}
+
+	id, err := res.LastInsertId()
+	if err != nil {
+		log.Errorf("repository initDB(): %v", err)
+		return 0, err
+	}
+
+	return id, nil
+}
+
+func (r *JobRepository) TransactionAddTag(t *Transaction, tag *schema.Tag) (int64, error) {
+	res, err := t.tx.Exec(`INSERT INTO tag (tag_name, tag_type) VALUES (?, ?)`, tag.Name, tag.Type)
+	if err != nil {
+		log.Errorf("Error while inserting tag into tag table: %v (Type %v)", tag.Name, tag.Type)
+		return 0, err
+	}
+	tagId, err := res.LastInsertId()
+	if err != nil {
+		log.Warn("Error while getting last insert ID")
+		return 0, err
+	}
+
+	return tagId, nil
+}
+
+func (r *JobRepository) TransactionSetTag(t *Transaction, jobId int64, tagId int64) error {
+	if _, err := t.tx.Exec(`INSERT INTO jobtag (job_id, tag_id) VALUES (?, ?)`, jobId, tagId); err != nil {
+		log.Errorf("Error while inserting jobtag into jobtag table: %v (TagID %v)", jobId, tagId)
+		return err
+	}
+
+	return nil
+}