Improve retention

Add db cleanup
Fixes #103

cmd/cc-backend/main.go

@@ -443,11 +443,24 @@ func main() {
 	switch cfg.Retention.Policy {
 	case "delete":
 		s.Every(1).Day().At("4:00").Do(func() {
-			jobs, err := jobRepo.FindJobsOlderThan(cfg.Retention.Age)
+			startTime := time.Now().Unix() - int64(cfg.Retention.Age*24*3600)
+			jobs, err := jobRepo.FindJobsBefore(startTime)
 			if err != nil {
 				log.Warnf("Error while looking for retention jobs: %s", err.Error())
 			}
 			archive.GetHandle().CleanUp(jobs)
+
+			if cfg.Retention.IncludeDB {
+				cnt, err := jobRepo.DeleteJobsBefore(startTime)
+				if err != nil {
+					log.Errorf("Error while deleting retention jobs from db: %s", err.Error())
+				} else {
+					log.Infof("Retention: Removed %d jobs from db", cnt)
+				}
+				if err = jobRepo.Optimize(); err != nil {
+					log.Errorf("Error occured in db optimization: %s", err.Error())
+				}
+			}
 		})
 	case "move":
 		log.Warn("Retention policy move not implemented")
@@ -455,7 +468,8 @@ func main() {
 
 	if cfg.Compression > 0 {
 		s.Every(1).Day().At("5:00").Do(func() {
-			jobs, err := jobRepo.FindJobsOlderThan(cfg.Compression)
+			startTime := time.Now().Unix() - int64(cfg.Compression*24*3600)
+			jobs, err := jobRepo.FindJobsBefore(startTime)
 			if err != nil {
 				log.Warnf("Error while looking for retention jobs: %s", err.Error())
 			}

internal/repository/job.go

@@ -96,6 +96,21 @@ func scanJob(row interface{ Scan(...interface{}) error }) (*schema.Job, error) {
 	return job, nil
 }
 
+func (r *JobRepository) Optimize() error {
+	var err error
+
+	switch r.driver {
+	case "sqlite3":
+		if _, err = r.DB.Exec(`VACUUM`); err != nil {
+			return err
+		}
+	case "mysql":
+		log.Info("Optimize currently not supported for mysql driver")
+	}
+
+	return nil
+}
+
 func (r *JobRepository) Flush() error {
 	var err error
 
@@ -707,10 +722,10 @@ func (r *JobRepository) StopJobsExceedingWalltimeBy(seconds int) error {
 	return nil
 }
 
-func (r *JobRepository) FindJobsOlderThan(days int) ([]*schema.Job, error) {
+func (r *JobRepository) FindJobsBefore(startTime int64) ([]*schema.Job, error) {
 
-	query := sq.Select(jobColumns...).From("job").Where(fmt.Sprintf("job.start_time < %d",
+	query := sq.Select(jobColumns...).From("job").Where(fmt.Sprintf(
-		time.Now().Unix()-int64(days*24*3600)))
+		"job.start_time < %d", startTime))
 
 	sql, args, err := query.ToSql()
 	if err != nil {

pkg/schema/config.go

@@ -58,9 +58,10 @@ type ClusterConfig struct {
 }
 
 type Retention struct {
-	Age      int    `json:"age"`
+	Age       int    `json:"age"`
-	Policy   string `json:"policy"`
+	IncludeDB bool   `json:"includeDB"`
-	Location string `json:"location"`
+	Policy    string `json:"policy"`
+	Location  string `json:"location"`
 }
 
 // Format of the configuration (file). See below for the defaults.

pkg/schema/schemas/config.schema.json

@@ -73,6 +73,10 @@
                                 "move"
                             ]
                         },
+                        "includeDB": {
+                            "description": "Also remove jobs from database",
+                            "type": "boolean"
+                        },
                         "age": {
                             "description": "Act on jobs with startTime older than age (in days)",
                             "type": "integer"