diff --git a/cmd/cc-backend/main.go b/cmd/cc-backend/main.go index c606ad0..7a90ed4 100644 --- a/cmd/cc-backend/main.go +++ b/cmd/cc-backend/main.go @@ -443,11 +443,24 @@ func main() { switch cfg.Retention.Policy { case "delete": s.Every(1).Day().At("4:00").Do(func() { - jobs, err := jobRepo.FindJobsOlderThan(cfg.Retention.Age) + startTime := time.Now().Unix() - int64(cfg.Retention.Age*24*3600) + jobs, err := jobRepo.FindJobsBefore(startTime) if err != nil { log.Warnf("Error while looking for retention jobs: %s", err.Error()) } archive.GetHandle().CleanUp(jobs) + + if cfg.Retention.IncludeDB { + cnt, err := jobRepo.DeleteJobsBefore(startTime) + if err != nil { + log.Errorf("Error while deleting retention jobs from db: %s", err.Error()) + } else { + log.Infof("Retention: Removed %d jobs from db", cnt) + } + if err = jobRepo.Optimize(); err != nil { + log.Errorf("Error occured in db optimization: %s", err.Error()) + } + } }) case "move": log.Warn("Retention policy move not implemented") @@ -455,7 +468,8 @@ func main() { if cfg.Compression > 0 { s.Every(1).Day().At("5:00").Do(func() { - jobs, err := jobRepo.FindJobsOlderThan(cfg.Compression) + startTime := time.Now().Unix() - int64(cfg.Compression*24*3600) + jobs, err := jobRepo.FindJobsBefore(startTime) if err != nil { log.Warnf("Error while looking for retention jobs: %s", err.Error()) } diff --git a/internal/repository/job.go b/internal/repository/job.go index b6b181e..4eee0d3 100644 --- a/internal/repository/job.go +++ b/internal/repository/job.go @@ -96,6 +96,21 @@ func scanJob(row interface{ Scan(...interface{}) error }) (*schema.Job, error) { return job, nil } +func (r *JobRepository) Optimize() error { + var err error + + switch r.driver { + case "sqlite3": + if _, err = r.DB.Exec(`VACUUM`); err != nil { + return err + } + case "mysql": + log.Info("Optimize currently not supported for mysql driver") + } + + return nil +} + func (r *JobRepository) Flush() error { var err error @@ -707,10 +722,10 @@ func (r *JobRepository) StopJobsExceedingWalltimeBy(seconds int) error { return nil } -func (r *JobRepository) FindJobsOlderThan(days int) ([]*schema.Job, error) { +func (r *JobRepository) FindJobsBefore(startTime int64) ([]*schema.Job, error) { - query := sq.Select(jobColumns...).From("job").Where(fmt.Sprintf("job.start_time < %d", - time.Now().Unix()-int64(days*24*3600))) + query := sq.Select(jobColumns...).From("job").Where(fmt.Sprintf( + "job.start_time < %d", startTime)) sql, args, err := query.ToSql() if err != nil { diff --git a/pkg/schema/config.go b/pkg/schema/config.go index 1f4eb85..9a88ea2 100644 --- a/pkg/schema/config.go +++ b/pkg/schema/config.go @@ -58,9 +58,10 @@ type ClusterConfig struct { } type Retention struct { - Age int `json:"age"` - Policy string `json:"policy"` - Location string `json:"location"` + Age int `json:"age"` + IncludeDB bool `json:"includeDB"` + Policy string `json:"policy"` + Location string `json:"location"` } // Format of the configuration (file). See below for the defaults. diff --git a/pkg/schema/schemas/config.schema.json b/pkg/schema/schemas/config.schema.json index 33d9ef2..6518281 100644 --- a/pkg/schema/schemas/config.schema.json +++ b/pkg/schema/schemas/config.schema.json @@ -73,6 +73,10 @@ "move" ] }, + "includeDB": { + "description": "Also remove jobs from database", + "type": "boolean" + }, "age": { "description": "Act on jobs with startTime older than age (in days)", "type": "integer"