cc-backend/internal/taskmanager/retentionService.go

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package taskmanager

import (
	"time"

	"github.com/ClusterCockpit/cc-backend/pkg/archive"
	pqarchive "github.com/ClusterCockpit/cc-backend/pkg/archive/parquet"
	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
	"github.com/go-co-op/gocron/v2"
)

func RegisterRetentionDeleteService(age int, includeDB bool, omitTagged bool) {
	cclog.Info("Register retention delete service")

	s.NewJob(gocron.DailyJob(1, gocron.NewAtTimes(gocron.NewAtTime(3, 0, 0))),
		gocron.NewTask(
			func() {
				startTime := time.Now().Unix() - int64(age*24*3600)
				jobs, err := jobRepo.FindJobsBetween(0, startTime, omitTagged)
				if err != nil {
					cclog.Warnf("Error while looking for retention jobs: %s", err.Error())
				}
				archive.GetHandle().CleanUp(jobs)

				if includeDB {
					cnt, err := jobRepo.DeleteJobsBefore(startTime, omitTagged)
					if err != nil {
						cclog.Errorf("Error while deleting retention jobs from db: %s", err.Error())
					} else {
						cclog.Infof("Retention: Removed %d jobs from db", cnt)
					}
					if err = jobRepo.Optimize(); err != nil {
						cclog.Errorf("Error occured in db optimization: %s", err.Error())
					}
				}
			}))
}

func RegisterRetentionMoveService(age int, includeDB bool, location string, omitTagged bool) {
	cclog.Info("Register retention move service")

	s.NewJob(gocron.DailyJob(1, gocron.NewAtTimes(gocron.NewAtTime(4, 0, 0))),
		gocron.NewTask(
			func() {
				startTime := time.Now().Unix() - int64(age*24*3600)
				jobs, err := jobRepo.FindJobsBetween(0, startTime, omitTagged)
				if err != nil {
					cclog.Warnf("Error while looking for retention jobs: %s", err.Error())
				}
				archive.GetHandle().Move(jobs, location)

				if includeDB {
					cnt, err := jobRepo.DeleteJobsBefore(startTime, omitTagged)
					if err != nil {
						cclog.Errorf("Error while deleting retention jobs from db: %v", err)
					} else {
						cclog.Infof("Retention: Removed %d jobs from db", cnt)
					}
					if err = jobRepo.Optimize(); err != nil {
						cclog.Errorf("Error occured in db optimization: %v", err)
					}
				}
			}))
}

func RegisterRetentionParquetService(retention Retention) {
	cclog.Info("Register retention parquet service")

	maxFileSizeMB := retention.MaxFileSizeMB
	if maxFileSizeMB <= 0 {
		maxFileSizeMB = 512
	}

	var target pqarchive.ParquetTarget
	var err error

	switch retention.TargetKind {
	case "s3":
		target, err = pqarchive.NewS3Target(pqarchive.S3TargetConfig{
			Endpoint:     retention.TargetEndpoint,
			Bucket:       retention.TargetBucket,
			AccessKey:    retention.TargetAccessKey,
			SecretKey:    retention.TargetSecretKey,
			Region:       retention.TargetRegion,
			UsePathStyle: retention.TargetUsePathStyle,
		})
	default:
		target, err = pqarchive.NewFileTarget(retention.TargetPath)
	}

	if err != nil {
		cclog.Errorf("Parquet retention: failed to create target: %v", err)
		return
	}

	s.NewJob(gocron.DailyJob(1, gocron.NewAtTimes(gocron.NewAtTime(5, 0, 0))),
		gocron.NewTask(
			func() {
				startTime := time.Now().Unix() - int64(retention.Age*24*3600)
				jobs, err := jobRepo.FindJobsBetween(0, startTime, retention.OmitTagged)
				if err != nil {
					cclog.Warnf("Parquet retention: error finding jobs: %v", err)
					return
				}
				if len(jobs) == 0 {
					return
				}

				cclog.Infof("Parquet retention: processing %d jobs", len(jobs))
				ar := archive.GetHandle()
				pw := pqarchive.NewParquetWriter(target, maxFileSizeMB)

				for _, job := range jobs {
					meta, err := ar.LoadJobMeta(job)
					if err != nil {
						cclog.Warnf("Parquet retention: load meta for job %d: %v", job.JobID, err)
						continue
					}

					data, err := ar.LoadJobData(job)
					if err != nil {
						cclog.Warnf("Parquet retention: load data for job %d: %v", job.JobID, err)
						continue
					}

					row, err := pqarchive.JobToParquetRow(meta, &data)
					if err != nil {
						cclog.Warnf("Parquet retention: convert job %d: %v", job.JobID, err)
						continue
					}

					if err := pw.AddJob(*row); err != nil {
						cclog.Errorf("Parquet retention: add job %d to writer: %v", job.JobID, err)
						continue
					}
				}

				if err := pw.Close(); err != nil {
					cclog.Errorf("Parquet retention: close writer: %v", err)
					return
				}

				ar.CleanUp(jobs)

				if retention.IncludeDB {
					cnt, err := jobRepo.DeleteJobsBefore(startTime, retention.OmitTagged)
					if err != nil {
						cclog.Errorf("Parquet retention: delete jobs from db: %v", err)
					} else {
						cclog.Infof("Parquet retention: removed %d jobs from db", cnt)
					}
					if err = jobRepo.Optimize(); err != nil {
						cclog.Errorf("Parquet retention: db optimization error: %v", err)
					}
				}
			}))
}