<persisted-output>
Output too large (49.5KB). Full output saved to: /Users/jan/.claude/projects/-Users-jan-prg-ClusterCockpit-cc-backend/3a41c7a4-abf8-484b-96ae-ee5618eea5ba/tool-results/REDACTED.txt

Preview (first 2KB):
     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→// Package repository provides the data access layer for cc-backend using the repository pattern.
     7→//
     8→// The repository pattern abstracts database operations and provides a clean interface for
     9→// data access. Each major entity (Job, User, Node, Tag) has its own repository with CRUD
    10→// operations and specialized queries.
    11→//
    12→// # Database Connection
    13→//
    14→// Initialize the database connection before using any repository:
    15→//
    16→//	repository.Connect("sqlite3", "./var/job.db")
    17→//
    18→// # Configuration
    19→//
    20→// Optional: Configure repository settings before initialization:
    21→//
    22→//	repository.SetConfig(&repository.RepositoryConfig{
    23→//	    CacheSize: 2 * 1024 * 1024,     // 2MB cache
    24→//	    MaxOpenConnections: 8,           // Connection pool size
    25→//	    MinRunningJobDuration: 300,      // Filter threshold
    26→//	})
    27→//
    28→// If not configured, sensible defaults are used automatically.
    29→//
    30→// # Repositories
    31→//
    32→//   - JobRepository: Job lifecycle management and querying
    33→//   - UserRepository: User management and authentication
    34→//   - NodeRepository: Cluster node state tracking
    35→//   - Tags: Job tagging and categorization
    36→//
    37→// # Caching
    38→//
    39→// Repositories use LRU caching to improve performance. Cache keys are constructed
    40→// as "type:id" (e.g., "metadata:123"). Cache is automatically invalidated on
    41→// mutations to maintain consistency.
    42→//
    43→// # Transaction Support
    44→//
    45→// For batch operations, use transactions:
    46→//
    47→//	t, err := jobRepo.TransactionInit()
...
</persisted-output>


     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→package taskmanager
     7→
     8→import (
     9→	"time"
    10→
    11→	"github.com/ClusterCockpit/cc-backend/pkg/archive"
    12→	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
    13→	"github.com/ClusterCockpit/cc-lib/v2/schema"
    14→	"github.com/go-co-op/gocron/v2"
    15→)
    16→
    17→func RegisterCompressionService(compressOlderThan int) {
    18→	cclog.Info("Register compression service")
    19→
    20→	s.NewJob(gocron.DailyJob(1, gocron.NewAtTimes(gocron.NewAtTime(5, 0, 0))),
    21→		gocron.NewTask(
    22→			func() {
    23→				var jobs []*schema.Job
    24→				var err error
    25→
    26→				ar := archive.GetHandle()
    27→				startTime := time.Now().Unix() - int64(compressOlderThan*24*3600)
    28→				lastTime := ar.CompressLast(startTime)
    29→				if startTime == lastTime {
    30→					cclog.Info("Compression Service - Complete archive run")
    31→					jobs, err = jobRepo.FindJobsBetween(0, startTime, "none")
    32→
    33→				} else {
    34→					jobs, err = jobRepo.FindJobsBetween(lastTime, startTime, "none")
    35→				}
    36→
    37→				if err != nil {
    38→					cclog.Warnf("Error while looking for compression jobs: %v", err)
    39→				}
    40→				ar.Compress(jobs)
    41→			}))
    42→}
    43→


     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→package taskmanager
     7→
     8→import (
     9→	"time"
    10→
    11→	"github.com/ClusterCockpit/cc-backend/internal/auth"
    12→	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
    13→	"github.com/go-co-op/gocron/v2"
    14→)
    15→
    16→func RegisterLdapSyncService(ds string) {
    17→	interval, err := parseDuration(ds)
    18→	if err != nil {
    19→		cclog.Warnf("Could not parse duration for sync interval: %v",
    20→			ds)
    21→		return
    22→	}
    23→
    24→	auth := auth.GetAuthInstance()
    25→
    26→	cclog.Infof("register ldap sync service with %s interval", ds)
    27→
    28→	s.NewJob(gocron.DurationJob(interval),
    29→		gocron.NewTask(
    30→			func() {
    31→				t := time.Now()
    32→				cclog.Infof("ldap sync started at %s", t.Format(time.RFC3339))
    33→				if err := auth.LdapAuth.Sync(); err != nil {
    34→					cclog.Errorf("ldap sync failed: %s", err.Error())
    35→				}
    36→			}))
    37→}
    38→


     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→package taskmanager
     7→
     8→import (
     9→	"time"
    10→
    11→	"github.com/ClusterCockpit/cc-backend/internal/config"
    12→	"github.com/ClusterCockpit/cc-backend/internal/repository"
    13→	pqarchive "github.com/ClusterCockpit/cc-backend/pkg/archive/parquet"
    14→	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
    15→	"github.com/go-co-op/gocron/v2"
    16→)
    17→
    18→func RegisterNodeStateRetentionDeleteService(ageHours int) {
    19→	cclog.Info("Register node state retention delete service")
    20→
    21→	s.NewJob(gocron.DailyJob(1, gocron.NewAtTimes(gocron.NewAtTime(2, 0, 0))),
    22→		gocron.NewTask(
    23→			func() {
    24→				cutoff := time.Now().Unix() - int64(ageHours*3600)
    25→				nodeRepo := repository.GetNodeRepository()
    26→				cnt, err := nodeRepo.DeleteNodeStatesBefore(cutoff)
    27→				if err != nil {
    28→					cclog.Errorf("NodeState retention: error deleting old rows: %v", err)
    29→				} else if cnt > 0 {
    30→					cclog.Infof("NodeState retention: deleted %d old rows", cnt)
    31→				}
    32→			}))
    33→}
    34→
    35→func RegisterNodeStateRetentionMoveService(cfg *config.NodeStateRetention) {
    36→	cclog.Info("Register node state retention move service")
    37→
    38→	maxFileSizeMB := cfg.MaxFileSizeMB
    39→	if maxFileSizeMB <= 0 {
    40→		maxFileSizeMB = 128
    41→	}
    42→
    43→	ageHours := cfg.Age
    44→	if ageHours <= 0 {
    45→		ageHours = 24
    46→	}
    47→
    48→	var target pqarchive.ParquetTarget
    49→	var err error
    50→
    51→	switch cfg.TargetKind {
    52→	case "s3":
    53→		target, err = pqarchive.NewS3Target(pqarchive.S3TargetConfig{
    54→			Endpoint:     cfg.TargetEndpoint,
    55→			Bucket:       cfg.TargetBucket,
    56→			AccessKey:    cfg.TargetAccessKey,
    57→			SecretKey:    cfg.TargetSecretKey,
    58→			Region:       cfg.TargetRegion,
    59→			UsePathStyle: cfg.TargetUsePathStyle,
    60→		})
    61→	default:
    62→		target, err = pqarchive.NewFileTarget(cfg.TargetPath)
    63→	}
    64→
    65→	if err != nil {
    66→		cclog.Errorf("NodeState move retention: failed to create target: %v", err)
    67→		return
    68→	}
    69→
    70→	s.NewJob(gocron.DailyJob(1, gocron.NewAtTimes(gocron.NewAtTime(2, 30, 0))),
    71→		gocron.NewTask(
    72→			func() {
    73→				cutoff := time.Now().Unix() - int64(ageHours*3600)
    74→				nodeRepo := repository.GetNodeRepository()
    75→
    76→				rows, err := nodeRepo.FindNodeStatesBefore(cutoff)
    77→				if err != nil {
    78→					cclog.Errorf("NodeState move retention: error finding rows: %v", err)
    79→					return
    80→				}
    81→				if len(rows) == 0 {
    82→					return
    83→				}
    84→
    85→				cclog.Infof("NodeState move retention: archiving %d rows", len(rows))
    86→				pw := pqarchive.NewNodeStateParquetWriter(target, maxFileSizeMB)
    87→
    88→				for _, ns := range rows {
    89→					row := pqarchive.ParquetNodeStateRow{
    90→						TimeStamp:       ns.TimeStamp,
    91→						NodeState:       ns.NodeState,
    92→						HealthState:     ns.HealthState,
    93→						HealthMetrics:   ns.HealthMetrics,
    94→						CpusAllocated:   int32(ns.CpusAllocated),
    95→						MemoryAllocated: ns.MemoryAllocated,
    96→						GpusAllocated:   int32(ns.GpusAllocated),
    97→						JobsRunning:     int32(ns.JobsRunning),
    98→						Hostname:        ns.Hostname,
    99→						Cluster:         ns.Cluster,
   100→						SubCluster:      ns.SubCluster,
   101→					}
   102→					if err := pw.AddRow(row); err != nil {
   103→						cclog.Errorf("NodeState move retention: add row: %v", err)
   104→						continue
   105→					}
   106→				}
   107→
   108→				if err := pw.Close(); err != nil {
   109→					cclog.Errorf("NodeState move retention: close writer: %v", err)
   110→					return
   111→				}
   112→
   113→				cnt, err := nodeRepo.DeleteNodeStatesBefore(cutoff)
   114→				if err != nil {
   115→					cclog.Errorf("NodeState move retention: error deleting rows: %v", err)
   116→				} else {
   117→					cclog.Infof("NodeState move retention: deleted %d rows from db", cnt)
   118→				}
   119→			}))
   120→}
   121→


     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→// Package archiver provides asynchronous job archiving functionality for ClusterCockpit.
     7→//
     8→// The archiver runs a background worker goroutine that processes job archiving requests
     9→// from a buffered channel. When jobs complete, their metric data is archived from the
    10→// metric store to the configured archive backend (filesystem, S3, etc.).
    11→//
    12→// # Architecture
    13→//
    14→// The archiver uses a producer-consumer pattern:
    15→//   - Producer: TriggerArchiving() sends jobs to archiveChannel
    16→//   - Consumer: archivingWorker() processes jobs from the channel
    17→//   - Coordination: sync.WaitGroup tracks pending archive operations
    18→//
    19→// # Lifecycle
    20→//
    21→//  1. Start(repo, ctx) - Initialize worker with context for cancellation
    22→//  2. TriggerArchiving(job) - Queue job for archiving (called when job stops)
    23→//  3. archivingWorker() - Background goroutine processes jobs
    24→//  4. Shutdown(timeout) - Graceful shutdown with timeout
    25→//
    26→// # Graceful Shutdown
    27→//
    28→// The archiver supports graceful shutdown with configurable timeout:
    29→//   - Closes channel to reject new jobs
    30→//   - Waits for pending jobs to complete (up to timeout)
    31→//   - Cancels context if timeout exceeded
    32→//   - Ensures worker goroutine exits cleanly
    33→//
    34→// # Example Usage
    35→//
    36→//	// Initialize archiver
    37→//	ctx, cancel := context.WithCancel(context.Background())
    38→//	defer cancel()
    39→//	archiver.Start(jobRepository, ctx)
    40→//
    41→//	// Trigger archiving when job completes
    42→//	archiver.TriggerArchiving(job)
    43→//
    44→//	// Graceful shutdown with 10 second timeout
    45→//	if err := archiver.Shutdown(10 * time.Second); err != nil {
    46→//	    log.Printf("Archiver shutdown timeout: %v", err)
    47→//	}
    48→package archiver
    49→
    50→import (
    51→	"context"
    52→	"fmt"
    53→	"sync"
    54→	"time"
    55→
    56→	"github.com/ClusterCockpit/cc-backend/internal/repository"
    57→	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
    58→	"github.com/ClusterCockpit/cc-lib/v2/schema"
    59→	sq "github.com/Masterminds/squirrel"
    60→)
    61→
    62→var (
    63→	archivePending sync.WaitGroup
    64→	archiveChannel chan *schema.Job
    65→	jobRepo        *repository.JobRepository
    66→	shutdownCtx    context.Context
    67→	shutdownCancel context.CancelFunc
    68→	workerDone     chan struct{}
    69→)
    70→
    71→// Start initializes the archiver and starts the background worker goroutine.
    72→//
    73→// The archiver processes job archiving requests asynchronously via a buffered channel.
    74→// Jobs are sent to the channel using TriggerArchiving() and processed by the worker.
    75→//
    76→// Parameters:
    77→//   - r: JobRepository instance for database operations
    78→//   - ctx: Context for cancellation (shutdown signal propagation)
    79→//
    80→// The worker goroutine will run until:
    81→//   - ctx is cancelled (via parent shutdown)
    82→//   - archiveChannel is closed (via Shutdown())
    83→//
    84→// Must be called before TriggerArchiving(). Safe to call only once.
    85→func Start(r *repository.JobRepository, ctx context.Context) {
    86→	shutdownCtx, shutdownCancel = context.WithCancel(ctx)
    87→	archiveChannel = make(chan *schema.Job, 128)
    88→	workerDone = make(chan struct{})
    89→	jobRepo = r
    90→
    91→	go archivingWorker()
    92→}
    93→
    94→// archivingWorker is the background goroutine that processes job archiving requests.
    95→//
    96→// The worker loop:
    97→//  1. Blocks waiting for jobs on archiveChannel or shutdown signal
    98→//  2. Fetches job metadata from repository
    99→//  3. Archives job data to configured backend (calls ArchiveJob)
   100→//  4. Updates job footprint and energy metrics in database
   101→//  5. Marks job as successfully archived
   102→//  6. Calls job stop hooks
   103→//
   104→// The worker exits when:
   105→//   - shutdownCtx is cancelled (timeout during shutdown)
   106→//   - archiveChannel is closed (normal shutdown)
   107→//
   108→// Errors during archiving are logged and the job is marked as failed,
   109→// but the worker continues processing other jobs.
   110→func archivingWorker() {
   111→	defer close(workerDone)
   112→
   113→	for {
   114→		select {
   115→		case <-shutdownCtx.Done():
   116→			cclog.Info("Archive worker received shutdown signal")
   117→			return
   118→
   119→		case job, ok := <-archiveChannel:
   120→			if !ok {
   121→				cclog.Info("Archive channel closed, worker exiting")
   122→				return
   123→			}
   124→
   125→			start := time.Now()
   126→			// not using meta data, called to load JobMeta into Cache?
   127→			// will fail if job meta not in repository
   128→			if _, err := jobRepo.FetchMetadata(job); err != nil {
   129→				cclog.Errorf("archiving job (dbid: %d) failed at check metadata step: %s", *job.ID, err.Error())
   130→				jobRepo.UpdateMonitoringStatus(*job.ID, schema.MonitoringStatusArchivingFailed)
   131→				archivePending.Done()
   132→				continue
   133→			}
   134→
   135→			// ArchiveJob will fetch all the data from a MetricDataRepository and push into configured archive backend
   136→			// Use shutdown context to allow cancellation
   137→			jobMeta, err := ArchiveJob(job, shutdownCtx)
   138→			if err != nil {
   139→				cclog.Errorf("archiving job (dbid: %d) failed at archiving job step: %s", *job.ID, err.Error())
   140→				jobRepo.UpdateMonitoringStatus(*job.ID, schema.MonitoringStatusArchivingFailed)
   141→				archivePending.Done()
   142→				continue
   143→			}
   144→
   145→			stmt := sq.Update("job").Where("job.id = ?", job.ID)
   146→
   147→			if stmt, err = jobRepo.UpdateFootprint(stmt, jobMeta); err != nil {
   148→				cclog.Errorf("archiving job (dbid: %d) failed at update Footprint step: %s", *job.ID, err.Error())
   149→				archivePending.Done()
   150→				continue


     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→package importer
     7→
     8→import (
     9→	"bytes"
    10→	"encoding/json"
    11→	"fmt"
    12→	"os"
    13→	"strings"
    14→
    15→	"github.com/ClusterCockpit/cc-backend/internal/config"
    16→	"github.com/ClusterCockpit/cc-backend/internal/repository"
    17→	"github.com/ClusterCockpit/cc-backend/pkg/archive"
    18→	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
    19→	"github.com/ClusterCockpit/cc-lib/v2/schema"
    20→)
    21→
    22→// HandleImportFlag imports jobs from file pairs specified in a comma-separated flag string.
    23→//
    24→// The flag format is: "<path-to-meta.json>:<path-to-data.json>[,<path-to-meta2.json>:<path-to-data2.json>,...]"
    25→//
    26→// For each job pair, this function:
    27→//  1. Reads and validates the metadata JSON file (schema.Job)
    28→//  2. Reads and validates the job data JSON file (schema.JobData)
    29→//  3. Enriches the job with calculated footprints and energy metrics
    30→//  4. Validates the job using SanityChecks()
    31→//  5. Imports the job into the archive
    32→//  6. Inserts the job into the database with associated tags
    33→//
    34→// Schema validation is performed if config.Keys.Validate is true.
    35→//
    36→// Returns an error if file reading, validation, enrichment, or database operations fail.
    37→// The function stops processing on the first error encountered.
    38→func HandleImportFlag(flag string) error {
    39→	r := repository.GetJobRepository()
    40→
    41→	for pair := range strings.SplitSeq(flag, ",") {
    42→		files := strings.Split(pair, ":")
    43→		if len(files) != 2 {
    44→			return fmt.Errorf("REPOSITORY/INIT > invalid import flag format")
    45→		}
    46→
    47→		raw, err := os.ReadFile(files[0])
    48→		if err != nil {
    49→			cclog.Warn("Error while reading metadata file for import")
    50→			return err
    51→		}
    52→
    53→		if config.Keys.Validate {
    54→			if err = schema.Validate(schema.Meta, bytes.NewReader(raw)); err != nil {
    55→				return fmt.Errorf("REPOSITORY/INIT > validate job meta: %v", err)
    56→			}
    57→		}
    58→		dec := json.NewDecoder(bytes.NewReader(raw))
    59→		dec.DisallowUnknownFields()
    60→		job := schema.Job{
    61→			Shared:           "none",
    62→			MonitoringStatus: schema.MonitoringStatusRunningOrArchiving,
    63→		}
    64→		if err = dec.Decode(&job); err != nil {
    65→			cclog.Warn("Error while decoding raw json metadata for import")
    66→			return err
    67→		}
    68→
    69→		raw, err = os.ReadFile(files[1])
    70→		if err != nil {
    71→			cclog.Warn("Error while reading jobdata file for import")
    72→			return err
    73→		}
    74→
    75→		if config.Keys.Validate {
    76→			if err = schema.Validate(schema.Data, bytes.NewReader(raw)); err != nil {
    77→				return fmt.Errorf("REPOSITORY/INIT > validate job data: %v", err)
    78→			}
    79→		}
    80→		dec = json.NewDecoder(bytes.NewReader(raw))
    81→		dec.DisallowUnknownFields()
    82→		jobData := schema.JobData{}
    83→		if err = dec.Decode(&jobData); err != nil {
    84→			cclog.Warn("Error while decoding raw json jobdata for import")
    85→			return err
    86→		}
    87→
    88→		job.MonitoringStatus = schema.MonitoringStatusArchivingSuccessful
    89→
    90→		if err = enrichJobMetadata(&job); err != nil {
    91→			cclog.Errorf("Error enriching job metadata: %v", err)
    92→			return err
    93→		}
    94→
    95→		if err = SanityChecks(&job); err != nil {
    96→			cclog.Warn("BaseJob SanityChecks failed")
    97→			return err
    98→		}
    99→
   100→		if err = archive.GetHandle().ImportJob(&job, &jobData); err != nil {
   101→			cclog.Error("Error while importing job")
   102→			return err
   103→		}
   104→
   105→		id, err := r.InsertJobDirect(&job)
   106→		if err != nil {
   107→			cclog.Warn("Error while job db insert")
   108→			return err
   109→		}
   110→
   111→		for _, tag := range job.Tags {
   112→			if err := r.ImportTag(id, tag.Type, tag.Name, tag.Scope); err != nil {
   113→				cclog.Error("Error while adding or creating tag on import")
   114→				return err
   115→			}
   116→		}
   117→
   118→		cclog.Infof("successfully imported a new job (jobId: %d, cluster: %s, dbid: %d)", job.JobID, job.Cluster, id)
   119→	}
   120→	return nil
   121→}
   122→


     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→package archiver
     7→
     8→import (
     9→	"context"
    10→	"math"
    11→
    12→	"github.com/ClusterCockpit/cc-backend/internal/metricdispatch"
    13→	"github.com/ClusterCockpit/cc-backend/pkg/archive"
    14→	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
    15→	"github.com/ClusterCockpit/cc-lib/v2/schema"
    16→)
    17→
    18→// ArchiveJob archives a completed job's metric data to the configured archive backend.
    19→//
    20→// This function performs the following operations:
    21→//  1. Loads all metric data for the job from the metric data repository
    22→//  2. Calculates job-level statistics (avg, min, max) for each metric
    23→//  3. Stores the job metadata and metric data to the archive backend
    24→//
    25→// Metric data is retrieved at the highest available resolution (typically 60s)
    26→// for the following scopes:
    27→//   - Node scope (always)
    28→//   - Core scope (for jobs with ≤8 nodes, to reduce data volume)
    29→//   - Accelerator scope (if job used accelerators)
    30→//
    31→// The function respects context cancellation. If ctx is cancelled (e.g., during
    32→// shutdown timeout), the operation will be interrupted and return an error.
    33→//
    34→// Parameters:
    35→//   - job: The job to archive (must be a completed job)
    36→//   - ctx: Context for cancellation and timeout control
    37→//
    38→// Returns:
    39→//   - *schema.Job with populated Statistics field
    40→//   - error if data loading or archiving fails
    41→//
    42→// If config.Keys.DisableArchive is true, only job statistics are calculated
    43→// and returned (no data is written to archive backend).
    44→func ArchiveJob(job *schema.Job, ctx context.Context) (*schema.Job, error) {
    45→	allMetrics := make([]string, 0)
    46→	metricConfigs := archive.GetCluster(job.Cluster).MetricConfig
    47→	for _, mc := range metricConfigs {
    48→		allMetrics = append(allMetrics, mc.Name)
    49→	}
    50→
    51→	scopes := []schema.MetricScope{schema.MetricScopeNode}
    52→	// FIXME: Add a config option for this
    53→	if job.NumNodes <= 8 {
    54→		// This will add the native scope if core scope is not available
    55→		scopes = append(scopes, schema.MetricScopeCore)
    56→	}
    57→
    58→	if job.NumAcc > 0 {
    59→		scopes = append(scopes, schema.MetricScopeAccelerator)
    60→	}
    61→
    62→	jobData, err := metricdispatch.LoadData(job, allMetrics, scopes, ctx, 0) // 0 Resulotion-Value retrieves highest res (60s)
    63→	if err != nil {
    64→		cclog.Error("Error wile loading job data for archiving")
    65→		return nil, err
    66→	}
    67→
    68→	job.Statistics = make(map[string]schema.JobStatistics)
    69→
    70→	for metric, data := range jobData {
    71→		avg, min, max := 0.0, math.MaxFloat32, -math.MaxFloat32
    72→		nodeData, ok := data["node"]
    73→		if !ok {
    74→			// This should never happen ?
    75→			continue
    76→		}
    77→
    78→		for _, series := range nodeData.Series {
    79→			avg += series.Statistics.Avg
    80→			min = math.Min(min, series.Statistics.Min)
    81→			max = math.Max(max, series.Statistics.Max)
    82→		}
    83→
    84→		// Round AVG Result to 2 Digits
    85→		job.Statistics[metric] = schema.JobStatistics{
    86→			Unit: schema.Unit{
    87→				Prefix: archive.GetMetricConfig(job.Cluster, metric).Unit.Prefix,
    88→				Base:   archive.GetMetricConfig(job.Cluster, metric).Unit.Base,
    89→			},
    90→			Avg: (math.Round((avg/float64(job.NumNodes))*100) / 100),
    91→			Min: min,
    92→			Max: max,
    93→		}
    94→	}
    95→
    96→	return job, archive.GetHandle().ImportJob(job, &jobData)
    97→}
    98→


     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→// Package importer provides functionality for importing job data into the ClusterCockpit database.
     7→//
     8→// The package supports two primary use cases:
     9→//  1. Bulk database initialization from archived jobs via InitDB()
    10→//  2. Individual job import from file pairs via HandleImportFlag()
    11→//
    12→// Both operations enrich job metadata by calculating footprints and energy metrics
    13→// before persisting to the database.
    14→package importer
    15→
    16→import (
    17→	"encoding/json"
    18→	"fmt"
    19→	"math"
    20→	"strings"
    21→	"time"
    22→
    23→	"github.com/ClusterCockpit/cc-backend/internal/repository"
    24→	"github.com/ClusterCockpit/cc-backend/pkg/archive"
    25→	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
    26→	"github.com/ClusterCockpit/cc-lib/v2/schema"
    27→)
    28→
    29→const (
    30→	addTagQuery = "INSERT INTO tag (tag_name, tag_type) VALUES (?, ?)"
    31→	setTagQuery = "INSERT INTO jobtag (job_id, tag_id) VALUES (?, ?)"
    32→)
    33→
    34→// InitDB reinitializes the job database from archived job data.
    35→//
    36→// This function performs the following operations:
    37→//  1. Flushes existing job, tag, and jobtag tables
    38→//  2. Iterates through all jobs in the archive
    39→//  3. Enriches each job with calculated footprints and energy metrics
    40→//  4. Inserts jobs and tags into the database in batched transactions
    41→//
    42→// Jobs are processed in batches of 100 for optimal performance. The function
    43→// continues processing even if individual jobs fail, logging errors and
    44→// returning a summary at the end.
    45→//
    46→// Returns an error if database initialization, transaction management, or
    47→// critical operations fail. Individual job failures are logged but do not
    48→// stop the overall import process.
    49→func InitDB() error {
    50→	r := repository.GetJobRepository()
    51→	if err := r.Flush(); err != nil {
    52→		cclog.Errorf("repository initDB(): %v", err)
    53→		return err
    54→	}
    55→	starttime := time.Now()
    56→	cclog.Print("Building job table...")
    57→
    58→	t, err := r.TransactionInit()
    59→	if err != nil {
    60→		cclog.Warn("Error while initializing SQL transactions")
    61→		return err
    62→	}
    63→	tags := make(map[string]int64)
    64→
    65→	// Not using cclog.Print because we want the line to end with `\r` and
    66→	// this function is only ever called when a special command line flag
    67→	// is passed anyways.
    68→	fmt.Printf("%d jobs inserted...\r", 0)
    69→
    70→	ar := archive.GetHandle()
    71→	i := 0
    72→	errorOccured := 0
    73→
    74→	for jobContainer := range ar.Iter(false) {
    75→
    76→		jobMeta := jobContainer.Meta
    77→		if jobMeta == nil {
    78→			cclog.Warn("skipping job with nil metadata")
    79→			errorOccured++
    80→			continue
    81→		}
    82→
    83→		// Bundle 100 inserts into one transaction for better performance
    84→		if i%100 == 0 {
    85→			if i > 0 {
    86→				if err := t.Commit(); err != nil {
    87→					cclog.Errorf("transaction commit error: %v", err)
    88→					return err
    89→				}
    90→				// Start a new transaction for the next batch
    91→				t, err = r.TransactionInit()
    92→				if err != nil {
    93→					cclog.Errorf("transaction init error: %v", err)
    94→					return err
    95→				}
    96→			}
    97→			fmt.Printf("%d jobs inserted...\r", i)
    98→		}
    99→
   100→		jobMeta.MonitoringStatus = schema.MonitoringStatusArchivingSuccessful
   101→
   102→		if err := enrichJobMetadata(jobMeta); err != nil {
   103→			cclog.Errorf("repository initDB(): %v", err)
   104→			errorOccured++
   105→			continue
   106→		}
   107→
   108→		if err := SanityChecks(jobMeta); err != nil {
   109→			cclog.Errorf("repository initDB(): %v", err)
   110→			errorOccured++
   111→			continue
   112→		}
   113→
   114→		id, jobErr := r.TransactionAddNamed(t,
   115→			repository.NamedJobInsert, jobMeta)
   116→		if jobErr != nil {
   117→			cclog.Errorf("repository initDB(): %v", jobErr)
   118→			errorOccured++
   119→			continue
   120→		}
   121→
   122→		// Job successfully inserted, increment counter
   123→		i += 1
   124→
   125→		for _, tag := range jobMeta.Tags {
   126→			tagstr := tag.Name + ":" + tag.Type
   127→			tagID, ok := tags[tagstr]
   128→			if !ok {
   129→				var err error
   130→				tagID, err = r.TransactionAdd(t,
   131→					addTagQuery,
   132→					tag.Name, tag.Type)
   133→				if err != nil {
   134→					cclog.Errorf("Error adding tag: %v", err)
   135→					errorOccured++
   136→					continue
   137→				}
   138→				tags[tagstr] = tagID
   139→			}
   140→
   141→			r.TransactionAdd(t,
   142→				setTagQuery,
   143→				id, tagID)
   144→		}
   145→	}
   146→
   147→	if errorOccured > 0 {
   148→		cclog.Warnf("Error in import of %d jobs!", errorOccured)
   149→	}
   150→
   151→	r.TransactionEnd(t)
   152→	cclog.Infof("A total of %d jobs have been registered in %.3f seconds.", i, time.Since(starttime).Seconds())
   153→	return nil
   154→}
   155→
   156→// enrichJobMetadata calculates and populates job footprints, energy metrics, and serialized fields.
   157→//
   158→// This function performs the following enrichment operations:
   159→//  1. Calculates job footprint metrics based on the subcluster configuration
   160→//  2. Computes energy footprint and total energy consumption in kWh
   161→//  3. Marshals footprints, resources, and metadata into JSON for database storage
   162→//
   163→// The function expects the job's MonitoringStatus and SubCluster to be already set.
   164→// Energy calculations convert power metrics (Watts) to energy (kWh) using the formula:
   165→//
   166→//	Energy (kWh) = (Power (W) * Duration (s) / 3600) / 1000
   167→//
   168→// Returns an error if subcluster retrieval, metric indexing, or JSON marshaling fails.
   169→func enrichJobMetadata(job *schema.Job) error {
   170→	sc, err := archive.GetSubCluster(job.Cluster, job.SubCluster)
   171→	if err != nil {
   172→		cclog.Errorf("cannot get subcluster: %s", err.Error())
   173→		return err
   174→	}
   175→
   176→	job.Footprint = make(map[string]float64)
   177→
   178→	for _, fp := range sc.Footprint {
   179→		statType := "avg"
   180→
   181→		if i, err := archive.MetricIndex(sc.MetricConfig, fp); err != nil {
   182→			statType = sc.MetricConfig[i].Footprint
   183→		}
   184→
   185→		name := fmt.Sprintf("%s_%s", fp, statType)
   186→
   187→		job.Footprint[name] = repository.LoadJobStat(job, fp, statType)
   188→	}
   189→
   190→	job.RawFootprint, err = json.Marshal(job.Footprint)
   191→	if err != nil {
   192→		cclog.Warn("Error while marshaling job footprint")
   193→		return err
   194→	}
   195→
   196→	job.EnergyFootprint = make(map[string]float64)
   197→
   198→	// Total Job Energy Outside Loop
   199→	totalEnergy := 0.0
   200→	for _, fp := range sc.EnergyFootprint {


     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→package repository
     6→
     7→import (
     8→	"fmt"
     9→
    10→	"github.com/jmoiron/sqlx"
    11→)
    12→
    13→// Transaction wraps a database transaction for job-related operations.
    14→type Transaction struct {
    15→	tx *sqlx.Tx
    16→}
    17→
    18→// TransactionInit begins a new transaction.
    19→func (r *JobRepository) TransactionInit() (*Transaction, error) {
    20→	tx, err := r.DB.Beginx()
    21→	if err != nil {
    22→		return nil, fmt.Errorf("beginning transaction: %w", err)
    23→	}
    24→	return &Transaction{tx: tx}, nil
    25→}
    26→
    27→// Commit commits the transaction.
    28→// After calling Commit, the transaction should not be used again.
    29→func (t *Transaction) Commit() error {
    30→	if t.tx == nil {
    31→		return fmt.Errorf("transaction already committed or rolled back")
    32→	}
    33→	err := t.tx.Commit()
    34→	t.tx = nil // Mark as completed
    35→	if err != nil {
    36→		return fmt.Errorf("committing transaction: %w", err)
    37→	}
    38→	return nil
    39→}
    40→
    41→// Rollback rolls back the transaction.
    42→// It's safe to call Rollback on an already committed or rolled back transaction.
    43→func (t *Transaction) Rollback() error {
    44→	if t.tx == nil {
    45→		return nil // Already committed/rolled back
    46→	}
    47→	err := t.tx.Rollback()
    48→	t.tx = nil // Mark as completed
    49→	if err != nil {
    50→		return fmt.Errorf("rolling back transaction: %w", err)
    51→	}
    52→	return nil
    53→}
    54→
    55→// TransactionEnd commits the transaction.
    56→// Deprecated: Use Commit() instead.
    57→func (r *JobRepository) TransactionEnd(t *Transaction) error {
    58→	return t.Commit()
    59→}
    60→
    61→// TransactionAddNamed executes a named query within the transaction.
    62→func (r *JobRepository) TransactionAddNamed(
    63→	t *Transaction,
    64→	query string,
    65→	args ...any,
    66→) (int64, error) {
    67→	if t.tx == nil {
    68→		return 0, fmt.Errorf("transaction is nil or already completed")
    69→	}
    70→
    71→	res, err := t.tx.NamedExec(query, args)
    72→	if err != nil {
    73→		return 0, fmt.Errorf("named exec: %w", err)
    74→	}
    75→
    76→	id, err := res.LastInsertId()
    77→	if err != nil {
    78→		return 0, fmt.Errorf("getting last insert id: %w", err)
    79→	}
    80→
    81→	return id, nil
    82→}
    83→
    84→// TransactionAdd executes a query within the transaction.
    85→func (r *JobRepository) TransactionAdd(t *Transaction, query string, args ...any) (int64, error) {
    86→	if t.tx == nil {
    87→		return 0, fmt.Errorf("transaction is nil or already completed")
    88→	}
    89→
    90→	res, err := t.tx.Exec(query, args...)
    91→	if err != nil {
    92→		return 0, fmt.Errorf("exec: %w", err)
    93→	}
    94→
    95→	id, err := res.LastInsertId()
    96→	if err != nil {
    97→		return 0, fmt.Errorf("getting last insert id: %w", err)
    98→	}
    99→
   100→	return id, nil
   101→}
   102→


     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→// Package api provides the REST API layer for ClusterCockpit.
     7→// It handles HTTP requests for job management, user administration,
     8→// cluster queries, node state updates, and metrics storage operations.
     9→// The API supports both JWT token authentication and session-based authentication.
    10→package api
    11→
    12→import (
    13→	"encoding/json"
    14→	"fmt"
    15→	"io"
    16→	"net/http"
    17→	"os"
    18→	"path/filepath"
    19→	"strings"
    20→	"sync"
    21→
    22→	"github.com/ClusterCockpit/cc-backend/internal/auth"
    23→	"github.com/ClusterCockpit/cc-backend/internal/config"
    24→	"github.com/ClusterCockpit/cc-backend/internal/repository"
    25→	"github.com/ClusterCockpit/cc-backend/internal/tagger"
    26→	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
    27→	"github.com/ClusterCockpit/cc-lib/v2/schema"
    28→	"github.com/ClusterCockpit/cc-lib/v2/util"
    29→	"github.com/go-chi/chi/v5"
    30→)
    31→
    32→// @title                      ClusterCockpit REST API
    33→// @version                    1.0.0
    34→// @description                API for batch job control.
    35→
    36→// @contact.name               ClusterCockpit Project
    37→// @contact.url                https://github.com/ClusterCockpit
    38→// @contact.email              support@clustercockpit.org
    39→
    40→// @license.name               MIT License
    41→// @license.url                https://opensource.org/licenses/MIT
    42→
    43→// @host                       localhost:8080
    44→
    45→// @securityDefinitions.apikey ApiKeyAuth
    46→// @in                         header
    47→// @name                       X-Auth-Token
    48→
    49→const (
    50→	noticeFilePath  = "./var/notice.txt"
    51→	noticeFilePerms = 0o644
    52→	maxNoticeLength = 10000 // Maximum allowed notice content length in characters
    53→)
    54→
    55→type RestAPI struct {
    56→	JobRepository   *repository.JobRepository
    57→	Authentication  *auth.Authentication
    58→	MachineStateDir string
    59→	// RepositoryMutex protects job creation operations from race conditions
    60→	// when checking for duplicate jobs during startJob API calls.
    61→	// It prevents concurrent job starts with the same jobId/cluster/startTime
    62→	// from creating duplicate entries in the database.
    63→	RepositoryMutex sync.Mutex
    64→}
    65→
    66→// New creates and initializes a new RestAPI instance with configured dependencies.
    67→func New() *RestAPI {
    68→	return &RestAPI{
    69→		JobRepository:   repository.GetJobRepository(),
    70→		MachineStateDir: config.Keys.MachineStateDir,
    71→		Authentication:  auth.GetAuthInstance(),
    72→	}
    73→}
    74→
    75→// MountAPIRoutes registers REST API endpoints for job and cluster management.
    76→// These routes use JWT token authentication via the X-Auth-Token header.
    77→func (api *RestAPI) MountAPIRoutes(r chi.Router) {
    78→	// REST API Uses TokenAuth
    79→	// User List
    80→	r.Get("/users/", api.getUsers)
    81→	// Cluster List
    82→	r.Get("/clusters/", api.getClusters)
    83→	// Slurm node state
    84→	r.Post("/nodestate/", api.updateNodeStates)
    85→	r.Put("/nodestate/", api.updateNodeStates)
    86→	// Job Handler
    87→	if config.Keys.APISubjects == nil {
    88→		cclog.Info("Enabling REST start/stop job API")
    89→		r.Post("/jobs/start_job/", api.startJob)
    90→		r.Put("/jobs/start_job/", api.startJob)
    91→		r.Post("/jobs/stop_job/", api.stopJobByRequest)
    92→		r.Put("/jobs/stop_job/", api.stopJobByRequest)
    93→	}
    94→	r.Get("/jobs/", api.getJobs)
    95→	r.Get("/jobs/used_nodes", api.getUsedNodes)
    96→	r.Post("/jobs/tag_job/{id}", api.tagJob)
    97→	r.Patch("/jobs/tag_job/{id}", api.tagJob)
    98→	r.Delete("/jobs/tag_job/{id}", api.removeTagJob)
    99→	r.Patch("/jobs/edit_meta/{id}", api.editMeta)
   100→	r.Patch("/jobs/edit_meta/", api.editMetaByRequest)
   101→	r.Get("/jobs/metrics/{id}", api.getJobMetrics)
   102→	r.Delete("/jobs/delete_job/", api.deleteJobByRequest)
   103→	r.Delete("/jobs/delete_job/{id}", api.deleteJobByID)
   104→	r.Delete("/jobs/delete_job_before/{ts}", api.deleteJobBefore)
   105→	r.Post("/jobs/{id}", api.getJobByID)
   106→	r.Get("/jobs/{id}", api.getCompleteJobByID)
   107→
   108→	r.Delete("/tags/", api.removeTags)
   109→
   110→	if api.MachineStateDir != "" {
   111→		r.Get("/machine_state/{cluster}/{host}", api.getMachineState)
   112→		r.Put("/machine_state/{cluster}/{host}", api.putMachineState)
   113→		r.Post("/machine_state/{cluster}/{host}", api.putMachineState)
   114→	}
   115→}
   116→
   117→// MountUserAPIRoutes registers user-accessible REST API endpoints.
   118→// These are limited endpoints for regular users with JWT token authentication.
   119→func (api *RestAPI) MountUserAPIRoutes(r chi.Router) {
   120→	// REST API Uses TokenAuth
   121→	r.Get("/jobs/", api.getJobs)
   122→	r.Post("/jobs/{id}", api.getJobByID)
   123→	r.Get("/jobs/{id}", api.getCompleteJobByID)
   124→	r.Get("/jobs/metrics/{id}", api.getJobMetrics)
   125→}
   126→
   127→// MountMetricStoreAPIRoutes registers metric storage API endpoints.
   128→// These endpoints handle metric data ingestion and health checks with JWT token authentication.
   129→func (api *RestAPI) MountMetricStoreAPIRoutes(r chi.Router) {
   130→	// REST API Uses TokenAuth
   131→	r.Post("/free", freeMetrics)
   132→	r.Post("/write", writeMetrics)
   133→	r.Get("/debug", debugMetrics)
   134→	r.Post("/healthcheck", api.updateNodeStates)
   135→	// Same endpoints but with trailing slash
   136→	r.Post("/free/", freeMetrics)
   137→	r.Post("/write/", writeMetrics)
   138→	r.Get("/debug/", debugMetrics)
   139→	r.Post("/healthcheck/", api.updateNodeStates)
   140→}
   141→
   142→// MountConfigAPIRoutes registers configuration and user management endpoints.
   143→// These routes use session-based authentication and require admin privileges.
   144→// Routes use full paths (including /config prefix) to avoid conflicting with
   145→// the /config page route when registered via Group instead of Route.
   146→func (api *RestAPI) MountConfigAPIRoutes(r chi.Router) {
   147→	// Settings Frontend Uses SessionAuth
   148→	if api.Authentication != nil {
   149→		r.Get("/config/roles/", api.getRoles)
   150→		r.Post("/config/users/", api.createUser)
   151→		r.Put("/config/users/", api.createUser)
   152→		r.Get("/config/users/", api.getUsers)
   153→		r.Delete("/config/users/", api.deleteUser)
   154→		r.Post("/config/user/{id}", api.updateUser)
   155→		r.Post("/config/notice/", api.editNotice)
   156→		r.Get("/config/taggers/", api.getTaggers)
   157→		r.Post("/config/taggers/run/", api.runTagger)
   158→	}
   159→}
   160→
   161→// MountFrontendAPIRoutes registers frontend-specific API endpoints.
   162→// These routes support JWT generation and user configuration updates with session authentication.
   163→func (api *RestAPI) MountFrontendAPIRoutes(r chi.Router) {
   164→	r.Get("/logs/", api.getJournalLog)
   165→	// Settings Frontend Uses SessionAuth
   166→	if api.Authentication != nil {
   167→		r.Get("/jwt/", api.getJWT)
   168→		r.Post("/configuration/", api.updateConfiguration)
   169→	}
   170→}
   171→
   172→// ErrorResponse model
   173→type ErrorResponse struct {
   174→	// Statustext of Errorcode
   175→	Status string `json:"status"`
   176→	Error  string `json:"error"` // Error Message
   177→}
   178→
   179→// DefaultAPIResponse model
   180→type DefaultAPIResponse struct {
   181→	Message string `json:"msg"`
   182→}
   183→
   184→// handleError writes a standardized JSON error response with the given status code.
   185→// It logs the error at WARN level and ensures proper Content-Type headers are set.
   186→func handleError(err error, statusCode int, rw http.ResponseWriter) {
   187→	cclog.Warnf("REST ERROR : %s", err.Error())
   188→	rw.Header().Add("Content-Type", "application/json")
   189→	rw.WriteHeader(statusCode)
   190→	if err := json.NewEncoder(rw).Encode(ErrorResponse{
   191→		Status: http.StatusText(statusCode),
   192→		Error:  err.Error(),
   193→	}); err != nil {
   194→		cclog.Errorf("Failed to encode error response: %v", err)
   195→	}
   196→}
   197→
   198→// decode reads JSON from r into val with strict validation that rejects unknown fields.
   199→func decode(r io.Reader, val any) error {
   200→	dec := json.NewDecoder(r)


     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→// Package archiver provides asynchronous job archiving functionality for ClusterCockpit.
     7→//
     8→// The archiver runs a background worker goroutine that processes job archiving requests
     9→// from a buffered channel. When jobs complete, their metric data is archived from the
    10→// metric store to the configured archive backend (filesystem, S3, etc.).
    11→//
    12→// # Architecture
    13→//
    14→// The archiver uses a producer-consumer pattern:
    15→//   - Producer: TriggerArchiving() sends jobs to archiveChannel
    16→//   - Consumer: archivingWorker() processes jobs from the channel
    17→//   - Coordination: sync.WaitGroup tracks pending archive operations
    18→//
    19→// # Lifecycle
    20→//
    21→//  1. Start(repo, ctx) - Initialize worker with context for cancellation
    22→//  2. TriggerArchiving(job) - Queue job for archiving (called when job stops)
    23→//  3. archivingWorker() - Background goroutine processes jobs
    24→//  4. Shutdown(timeout) - Graceful shutdown with timeout
    25→//
    26→// # Graceful Shutdown
    27→//
    28→// The archiver supports graceful shutdown with configurable timeout:
    29→//   - Closes channel to reject new jobs
    30→//   - Waits for pending jobs to complete (up to timeout)
    31→//   - Cancels context if timeout exceeded
    32→//   - Ensures worker goroutine exits cleanly
    33→//
    34→// # Example Usage
    35→//
    36→//	// Initialize archiver
    37→//	ctx, cancel := context.WithCancel(context.Background())
    38→//	defer cancel()
    39→//	archiver.Start(jobRepository, ctx)
    40→//
    41→//	// Trigger archiving when job completes
    42→//	archiver.TriggerArchiving(job)
    43→//
    44→//	// Graceful shutdown with 10 second timeout
    45→//	if err := archiver.Shutdown(10 * time.Second); err != nil {
    46→//	    log.Printf("Archiver shutdown timeout: %v", err)
    47→//	}
    48→package archiver
    49→
    50→import (
    51→	"context"
    52→	"fmt"
    53→	"sync"
    54→	"time"
    55→
    56→	"github.com/ClusterCockpit/cc-backend/internal/repository"
    57→	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
    58→	"github.com/ClusterCockpit/cc-lib/v2/schema"
    59→	sq "github.com/Masterminds/squirrel"
    60→)
    61→
    62→var (
    63→	archivePending sync.WaitGroup
    64→	archiveChannel chan *schema.Job
    65→	jobRepo        *repository.JobRepository
    66→	shutdownCtx    context.Context
    67→	shutdownCancel context.CancelFunc
    68→	workerDone     chan struct{}
    69→)
    70→
    71→// Start initializes the archiver and starts the background worker goroutine.
    72→//
    73→// The archiver processes job archiving requests asynchronously via a buffered channel.
    74→// Jobs are sent to the channel using TriggerArchiving() and processed by the worker.
    75→//
    76→// Parameters:
    77→//   - r: JobRepository instance for database operations
    78→//   - ctx: Context for cancellation (shutdown signal propagation)
    79→//
    80→// The worker goroutine will run until:
    81→//   - ctx is cancelled (via parent shutdown)
    82→//   - archiveChannel is closed (via Shutdown())
    83→//
    84→// Must be called before TriggerArchiving(). Safe to call only once.
    85→func Start(r *repository.JobRepository, ctx context.Context) {
    86→	shutdownCtx, shutdownCancel = context.WithCancel(ctx)
    87→	archiveChannel = make(chan *schema.Job, 128)
    88→	workerDone = make(chan struct{})
    89→	jobRepo = r
    90→
    91→	go archivingWorker()
    92→}
    93→
    94→// archivingWorker is the background goroutine that processes job archiving requests.
    95→//
    96→// The worker loop:
    97→//  1. Blocks waiting for jobs on archiveChannel or shutdown signal
    98→//  2. Fetches job metadata from repository
    99→//  3. Archives job data to configured backend (calls ArchiveJob)
   100→//  4. Updates job footprint and energy metrics in database
   101→//  5. Marks job as successfully archived
   102→//  6. Calls job stop hooks
   103→//
   104→// The worker exits when:
   105→//   - shutdownCtx is cancelled (timeout during shutdown)
   106→//   - archiveChannel is closed (normal shutdown)
   107→//
   108→// Errors during archiving are logged and the job is marked as failed,
   109→// but the worker continues processing other jobs.
   110→func archivingWorker() {
   111→	defer close(workerDone)
   112→
   113→	for {
   114→		select {
   115→		case <-shutdownCtx.Done():
   116→			cclog.Info("Archive worker received shutdown signal")
   117→			return
   118→
   119→		case job, ok := <-archiveChannel:
   120→			if !ok {
   121→				cclog.Info("Archive channel closed, worker exiting")
   122→				return
   123→			}
   124→
   125→			start := time.Now()
   126→			// not using meta data, called to load JobMeta into Cache?
   127→			// will fail if job meta not in repository
   128→			if _, err := jobRepo.FetchMetadata(job); err != nil {
   129→				cclog.Errorf("archiving job (dbid: %d) failed at check metadata step: %s", *job.ID, err.Error())
   130→				jobRepo.UpdateMonitoringStatus(*job.ID, schema.MonitoringStatusArchivingFailed)
   131→				archivePending.Done()
   132→				continue
   133→			}
   134→
   135→			// ArchiveJob will fetch all the data from a MetricDataRepository and push into configured archive backend
   136→			// Use shutdown context to allow cancellation
   137→			jobMeta, err := ArchiveJob(job, shutdownCtx)
   138→			if err != nil {
   139→				cclog.Errorf("archiving job (dbid: %d) failed at archiving job step: %s", *job.ID, err.Error())
   140→				jobRepo.UpdateMonitoringStatus(*job.ID, schema.MonitoringStatusArchivingFailed)
   141→				archivePending.Done()
   142→				continue
   143→			}
   144→
   145→			stmt := sq.Update("job").Where("job.id = ?", job.ID)
   146→
   147→			if stmt, err = jobRepo.UpdateFootprint(stmt, jobMeta); err != nil {
   148→				cclog.Errorf("archiving job (dbid: %d) failed at update Footprint step: %s", *job.ID, err.Error())
   149→				archivePending.Done()
   150→				continue
   151→			}
   152→			if stmt, err = jobRepo.UpdateEnergy(stmt, jobMeta); err != nil {
   153→				cclog.Errorf("archiving job (dbid: %d) failed at update Energy step: %s", *job.ID, err.Error())
   154→				archivePending.Done()
   155→				continue
   156→			}
   157→			// Update the jobs database entry one last time:
   158→			stmt = jobRepo.MarkArchived(stmt, schema.MonitoringStatusArchivingSuccessful)
   159→			if err := jobRepo.Execute(stmt); err != nil {
   160→				cclog.Errorf("archiving job (dbid: %d) failed at db execute: %s", *job.ID, err.Error())
   161→				archivePending.Done()
   162→				continue
   163→			}
   164→			cclog.Debugf("archiving job %d took %s", job.JobID, time.Since(start))
   165→			cclog.Infof("archiving job (dbid: %d) successful", *job.ID)
   166→
   167→			repository.CallJobStopHooks(job)
   168→			archivePending.Done()
   169→		}
   170→	}
   171→}
   172→
   173→// TriggerArchiving queues a job for asynchronous archiving.
   174→//
   175→// This function should be called when a job completes (stops) to archive its
   176→// metric data from the metric store to the configured archive backend.
   177→//
   178→// The function:
   179→//  1. Increments the pending job counter (WaitGroup)
   180→//  2. Sends the job to the archiving channel (buffered, capacity 128)
   181→//  3. Returns immediately (non-blocking unless channel is full)
   182→//
   183→// The actual archiving is performed asynchronously by the worker goroutine.
   184→// Upon completion, the worker will decrement the pending counter.
   185→//
   186→// Panics if Start() has not been called first.
   187→func TriggerArchiving(job *schema.Job) {
   188→	if archiveChannel == nil {
   189→		cclog.Fatal("Cannot archive without archiving channel. Did you Start the archiver?")
   190→	}
   191→
   192→	archivePending.Add(1)
   193→	archiveChannel <- job
   194→}
   195→
   196→// Shutdown performs a graceful shutdown of the archiver with a configurable timeout.
   197→//
   198→// The shutdown process:
   199→//  1. Closes archiveChannel - no new jobs will be accepted
   200→//  2. Waits for pending jobs to complete (up to timeout duration)
   201→//  3. If timeout is exceeded:
   202→//     - Cancels shutdownCtx to interrupt ongoing ArchiveJob operations
   203→//     - Returns error indicating timeout
   204→//  4. Waits for worker goroutine to exit cleanly
   205→//
   206→// Parameters:
   207→//   - timeout: Maximum duration to wait for pending jobs to complete
   208→//     (recommended: 10-30 seconds for production)
   209→//
   210→// Returns:
   211→//   - nil if all jobs completed within timeout
   212→//   - error if timeout was exceeded (some jobs may not have been archived)
   213→//
   214→// Jobs that don't complete within the timeout will be marked as failed.
   215→// The function always ensures the worker goroutine exits before returning.
   216→//
   217→// Example:
   218→//
   219→//	if err := archiver.Shutdown(10 * time.Second); err != nil {
   220→//	    log.Printf("Some jobs did not complete: %v", err)
   221→//	}
   222→func Shutdown(timeout time.Duration) error {
   223→	cclog.Info("Initiating archiver shutdown...")
   224→
   225→	// Close channel to signal no more jobs will be accepted
   226→	close(archiveChannel)
   227→
   228→	// Create a channel to signal when all jobs are done
   229→	done := make(chan struct{})
   230→	go func() {
   231→		archivePending.Wait()
   232→		close(done)
   233→	}()
   234→
   235→	// Wait for jobs to complete or timeout
   236→	select {
   237→	case <-done:
   238→		cclog.Info("All archive jobs completed successfully")
   239→		// Wait for worker to exit
   240→		<-workerDone
   241→		return nil
   242→	case <-time.After(timeout):
   243→		cclog.Warn("Archiver shutdown timeout exceeded, cancelling remaining operations")
   244→		// Cancel any ongoing operations
   245→		shutdownCancel()
   246→		// Wait for worker to exit
   247→		<-workerDone
   248→		return fmt.Errorf("archiver shutdown timeout after %v", timeout)
   249→	}
   250→}
   251→


Found 12 files
internal/repository/jobCreate.go
internal/repository/node.go
internal/repository/user.go
internal/repository/dbConnection.go
internal/repository/job.go
internal/repository/tags.go
internal/repository/userConfig.go
internal/repository/transaction_test.go
internal/repository/transaction.go
internal/repository/repository_test.go
internal/repository/job_test.go
internal/repository/jobCreate_test.go


No files found


     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→// Package metricstore provides an efficient in-memory time-series metric storage system
     7→// with support for hierarchical data organization, checkpointing, and archiving.
     8→//
     9→// The package organizes metrics in a tree structure (cluster → host → component) and
    10→// provides concurrent read/write access to metric data with configurable aggregation strategies.
    11→// Background goroutines handle periodic checkpointing (JSON or Avro format), archiving old data,
    12→// and enforcing retention policies.
    13→//
    14→// Key features:
    15→//   - In-memory metric storage with configurable retention
    16→//   - Hierarchical data organization (selectors)
    17→//   - Concurrent checkpoint/archive workers
    18→//   - Support for sum and average aggregation
    19→//   - NATS integration for metric ingestion
    20→package metricstore
    21→
    22→import (
    23→	"bytes"
    24→	"context"
    25→	"encoding/json"
    26→	"errors"
    27→	"fmt"
    28→	"runtime"
    29→	"runtime/debug"
    30→	"slices"
    31→	"sync"
    32→	"time"
    33→
    34→	"github.com/ClusterCockpit/cc-backend/internal/config"
    35→	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
    36→	"github.com/ClusterCockpit/cc-lib/v2/resampler"
    37→	"github.com/ClusterCockpit/cc-lib/v2/schema"
    38→	"github.com/ClusterCockpit/cc-lib/v2/util"
    39→)
    40→
    41→// GlobalState holds the global state for the metric store with thread-safe access.
    42→type GlobalState struct {
    43→	mu                sync.RWMutex
    44→	lastRetentionTime int64
    45→	selectorsExcluded bool
    46→}
    47→
    48→var (
    49→	singleton  sync.Once
    50→	msInstance *MemoryStore
    51→	// shutdownFunc stores the context cancellation function created in Init
    52→	// and is called during Shutdown to cancel all background goroutines
    53→	shutdownFunc   context.CancelFunc
    54→	shutdownFuncMu sync.Mutex // Protects shutdownFunc from concurrent access
    55→	// Create a global instance
    56→	state = &GlobalState{}
    57→)
    58→
    59→// NodeProvider provides information about nodes currently in use by running jobs.
    60→//
    61→// This interface allows metricstore to query job information without directly
    62→// depending on the repository package, breaking the import cycle.
    63→//
    64→// Implementations should return nodes that are actively processing jobs started
    65→// before the given timestamp. These nodes will be excluded from retention-based
    66→// garbage collection to prevent data loss for jobs that are still running or
    67→// recently completed.
    68→type NodeProvider interface {
    69→	// GetUsedNodes returns a map of cluster names to sorted lists of unique hostnames
    70→	// that are currently in use by jobs that started before the given timestamp.
    71→	//
    72→	// Parameters:
    73→	//   - ts: Unix timestamp threshold - returns nodes with jobs started before this time
    74→	//
    75→	// Returns:
    76→	//   - Map of cluster names to lists of node hostnames that should be excluded from garbage collection
    77→	//   - Error if the query fails
    78→	GetUsedNodes(ts int64) (map[string][]string, error)
    79→}
    80→
    81→// Metric represents a single metric data point to be written to the store.
    82→type Metric struct {
    83→	Name  string
    84→	Value schema.Float
    85→	// MetricConfig contains frequency and aggregation settings for this metric.
    86→	// If Frequency is 0, configuration will be looked up from MemoryStore.Metrics during Write().
    87→	MetricConfig MetricConfig
    88→}
    89→
    90→// MemoryStore is the main in-memory time-series metric storage implementation.
    91→//
    92→// It organizes metrics in a hierarchical tree structure where each level represents
    93→// a component of the system hierarchy (e.g., cluster → host → CPU). Each level can
    94→// store multiple metrics as time-series buffers.
    95→//
    96→// The store is initialized as a singleton via InitMetrics() and accessed via GetMemoryStore().
    97→// All public methods are safe for concurrent use.
    98→type MemoryStore struct {
    99→	Metrics      map[string]MetricConfig
   100→	root         Level
   101→	nodeProvider NodeProvider
   102→}
   103→
   104→// Init initializes the metric store from configuration and starts background workers.
   105→//
   106→// This function must be called exactly once before any other metricstore operations.
   107→// It performs the following initialization steps:
   108→//  1. Validates and decodes the metric store configuration
   109→//  2. Configures worker pool size (defaults to NumCPU/2+1, max 10)
   110→//  3. Loads metric configurations from all registered clusters
   111→//  4. Restores checkpoints within the retention window
   112→//  5. Starts background workers for retention, checkpointing, archiving, and monitoring
   113→//  6. Optionally subscribes to NATS for real-time metric ingestion
   114→//
   115→// Parameters:
   116→//   - rawConfig: JSON configuration for the metric store (see MetricStoreConfig)
   117→//   - wg: WaitGroup that will be incremented for each background goroutine started
   118→//
   119→// The function will call cclog.Fatal on critical errors during initialization.
   120→// Use Shutdown() to cleanly stop all background workers started by Init().
   121→//
   122→// Note: Signal handling must be implemented by the caller. Call Shutdown() when
   123→// receiving termination signals to ensure checkpoint data is persisted.
   124→func Init(rawConfig json.RawMessage, metrics map[string]MetricConfig, wg *sync.WaitGroup) {
   125→	startupTime := time.Now()
   126→
   127→	if rawConfig != nil {
   128→		config.Validate(configSchema, rawConfig)
   129→		dec := json.NewDecoder(bytes.NewReader(rawConfig))
   130→		dec.DisallowUnknownFields()
   131→		if err := dec.Decode(&Keys); err != nil {
   132→			cclog.Abortf("[METRICSTORE]> Metric Store Config Init: Could not decode config file '%s'.\nError: %s\n", rawConfig, err.Error())
   133→		}
   134→	}
   135→
   136→	// Set NumWorkers from config or use default
   137→	if Keys.NumWorkers <= 0 {
   138→		Keys.NumWorkers = min(runtime.NumCPU()/2+1, DefaultMaxWorkers)
   139→	}
   140→	cclog.Debugf("[METRICSTORE]> Using %d workers for checkpoint/archive operations\n", Keys.NumWorkers)
   141→
   142→	// Pass the config.MetricStoreKeys
   143→	InitMetrics(metrics)
   144→
   145→	ms := GetMemoryStore()
   146→
   147→	d, err := time.ParseDuration(Keys.RetentionInMemory)
   148→	if err != nil {
   149→		cclog.Fatal(err)
   150→	}
   151→
   152→	restoreFrom := startupTime.Add(-d)
   153→	cclog.Infof("[METRICSTORE]> Loading checkpoints newer than %s\n", restoreFrom.Format(time.RFC3339))
   154→	files, err := ms.FromCheckpointFiles(Keys.Checkpoints.RootDir, restoreFrom.Unix())
   155→	loadedData := ms.SizeInBytes() / 1024 / 1024 // In MB
   156→	if err != nil {
   157→		cclog.Fatalf("[METRICSTORE]> Loading checkpoints failed: %s\n", err.Error())
   158→	} else {
   159→		cclog.Infof("[METRICSTORE]> Checkpoints loaded (%d files, %d MB, that took %fs)\n", files, loadedData, time.Since(startupTime).Seconds())
   160→	}
   161→
   162→	// Try to use less memory by forcing a GC run here and then
   163→	// lowering the target percentage. The default of 100 means
   164→	// that only once the ratio of new allocations execeds the
   165→	// previously active heap, a GC is triggered.
   166→	// Forcing a GC here will set the "previously active heap"
   167→	// to a minumum.
   168→	// runtime.GC()
   169→
   170→	ctx, shutdown := context.WithCancel(context.Background())
   171→
   172→	Retention(wg, ctx)
   173→	Checkpointing(wg, ctx)
   174→	CleanUp(wg, ctx)
   175→	WALStaging(wg, ctx)
   176→	MemoryUsageTracker(wg, ctx)
   177→
   178→	// Note: Signal handling has been removed from this function.
   179→	// The caller is responsible for handling shutdown signals and calling
   180→	// the shutdown() function when appropriate.
   181→	// Store the shutdown function for later use by Shutdown()
   182→	shutdownFuncMu.Lock()
   183→	shutdownFunc = shutdown
   184→	shutdownFuncMu.Unlock()
   185→
   186→	if Keys.Subscriptions != nil {
   187→		err = ReceiveNats(ms, 1, ctx)
   188→		if err != nil {
   189→			cclog.Fatal(err)
   190→		}
   191→	}
   192→}
   193→
   194→// InitMetrics initializes the singleton MemoryStore instance with the given metric configurations.
   195→//
   196→// This function must be called before GetMemoryStore() and can only be called once due to
   197→// the singleton pattern. It assigns each metric an internal offset for efficient buffer indexing.
   198→//
   199→// Parameters:
   200→//   - metrics: Map of metric names to their configurations (frequency and aggregation strategy)


Found 9 files
pkg/metricstore/config.go
pkg/metricstore/walCheckpoint.go
pkg/metricstore/configSchema.go
pkg/metricstore/checkpoint.go
pkg/metricstore/metricstore.go
pkg/metricstore/lineprotocol.go
pkg/metricstore/parquetArchive_test.go
pkg/metricstore/parquetArchive.go
pkg/metricstore/archive.go


     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→// Package api provides the REST API layer for ClusterCockpit.
     7→// It handles HTTP requests for job management, user administration,
     8→// cluster queries, node state updates, and metrics storage operations.
     9→// The API supports both JWT token authentication and session-based authentication.
    10→package api
    11→
    12→import (
    13→	"encoding/json"
    14→	"fmt"
    15→	"io"
    16→	"net/http"
    17→	"os"
    18→	"path/filepath"
    19→	"strings"
    20→	"sync"
    21→
    22→	"github.com/ClusterCockpit/cc-backend/internal/auth"
    23→	"github.com/ClusterCockpit/cc-backend/internal/config"
    24→	"github.com/ClusterCockpit/cc-backend/internal/repository"
    25→	"github.com/ClusterCockpit/cc-backend/internal/tagger"
    26→	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
    27→	"github.com/ClusterCockpit/cc-lib/v2/schema"
    28→	"github.com/ClusterCockpit/cc-lib/v2/util"
    29→	"github.com/go-chi/chi/v5"
    30→)
    31→
    32→// @title                      ClusterCockpit REST API
    33→// @version                    1.0.0
    34→// @description                API for batch job control.
    35→
    36→// @contact.name               ClusterCockpit Project
    37→// @contact.url                https://github.com/ClusterCockpit
    38→// @contact.email              support@clustercockpit.org
    39→
    40→// @license.name               MIT License
    41→// @license.url                https://opensource.org/licenses/MIT
    42→
    43→// @host                       localhost:8080
    44→
    45→// @securityDefinitions.apikey ApiKeyAuth
    46→// @in                         header
    47→// @name                       X-Auth-Token
    48→
    49→const (
    50→	noticeFilePath  = "./var/notice.txt"
    51→	noticeFilePerms = 0o644
    52→	maxNoticeLength = 10000 // Maximum allowed notice content length in characters
    53→)
    54→
    55→type RestAPI struct {
    56→	JobRepository   *repository.JobRepository
    57→	Authentication  *auth.Authentication
    58→	MachineStateDir string
    59→	// RepositoryMutex protects job creation operations from race conditions
    60→	// when checking for duplicate jobs during startJob API calls.
    61→	// It prevents concurrent job starts with the same jobId/cluster/startTime
    62→	// from creating duplicate entries in the database.
    63→	RepositoryMutex sync.Mutex
    64→}
    65→
    66→// New creates and initializes a new RestAPI instance with configured dependencies.
    67→func New() *RestAPI {
    68→	return &RestAPI{
    69→		JobRepository:   repository.GetJobRepository(),
    70→		MachineStateDir: config.Keys.MachineStateDir,
    71→		Authentication:  auth.GetAuthInstance(),
    72→	}
    73→}
    74→
    75→// MountAPIRoutes registers REST API endpoints for job and cluster management.
    76→// These routes use JWT token authentication via the X-Auth-Token header.
    77→func (api *RestAPI) MountAPIRoutes(r chi.Router) {
    78→	// REST API Uses TokenAuth
    79→	// User List
    80→	r.Get("/users/", api.getUsers)
    81→	// Cluster List
    82→	r.Get("/clusters/", api.getClusters)
    83→	// Slurm node state
    84→	r.Post("/nodestate/", api.updateNodeStates)
    85→	r.Put("/nodestate/", api.updateNodeStates)
    86→	// Job Handler
    87→	if config.Keys.APISubjects == nil {
    88→		cclog.Info("Enabling REST start/stop job API")
    89→		r.Post("/jobs/start_job/", api.startJob)
    90→		r.Put("/jobs/start_job/", api.startJob)
    91→		r.Post("/jobs/stop_job/", api.stopJobByRequest)
    92→		r.Put("/jobs/stop_job/", api.stopJobByRequest)
    93→	}
    94→	r.Get("/jobs/", api.getJobs)
    95→	r.Get("/jobs/used_nodes", api.getUsedNodes)
    96→	r.Post("/jobs/tag_job/{id}", api.tagJob)
    97→	r.Patch("/jobs/tag_job/{id}", api.tagJob)
    98→	r.Delete("/jobs/tag_job/{id}", api.removeTagJob)
    99→	r.Patch("/jobs/edit_meta/{id}", api.editMeta)
   100→	r.Patch("/jobs/edit_meta/", api.editMetaByRequest)
   101→	r.Get("/jobs/metrics/{id}", api.getJobMetrics)
   102→	r.Delete("/jobs/delete_job/", api.deleteJobByRequest)
   103→	r.Delete("/jobs/delete_job/{id}", api.deleteJobByID)
   104→	r.Delete("/jobs/delete_job_before/{ts}", api.deleteJobBefore)
   105→	r.Post("/jobs/{id}", api.getJobByID)
   106→	r.Get("/jobs/{id}", api.getCompleteJobByID)
   107→
   108→	r.Delete("/tags/", api.removeTags)
   109→
   110→	if api.MachineStateDir != "" {
   111→		r.Get("/machine_state/{cluster}/{host}", api.getMachineState)
   112→		r.Put("/machine_state/{cluster}/{host}", api.putMachineState)
   113→		r.Post("/machine_state/{cluster}/{host}", api.putMachineState)
   114→	}
   115→}
   116→
   117→// MountUserAPIRoutes registers user-accessible REST API endpoints.
   118→// These are limited endpoints for regular users with JWT token authentication.
   119→func (api *RestAPI) MountUserAPIRoutes(r chi.Router) {
   120→	// REST API Uses TokenAuth
   121→	r.Get("/jobs/", api.getJobs)
   122→	r.Post("/jobs/{id}", api.getJobByID)
   123→	r.Get("/jobs/{id}", api.getCompleteJobByID)
   124→	r.Get("/jobs/metrics/{id}", api.getJobMetrics)
   125→}
   126→
   127→// MountMetricStoreAPIRoutes registers metric storage API endpoints.
   128→// These endpoints handle metric data ingestion and health checks with JWT token authentication.
   129→func (api *RestAPI) MountMetricStoreAPIRoutes(r chi.Router) {
   130→	// REST API Uses TokenAuth
   131→	r.Post("/free", freeMetrics)
   132→	r.Post("/write", writeMetrics)
   133→	r.Get("/debug", debugMetrics)
   134→	r.Post("/healthcheck", api.updateNodeStates)
   135→	// Same endpoints but with trailing slash
   136→	r.Post("/free/", freeMetrics)
   137→	r.Post("/write/", writeMetrics)
   138→	r.Get("/debug/", debugMetrics)
   139→	r.Post("/healthcheck/", api.updateNodeStates)
   140→}
   141→
   142→// MountConfigAPIRoutes registers configuration and user management endpoints.
   143→// These routes use session-based authentication and require admin privileges.
   144→// Routes use full paths (including /config prefix) to avoid conflicting with
   145→// the /config page route when registered via Group instead of Route.
   146→func (api *RestAPI) MountConfigAPIRoutes(r chi.Router) {
   147→	// Settings Frontend Uses SessionAuth
   148→	if api.Authentication != nil {
   149→		r.Get("/config/roles/", api.getRoles)
   150→		r.Post("/config/users/", api.createUser)


Found 3 files
internal/api/nats.go
internal/api/nats_test.go
internal/api/api_test.go


     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→package repository
     7→
     8→import (
     9→	"encoding/json"
    10→	"fmt"
    11→
    12→	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
    13→	"github.com/ClusterCockpit/cc-lib/v2/schema"
    14→	sq "github.com/Masterminds/squirrel"
    15→)
    16→
    17→const NamedJobCacheInsert string = `INSERT INTO job_cache (
    18→	job_id, hpc_user, project, cluster, subcluster, cluster_partition, array_job_id, num_nodes, num_hwthreads, num_acc,
    19→	shared, monitoring_status, smt, job_state, start_time, duration, walltime, footprint, energy, energy_footprint, resources, meta_data
    20→) VALUES (
    21→	:job_id, :hpc_user, :project, :cluster, :subcluster, :cluster_partition, :array_job_id, :num_nodes, :num_hwthreads, :num_acc,
    22→  :shared, :monitoring_status, :smt, :job_state, :start_time, :duration, :walltime, :footprint,  :energy, :energy_footprint, :resources, :meta_data
    23→);`
    24→
    25→const NamedJobInsert string = `INSERT INTO job (
    26→	job_id, hpc_user, project, cluster, subcluster, cluster_partition, array_job_id, num_nodes, num_hwthreads, num_acc,
    27→	shared, monitoring_status, smt, job_state, start_time, duration, walltime, footprint, energy, energy_footprint, resources, meta_data
    28→) VALUES (
    29→	:job_id, :hpc_user, :project, :cluster, :subcluster, :cluster_partition, :array_job_id, :num_nodes, :num_hwthreads, :num_acc,
    30→  :shared, :monitoring_status, :smt, :job_state, :start_time, :duration, :walltime, :footprint,  :energy, :energy_footprint, :resources, :meta_data
    31→);`
    32→
    33→// InsertJobDirect inserts a job directly into the job table (not job_cache).
    34→// Use this when the returned ID will be used for operations on the job table
    35→// (e.g., adding tags), or for imported jobs that are already completed.
    36→func (r *JobRepository) InsertJobDirect(job *schema.Job) (int64, error) {
    37→	r.Mutex.Lock()
    38→	defer r.Mutex.Unlock()
    39→
    40→	res, err := r.DB.NamedExec(NamedJobInsert, job)
    41→	if err != nil {
    42→		cclog.Warn("Error while NamedJobInsert (direct)")
    43→		return 0, err
    44→	}
    45→	id, err := res.LastInsertId()
    46→	if err != nil {
    47→		cclog.Warn("Error while getting last insert ID (direct)")
    48→		return 0, err
    49→	}
    50→
    51→	return id, nil
    52→}
    53→
    54→func (r *JobRepository) InsertJob(job *schema.Job) (int64, error) {
    55→	r.Mutex.Lock()
    56→	defer r.Mutex.Unlock()
    57→
    58→	res, err := r.DB.NamedExec(NamedJobCacheInsert, job)
    59→	if err != nil {
    60→		cclog.Warn("Error while NamedJobInsert")
    61→		return 0, err
    62→	}
    63→	id, err := res.LastInsertId()
    64→	if err != nil {
    65→		cclog.Warn("Error while getting last insert ID")
    66→		return 0, err
    67→	}
    68→
    69→	return id, nil
    70→}
    71→
    72→func (r *JobRepository) SyncJobs() ([]*schema.Job, error) {
    73→	r.Mutex.Lock()
    74→	defer r.Mutex.Unlock()
    75→
    76→	query := sq.Select(jobCacheColumns...).From("job_cache")
    77→
    78→	rows, err := query.RunWith(r.stmtCache).Query()
    79→	if err != nil {
    80→		cclog.Errorf("Error while running query %v", err)
    81→		return nil, err
    82→	}
    83→	defer rows.Close()
    84→
    85→	jobs := make([]*schema.Job, 0, 50)
    86→	for rows.Next() {
    87→		job, err := scanJob(rows)
    88→		if err != nil {
    89→			cclog.Warn("Error while scanning rows")
    90→			return nil, err
    91→		}
    92→		jobs = append(jobs, job)
    93→	}
    94→
    95→	// Transfer cached jobs to main table and clear cache in a single transaction.
    96→	tx, err := r.DB.Beginx()
    97→	if err != nil {
    98→		cclog.Errorf("SyncJobs: begin transaction: %v", err)
    99→		return nil, err
   100→	}
   101→	defer tx.Rollback()
   102→
   103→	// Use INSERT OR IGNORE to skip jobs already transferred by the stop path
   104→	_, err = tx.Exec(
   105→		"INSERT OR IGNORE INTO job (job_id, cluster, subcluster, start_time, hpc_user, project, cluster_partition, array_job_id, num_nodes, num_hwthreads, num_acc, shared, monitoring_status, smt, job_state, duration, walltime, footprint, energy, energy_footprint, resources, meta_data) SELECT job_id, cluster, subcluster, start_time, hpc_user, project, cluster_partition, array_job_id, num_nodes, num_hwthreads, num_acc, shared, monitoring_status, smt, job_state, duration, walltime, footprint, energy, energy_footprint, resources, meta_data FROM job_cache")
   106→	if err != nil {
   107→		cclog.Errorf("Error while Job sync: %v", err)
   108→		return nil, err
   109→	}
   110→
   111→	_, err = tx.Exec("DELETE FROM job_cache")
   112→	if err != nil {
   113→		cclog.Errorf("Error while Job cache clean: %v", err)
   114→		return nil, err
   115→	}
   116→
   117→	if err := tx.Commit(); err != nil {
   118→		cclog.Errorf("SyncJobs: commit transaction: %v", err)
   119→		return nil, err
   120→	}
   121→
   122→	// Resolve correct job.id from the job table. The IDs read from job_cache
   123→	// are from a different auto-increment sequence and must not be used to
   124→	// query the job table.
   125→	for _, job := range jobs {
   126→		var newID int64
   127→		if err := sq.Select("job.id").From("job").
   128→			Where("job.job_id = ? AND job.cluster = ? AND job.start_time = ?",
   129→				job.JobID, job.Cluster, job.StartTime).
   130→			RunWith(r.stmtCache).QueryRow().Scan(&newID); err != nil {
   131→			cclog.Warnf("SyncJobs: could not resolve job table id for job %d on %s: %v",
   132→				job.JobID, job.Cluster, err)
   133→			continue
   134→		}
   135→		job.ID = &newID
   136→	}
   137→
   138→	return jobs, nil
   139→}
   140→
   141→// TransferCachedJobToMain moves a job from job_cache to the job table.
   142→// Caller must hold r.Mutex. Returns the new job table ID.
   143→func (r *JobRepository) TransferCachedJobToMain(cacheID int64) (int64, error) {
   144→	res, err := r.DB.Exec(
   145→		"INSERT INTO job (job_id, cluster, subcluster, start_time, hpc_user, project, cluster_partition, array_job_id, num_nodes, num_hwthreads, num_acc, shared, monitoring_status, smt, job_state, duration, walltime, footprint, energy, energy_footprint, resources, meta_data) SELECT job_id, cluster, subcluster, start_time, hpc_user, project, cluster_partition, array_job_id, num_nodes, num_hwthreads, num_acc, shared, monitoring_status, smt, job_state, duration, walltime, footprint, energy, energy_footprint, resources, meta_data FROM job_cache WHERE id = ?",
   146→		cacheID)
   147→	if err != nil {
   148→		return 0, fmt.Errorf("transferring cached job %d to main table failed: %w", cacheID, err)
   149→	}
   150→
   151→	newID, err := res.LastInsertId()
   152→	if err != nil {
   153→		return 0, fmt.Errorf("getting new job ID after transfer failed: %w", err)
   154→	}
   155→
   156→	_, err = r.DB.Exec("DELETE FROM job_cache WHERE id = ?", cacheID)
   157→	if err != nil {
   158→		return 0, fmt.Errorf("deleting cached job %d after transfer failed: %w", cacheID, err)
   159→	}
   160→
   161→	return newID, nil
   162→}
   163→
   164→// Start inserts a new job in the table, returning the unique job ID.
   165→// Statistics are not transfered!
   166→func (r *JobRepository) Start(job *schema.Job) (id int64, err error) {
   167→	job.RawFootprint, err = json.Marshal(job.Footprint)
   168→	if err != nil {
   169→		return -1, fmt.Errorf("REPOSITORY/JOB > encoding footprint field failed: %w", err)
   170→	}
   171→
   172→	job.RawResources, err = json.Marshal(job.Resources)
   173→	if err != nil {
   174→		return -1, fmt.Errorf("REPOSITORY/JOB > encoding resources field failed: %w", err)
   175→	}
   176→
   177→	job.RawMetaData, err = json.Marshal(job.MetaData)
   178→	if err != nil {
   179→		return -1, fmt.Errorf("REPOSITORY/JOB > encoding metaData field failed: %w", err)
   180→	}
   181→
   182→	return r.InsertJob(job)
   183→}
   184→
   185→// StartDirect inserts a new job directly into the job table (not job_cache).
   186→// Use this when the returned ID will immediately be used for job table
   187→// operations such as adding tags.
   188→func (r *JobRepository) StartDirect(job *schema.Job) (id int64, err error) {
   189→	job.RawFootprint, err = json.Marshal(job.Footprint)
   190→	if err != nil {
   191→		return -1, fmt.Errorf("REPOSITORY/JOB > encoding footprint field failed: %w", err)
   192→	}
   193→
   194→	job.RawResources, err = json.Marshal(job.Resources)
   195→	if err != nil {
   196→		return -1, fmt.Errorf("REPOSITORY/JOB > encoding resources field failed: %w", err)
   197→	}
   198→
   199→	job.RawMetaData, err = json.Marshal(job.MetaData)
   200→	if err != nil {
   201→		return -1, fmt.Errorf("REPOSITORY/JOB > encoding metaData field failed: %w", err)
   202→	}
   203→
   204→	return r.InsertJobDirect(job)
   205→}
   206→
   207→// Stop updates the job with the database id jobId using the provided arguments.
   208→func (r *JobRepository) Stop(
   209→	jobID int64,
   210→	duration int32,
   211→	state schema.JobState,
   212→	monitoringStatus int32,
   213→) (err error) {
   214→	// Invalidate cache entries as job state is changing
   215→	r.cache.Del(fmt.Sprintf("metadata:%d", jobID))
   216→	r.cache.Del(fmt.Sprintf("energyFootprint:%d", jobID))
   217→
   218→	stmt := sq.Update("job").
   219→		Set("job_state", state).
   220→		Set("duration", duration).
   221→		Set("monitoring_status", monitoringStatus).
   222→		Where("job.id = ?", jobID)
   223→
   224→	_, err = stmt.RunWith(r.stmtCache).Exec()
   225→	return err
   226→}
   227→


     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→package repository
     7→
     8→import (
     9→	"sync"
    10→
    11→	"github.com/ClusterCockpit/cc-lib/v2/schema"
    12→)
    13→
    14→// JobHook interface allows external components to hook into job lifecycle events.
    15→// Implementations can perform actions when jobs start or stop, such as tagging,
    16→// logging, notifications, or triggering external workflows.
    17→//
    18→// Example implementation:
    19→//
    20→//	type MyJobTagger struct{}
    21→//
    22→//	func (t *MyJobTagger) JobStartCallback(job *schema.Job) {
    23→//	    if job.NumNodes > 100 {
    24→//	        // Tag large jobs automatically
    25→//	    }
    26→//	}
    27→//
    28→//	func (t *MyJobTagger) JobStopCallback(job *schema.Job) {
    29→//	    if job.State == schema.JobStateFailed {
    30→//	        // Log or alert on failed jobs
    31→//	    }
    32→//	}
    33→//
    34→// Register hooks during application initialization:
    35→//
    36→//	repository.RegisterJobHook(&MyJobTagger{})
    37→type JobHook interface {
    38→	// JobStartCallback is invoked when one or more jobs start.
    39→	// This is called synchronously, so implementations should be fast.
    40→	JobStartCallback(job *schema.Job)
    41→
    42→	// JobStopCallback is invoked when a job completes.
    43→	// This is called synchronously, so implementations should be fast.
    44→	JobStopCallback(job *schema.Job)
    45→}
    46→
    47→var (
    48→	initOnce sync.Once
    49→	hooks    []JobHook
    50→)
    51→
    52→// RegisterJobHook registers a JobHook to receive job lifecycle callbacks.
    53→// Multiple hooks can be registered and will be called in registration order.
    54→// This function is safe to call multiple times and is typically called during
    55→// application initialization.
    56→//
    57→// Nil hooks are silently ignored to simplify conditional registration.
    58→func RegisterJobHook(hook JobHook) {
    59→	initOnce.Do(func() {
    60→		hooks = make([]JobHook, 0)
    61→	})
    62→
    63→	if hook != nil {
    64→		hooks = append(hooks, hook)
    65→	}
    66→}
    67→
    68→// CallJobStartHooks invokes all registered JobHook.JobStartCallback methods
    69→// for each job in the provided slice. This is called internally by the repository
    70→// when jobs are started (e.g., via StartJob or batch job imports).
    71→//
    72→// Hooks are called synchronously in registration order. If a hook panics,
    73→// the panic will propagate to the caller.
    74→func CallJobStartHooks(jobs []*schema.Job) {
    75→	if hooks == nil {
    76→		return
    77→	}
    78→
    79→	for _, hook := range hooks {
    80→		if hook != nil {
    81→			for _, job := range jobs {
    82→				hook.JobStartCallback(job)
    83→			}
    84→		}
    85→	}
    86→}
    87→
    88→// CallJobStopHooks invokes all registered JobHook.JobStopCallback methods
    89→// for the provided job. This is called internally by the repository when a
    90→// job completes (e.g., via StopJob or job state updates).
    91→//
    92→// Hooks are called synchronously in registration order. If a hook panics,
    93→// the panic will propagate to the caller.
    94→func CallJobStopHooks(job *schema.Job) {
    95→	if hooks == nil {
    96→		return
    97→	}
    98→
    99→	for _, hook := range hooks {
   100→		if hook != nil {
   101→			hook.JobStopCallback(job)
   102→		}
   103→	}
   104→}
   105→


     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→package repository
     7→
     8→import "time"
     9→
    10→// RepositoryConfig holds configuration for repository operations.
    11→// All fields have sensible defaults, so this configuration is optional.
    12→type RepositoryConfig struct {
    13→	// CacheSize is the LRU cache size in bytes for job metadata and energy footprints.
    14→	// Default: 1MB (1024 * 1024 bytes)
    15→	CacheSize int
    16→
    17→	// MaxOpenConnections is the maximum number of open database connections.
    18→	// Default: 4
    19→	MaxOpenConnections int
    20→
    21→	// MaxIdleConnections is the maximum number of idle database connections.
    22→	// Default: 4
    23→	MaxIdleConnections int
    24→
    25→	// ConnectionMaxLifetime is the maximum amount of time a connection may be reused.
    26→	// Default: 1 hour
    27→	ConnectionMaxLifetime time.Duration
    28→
    29→	// ConnectionMaxIdleTime is the maximum amount of time a connection may be idle.
    30→	// Default: 10 minutes
    31→	ConnectionMaxIdleTime time.Duration
    32→
    33→	// MinRunningJobDuration is the minimum duration in seconds for a job to be
    34→	// considered in "running jobs" queries. This filters out very short jobs.
    35→	// Default: 600 seconds (10 minutes)
    36→	MinRunningJobDuration int
    37→
    38→	// DbCacheSizeMB is the SQLite page cache size per connection in MB.
    39→	// Uses negative PRAGMA cache_size notation (KiB). With MaxOpenConnections=4
    40→	// and DbCacheSizeMB=2048, total page cache is up to 8GB.
    41→	// Default: 2048 (2GB)
    42→	DbCacheSizeMB int
    43→
    44→	// DbSoftHeapLimitMB is the process-wide SQLite soft heap limit in MB.
    45→	// SQLite will try to release cache pages to stay under this limit.
    46→	// It's a soft limit — queries won't fail, but cache eviction becomes more aggressive.
    47→	// Default: 16384 (16GB)
    48→	DbSoftHeapLimitMB int
    49→
    50→	// BusyTimeoutMs is the SQLite busy_timeout in milliseconds.
    51→	// When a write is blocked by another writer, SQLite retries internally
    52→	// using a backoff mechanism for up to this duration before returning SQLITE_BUSY.
    53→	// Default: 60000 (60 seconds)
    54→	BusyTimeoutMs int
    55→}
    56→
    57→// DefaultConfig returns the default repository configuration.
    58→// These values are optimized for typical deployments.
    59→func DefaultConfig() *RepositoryConfig {
    60→	return &RepositoryConfig{
    61→		CacheSize:             1 * 1024 * 1024, // 1MB
    62→		MaxOpenConnections:    4,
    63→		MaxIdleConnections:    4,
    64→		ConnectionMaxLifetime: time.Hour,
    65→		ConnectionMaxIdleTime: 10 * time.Minute,
    66→		MinRunningJobDuration: 600, // 10 minutes
    67→		DbCacheSizeMB:        2048,  // 2GB per connection
    68→		DbSoftHeapLimitMB:    16384, // 16GB process-wide
    69→		BusyTimeoutMs:        60000, // 60 seconds
    70→	}
    71→}
    72→
    73→// repoConfig is the package-level configuration instance.
    74→// It is initialized with defaults and can be overridden via SetConfig.
    75→var repoConfig *RepositoryConfig = DefaultConfig()
    76→
    77→// SetConfig sets the repository configuration.
    78→// This must be called before any repository initialization (Connect, GetJobRepository, etc.).
    79→// If not called, default values from DefaultConfig() are used.
    80→func SetConfig(cfg *RepositoryConfig) {
    81→	if cfg != nil {
    82→		repoConfig = cfg
    83→	}
    84→}
    85→
    86→// GetConfig returns the current repository configuration.
    87→func GetConfig() *RepositoryConfig {
    88→	return repoConfig
    89→}
    90→


     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→// This file implements checkpoint persistence for the in-memory metric store.
     7→//
     8→// Checkpoints enable graceful restarts by periodically saving in-memory metric
     9→// data to disk in JSON or binary format. The checkpoint system:
    10→//
    11→// Key Features:
    12→//   - Periodic background checkpointing via the Checkpointing() worker
    13→//   - Two format families: JSON (human-readable) and WAL+binary (compact, crash-safe)
    14→//   - Parallel checkpoint creation and loading using worker pools
    15→//   - Hierarchical file organization: checkpoint_dir/cluster/host/timestamp.{json|bin}
    16→//   - WAL file: checkpoint_dir/cluster/host/current.wal (append-only, per-entry)
    17→//   - Only saves unarchived data (archived data is already persisted elsewhere)
    18→//   - GC optimization during loading to prevent excessive heap growth
    19→//
    20→// Checkpoint Workflow:
    21→//  1. Init() loads checkpoints within retention window at startup
    22→//  2. Checkpointing() worker periodically saves new data
    23→//  3. Shutdown() writes final checkpoint before exit
    24→//
    25→// File Organization:
    26→//
    27→//	checkpoints/
    28→//	  cluster1/
    29→//	    host001/
    30→//	      1234567890.json  (JSON format: full subtree snapshot)
    31→//	      1234567890.bin   (binary format: full subtree snapshot)
    32→//	      current.wal      (WAL format: append-only per-entry log)
    33→//	    host002/
    34→//	      ...
    35→package metricstore
    36→
    37→import (
    38→	"bufio"
    39→	"context"
    40→	"encoding/json"
    41→	"errors"
    42→	"fmt"
    43→	"io/fs"
    44→	"os"
    45→	"path"
    46→	"path/filepath"
    47→	"sort"
    48→	"strconv"
    49→	"strings"
    50→	"sync"
    51→	"sync/atomic"
    52→	"time"
    53→
    54→	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
    55→	"github.com/ClusterCockpit/cc-lib/v2/schema"
    56→)
    57→
    58→const (
    59→	CheckpointFilePerms = 0o644                    // File permissions for checkpoint files
    60→	CheckpointDirPerms  = 0o755                    // Directory permissions for checkpoint directories
    61→	GCTriggerInterval   = DefaultGCTriggerInterval // Interval for triggering GC during checkpoint loading
    62→)
    63→
    64→// CheckpointMetrics represents metric data in a checkpoint file.
    65→// Whenever the structure changes, update MarshalJSON as well!
    66→type CheckpointMetrics struct {
    67→	Data      []schema.Float `json:"data"`
    68→	Frequency int64          `json:"frequency"`
    69→	Start     int64          `json:"start"`
    70→}
    71→
    72→// CheckpointFile represents the hierarchical structure of a checkpoint file.
    73→// It mirrors the Level tree structure from the MemoryStore.
    74→type CheckpointFile struct {
    75→	Metrics  map[string]*CheckpointMetrics `json:"metrics"`
    76→	Children map[string]*CheckpointFile    `json:"children"`
    77→	From     int64                         `json:"from"`
    78→	To       int64                         `json:"to"`
    79→}
    80→
    81→// lastCheckpoint tracks the timestamp of the last checkpoint creation.
    82→var (
    83→	lastCheckpoint   time.Time
    84→	lastCheckpointMu sync.Mutex
    85→)
    86→
    87→// Checkpointing starts a background worker that periodically saves metric data to disk.
    88→//
    89→// Checkpoints are written every 12 hours (hardcoded).
    90→//
    91→// Format behaviour:
    92→//   - "json": Periodic checkpointing every checkpointInterval
    93→//   - "wal":  Periodic binary snapshots + WAL rotation every checkpointInterval
    94→func Checkpointing(wg *sync.WaitGroup, ctx context.Context) {
    95→	lastCheckpointMu.Lock()
    96→	lastCheckpoint = time.Now()
    97→	lastCheckpointMu.Unlock()
    98→
    99→	ms := GetMemoryStore()
   100→
   101→	wg.Go(func() {
   102→
   103→		d := 12 * time.Hour // default checkpoint interval
   104→		if Keys.CheckpointInterval != "" {
   105→			parsed, err := time.ParseDuration(Keys.CheckpointInterval)
   106→			if err != nil {
   107→				cclog.Errorf("[METRICSTORE]> invalid checkpoint-interval %q: %s, using default 12h", Keys.CheckpointInterval, err)
   108→			} else {
   109→				d = parsed
   110→			}
   111→		}
   112→
   113→		ticker := time.NewTicker(d)
   114→		defer ticker.Stop()
   115→
   116→		for {
   117→			select {
   118→			case <-ctx.Done():
   119→				return
   120→			case <-ticker.C:
   121→				lastCheckpointMu.Lock()
   122→				from := lastCheckpoint
   123→				lastCheckpointMu.Unlock()
   124→
   125→				now := time.Now()
   126→				cclog.Infof("[METRICSTORE]> start checkpointing (starting at %s)...", from.Format(time.RFC3339))
   127→
   128→				if Keys.Checkpoints.FileFormat == "wal" {
   129→					n, hostDirs, err := ms.ToCheckpointWAL(Keys.Checkpoints.RootDir, from.Unix(), now.Unix())
   130→					if err != nil {
   131→						cclog.Errorf("[METRICSTORE]> binary checkpointing failed: %s", err.Error())
   132→					} else {
   133→						cclog.Infof("[METRICSTORE]> done: %d binary snapshot files created", n)
   134→						lastCheckpointMu.Lock()
   135→						lastCheckpoint = now
   136→						lastCheckpointMu.Unlock()
   137→						// Rotate WAL files for successfully checkpointed hosts.
   138→						RotateWALFiles(hostDirs)
   139→					}
   140→				} else {
   141→					n, err := ms.ToCheckpoint(Keys.Checkpoints.RootDir, from.Unix(), now.Unix())
   142→					if err != nil {
   143→						cclog.Errorf("[METRICSTORE]> checkpointing failed: %s", err.Error())
   144→					} else {
   145→						cclog.Infof("[METRICSTORE]> done: %d checkpoint files created", n)
   146→						lastCheckpointMu.Lock()
   147→						lastCheckpoint = now
   148→						lastCheckpointMu.Unlock()
   149→					}
   150→				}


     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→package metricstore
     7→
     8→import (
     9→	"context"
    10→	"errors"
    11→	"fmt"
    12→	"os"
    13→	"path/filepath"
    14→	"sync"
    15→	"sync/atomic"
    16→	"time"
    17→
    18→	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
    19→)
    20→
    21→// Worker for either Archiving or Deleting files
    22→
    23→func CleanUp(wg *sync.WaitGroup, ctx context.Context) {
    24→	if Keys.Cleanup.Mode == "archive" {
    25→		// Run as Archiver
    26→		cleanUpWorker(wg, ctx,
    27→			Keys.RetentionInMemory,
    28→			"archiving",
    29→			Keys.Cleanup.RootDir,
    30→			false,
    31→		)
    32→	} else {
    33→		// Run as Deleter
    34→		cleanUpWorker(wg, ctx,
    35→			Keys.RetentionInMemory,
    36→			"deleting",
    37→			"",
    38→			true,
    39→		)
    40→	}
    41→}
    42→
    43→// cleanUpWorker takes simple values to configure what it does
    44→func cleanUpWorker(wg *sync.WaitGroup, ctx context.Context, interval string, mode string, cleanupDir string, delete bool) {
    45→	wg.Go(func() {
    46→
    47→		d, err := time.ParseDuration(interval)
    48→		if err != nil {
    49→			cclog.Fatalf("[METRICSTORE]> error parsing %s interval duration: %v\n", mode, err)
    50→		}
    51→		if d <= 0 {
    52→			return
    53→		}
    54→
    55→		ticker := time.NewTicker(d)
    56→		defer ticker.Stop()
    57→
    58→		for {
    59→			select {
    60→			case <-ctx.Done():
    61→				return
    62→			case <-ticker.C:
    63→				t := time.Now().Add(-d)
    64→				cclog.Infof("[METRICSTORE]> start %s checkpoints (older than %s)...", mode, t.Format(time.RFC3339))
    65→
    66→				n, err := CleanupCheckpoints(Keys.Checkpoints.RootDir, cleanupDir, t.Unix(), delete)
    67→
    68→				if err != nil {
    69→					cclog.Errorf("[METRICSTORE]> %s failed: %s", mode, err.Error())
    70→				} else {
    71→					if delete {
    72→						cclog.Infof("[METRICSTORE]> done: %d checkpoints deleted", n)
    73→					} else {
    74→						cclog.Infof("[METRICSTORE]> done: %d checkpoint files archived to parquet", n)
    75→					}
    76→				}
    77→			}
    78→		}
    79→	})
    80→}
    81→
    82→var ErrNoNewArchiveData error = errors.New("all data already archived")
    83→
    84→// CleanupCheckpoints deletes or archives all checkpoint files older than `from`.
    85→// When archiving, consolidates all hosts per cluster into a single Parquet file.
    86→func CleanupCheckpoints(checkpointsDir, cleanupDir string, from int64, deleteInstead bool) (int, error) {
    87→	if deleteInstead {
    88→		return deleteCheckpoints(checkpointsDir, from)
    89→	}
    90→
    91→	return archiveCheckpoints(checkpointsDir, cleanupDir, from)
    92→}
    93→
    94→// deleteCheckpoints removes checkpoint files older than `from` across all clusters/hosts.
    95→func deleteCheckpoints(checkpointsDir string, from int64) (int, error) {
    96→	entries1, err := os.ReadDir(checkpointsDir)
    97→	if err != nil {
    98→		return 0, err
    99→	}
   100→
   101→	type workItem struct {
   102→		dir            string
   103→		cluster, host  string
   104→	}
   105→
   106→	var wg sync.WaitGroup
   107→	n, errs := int32(0), int32(0)
   108→	work := make(chan workItem, Keys.NumWorkers)
   109→
   110→	wg.Add(Keys.NumWorkers)
   111→	for worker := 0; worker < Keys.NumWorkers; worker++ {
   112→		go func() {
   113→			defer wg.Done()
   114→			for item := range work {
   115→				entries, err := os.ReadDir(item.dir)
   116→				if err != nil {
   117→					cclog.Errorf("error reading %s/%s: %s", item.cluster, item.host, err.Error())
   118→					atomic.AddInt32(&errs, 1)
   119→					continue
   120→				}
   121→
   122→				files, err := findFiles(entries, from, false)
   123→				if err != nil {
   124→					cclog.Errorf("error finding files in %s/%s: %s", item.cluster, item.host, err.Error())
   125→					atomic.AddInt32(&errs, 1)
   126→					continue
   127→				}
   128→
   129→				for _, checkpoint := range files {
   130→					if err := os.Remove(filepath.Join(item.dir, checkpoint)); err != nil {
   131→						cclog.Errorf("error deleting %s/%s/%s: %s", item.cluster, item.host, checkpoint, err.Error())
   132→						atomic.AddInt32(&errs, 1)
   133→					} else {
   134→						atomic.AddInt32(&n, 1)
   135→					}
   136→				}
   137→			}
   138→		}()
   139→	}
   140→
   141→	for _, de1 := range entries1 {
   142→		entries2, e := os.ReadDir(filepath.Join(checkpointsDir, de1.Name()))
   143→		if e != nil {
   144→			err = e
   145→			continue
   146→		}
   147→
   148→		for _, de2 := range entries2 {
   149→			work <- workItem{
   150→				dir:     filepath.Join(checkpointsDir, de1.Name(), de2.Name()),


Bash

grep -n "func.*startJob\|func.*stopJob" /Users/jan/prg/ClusterCockpit/cc-backend/internal/api/rest.go


(Bash completed with no output)


grep -n "func.*startJob\|func.*stopJob\|func.*tagJob" /Users/jan/prg/ClusterCockpit/cc-backend/internal/api/*.go


/Users/jan/prg/ClusterCockpit/cc-backend/internal/api/job.go:545:func (api *RestAPI) tagJob(rw http.ResponseWriter, r *http.Request) {
/Users/jan/prg/ClusterCockpit/cc-backend/internal/api/job.go:717:func (api *RestAPI) startJob(rw http.ResponseWriter, r *http.Request) {
/Users/jan/prg/ClusterCockpit/cc-backend/internal/api/job.go:805:func (api *RestAPI) stopJobByRequest(rw http.ResponseWriter, r *http.Request) {