Add documentation for importer

internal/importer/handleImport.go

@@ -8,7 +8,6 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
-	"math"
 	"os"
 	"strings"
 
@@ -19,7 +18,22 @@ import (
 	"github.com/ClusterCockpit/cc-lib/schema"
 )
 
-// Import all jobs specified as `<path-to-meta.json>:<path-to-data.json>,...`
+// HandleImportFlag imports jobs from file pairs specified in a comma-separated flag string.
+//
+// The flag format is: "<path-to-meta.json>:<path-to-data.json>[,<path-to-meta2.json>:<path-to-data2.json>,...]"
+//
+// For each job pair, this function:
+//  1. Reads and validates the metadata JSON file (schema.Job)
+//  2. Reads and validates the job data JSON file (schema.JobData)
+//  3. Enriches the job with calculated footprints and energy metrics
+//  4. Validates the job using SanityChecks()
+//  5. Imports the job into the archive
+//  6. Inserts the job into the database with associated tags
+//
+// Schema validation is performed if config.Keys.Validate is true.
+//
+// Returns an error if file reading, validation, enrichment, or database operations fail.
+// The function stops processing on the first error encountered.
 func HandleImportFlag(flag string) error {
 	r := repository.GetJobRepository()
 
@@ -72,75 +86,8 @@ func HandleImportFlag(flag string) error {
 
 		job.MonitoringStatus = schema.MonitoringStatusArchivingSuccessful
 
-		sc, err := archive.GetSubCluster(job.Cluster, job.SubCluster)
-		if err != nil {
-			cclog.Errorf("cannot get subcluster: %s", err.Error())
-			return err
-		}
-
-		job.Footprint = make(map[string]float64)
-
-		for _, fp := range sc.Footprint {
-			statType := "avg"
-
-			if i, err := archive.MetricIndex(sc.MetricConfig, fp); err != nil {
-				statType = sc.MetricConfig[i].Footprint
-			}
-
-			name := fmt.Sprintf("%s_%s", fp, statType)
-
-			job.Footprint[name] = repository.LoadJobStat(&job, fp, statType)
-		}
-
-		job.RawFootprint, err = json.Marshal(job.Footprint)
-		if err != nil {
-			cclog.Warn("Error while marshaling job footprint")
-			return err
-		}
-
-		job.EnergyFootprint = make(map[string]float64)
-
-		// Total Job Energy Outside Loop
-		totalEnergy := 0.0
-		for _, fp := range sc.EnergyFootprint {
-			// Always Init Metric Energy Inside Loop
-			metricEnergy := 0.0
-			if i, err := archive.MetricIndex(sc.MetricConfig, fp); err == nil {
-				// Note: For DB data, calculate and save as kWh
-				if sc.MetricConfig[i].Energy == "energy" { // this metric has energy as unit (Joules)
-					cclog.Warnf("Update EnergyFootprint for Job %d and Metric %s on cluster %s: Set to 'energy' in cluster.json: Not implemented, will return 0.0", job.JobID, job.Cluster, fp)
-					// FIXME: Needs sum as stats type
-				} else if sc.MetricConfig[i].Energy == "power" { // this metric has power as unit (Watt)
-					// Energy: Power (in Watts) * Time (in Seconds)
-					// Unit: (W * (s / 3600)) / 1000 = kWh
-					// Round 2 Digits: round(Energy * 100) / 100
-					// Here: (All-Node Metric Average * Number of Nodes) * (Job Duration in Seconds / 3600) / 1000
-					// Note: Shared Jobs handled correctly since "Node Average" is based on partial resources, while "numNodes" factor is 1
-					rawEnergy := ((repository.LoadJobStat(&job, fp, "avg") * float64(job.NumNodes)) * (float64(job.Duration) / 3600.0)) / 1000.0
-					metricEnergy = math.Round(rawEnergy*100.0) / 100.0
-				}
-			} else {
-				cclog.Warnf("Error while collecting energy metric %s for job, DB ID '%v', return '0.0'", fp, job.ID)
-			}
-
-			job.EnergyFootprint[fp] = metricEnergy
-			totalEnergy += metricEnergy
-		}
-
-		job.Energy = (math.Round(totalEnergy*100.0) / 100.0)
-		if job.RawEnergyFootprint, err = json.Marshal(job.EnergyFootprint); err != nil {
-			cclog.Warnf("Error while marshaling energy footprint for job INTO BYTES, DB ID '%v'", job.ID)
-			return err
-		}
-
-		job.RawResources, err = json.Marshal(job.Resources)
-		if err != nil {
-			cclog.Warn("Error while marshaling job resources")
-			return err
-		}
-		job.RawMetaData, err = json.Marshal(job.MetaData)
-		if err != nil {
-			cclog.Warn("Error while marshaling job metadata")
+		if err = enrichJobMetadata(&job); err != nil {
+			cclog.Errorf("Error enriching job metadata: %v", err)
 			return err
 		}