Add S3 loadJobMeta

pkg/archive/s3Backend.go

@@ -1,11 +1,17 @@
-// Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg.
+// Copyright (C) 2023 NHR@FAU, University Erlangen-Nuremberg.
 // All rights reserved.
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 package archive
 
 import (
+	"context"
 	"encoding/json"
+	"fmt"
+	"io"
+	"strconv"
+	"strings"
+	"time"
 
 	"github.com/ClusterCockpit/cc-backend/pkg/log"
 	"github.com/ClusterCockpit/cc-backend/pkg/schema"
@@ -17,60 +23,171 @@ type S3ArchiveConfig struct {
 	Endpoint        string `json:"endpoint"`
 	AccessKeyID     string `json:"accessKeyID"`
 	SecretAccessKey string `json:"secretAccessKey"`
+	Bucket          string `json:"bucket"`
 	UseSSL          bool   `json:"useSSL"`
 }
 
 type S3Archive struct {
-	path string
+	client   *minio.Client
+	bucket   string
+	clusters []string
 }
 
 func (s3a *S3Archive) Init(rawConfig json.RawMessage) (uint64, error) {
 	var config S3ArchiveConfig
-	if err := json.Unmarshal(rawConfig, &config); err != nil {
+	var err error
+	if err = json.Unmarshal(rawConfig, &config); err != nil {
 		log.Warnf("Init() > Unmarshal error: %#v", err)
 		return 0, err
 	}
 
-	client, err := minio.New(config.Endpoint, &minio.Options{
+	fmt.Printf("Endpoint: %s Bucket: %s\n", config.Endpoint, config.Bucket)
+
+	s3a.client, err = minio.New(config.Endpoint, &minio.Options{
 		Creds:  credentials.NewStaticV4(config.AccessKeyID, config.SecretAccessKey, ""),
 		Secure: config.UseSSL,
 	})
 	if err != nil {
-		log.Fatalln(err)
+		err = fmt.Errorf("Init() : Initialize minio client failed")
+		return 0, err
 	}
 
-	return 0, err
+	s3a.bucket = config.Bucket
+
+	found, err := s3a.client.BucketExists(context.Background(), s3a.bucket)
+	if err != nil {
+		err = fmt.Errorf("Init() : %v", err)
+		return 0, err
+	}
+
+	if found {
+		log.Infof("Bucket found.")
+	} else {
+		log.Infof("Bucket not found.")
+	}
+
+	r, err := s3a.client.GetObject(context.Background(),
+		s3a.bucket, "version.txt", minio.GetObjectOptions{})
+	if err != nil {
+		err = fmt.Errorf("Init() : Get version object failed")
+		return 0, err
+	}
+	defer r.Close()
+
+	b, err := io.ReadAll(r)
+	if err != nil {
+		log.Errorf("Init() : %v", err)
+		return 0, err
+	}
+	version, err := strconv.ParseUint(strings.TrimSuffix(string(b), "\n"), 10, 64)
+	if err != nil {
+		log.Errorf("Init() : %v", err)
+		return 0, err
+	}
+
+	if version != Version {
+		return 0, fmt.Errorf("unsupported version %d, need %d", version, Version)
+	}
+
+	for object := range s3a.client.ListObjects(
+		context.Background(),
+		s3a.bucket, minio.ListObjectsOptions{
+			Recursive: false,
+		}) {
+
+		if object.Err != nil {
+			log.Errorf("listObject: %v", object.Err)
+		}
+		if strings.HasSuffix(object.Key, "/") {
+			s3a.clusters = append(s3a.clusters, strings.TrimSuffix(object.Key, "/"))
+		}
+	}
+
+	return version, err
 }
 
 func (s3a *S3Archive) Info() {
+	fmt.Printf("Job archive %s\n", s3a.bucket)
+	var clusters []string
+
+	for object := range s3a.client.ListObjects(
+		context.Background(),
+		s3a.bucket, minio.ListObjectsOptions{
+			Recursive: false,
+		}) {
+
+		if object.Err != nil {
+			log.Errorf("listObject: %v", object.Err)
+		}
+		if strings.HasSuffix(object.Key, "/") {
+			clusters = append(clusters, object.Key)
+		}
+	}
+	ci := make(map[string]*clusterInfo)
+	for _, cluster := range clusters {
+		ci[cluster] = &clusterInfo{dateFirst: time.Now().Unix()}
+		for d := range s3a.client.ListObjects(
+			context.Background(),
+			s3a.bucket, minio.ListObjectsOptions{
+				Recursive: true,
+				Prefix:    cluster,
+			}) {
+			log.Errorf("%s", d.Key)
+			ci[cluster].diskSize += (float64(d.Size) * 1e-6)
+		}
+	}
 }
 
-func (s3a *S3Archive) Exists(job *schema.Job) bool {
-}
+//	func (s3a *S3Archive) Exists(job *schema.Job) bool {
+//		return true
+//	}
 
 func (s3a *S3Archive) LoadJobMeta(job *schema.Job) (*schema.JobMeta, error) {
+	filename := getPath(job, "/", "meta.json")
+	log.Infof("Init() : %s", filename)
+
+	r, err := s3a.client.GetObject(context.Background(),
+		s3a.bucket, filename, minio.GetObjectOptions{})
+	if err != nil {
+		err = fmt.Errorf("Init() : Get version object failed")
+		return nil, err
+	}
+	b, err := io.ReadAll(r)
+	if err != nil {
+		log.Errorf("Init() : %v", err)
+		return nil, err
+	}
+
+	return loadJobMeta(b)
 }
 
 func (s3a *S3Archive) LoadJobData(job *schema.Job) (schema.JobData, error) {
+	var err error
+	return schema.JobData{}, err
 }
 
-func (s3a *S3Archive) LoadClusterCfg(name string) (*schema.Cluster, error) {
+//	func (s3a *S3Archive) LoadClusterCfg(name string) (*schema.Cluster, error) {
+//		var err error
+//		return &schema.Cluster{}, err
+//	}
+//
+// func (s3a *S3Archive) StoreJobMeta(jobMeta *schema.JobMeta) error
+func (s3a *S3Archive) ImportJob(jobMeta *schema.JobMeta, jobData *schema.JobData) error {
+	var err error
+	return err
 }
 
-func (s3a *S3Archive) StoreJobMeta(jobMeta *schema.JobMeta) error
-
-func (s3a *S3Archive) ImportJob(jobMeta *schema.JobMeta, jobData *schema.JobData) error
-
-func (s3a *S3Archive) GetClusters() []string
-
-func (s3a *S3Archive) CleanUp(jobs []*schema.Job)
-
-func (s3a *S3Archive) Move(jobs []*schema.Job, path string)
-
-func (s3a *S3Archive) Clean(before int64, after int64)
-
-func (s3a *S3Archive) Compress(jobs []*schema.Job)
-
-func (s3a *S3Archive) CompressLast(starttime int64) int64
-
-func (s3a *S3Archive) Iter(loadMetricData bool) <-chan JobContainer
+//
+// func (s3a *S3Archive) GetClusters() []string
+//
+// func (s3a *S3Archive) CleanUp(jobs []*schema.Job)
+//
+// func (s3a *S3Archive) Move(jobs []*schema.Job, path string)
+//
+// func (s3a *S3Archive) Clean(before int64, after int64)
+//
+// func (s3a *S3Archive) Compress(jobs []*schema.Job)
+//
+// func (s3a *S3Archive) CompressLast(starttime int64) int64
+//
+// func (s3a *S3Archive) Iter(loadMetricData bool) <-chan JobContainer