From c274e979f4a09d1521308d16fe30cc6318245bf3 Mon Sep 17 00:00:00 2001 From: Michael Schwarz Date: Mon, 12 Jun 2023 09:09:40 +0200 Subject: [PATCH 1/4] Skip files with size of 0 Bytes --- tools/archive-migration/main.go | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/tools/archive-migration/main.go b/tools/archive-migration/main.go index dce44de..3e82c4e 100644 --- a/tools/archive-migration/main.go +++ b/tools/archive-migration/main.go @@ -308,6 +308,17 @@ func main() { go func() { defer wg.Done() + // check if source data is available, otherwise skip job + src_data_path := getPath(job, srcPath, "data.json") + info, err := os.Stat(src_data_path) + if err != nil { + log.Fatal(err) + } + if info.Size() == 0 { + fmt.Printf("Skip path %s, filesize is 0 Bytes.", src_data_path) + return + } + path := getPath(job, dstPath, "meta.json") err = os.MkdirAll(filepath.Dir(path), 0750) if err != nil { @@ -332,7 +343,7 @@ func main() { } var jd *JobData - jd, err = loadJobData(getPath(job, srcPath, "data.json")) + jd, err = loadJobData(src_data_path) if err != nil { log.Fatal(err) } From d4a26e71cdf8871857458cc0da46c4ad3b6fa3ce Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Mon, 12 Jun 2023 14:44:54 +0200 Subject: [PATCH 2/4] Introduce debug mode --- tools/archive-migration/main.go | 123 +++++++++++++++++--------------- 1 file changed, 66 insertions(+), 57 deletions(-) diff --git a/tools/archive-migration/main.go b/tools/archive-migration/main.go index 3e82c4e..93cdad5 100644 --- a/tools/archive-migration/main.go +++ b/tools/archive-migration/main.go @@ -23,6 +23,8 @@ import ( const Version = 1 var ar FsArchive +var srcPath string +var dstPath string func loadJobData(filename string) (*JobData, error) { @@ -243,17 +245,65 @@ func deepCopyClusterConfig(co *Cluster) schema.Cluster { return cn } +func convertJob(job *JobMeta) { + // check if source data is available, otherwise skip job + src_data_path := getPath(job, srcPath, "data.json") + info, err := os.Stat(src_data_path) + if err != nil { + log.Fatal(err) + } + if info.Size() == 0 { + fmt.Printf("Skip path %s, filesize is 0 Bytes.", src_data_path) + return + } + + path := getPath(job, dstPath, "meta.json") + err = os.MkdirAll(filepath.Dir(path), 0750) + if err != nil { + log.Fatal(err) + } + f, err := os.Create(path) + if err != nil { + log.Fatal(err) + } + + jmn := deepCopyJobMeta(job) + if err = EncodeJobMeta(f, &jmn); err != nil { + log.Fatal(err) + } + if err = f.Close(); err != nil { + log.Fatal(err) + } + + f, err = os.Create(getPath(job, dstPath, "data.json")) + if err != nil { + log.Fatal(err) + } + + var jd *JobData + jd, err = loadJobData(src_data_path) + if err != nil { + log.Fatal(err) + } + jdn := deepCopyJobData(jd, job.Cluster, job.SubCluster) + if err := EncodeJobData(f, jdn); err != nil { + log.Fatal(err) + } + if err := f.Close(); err != nil { + log.Fatal(err) + } +} + func main() { var flagLogLevel, flagConfigFile string - var flagLogDateTime bool - var srcPath string - var dstPath string + var flagLogDateTime, debug bool flag.BoolVar(&flagLogDateTime, "logdate", false, "Set this flag to add date and time to log messages") + flag.BoolVar(&debug, "debug", false, "Set this flag to force sequential execution for debugging") flag.StringVar(&flagLogLevel, "loglevel", "warn", "Sets the logging level: `[debug,info,warn (default),err,fatal,crit]`") flag.StringVar(&flagConfigFile, "config", "./config.json", "Specify alternative path to `config.json`") - flag.StringVar(&srcPath, "s", "./var/job-archive", "Specify the source job archive path") - flag.StringVar(&dstPath, "d", "./var/job-archive-new", "Specify the destination job archive path") + flag.StringVar(&srcPath, "src", "./var/job-archive", "Specify the source job archive path") + flag.StringVar(&dstPath, "dst", "./var/job-archive-new", "Specify the destination job archive path") flag.Parse() if _, err := os.Stat(filepath.Join(srcPath, "version.txt")); !errors.Is(err, os.ErrNotExist) { @@ -302,59 +352,18 @@ func main() { var wg sync.WaitGroup for job := range ar.Iter() { - // fmt.Printf("Job %d\n", job.JobID) - job := job - wg.Add(1) + if debug { + fmt.Printf("Job %d\n", job.JobID) + convertJob(job) + } else { + job := job + wg.Add(1) - go func() { - defer wg.Done() - // check if source data is available, otherwise skip job - src_data_path := getPath(job, srcPath, "data.json") - info, err := os.Stat(src_data_path) - if err != nil { - log.Fatal(err) - } - if info.Size() == 0 { - fmt.Printf("Skip path %s, filesize is 0 Bytes.", src_data_path) - return - } - - path := getPath(job, dstPath, "meta.json") - err = os.MkdirAll(filepath.Dir(path), 0750) - if err != nil { - log.Fatal(err) - } - f, err := os.Create(path) - if err != nil { - log.Fatal(err) - } - - jmn := deepCopyJobMeta(job) - if err = EncodeJobMeta(f, &jmn); err != nil { - log.Fatal(err) - } - if err = f.Close(); err != nil { - log.Fatal(err) - } - - f, err = os.Create(getPath(job, dstPath, "data.json")) - if err != nil { - log.Fatal(err) - } - - var jd *JobData - jd, err = loadJobData(src_data_path) - if err != nil { - log.Fatal(err) - } - jdn := deepCopyJobData(jd, job.Cluster, job.SubCluster) - if err := EncodeJobData(f, jdn); err != nil { - log.Fatal(err) - } - if err := f.Close(); err != nil { - log.Fatal(err) - } - }() + go func() { + defer wg.Done() + convertJob(job) + }() + } } wg.Wait() From c7c83fc08c6875aa9a45d055e3fd0735916ae276 Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Mon, 12 Jun 2023 14:58:33 +0200 Subject: [PATCH 3/4] Replace Analysis with Status view in Homepage --- web/templates/home.tmpl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/web/templates/home.tmpl b/web/templates/home.tmpl index ff49e1e..fbe1237 100644 --- a/web/templates/home.tmpl +++ b/web/templates/home.tmpl @@ -9,8 +9,8 @@ Running Jobs Total Jobs {{if .User.HasRole .Roles.admin}} + Status View System View - Analysis View {{end}} @@ -21,8 +21,8 @@ {{.ID}} {{.RunningJobs}} jobs {{.TotalJobs}} jobs + Status View System View - Analysis View {{end}} {{else}} From 29b6022f976355112b9779ae996a32876cc3b477 Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Mon, 12 Jun 2023 16:45:23 +0200 Subject: [PATCH 4/4] Allow to overwrite legal texts --- web/web.go | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/web/web.go b/web/web.go index 452ed48..3547b85 100644 --- a/web/web.go +++ b/web/web.go @@ -9,6 +9,7 @@ import ( "html/template" "io/fs" "net/http" + "os" "strings" "github.com/ClusterCockpit/cc-backend/internal/auth" @@ -46,6 +47,23 @@ func init() { return nil } + if path == "templates/imprint.tmpl" { + if _, err := os.Stat("./var/imprint.tmpl"); err == nil { + log.Info("overwrite imprint.tmpl with local file") + templates[strings.TrimPrefix(path, "templates/")] = + template.Must(template.Must(base.Clone()).ParseFiles("./var/imprint.tmpl")) + return nil + } + } + if path == "templates/privacy.tmpl" { + if _, err := os.Stat("./var/privacy.tmpl"); err == nil { + log.Info("overwrite privacy.tmpl with local file") + templates[strings.TrimPrefix(path, "templates/")] = + template.Must(template.Must(base.Clone()).ParseFiles("./var/privacy.tmpl")) + return nil + } + } + templates[strings.TrimPrefix(path, "templates/")] = template.Must(template.Must(base.Clone()).ParseFS(templateFiles, path)) return nil }); err != nil {