Merge pull request #165 from ClusterCockpit/hotfix

Hotfix

.gitignore

@@ -16,3 +16,4 @@ var/job.db-shm
 var/job.db-wal
 
 dist/
+*.db

.goreleaser.yaml

@@ -1,4 +1,3 @@
-# This is an example .goreleaser.yml file with some sensible defaults.
 before:
   hooks:
     - go mod tidy
@@ -12,7 +11,12 @@ builds:
     goamd64:
       - v3
     id: "cc-backend"
+    binary: cc-backend
     main: ./cmd/cc-backend
+    ldflags:
+      - -s -w -X main.version={{.Version}}
+      - -X main.commit={{.Commit}} -X main.date={{.Date}}
+      - -linkmode external -extldflags -static
     tags:
       - static_build
     hooks:
@@ -26,6 +30,7 @@ builds:
     goamd64:
       - v3
     id: "archive-manager"
+    binary: archive-manager
     main: ./tools/archive-manager
     tags:
       - static_build
@@ -38,6 +43,7 @@ builds:
     goamd64:
       - v3
     id: "archive-migration"
+    binary: archive-migration
     main: ./tools/archive-migration
     tags:
       - static_build
@@ -50,6 +56,7 @@ builds:
     goamd64:
       - v3
     id: "gen-keypair"
+    binary: gen-keypair
     main: ./tools/gen-keypair
     tags:
       - static_build
@@ -97,6 +104,6 @@ changelog:
 release:
   draft: true
   footer: |
-    Supports job archive version 1 and database version 4.
+    Supports job archive version 1 and database version 5.
 
 # vim: set ts=2 sw=2 tw=0 fo=cnqoj

cmd/cc-backend/main.go

@@ -520,7 +520,7 @@ func main() {
 			}
 
 			if err != nil {
-				log.Warnf("Error while looking for retention jobs: %v", err)
+				log.Warnf("Error while looking for compression jobs: %v", err)
 			}
 			ar.Compress(jobs)
 		})

internal/repository/job.go

@@ -801,9 +801,11 @@ func (r *JobRepository) FindJobsBetween(startTimeBegin int64, startTimeEnd int64
 	}
 
 	if startTimeBegin == 0 {
+		log.Infof("Find jobs before %d", startTimeEnd)
 		query = sq.Select(jobColumns...).From("job").Where(fmt.Sprintf(
 			"job.start_time < %d", startTimeEnd))
 	} else {
+		log.Infof("Find jobs between %d and %d", startTimeBegin, startTimeEnd)
 		query = sq.Select(jobColumns...).From("job").Where(fmt.Sprintf(
 			"job.start_time BETWEEN %d AND %d", startTimeBegin, startTimeEnd))
 	}
@@ -825,6 +827,7 @@ func (r *JobRepository) FindJobsBetween(startTimeBegin int64, startTimeEnd int64
 		jobs = append(jobs, job)
 	}
 
+	log.Infof("Return job count %d", len(jobs))
 	return jobs, nil
 }
 

internal/repository/migration.go

@@ -65,7 +65,7 @@ func checkDBVersion(backend string, db *sql.DB) error {
 	}
 
 	if v < Version {
-		return fmt.Errorf("unsupported database version %d, need %d.\nPlease backup your database file and run cc-backend --migrate-db", v, Version)
+		return fmt.Errorf("unsupported database version %d, need %d.\nPlease backup your database file and run cc-backend -migrate-db", v, Version)
 	}
 
 	if v > Version {

internal/repository/migrations/sqlite3/04_add-constraints.up.sql

@@ -30,6 +30,8 @@ file_bw_avg         REAL NOT NULL DEFAULT 0.0,
 file_data_vol_total REAL NOT NULL DEFAULT 0.0,
 UNIQUE (job_id, cluster, start_time));
 
+
+UPDATE job SET job_state='cancelled' WHERE job_state='canceled';
 INSERT INTO job_new SELECT * FROM job;
 DROP TABLE job;
 ALTER TABLE job_new RENAME TO job;

internal/util/fstat.go

@@ -1,4 +1,4 @@
-// Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg.
+// Copyright (C) 2023 NHR@FAU, University Erlangen-Nuremberg.
 // All rights reserved.
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
@@ -20,6 +20,7 @@ func GetFilesize(filePath string) int64 {
 	fileInfo, err := os.Stat(filePath)
 	if err != nil {
 		log.Errorf("Error on Stat %s: %v", filePath, err)
+		return 0
 	}
 	return fileInfo.Size()
 }
@@ -28,6 +29,7 @@ func GetFilecount(path string) int {
 	files, err := os.ReadDir(path)
 	if err != nil {
 		log.Errorf("Error on ReadDir %s: %v", path, err)
+		return 0
 	}
 
 	return len(files)

internal/util/util_test.go

@@ -0,0 +1,71 @@
+// Copyright (C) 2023 NHR@FAU, University Erlangen-Nuremberg.
+// All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+package util_test
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/ClusterCockpit/cc-backend/internal/util"
+)
+
+func TestCheckFileExists(t *testing.T) {
+	tmpdir := t.TempDir()
+	filePath := filepath.Join(tmpdir, "version.txt")
+
+	if err := os.WriteFile(filePath, []byte(fmt.Sprintf("%d", 1)), 0666); err != nil {
+		t.Fatal(err)
+	}
+	if !util.CheckFileExists(filePath) {
+		t.Fatal("expected true, got false")
+	}
+
+	filePath = filepath.Join(tmpdir, "version-test.txt")
+	if util.CheckFileExists(filePath) {
+		t.Fatal("expected false, got true")
+	}
+}
+
+func TestGetFileSize(t *testing.T) {
+	tmpdir := t.TempDir()
+	filePath := filepath.Join(tmpdir, "data.json")
+
+	if s := util.GetFilesize(filePath); s > 0 {
+		t.Fatalf("expected 0, got %d", s)
+	}
+
+	if err := os.WriteFile(filePath, []byte(fmt.Sprintf("%d", 1)), 0666); err != nil {
+		t.Fatal(err)
+	}
+	if s := util.GetFilesize(filePath); s == 0 {
+		t.Fatal("expected not 0, got 0")
+	}
+}
+
+func TestGetFileCount(t *testing.T) {
+	tmpdir := t.TempDir()
+
+	if c := util.GetFilecount(tmpdir); c != 0 {
+		t.Fatalf("expected 0, got %d", c)
+	}
+
+	filePath := filepath.Join(tmpdir, "data-1.json")
+	if err := os.WriteFile(filePath, []byte(fmt.Sprintf("%d", 1)), 0666); err != nil {
+		t.Fatal(err)
+	}
+	filePath = filepath.Join(tmpdir, "data-2.json")
+	if err := os.WriteFile(filePath, []byte(fmt.Sprintf("%d", 1)), 0666); err != nil {
+		t.Fatal(err)
+	}
+	if c := util.GetFilecount(tmpdir); c != 2 {
+		t.Fatalf("expected 2, got %d", c)
+	}
+
+	if c := util.GetFilecount(filePath); c != 0 {
+		t.Fatalf("expected 0, got %d", c)
+	}
+}

pkg/archive/archive_test.go

@@ -1,4 +1,4 @@
-// Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg.
+// Copyright (C) 2023 NHR@FAU, University Erlangen-Nuremberg.
 // All rights reserved.
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.

pkg/archive/fsBackend.go

@@ -348,7 +348,7 @@ func (fsa *FsArchive) Compress(jobs []*schema.Job) {
 
 	for _, job := range jobs {
 		fileIn := getPath(job, fsa.path, "data.json")
-		if !util.CheckFileExists(fileIn) && util.GetFilesize(fileIn) > 2000 {
+		if util.CheckFileExists(fileIn) && util.GetFilesize(fileIn) > 2000 {
 			util.CompressFile(fileIn, getPath(job, fsa.path, "data.json.gz"))
 			cnt++
 		}
@@ -372,6 +372,7 @@ func (fsa *FsArchive) CompressLast(starttime int64) int64 {
 		return starttime
 	}
 
+	log.Infof("fsBackend Compress - start %d last %d", starttime, last)
 	os.WriteFile(filename, []byte(fmt.Sprintf("%d", starttime)), 0644)
 	return last
 }

pkg/schema/schemas/job-metric-statistics.schema.json

@@ -1,6 +1,6 @@
 {
     "$schema": "http://json-schema.org/draft/2020-12/schema",
-    "$id": "schemafs://job-metric-statistics.schema.json",
+    "$id": "embedfs://job-metric-statistics.schema.json",
     "title": "Job statistics",
     "description": "Format specification for job metric statistics",
     "type": "object",

startDemo.sh

@@ -2,7 +2,7 @@
 
 if [ -d './var' ]; then
     echo 'Directory ./var already exists! Skipping initialization.'
-    ./cc-backend --server --dev
+    ./cc-backend -server -dev
 else
     make
     wget https://hpc-mover.rrze.uni-erlangen.de/HPC-Data/0x7b58aefb/eig7ahyo6fo2bais0ephuf2aitohv1ai/job-archive-demo.tar
@@ -12,6 +12,6 @@ else
     cp ./configs/env-template.txt .env
     cp ./configs/config-demo.json config.json
 
-    ./cc-backend --migrate-db
+    ./cc-backend -migrate-db
-    ./cc-backend --server --dev --init-db --add-user demo:admin:demo
+    ./cc-backend -server -dev -init-db -add-user demo:admin:demo
 fi