From eb626db9c211ea59eb7e79610b596183827a73bd Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Tue, 27 Jun 2023 14:29:56 +0200 Subject: [PATCH 1/5] Debug compression service --- cmd/cc-backend/main.go | 2 +- internal/repository/job.go | 3 +++ .../repository/migrations/sqlite3/04_add-constraints.up.sql | 2 ++ pkg/archive/fsBackend.go | 1 + 4 files changed, 7 insertions(+), 1 deletion(-) diff --git a/cmd/cc-backend/main.go b/cmd/cc-backend/main.go index 438c9ab..af0ce33 100644 --- a/cmd/cc-backend/main.go +++ b/cmd/cc-backend/main.go @@ -520,7 +520,7 @@ func main() { } if err != nil { - log.Warnf("Error while looking for retention jobs: %v", err) + log.Warnf("Error while looking for compression jobs: %v", err) } ar.Compress(jobs) }) diff --git a/internal/repository/job.go b/internal/repository/job.go index 762ba3f..e060551 100644 --- a/internal/repository/job.go +++ b/internal/repository/job.go @@ -801,9 +801,11 @@ func (r *JobRepository) FindJobsBetween(startTimeBegin int64, startTimeEnd int64 } if startTimeBegin == 0 { + log.Infof("Find jobs before %d", startTimeEnd) query = sq.Select(jobColumns...).From("job").Where(fmt.Sprintf( "job.start_time < %d", startTimeEnd)) } else { + log.Infof("Find jobs between %d and %d", startTimeBegin, startTimeEnd) query = sq.Select(jobColumns...).From("job").Where(fmt.Sprintf( "job.start_time BETWEEN %d AND %d", startTimeBegin, startTimeEnd)) } @@ -825,6 +827,7 @@ func (r *JobRepository) FindJobsBetween(startTimeBegin int64, startTimeEnd int64 jobs = append(jobs, job) } + log.Infof("Return job count %d", len(jobs)) return jobs, nil } diff --git a/internal/repository/migrations/sqlite3/04_add-constraints.up.sql b/internal/repository/migrations/sqlite3/04_add-constraints.up.sql index aec4898..06b1a9b 100644 --- a/internal/repository/migrations/sqlite3/04_add-constraints.up.sql +++ b/internal/repository/migrations/sqlite3/04_add-constraints.up.sql @@ -30,6 +30,8 @@ file_bw_avg REAL NOT NULL DEFAULT 0.0, file_data_vol_total REAL NOT NULL DEFAULT 0.0, UNIQUE (job_id, cluster, start_time)); + +UPDATE job SET job_state='cancelled' WHERE job_state='canceled'; INSERT INTO job_new SELECT * FROM job; DROP TABLE job; ALTER TABLE job_new RENAME TO job; diff --git a/pkg/archive/fsBackend.go b/pkg/archive/fsBackend.go index 0a9c224..b58891f 100644 --- a/pkg/archive/fsBackend.go +++ b/pkg/archive/fsBackend.go @@ -372,6 +372,7 @@ func (fsa *FsArchive) CompressLast(starttime int64) int64 { return starttime } + log.Infof("fsBackend Compress - start %d last %d", starttime, last) os.WriteFile(filename, []byte(fmt.Sprintf("%d", starttime)), 0644) return last } From 34050c8ce068520b26c0069c1c9737c397b39482 Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Tue, 27 Jun 2023 14:30:09 +0200 Subject: [PATCH 2/5] Fix argument dashes --- internal/repository/migration.go | 2 +- startDemo.sh | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/internal/repository/migration.go b/internal/repository/migration.go index 7582156..b92a68b 100644 --- a/internal/repository/migration.go +++ b/internal/repository/migration.go @@ -65,7 +65,7 @@ func checkDBVersion(backend string, db *sql.DB) error { } if v < Version { - return fmt.Errorf("unsupported database version %d, need %d.\nPlease backup your database file and run cc-backend --migrate-db", v, Version) + return fmt.Errorf("unsupported database version %d, need %d.\nPlease backup your database file and run cc-backend -migrate-db", v, Version) } if v > Version { diff --git a/startDemo.sh b/startDemo.sh index 1cfe11c..faf6d35 100755 --- a/startDemo.sh +++ b/startDemo.sh @@ -2,7 +2,7 @@ if [ -d './var' ]; then echo 'Directory ./var already exists! Skipping initialization.' - ./cc-backend --server --dev + ./cc-backend -server -dev else make wget https://hpc-mover.rrze.uni-erlangen.de/HPC-Data/0x7b58aefb/eig7ahyo6fo2bais0ephuf2aitohv1ai/job-archive-demo.tar @@ -12,6 +12,6 @@ else cp ./configs/env-template.txt .env cp ./configs/config-demo.json config.json - ./cc-backend --migrate-db - ./cc-backend --server --dev --init-db --add-user demo:admin:demo + ./cc-backend -migrate-db + ./cc-backend -server -dev -init-db -add-user demo:admin:demo fi From cf1b7e2db4a5d7b4be54a14f484c41ec323b04cf Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Tue, 27 Jun 2023 15:35:29 +0200 Subject: [PATCH 3/5] Fix typo in schema id --- pkg/schema/schemas/job-metric-statistics.schema.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/schema/schemas/job-metric-statistics.schema.json b/pkg/schema/schemas/job-metric-statistics.schema.json index 8f14f72..3412c23 100644 --- a/pkg/schema/schemas/job-metric-statistics.schema.json +++ b/pkg/schema/schemas/job-metric-statistics.schema.json @@ -1,6 +1,6 @@ { "$schema": "http://json-schema.org/draft/2020-12/schema", - "$id": "schemafs://job-metric-statistics.schema.json", + "$id": "embedfs://job-metric-statistics.schema.json", "title": "Job statistics", "description": "Format specification for job metric statistics", "type": "object", From ab1a9fa781bb019740a1ff279e4c37ef1937a34a Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Tue, 27 Jun 2023 15:35:57 +0200 Subject: [PATCH 4/5] Update goreleaser config --- .goreleaser.yaml | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/.goreleaser.yaml b/.goreleaser.yaml index 78dd938..3bd6c7e 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -1,4 +1,3 @@ -# This is an example .goreleaser.yml file with some sensible defaults. before: hooks: - go mod tidy @@ -12,7 +11,12 @@ builds: goamd64: - v3 id: "cc-backend" + binary: cc-backend main: ./cmd/cc-backend + ldflags: + - -s -w -X main.version={{.Version}} + - -X main.commit={{.Commit}} -X main.date={{.Date}} + - -linkmode external -extldflags -static tags: - static_build hooks: @@ -26,6 +30,7 @@ builds: goamd64: - v3 id: "archive-manager" + binary: archive-manager main: ./tools/archive-manager tags: - static_build @@ -38,6 +43,7 @@ builds: goamd64: - v3 id: "archive-migration" + binary: archive-migration main: ./tools/archive-migration tags: - static_build @@ -50,6 +56,7 @@ builds: goamd64: - v3 id: "gen-keypair" + binary: gen-keypair main: ./tools/gen-keypair tags: - static_build @@ -97,6 +104,6 @@ changelog: release: draft: true footer: | - Supports job archive version 1 and database version 4. + Supports job archive version 1 and database version 5. # vim: set ts=2 sw=2 tw=0 fo=cnqoj From 6d8a3aa256af5ead8c2702cbd4c783b04f9ffb2a Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Wed, 28 Jun 2023 07:39:39 +0200 Subject: [PATCH 5/5] Fix bug in fsBackend and add tests for file utils --- .gitignore | 1 + internal/util/fstat.go | 4 ++- internal/util/util_test.go | 71 +++++++++++++++++++++++++++++++++++++ pkg/archive/archive_test.go | 2 +- pkg/archive/fsBackend.go | 2 +- 5 files changed, 77 insertions(+), 3 deletions(-) create mode 100644 internal/util/util_test.go diff --git a/.gitignore b/.gitignore index 0e18666..2f7c206 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,4 @@ var/job.db-shm var/job.db-wal dist/ +*.db diff --git a/internal/util/fstat.go b/internal/util/fstat.go index 3361e39..0c84014 100644 --- a/internal/util/fstat.go +++ b/internal/util/fstat.go @@ -1,4 +1,4 @@ -// Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. +// Copyright (C) 2023 NHR@FAU, University Erlangen-Nuremberg. // All rights reserved. // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. @@ -20,6 +20,7 @@ func GetFilesize(filePath string) int64 { fileInfo, err := os.Stat(filePath) if err != nil { log.Errorf("Error on Stat %s: %v", filePath, err) + return 0 } return fileInfo.Size() } @@ -28,6 +29,7 @@ func GetFilecount(path string) int { files, err := os.ReadDir(path) if err != nil { log.Errorf("Error on ReadDir %s: %v", path, err) + return 0 } return len(files) diff --git a/internal/util/util_test.go b/internal/util/util_test.go new file mode 100644 index 0000000..a318649 --- /dev/null +++ b/internal/util/util_test.go @@ -0,0 +1,71 @@ +// Copyright (C) 2023 NHR@FAU, University Erlangen-Nuremberg. +// All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. +package util_test + +import ( + "fmt" + "os" + "path/filepath" + "testing" + + "github.com/ClusterCockpit/cc-backend/internal/util" +) + +func TestCheckFileExists(t *testing.T) { + tmpdir := t.TempDir() + filePath := filepath.Join(tmpdir, "version.txt") + + if err := os.WriteFile(filePath, []byte(fmt.Sprintf("%d", 1)), 0666); err != nil { + t.Fatal(err) + } + if !util.CheckFileExists(filePath) { + t.Fatal("expected true, got false") + } + + filePath = filepath.Join(tmpdir, "version-test.txt") + if util.CheckFileExists(filePath) { + t.Fatal("expected false, got true") + } +} + +func TestGetFileSize(t *testing.T) { + tmpdir := t.TempDir() + filePath := filepath.Join(tmpdir, "data.json") + + if s := util.GetFilesize(filePath); s > 0 { + t.Fatalf("expected 0, got %d", s) + } + + if err := os.WriteFile(filePath, []byte(fmt.Sprintf("%d", 1)), 0666); err != nil { + t.Fatal(err) + } + if s := util.GetFilesize(filePath); s == 0 { + t.Fatal("expected not 0, got 0") + } +} + +func TestGetFileCount(t *testing.T) { + tmpdir := t.TempDir() + + if c := util.GetFilecount(tmpdir); c != 0 { + t.Fatalf("expected 0, got %d", c) + } + + filePath := filepath.Join(tmpdir, "data-1.json") + if err := os.WriteFile(filePath, []byte(fmt.Sprintf("%d", 1)), 0666); err != nil { + t.Fatal(err) + } + filePath = filepath.Join(tmpdir, "data-2.json") + if err := os.WriteFile(filePath, []byte(fmt.Sprintf("%d", 1)), 0666); err != nil { + t.Fatal(err) + } + if c := util.GetFilecount(tmpdir); c != 2 { + t.Fatalf("expected 2, got %d", c) + } + + if c := util.GetFilecount(filePath); c != 0 { + t.Fatalf("expected 0, got %d", c) + } +} diff --git a/pkg/archive/archive_test.go b/pkg/archive/archive_test.go index b41a033..5523a21 100644 --- a/pkg/archive/archive_test.go +++ b/pkg/archive/archive_test.go @@ -1,4 +1,4 @@ -// Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. +// Copyright (C) 2023 NHR@FAU, University Erlangen-Nuremberg. // All rights reserved. // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. diff --git a/pkg/archive/fsBackend.go b/pkg/archive/fsBackend.go index b58891f..cb8ed69 100644 --- a/pkg/archive/fsBackend.go +++ b/pkg/archive/fsBackend.go @@ -348,7 +348,7 @@ func (fsa *FsArchive) Compress(jobs []*schema.Job) { for _, job := range jobs { fileIn := getPath(job, fsa.path, "data.json") - if !util.CheckFileExists(fileIn) && util.GetFilesize(fileIn) > 2000 { + if util.CheckFileExists(fileIn) && util.GetFilesize(fileIn) > 2000 { util.CompressFile(fileIn, getPath(job, fsa.path, "data.json.gz")) cnt++ }