ClusterCockpit/cc-backend
1
0
Fork 0
mirror of https://github.com/ClusterCockpit/cc-backend synced 2025-07-23 12:51:40 +02:00
Code Issues Packages Releases Activity

Restructure black box test

Browse Source 
Remove string ptr from unit prefix
This commit is contained in:
Jan Eitzinger
2023-04-25 09:26:48 +02:00
parent 500ae29d25
commit 5a09277d1c
16 changed files with 1034 additions and 128 deletions
Download Patch File Download Diff File Expand all files Collapse all files

41
internal/repository/init.go
View File

@@ -77,8 +77,8 @@ func HandleImportFlag(flag string) error {
return err
}
checkJobData(&jobData)
SanityChecks(&jobMeta.BaseJob)
//checkJobData(&jobData)
// SanityChecks(&jobMeta.BaseJob)
jobMeta.MonitoringStatus = schema.MonitoringStatusArchivingSuccessful
if job, err := GetJobRepository().Find(&jobMeta.JobID, &jobMeta.Cluster, &jobMeta.StartTime); err != sql.ErrNoRows {
if err != nil {
@@ -339,17 +339,15 @@ func getNormalizationFactor(v float64) (float64, int) {
return math.Pow10(count * scale), count * scale
}
func normalize(avg float64, u schema.Unit) (float64, schema.Unit) {
func normalize(avg float64, p string) (float64, string) {
f, e := getNormalizationFactor(avg)
if e != 0 {
p := units.NewPrefixFromFactor(units.NewPrefix(*u.Prefix), e)
np := p.Prefix()
return f, schema.Unit{Prefix: &np, Base: u.Base}
np := units.NewPrefixFromFactor(units.NewPrefix(p), e)
return f, np.Prefix()
}
return f, u
return f, p
}
func checkJobData(d *schema.JobData) error {
@@ -368,22 +366,23 @@ func checkJobData(d *schema.JobData) error {
}
avg := sum / float64(len(metric.Series))
f, u := normalize(avg, metric.Unit)
f, p := normalize(avg, metric.Unit.Prefix)
if u.Prefix != metric.Unit.Prefix {
if p != metric.Unit.Prefix {
for _, s := range metric.Series {
fp := schema.ConvertFloatToFloat64(s.Data)
fmt.Printf("Convert %e", f)
// for _, s := range metric.Series {
// fp := schema.ConvertFloatToFloat64(s.Data)
//
// for i := 0; i < len(fp); i++ {
// fp[i] *= f
// fp[i] = math.Ceil(fp[i])
// }
//
// s.Data = schema.GetFloat64ToFloat(fp)
// }
for i := 0; i < len(fp); i++ {
fp[i] *= f
fp[i] = math.Ceil(fp[i])
}
s.Data = schema.GetFloat64ToFloat(fp)
}
metric.Unit.Prefix = u.Prefix
metric.Unit.Prefix = p
}
}
}

22
internal/repository/init_test.go
View File

@@ -77,25 +77,3 @@ func TestNormalizeKeep(t *testing.T) {
t.Errorf("Failed Prefix or unit: Want G, Got %s", p.Prefix())
}
}
//
// func TestHandleImportFlag(t *testing.T) {
// t.Error("wrong summary for diagnostic ")
// r := setupRepo(t)
//
// s := "../../test/repo/meta1.json:../../test/repo/data1.json"
// err := HandleImportFlag(s)
// if err != nil {
// t.Fatal(err)
// }
//
// jobId, cluster, startTime := int64(1404396), "emmy", int64(1609299584)
// job, err := r.Find(&jobId, &cluster, &startTime)
// if err != nil {
// t.Fatal(err)
// }
//
// if job.ID != 1366 {
// t.Errorf("wrong summary for diagnostic 3\ngot: %d \nwant: 1366", job.JobID)
// }
// }

69
internal/repository_test/importFlag.go Normal file
View File

@@ -0,0 +1,69 @@
// Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package repository_test
import "testing"
func TestHandleImportFlag(t *testing.T) {
r := setupRepo(t)
paths, err := filepath.Glob(filepath.Join("testdata", "*.input"))
if err != nil {
t.Fatal(err)
}
for _, path := range paths {
_, filename := filepath.Split(path)
testname := filename[:len(filename)-len(filepath.Ext(path))]
// Each path turns into a test: the test name is the filename without the
// extension.
t.Run(testname, func(t *testing.T) {
source, err := os.ReadFile(path)
if err != nil {
t.Fatal("error reading source file:", err)
}
// >>> This is the actual code under test.
output, err := format.Source(source)
if err != nil {
t.Fatal("error formatting:", err)
}
// <<<
// Each input file is expected to have a "golden output" file, with the
// same path except the .input extension is replaced by .golden
goldenfile := filepath.Join("testdata", testname+".golden")
want, err := os.ReadFile(goldenfile)
if err != nil {
t.Fatal("error reading golden file:", err)
}
if !bytes.Equal(output, want) {
t.Errorf("\n==== got:\n%s\n==== want:\n%s\n", output, want)
}
})
}
s := "../../test/repo/meta1.json:../../test/repo/data1.json"
err := HandleImportFlag(s)
if err != nil {
t.Fatal(err)
}
jobId, cluster, startTime := int64(398764), "fritz", int64(1675954353)
job, err := r.Find(&jobId, &cluster, &startTime)
if err != nil {
t.Fatal(err)
}
if job.ID != 2 {
t.Errorf("wrong summary for diagnostic 3\ngot: %d \nwant: 1366", job.JobID)
}
}

112
internal/repository_test/setup.go Normal file
View File

@@ -0,0 +1,112 @@
// Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package repository_test
import (
"encoding/json"
"fmt"
"os"
"path/filepath"
"testing"
"github.com/ClusterCockpit/cc-backend/internal/config"
"github.com/ClusterCockpit/cc-backend/internal/repository"
"github.com/ClusterCockpit/cc-backend/pkg/archive"
"github.com/ClusterCockpit/cc-backend/pkg/log"
)
func copyFile(s string, d string) error {
r, err := os.Open(s)
if err != nil {
return err
}
defer r.Close()
w, err := os.Create(d)
if err != nil {
return err
}
defer w.Close()
w.ReadFrom(r)
return nil
}
func setupRepo(t *testing.T) *repository.JobRepository {
const testconfig = `{
"addr": "0.0.0.0:8080",
"validate": false,
"archive": {
"kind": "file",
"path": "./var/job-archive"
},
"clusters": [
{
"name": "testcluster",
"metricDataRepository": {"kind": "test", "url": "bla:8081"},
"filterRanges": {
"numNodes": { "from": 1, "to": 64 },
"duration": { "from": 0, "to": 86400 },
"startTime": { "from": "2022-01-01T00:00:00Z", "to": null }
}
},
{
"name": "fritz",
"metricDataRepository": {"kind": "test", "url": "bla:8081"},
"filterRanges": {
"numNodes": { "from": 1, "to": 944 },
"duration": { "from": 0, "to": 86400 },
"startTime": { "from": "2022-01-01T00:00:00Z", "to": null }
}
},
{
"name": "taurus",
"metricDataRepository": {"kind": "test", "url": "bla:8081"},
"filterRanges": {
"numNodes": { "from": 1, "to": 4000 },
"duration": { "from": 0, "to": 604800 },
"startTime": { "from": "2010-01-01T00:00:00Z", "to": null }
}
}
]}`
log.Init("info", true)
tmpdir := t.TempDir()
jobarchive := filepath.Join(tmpdir, "job-archive")
if err := os.Mkdir(jobarchive, 0777); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(jobarchive, "version.txt"), []byte(fmt.Sprintf("%d", 1)), 0666); err != nil {
t.Fatal(err)
}
fritzArchive := filepath.Join(tmpdir, "job-archive", "fritz")
if err := os.Mkdir(fritzArchive, 0777); err != nil {
t.Fatal(err)
}
if err := copyFile(filepath.Join("testdata", "cluster-fritz.json"),
filepath.Join(fritzArchive, "cluster.json")); err != nil {
t.Fatal(err)
}
dbfilepath := filepath.Join(tmpdir, "test.db")
err := repository.MigrateDB("sqlite3", dbfilepath)
if err != nil {
t.Fatal(err)
}
cfgFilePath := filepath.Join(tmpdir, "config.json")
if err := os.WriteFile(cfgFilePath, []byte(testconfig), 0666); err != nil {
t.Fatal(err)
}
config.Init(cfgFilePath)
archiveCfg := fmt.Sprintf("{\"kind\": \"file\",\"path\": \"%s\"}", jobarchive)
if err := archive.Init(json.RawMessage(archiveCfg), config.Keys.DisableArchive); err != nil {
t.Fatal(err)
}
repository.Connect("sqlite3", dbfilepath)
return repository.GetJobRepository()
}

746
internal/repository_test/testdata/cluster-fritz.json vendored Normal file
View File

@@ -0,0 +1,746 @@
{
"name": "fritz",
"metricConfig": [
{
"name": "cpu_load",
"unit": {
"base": ""
},
"scope": "node",
"aggregation": "avg",
"timestep": 60,
"peak": 72,
"normal": 72,
"caution": 36,
"alert": 20
},
{
"name": "cpu_user",
"unit": {
"base": ""
},
"scope": "hwthread",
"aggregation": "avg",
"timestep": 60,
"peak": 100,
"normal": 50,
"caution": 20,
"alert": 10
},
{
"name": "mem_used",
"unit": {
"base": "B",
"prefix": "G"
},
"scope": "node",
"aggregation": "sum",
"timestep": 60,
"peak": 256,
"normal": 128,
"caution": 200,
"alert": 240
},
{
"name": "flops_any",
"unit": {
"base": "F/s",
"prefix": "G"
},
"scope": "hwthread",
"aggregation": "sum",
"timestep": 60,
"peak": 5600,
"normal": 1000,
"caution": 200,
"alert": 50
},
{
"name": "flops_sp",
"unit": {
"base": "F/s",
"prefix": "G"
},
"scope": "hwthread",
"aggregation": "sum",
"timestep": 60,
"peak": 5600,
"normal": 1000,
"caution": 200,
"alert": 50
},
{
"name": "flops_dp",
"unit": {
"base": "F/s",
"prefix": "G"
},
"scope": "hwthread",
"aggregation": "sum",
"timestep": 60,
"peak": 2300,
"normal": 500,
"caution": 100,
"alert": 50
},
{
"name": "mem_bw",
"unit": {
"base": "B/s",
"prefix": "G"
},
"scope": "socket",
"aggregation": "sum",
"timestep": 60,
"peak": 350,
"normal": 100,
"caution": 50,
"alert": 10
},
{
"name": "clock",
"unit": {
"base": "Hz",
"prefix": "M"
},
"scope": "hwthread",
"aggregation": "avg",
"timestep": 60,
"peak": 3000,
"normal": 2400,
"caution": 1800,
"alert": 1200
},
{
"name": "cpu_power",
"unit": {
"base": "W"
},
"scope": "socket",
"aggregation": "sum",
"timestep": 60,
"peak": 500,
"normal": 250,
"caution": 100,
"alert": 50
},
{
"name": "mem_power",
"unit": {
"base": "W"
},
"scope": "socket",
"aggregation": "sum",
"timestep": 60,
"peak": 100,
"normal": 50,
"caution": 20,
"alert": 10
},
{
"name": "ipc",
"unit": {
"base": "IPC"
},
"scope": "hwthread",
"aggregation": "avg",
"timestep": 60,
"peak": 4,
"normal": 2,
"caution": 1,
"alert": 0.5
},
{
"name": "vectorization_ratio",
"unit": {
"base": ""
},
"scope": "hwthread",
"aggregation": "avg",
"timestep": 60,
"peak": 100,
"normal": 60,
"caution": 40,
"alert": 10
},
{
"name": "ib_recv",
"unit": {
"base": "B/s"
},
"scope": "node",
"aggregation": "sum",
"timestep": 60,
"peak": 1250000,
"normal": 6000000,
"caution": 200,
"alert": 1
},
{
"name": "ib_xmit",
"unit": {
"base": "B/s"
},
"scope": "node",
"aggregation": "sum",
"timestep": 60,
"peak": 1250000,
"normal": 6000000,
"caution": 200,
"alert": 1
},
{
"name": "ib_recv_pkts",
"unit": {
"base": ""
},
"scope": "node",
"aggregation": "sum",
"timestep": 60,
"peak": 6,
"normal": 4,
"caution": 2,
"alert": 1
},
{
"name": "ib_xmit_pkts",
"unit": {
"base": ""
},
"scope": "node",
"aggregation": "sum",
"timestep": 60,
"peak": 6,
"normal": 4,
"caution": 2,
"alert": 1
},
{
"name": "nfs4_read",
"unit": {
"base": "B/s",
"prefix": "M"
},
"scope": "node",
"aggregation": "sum",
"timestep": 60,
"peak": 6,
"normal": 4,
"caution": 2,
"alert": 1
},
{
"name": "nfs4_write",
"unit": {
"base": "B/s",
"prefix": "M"
},
"scope": "node",
"aggregation": "sum",
"timestep": 60,
"peak": 6,
"normal": 4,
"caution": 2,
"alert": 1
},
{
"name": "nfs4_total",
"unit": {
"base": "B/s",
"prefix": "M"
},
"scope": "node",
"aggregation": "sum",
"timestep": 60,
"peak": 6,
"normal": 4,
"caution": 2,
"alert": 1
}
],
"subClusters": [
{
"name": "main",
"nodes": "f01[01-88],f02[01-88],f03[01-88],f03[01-88],f04[01-88],f05[01-88],f06[01-88],f07[01-88],f08[01-88],f09[01-88],f10[01-88],f11[01-56],f12[01-56]",
"processorType": "Intel Icelake",
"socketsPerNode": 2,
"coresPerSocket": 36,
"threadsPerCore": 1,
"flopRateScalar": {
"unit": {
"base": "F/s",
"prefix": "G"
},
"value": 432
},
"flopRateSimd": {
"unit": {
"base": "F/s",
"prefix": "G"
},
"value": 9216
},
"memoryBandwidth": {
"unit": {
"base": "B/s",
"prefix": "G"
},
"value": 350
},
"topology": {
"node": [
0,
1,
2,
3,
4,
5,
6,
7,
8,
9,
10,
11,
12,
13,
14,
15,
16,
17,
18,
19,
20,
21,
22,
23,
24,
25,
26,
27,
28,
29,
30,
31,
32,
33,
34,
35,
36,
37,
38,
39,
40,
41,
42,
43,
44,
45,
46,
47,
48,
49,
50,
51,
52,
53,
54,
55,
56,
57,
58,
59,
60,
61,
62,
63,
64,
65,
66,
67,
68,
69,
70,
71
],
"socket": [
[
0,
1,
2,
3,
4,
5,
6,
7,
8,
9,
10,
11,
12,
13,
14,
15,
16,
17,
18,
19,
20,
21,
22,
23,
24,
25,
26,
27,
28,
29,
30,
31,
32,
33,
34,
35
],
[
36,
37,
38,
39,
40,
41,
42,
43,
44,
45,
46,
47,
48,
49,
50,
51,
52,
53,
54,
55,
56,
57,
58,
59,
60,
61,
62,
63,
64,
65,
66,
67,
68,
69,
70,
71
]
],
"memoryDomain": [
[
0,
1,
2,
3,
4,
5,
6,
7,
8,
9,
10,
11,
12,
13,
14,
15,
16,
17
],
[
18,
19,
20,
21,
22,
23,
24,
25,
26,
27,
28,
29,
30,
31,
32,
33,
34,
35
],
[
36,
37,
38,
39,
40,
41,
42,
43,
44,
45,
46,
47,
48,
49,
50,
51,
52,
53
],
[
54,
55,
56,
57,
58,
59,
60,
61,
62,
63,
64,
65,
66,
67,
68,
69,
70,
71
]
],
"core": [
[
0
],
[
1
],
[
2
],
[
3
],
[
4
],
[
5
],
[
6
],
[
7
],
[
8
],
[
9
],
[
10
],
[
11
],
[
12
],
[
13
],
[
14
],
[
15
],
[
16
],
[
17
],
[
18
],
[
19
],
[
20
],
[
21
],
[
22
],
[
23
],
[
24
],
[
25
],
[
26
],
[
27
],
[
28
],
[
29
],
[
30
],
[
31
],
[
32
],
[
33
],
[
34
],
[
35
],
[
36
],
[
37
],
[
38
],
[
39
],
[
40
],
[
41
],
[
42
],
[
43
],
[
44
],
[
45
],
[
46
],
[
47
],
[
48
],
[
49
],
[
50
],
[
51
],
[
52
],
[
53
],
[
54
],
[
55
],
[
56
],
[
57
],
[
58
],
[
59
],
[
60
],
[
61
],
[
62
],
[
63
],
[
64
],
[
65
],
[
66
],
[
67
],
[
68
],
[
69
],
[
70
],
[
71
]
]
}
}
]
}

1
internal/repository_test/testdata/data-fritzError.json vendored Normal file
View File

File diff suppressed because one or more lines are too long

1
internal/repository_test/testdata/data-fritzMinimal.json vendored Normal file
View File

File diff suppressed because one or more lines are too long

496
internal/repository_test/testdata/data-taurus.json vendored Normal file
View File

@@ -0,0 +1,496 @@
{
"cpu_used": {
"core": {
"unit": {
"base": ""
},
"timestep": 30,
"series": [
{
"hostname": "taurusi6489",
"id": "0",
"statistics": {
"min": 0.09090909090909093,
"avg": 0.9173553719008265,
"max": 1.0000000000000002
},
"data": [
0.09090909090909093,
0.9999999999999999,
1.0,
1.0000000000000002,
1.0,
1.0000000000000002,
0.9999999999999999,
1.0,
1.0,
1.0,
1.0
]
},
{
"hostname": "taurusi6489",
"id": "1",
"statistics": {
"min": 0.03694102397926118,
"avg": 0.045968409230268584,
"max": 0.08809840425531917
},
"data": [
0.08809840425531917,
0.05710659898477157,
0.04034861200774694,
0.037962362102530824,
0.03976721629485936,
0.04163976759199483,
0.03694102397926118,
0.03821243523316062,
0.03851132686084142,
0.044752092723760455,
0.04231266149870802
]
},
{
"hostname": "taurusi6490",
"id": "10",
"statistics": {
"min": 0.10505319148936171,
"avg": 0.9186411992263056,
"max": 1.0000000000000002
},
"data": [
0.10505319148936171,
1.0000000000000002,
1.0,
1.0,
1.0,
0.9999999999999999,
1.0,
0.9999999999999999,
1.0,
1.0,
1.0
]
},
{
"hostname": "taurusi6490",
"id": "11",
"statistics": {
"min": 0.05286048845767815,
"avg": 0.07053823838706144,
"max": 0.075148113501715
},
"data": [
0.05286048845767815,
0.06936597614563718,
0.07254534083802376,
0.075148113501715,
0.06909547738693468,
0.07372696032489846,
0.07077983088005012,
0.07082419304293325,
0.07424812030075188,
0.07285803627267043,
0.07446808510638298
]
}
],
"statisticsSeries": null
}
},
"ipc": {
"core": {
"unit": {
"base": "IPC"
},
"timestep": 60,
"series": [
{
"hostname": "taurusi6489",
"id": "0",
"statistics": {
"min": 1.3808406263195592,
"avg": 1.3960848578375105,
"max": 1.4485575599350569
},
"data": [
1.4485575599350569,
1.3808406263195592,
1.3830284413690626,
1.3836692663348698,
1.3843283952290035
]
},
{
"hostname": "taurusi6489",
"id": "1",
"statistics": {
"min": 0.30469640475234366,
"avg": 0.8816944294664065,
"max": 1.797623522191001
},
"data": [
1.797623522191001,
0.954395633726228,
1.0019972349956185,
0.30469640475234366,
0.3497593516668412
]
},
{
"hostname": "taurusi6490",
"id": "10",
"statistics": {
"min": 1.3791232173760588,
"avg": 1.3850247295506815,
"max": 1.386710405495511
},
"data": [
1.3791232173760588,
1.38619977419787,
1.386397917938246,
1.3866923327457215,
1.386710405495511
]
},
{
"hostname": "taurusi6490",
"id": "11",
"statistics": {
"min": 0.6424094604392216,
"avg": 0.9544442638400293,
"max": 1.2706704244636826
},
"data": [
1.2706704244636826,
0.6424094604392216,
0.9249973908234796,
0.6940110823242276,
1.2401329611495353
]
}
],
"statisticsSeries": null
}
},
"flops_any": {
"core": {
"unit": {
"base": "F/s"
},
"timestep": 60,
"series": [
{
"hostname": "taurusi6489",
"id": "0",
"statistics": {
"min": 0.0,
"avg": 184.2699002412084,
"max": 921.3495012060421
},
"data": [
921.3495012060421,
0.0,
0.0,
0.0,
0.0
]
},
{
"hostname": "taurusi6489",
"id": "1",
"statistics": {
"min": 0.13559227208748068,
"avg": 273.2997868356056,
"max": 1355.9227390817396
},
"data": [
1355.9227390817396,
8.94908797747172,
0.6779613312519499,
0.13559227208748068,
0.8135535154771758
]
},
{
"hostname": "taurusi6490",
"id": "10",
"statistics": {
"min": 0.0,
"avg": 1678.8419461262179,
"max": 4346.591400350933
},
"data": [
4346.591400350933,
0.0,
578.4248288199713,
0.0,
3469.193501460185
]
},
{
"hostname": "taurusi6490",
"id": "11",
"statistics": {
"min": 45.28689133054866,
"avg": 609.6644949204072,
"max": 2582.7080822873186
},
"data": [
2582.7080822873186,
45.28689133054866,
48.67663233623293,
47.591911855555026,
324.0589567923803
]
}
],
"statisticsSeries": null
}
},
"mem_bw": {
"socket": {
"unit": {
"base": "B/s"
},
"timestep": 60,
"series": [
{
"hostname": "taurusi6489",
"id": "0",
"statistics": {
"min": 653671812.1661415,
"avg": 1637585527.5854635,
"max": 2614718291.9554267
},
"data": [
653671812.1661415,
2614718291.9554267,
1732453371.7073724,
1612865229.8704093,
1574218932.2279677
]
},
{
"hostname": "taurusi6490",
"id": "0",
"statistics": {
"min": 1520190251.61048,
"avg": 1572477682.3850098,
"max": 1688960732.2760606
},
"data": [
1688960732.2760606,
1580140679.8216474,
1520190251.61048,
1541841829.6250021,
1531254918.591859
]
}
],
"statisticsSeries": null
}
},
"file_bw": {
"node": {
"unit": {
"base": "B/s"
},
"timestep": 30,
"series": [
{
"hostname": "taurusi6489",
"statistics": {
"min": 0.0,
"avg": 190352.6328851857,
"max": 2093878.361723524
},
"data": [
0.0,
0.0,
0.0,
0.6000135186380174,
0.0,
0.0,
2093878.361723524,
0.0,
0.0,
0.0,
0.0
]
},
{
"hostname": "taurusi6490",
"statistics": {
"min": 0.0,
"avg": 1050832.4509396513,
"max": 11559156.360352296
},
"data": [
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
11559156.360352296,
0.0,
0.5999838690326298,
0.0,
0.0
]
}
],
"statisticsSeries": null
}
},
"net_bw": {
"node": {
"unit": {
"base": "B/s"
},
"timestep": 30,
"series": [
{
"hostname": "taurusi6489",
"statistics": {
"min": 126779.89655880642,
"avg": 653834.5091507058,
"max": 1285639.5107541133
},
"data": [
1158202.7403032137,
126779.89655880642,
419017.91939583793,
345766.3974972795,
645419.3296982117,
644667.7333333333,
1285639.5107541133,
643481.2108874657,
640025.3562553325,
643241.4875354709,
639938.0184386979
]
},
{
"hostname": "taurusi6490",
"statistics": {
"min": 640156.9862985397,
"avg": 872367.6551257868,
"max": 1916309.7075416835
},
"data": [
1774843.146788355,
643218.3646426039,
641681.1031071587,
644690.1512268113,
647183.5650609672,
644439.3303402043,
1916309.7075416835,
643748.3241006166,
757189.8273227927,
642583.6999539217,
640156.9862985397
]
}
],
"statisticsSeries": null
}
},
"mem_used": {
"node": {
"unit": {
"base": "B"
},
"timestep": 30,
"series": [
{
"hostname": "taurusi6489",
"statistics": {
"min": 2779066368.0,
"avg": 9282117259.636364,
"max": 10202595328.0
},
"data": [
2779066368.0,
8518217728.0,
9852760064.0,
9979805696.0,
10039619584.0,
10087104512.0,
10136084480.0,
10202595328.0,
10154196992.0,
10177409024.0,
10176430080.0
]
},
{
"hostname": "taurusi6490",
"statistics": {
"min": 9993277440.0,
"avg": 10013080110.545454,
"max": 10039676928.0
},
"data": [
10001317888.0,
10013028352.0,
10006728704.0,
10039676928.0,
10035838976.0,
10033356800.0,
10006577152.0,
10005659648.0,
9993277440.0,
9993564160.0,
10014855168.0
]
}
],
"statisticsSeries": null
}
},
"cpu_power": {
"socket": {
"unit": {
"base": "W"
},
"timestep": 60,
"series": [
{
"hostname": "taurusi6489",
"id": "0",
"statistics": {
"min": 35.50647456742635,
"avg": 72.08313211552377,
"max": 83.33799371150049
},
"data": [
35.50647456742635,
75.65022009482759,
83.33799371150049,
83.00405043233219,
82.9169217715322
]
},
{
"hostname": "taurusi6490",
"id": "0",
"statistics": {
"min": 83.8466923147859,
"avg": 85.18572681122097,
"max": 85.83909286117324
},
"data": [
83.8466923147859,
85.58816979864088,
85.31266819129794,
85.83909286117324,
85.34201089020692
]
}
],
"statisticsSeries": null
}
}
}

1
internal/repository_test/testdata/meta-fritzError.input vendored Normal file
View File

@@ -0,0 +1 @@
{"jobId":398955,"user":"k106eb10","project":"k106eb","cluster":"fritz","subCluster":"main","partition":"singlenode","arrayJobId":0,"numNodes":1,"numHwthreads":72,"numAcc":0,"exclusive":1,"monitoringStatus":1,"smt":0,"jobState":"completed","duration":260,"walltime":86340,"resources":[{"hostname":"f0720"}],"metaData":{"jobName":"ams_pipeline","jobScript":"#!/bin/bash -l\n#SBATCH --job-name=ams_pipeline\n#SBATCH --time=23:59:00\n#SBATCH --partition=singlenode\n#SBATCH --ntasks=72\n#SBATCH --hint=multithread\n#SBATCH --chdir=/home/atuin/k106eb/k106eb10/ACE/Ni-Al/DFT/VASP_PBE_500_0.125_0.1_NM/AlNi/binaries/bulk/base-hcp/occ-shaken/hcp16.occ.4.shake.0/cfg/NiAl3NiAl11\n#SBATCH --export=NONE\nunset SLURM_EXPORT_ENV\nuss=$(whoami)\nfind /dev/shm/ -user $uss -type f -mmin +30 -delete\ncd \"/home/atuin/k106eb/k106eb10/ACE/Ni-Al/DFT/VASP_PBE_500_0.125_0.1_NM/AlNi/binaries/bulk/base-hcp/occ-shaken/hcp16.occ.4.shake.0/cfg/NiAl3NiAl11\"\nams_pipeline pipeline.json \u003e \"/home/atuin/k106eb/k106eb10/ACE/Ni-Al/DFT/VASP_PBE_500_0.125_0.1_NM/AlNi/binaries/bulk/base-hcp/occ-shaken/hcp16.occ.4.shake.0/cfg/NiAl3NiAl11/ams_pipeline_job.sh.out\" 2\u003e \"/home/atuin/k106eb/k106eb10/ACE/Ni-Al/DFT/VASP_PBE_500_0.125_0.1_NM/AlNi/binaries/bulk/base-hcp/occ-shaken/hcp16.occ.4.shake.0/cfg/NiAl3NiAl11/ams_pipeline_job.sh.err\"\n","slurmInfo":"\nJobId=398955 JobName=ams_pipeline\n UserId=k106eb10(210387) GroupId=80111\n Account=k106eb QOS=normal \n Requeue=False Restarts=0 BatchFlag=True \n TimeLimit=1439\n SubmitTime=2023-02-09T14:11:22\n Partition=singlenode \n NodeList=f0720\n NumNodes=1 NumCPUs=72 NumTasks=72 CPUs/Task=1\n NTasksPerNode:Socket:Core=0:None:None\n TRES_req=cpu=72,mem=250000M,node=1,billing=72\n TRES_alloc=cpu=72,node=1,billing=72\n Command=/home/atuin/k106eb/k106eb10/ACE/Ni-Al/DFT/VASP_PBE_500_0.125_0.1_NM/AlNi/binaries/bulk/base-hcp/occ-shaken/hcp16.occ.4.shake.0/cfg/NiAl3NiAl11/ams_pipeline_job.sh\n WorkDir=/home/atuin/k106eb/k106eb10/ACE/Ni-Al/DFT/VASP_PBE_500_0.125_0.1_NM/AlNi/binaries/bulk/base-hcp/occ-shaken/hcp16.occ.4.shake.0/cfg/NiAl3NiAl11\n StdErr=\n StdOut=ams_pipeline.o%j\n"},"startTime":1675956725,"statistics":{"clock":{"unit":{"base":"Hz","prefix":"M"},"avg":2335.254,"min":800.418,"max":2734.922},"cpu_load":{"unit":{"base":""},"avg":52.72,"min":34.46,"max":71.91},"cpu_power":{"unit":{"base":"W"},"avg":407.767,"min":93.932,"max":497.636},"cpu_user":{"unit":{"base":""},"avg":63.678,"min":19.872,"max":96.633},"flops_any":{"unit":{"base":"F/s","prefix":"G"},"avg":635.672,"min":0,"max":1332.874},"flops_dp":{"unit":{"base":"F/s","prefix":"G"},"avg":261.006,"min":0,"max":382.294},"flops_sp":{"unit":{"base":"F/s","prefix":"G"},"avg":113.659,"min":0,"max":568.286},"ib_recv":{"unit":{"base":"B/s"},"avg":27981.111,"min":69.4,"max":48084.589},"ib_recv_pkts":{"unit":{"base":"packets/s"},"avg":398.939,"min":0.5,"max":693.817},"ib_xmit":{"unit":{"base":"B/s"},"avg":188.513,"min":39.597,"max":724.568},"ib_xmit_pkts":{"unit":{"base":"packets/s"},"avg":0.867,"min":0.2,"max":2.933},"ipc":{"unit":{"base":"IPC"},"avg":0.944,"min":0.564,"max":1.291},"mem_bw":{"unit":{"base":"B/s","prefix":"G"},"avg":79.565,"min":0.021,"max":116.02},"mem_power":{"unit":{"base":"W"},"avg":24.692,"min":7.883,"max":31.318},"mem_used":{"unit":{"base":"B","prefix":"G"},"avg":22.566,"min":8.225,"max":27.613},"nfs4_read":{"unit":{"base":"B/s","prefix":"M"},"avg":647,"min":0,"max":1946},"nfs4_total":{"unit":{"base":"B/s","prefix":"M"},"avg":6181.6,"min":1270,"max":11411},"nfs4_write":{"unit":{"base":"B/s","prefix":"M"},"avg":22.4,"min":11,"max":29},"vectorization_ratio":{"unit":{"base":"%"},"avg":77.351,"min":0,"max":98.837}}}

1
internal/repository_test/testdata/meta-fritzMinimal.input vendored Normal file
View File

@@ -0,0 +1 @@
{"jobId":398764,"user":"k106eb10","project":"k106eb","cluster":"fritz","subCluster":"main","partition":"singlenode","arrayJobId":0,"numNodes":1,"numHwthreads":72,"numAcc":0,"exclusive":1,"monitoringStatus":1,"smt":0,"jobState":"completed","duration":177,"walltime":86340,"resources":[{"hostname":"f0649"}],"metaData":{"jobName":"ams_pipeline","jobScript":"#!/bin/bash -l\n#SBATCH --job-name=ams_pipeline\n#SBATCH --time=23:59:00\n#SBATCH --partition=singlenode\n#SBATCH --ntasks=72\n#SBATCH --hint=multithread\n#SBATCH --chdir=/home/atuin/k106eb/k106eb10/ACE/Ni-Al/DFT/VASP_PBE_500_0.125_0.1_NM/AlNi/binaries/bulk/moreprototypesYury/gen_131_CuTe/cfg/Ni2Al2\n#SBATCH --export=NONE\nunset SLURM_EXPORT_ENV\nuss=$(whoami)\nfind /dev/shm/ -user $uss -type f -mmin +30 -delete\ncd \"/home/atuin/k106eb/k106eb10/ACE/Ni-Al/DFT/VASP_PBE_500_0.125_0.1_NM/AlNi/binaries/bulk/moreprototypesYury/gen_131_CuTe/cfg/Ni2Al2\"\nams_pipeline pipeline.json \u003e \"/home/atuin/k106eb/k106eb10/ACE/Ni-Al/DFT/VASP_PBE_500_0.125_0.1_NM/AlNi/binaries/bulk/moreprototypesYury/gen_131_CuTe/cfg/Ni2Al2/ams_pipeline_job.sh.out\" 2\u003e \"/home/atuin/k106eb/k106eb10/ACE/Ni-Al/DFT/VASP_PBE_500_0.125_0.1_NM/AlNi/binaries/bulk/moreprototypesYury/gen_131_CuTe/cfg/Ni2Al2/ams_pipeline_job.sh.err\"\n","slurmInfo":"\nJobId=398764 JobName=ams_pipeline\n UserId=k106eb10(210387) GroupId=80111\n Account=k106eb QOS=normal \n Requeue=False Restarts=0 BatchFlag=True \n TimeLimit=1439\n SubmitTime=2023-02-09T14:10:20\n Partition=singlenode \n NodeList=f0649\n NumNodes=1 NumCPUs=72 NumTasks=72 CPUs/Task=1\n NTasksPerNode:Socket:Core=0:None:None\n TRES_req=cpu=72,mem=250000M,node=1,billing=72\n TRES_alloc=cpu=72,node=1,billing=72\n Command=/home/atuin/k106eb/k106eb10/ACE/Ni-Al/DFT/VASP_PBE_500_0.125_0.1_NM/AlNi/binaries/bulk/moreprototypesYury/gen_131_CuTe/cfg/Ni2Al2/ams_pipeline_job.sh\n WorkDir=/home/atuin/k106eb/k106eb10/ACE/Ni-Al/DFT/VASP_PBE_500_0.125_0.1_NM/AlNi/binaries/bulk/moreprototypesYury/gen_131_CuTe/cfg/Ni2Al2\n StdErr=\n StdOut=ams_pipeline.o%j\n"},"startTime":1675954353,"statistics":{"clock":{"unit":{"base":"Hz","prefix":"M"},"avg":1336.519,"min":801.564,"max":2348.215},"cpu_load":{"unit":{"base":""},"avg":31.64,"min":17.36,"max":45.54},"cpu_power":{"unit":{"base":"W"},"avg":150.018,"min":93.672,"max":261.592},"cpu_user":{"unit":{"base":""},"avg":28.518,"min":0.09,"max":57.343},"flops_any":{"unit":{"base":"F/s","prefix":"G"},"avg":45.012,"min":0,"max":135.037},"flops_dp":{"unit":{"base":"F/s","prefix":"G"},"avg":22.496,"min":0,"max":67.488},"flops_sp":{"unit":{"base":"F/s","prefix":"G"},"avg":0.02,"min":0,"max":0.061},"ib_recv":{"unit":{"base":"B/s"},"avg":14442.82,"min":219.998,"max":42581.368},"ib_recv_pkts":{"unit":{"base":"packets/s"},"avg":201.532,"min":1.25,"max":601.345},"ib_xmit":{"unit":{"base":"B/s"},"avg":282.098,"min":56.2,"max":569.363},"ib_xmit_pkts":{"unit":{"base":"packets/s"},"avg":1.228,"min":0.433,"max":2},"ipc":{"unit":{"base":"IPC"},"avg":0.77,"min":0.564,"max":0.906},"mem_bw":{"unit":{"base":"B/s","prefix":"G"},"avg":4.872,"min":0.025,"max":14.552},"mem_power":{"unit":{"base":"W"},"avg":7.725,"min":6.286,"max":10.556},"mem_used":{"unit":{"base":"B","prefix":"G"},"avg":6.162,"min":6.103,"max":6.226},"nfs4_read":{"unit":{"base":"B/s","prefix":"M"},"avg":1045.333,"min":311,"max":1525},"nfs4_total":{"unit":{"base":"B/s","prefix":"M"},"avg":6430,"min":2796,"max":11518},"nfs4_write":{"unit":{"base":"B/s","prefix":"M"},"avg":24.333,"min":0,"max":38},"vectorization_ratio":{"unit":{"base":"%"},"avg":25.528,"min":0,"max":76.585}}}

98
internal/repository_test/testdata/meta-taurus.input vendored Normal file
View File

@@ -0,0 +1,98 @@
{
"jobId": 20639587,
"user": "s3804552",
"project": "p_speichersysteme",
"cluster": "taurus",
"subCluster": "haswell",
"partition": "haswell64",
"numNodes": 2,
"numHwthreads": 4,
"exclusive": 0,
"startTime": 1635856524,
"jobState": "completed",
"duration": 310,
"walltime": 3600,
"smt": 0,
"resources": [
{
"hostname": "taurusi6489",
"hwthreads": [
0,
1
]
},
{
"hostname": "taurusi6490",
"hwthreads": [
10,
11
]
}
],
"statistics": {
"cpu_used": {
"min": 0.03694102397926118,
"avg": 0.48812580468611544,
"max": 1.0000000000000002,
"unit": {
"base": ""
}
},
"ipc": {
"min": 0.30469640475234366,
"avg": 1.154312070173657,
"max": 1.797623522191001,
"unit": {
"base": "IPC"
}
},
"flops_any": {
"min": 0.0,
"avg": 686.5190320308598,
"max": 4346.591400350933,
"unit": {
"base": "F/s"
}
},
"mem_bw": {
"min": 653671812.1661415,
"avg": 1605031604.9852366,
"max": 2614718291.9554267,
"unit": {
"base": "B/s"
}
},
"file_bw": {
"min": 0.0,
"avg": 620592.5419124186,
"max": 11559156.360352296,
"unit": {
"base": "B/s"
}
},
"net_bw": {
"min": 126779.89655880642,
"avg": 763101.082138246,
"max": 1916309.7075416835,
"unit": {
"base": "B/s"
}
},
"mem_used": {
"min": 2779066368.0,
"avg": 9647598685.09091,
"max": 10202595328.0,
"unit": {
"base": "B"
}
},
"cpu_power": {
"min": 35.50647456742635,
"avg": 78.63442946337237,
"max": 85.83909286117324,
"unit": {
"base": "W"
}
}
}
}