diff --git a/pkg/archive/fsBackend_test.go b/pkg/archive/fsBackend_test.go index 6e3cfc0..95d94cf 100644 --- a/pkg/archive/fsBackend_test.go +++ b/pkg/archive/fsBackend_test.go @@ -20,7 +20,7 @@ func init() { func TestInitEmptyPath(t *testing.T) { var fsa FsArchive - _, err := fsa.Init(json.RawMessage("{\"kind\":\"../../test/archive\"}")) + _, err := fsa.Init(json.RawMessage("{\"kind\":\"testdata/archive\"}")) if err == nil { t.Fatal(err) } @@ -28,14 +28,14 @@ func TestInitEmptyPath(t *testing.T) { func TestInitNoJson(t *testing.T) { var fsa FsArchive - _, err := fsa.Init(json.RawMessage("\"path\":\"../../test/archive\"}")) + _, err := fsa.Init(json.RawMessage("\"path\":\"testdata/archive\"}")) if err == nil { t.Fatal(err) } } func TestInitNotExists(t *testing.T) { var fsa FsArchive - _, err := fsa.Init(json.RawMessage("{\"path\":\"../../test/job-archive\"}")) + _, err := fsa.Init(json.RawMessage("{\"path\":\"testdata/job-archive\"}")) if err == nil { t.Fatal(err) } @@ -43,11 +43,11 @@ func TestInitNotExists(t *testing.T) { func TestInit(t *testing.T) { var fsa FsArchive - version, err := fsa.Init(json.RawMessage("{\"path\":\"../../test/archive\"}")) + version, err := fsa.Init(json.RawMessage("{\"path\":\"testdata/archive\"}")) if err != nil { t.Fatal(err) } - if fsa.path != "../../test/archive" { + if fsa.path != "testdata/archive" { t.Fail() } if version != 1 { @@ -60,12 +60,12 @@ func TestInit(t *testing.T) { func TestLoadJobMetaInternal(t *testing.T) { var fsa FsArchive - _, err := fsa.Init(json.RawMessage("{\"path\":\"../../test/archive\"}")) + _, err := fsa.Init(json.RawMessage("{\"path\":\"testdata/archive\"}")) if err != nil { t.Fatal(err) } - job, err := loadJobMeta("../../test/archive/emmy/1404/397/1609300556/meta.json") + job, err := loadJobMeta("testdata/archive/emmy/1404/397/1609300556/meta.json") if err != nil { t.Fatal(err) } @@ -83,7 +83,7 @@ func TestLoadJobMetaInternal(t *testing.T) { func TestLoadJobMeta(t *testing.T) { var fsa FsArchive - _, err := fsa.Init(json.RawMessage("{\"path\":\"../../test/archive\"}")) + _, err := fsa.Init(json.RawMessage("{\"path\":\"testdata/archive\"}")) if err != nil { t.Fatal(err) } @@ -111,7 +111,7 @@ func TestLoadJobMeta(t *testing.T) { func TestLoadJobData(t *testing.T) { var fsa FsArchive - _, err := fsa.Init(json.RawMessage("{\"path\":\"../../test/archive\"}")) + _, err := fsa.Init(json.RawMessage("{\"path\":\"testdata/archive\"}")) if err != nil { t.Fatal(err) } @@ -137,7 +137,7 @@ func TestLoadJobData(t *testing.T) { func TestLoadCluster(t *testing.T) { var fsa FsArchive - _, err := fsa.Init(json.RawMessage("{\"path\":\"../../test/archive\"}")) + _, err := fsa.Init(json.RawMessage("{\"path\":\"testdata/archive\"}")) if err != nil { t.Fatal(err) } @@ -154,7 +154,7 @@ func TestLoadCluster(t *testing.T) { func TestIter(t *testing.T) { var fsa FsArchive - _, err := fsa.Init(json.RawMessage("{\"path\":\"../../test/archive\"}")) + _, err := fsa.Init(json.RawMessage("{\"path\":\"testdata/archive\"}")) if err != nil { t.Fatal(err) } diff --git a/pkg/archive/testdata/archive/emmy/1403/244/1608923076/data.json.gz b/pkg/archive/testdata/archive/emmy/1403/244/1608923076/data.json.gz new file mode 100644 index 0000000..8f7df00 Binary files /dev/null and b/pkg/archive/testdata/archive/emmy/1403/244/1608923076/data.json.gz differ diff --git a/pkg/archive/testdata/archive/emmy/1403/244/1608923076/meta.json b/pkg/archive/testdata/archive/emmy/1403/244/1608923076/meta.json new file mode 100644 index 0000000..1ce3f87 --- /dev/null +++ b/pkg/archive/testdata/archive/emmy/1403/244/1608923076/meta.json @@ -0,0 +1,194 @@ +{ + "exclusive": 1, + "jobId": 1403244, + "statistics": { + "mem_bw": { + "avg": 63.57, + "min": 0, + "unit": { + "base": "B/s", + "prefix": "G" + }, + "max": 74.5 + }, + "rapl_power": { + "avg": 228.07, + "min": 0, + "unit": { + "base": "W" + }, + "max": 258.56 + }, + "ipc": { + "unit": { + "base": "IPC" + }, + "max": 0.510204081632653, + "avg": 1.53846153846154, + "min": 0.0 + }, + "clock": { + "min": 1380.32, + "avg": 2599.39, + "unit": { + "base": "Hz", + "prefix": "M" + }, + "max": 2634.46 + }, + "cpu_load": { + "avg": 18.4, + "min": 0, + "max": 23.58, + "unit": { + "base": "load" + } + }, + "flops_any": { + "max": 404.62, + "unit": { + "base": "F/s", + "prefix": "G" + }, + "avg": 225.59, + "min": 0 + }, + "flops_dp": { + "max": 0.24, + "unit": { + "base": "F/s", + "prefix": "G" + }, + "min": 0, + "avg": 0 + }, + "mem_used": { + "min": 1.55, + "avg": 27.84, + "unit": { + "base": "B", + "prefix": "G" + }, + "max": 37.5 + }, + "flops_sp": { + "min": 0, + "avg": 225.59, + "max": 404.62, + "unit": { + "base": "F/s", + "prefix": "G" + } + } + }, + "resources": [ + { + "hostname": "e0102" + }, + { + "hostname": "e0103" + }, + { + "hostname": "e0105" + }, + { + "hostname": "e0106" + }, + { + "hostname": "e0107" + }, + { + "hostname": "e0108" + }, + { + "hostname": "e0114" + }, + { + "hostname": "e0320" + }, + { + "hostname": "e0321" + }, + { + "hostname": "e0325" + }, + { + "hostname": "e0404" + }, + { + "hostname": "e0415" + }, + { + "hostname": "e0433" + }, + { + "hostname": "e0437" + }, + { + "hostname": "e0439" + }, + { + "hostname": "e0501" + }, + { + "hostname": "e0503" + }, + { + "hostname": "e0505" + }, + { + "hostname": "e0506" + }, + { + "hostname": "e0512" + }, + { + "hostname": "e0513" + }, + { + "hostname": "e0514" + }, + { + "hostname": "e0653" + }, + { + "hostname": "e0701" + }, + { + "hostname": "e0716" + }, + { + "hostname": "e0727" + }, + { + "hostname": "e0728" + }, + { + "hostname": "e0925" + }, + { + "hostname": "e0926" + }, + { + "hostname": "e0929" + }, + { + "hostname": "e0934" + }, + { + "hostname": "e0951" + } + ], + "walltime": 10, + "jobState": "completed", + "cluster": "emmy", + "subCluster": "haswell", + "stopTime": 1609009562, + "user": "emmyUser6", + "startTime": 1608923076, + "partition": "work", + "tags": [], + "project": "no project", + "numNodes": 32, + "duration": 86486 +} diff --git a/pkg/archive/testdata/archive/emmy/1404/397/1609300556/data.json.gz b/pkg/archive/testdata/archive/emmy/1404/397/1609300556/data.json.gz new file mode 100644 index 0000000..1bfef4f Binary files /dev/null and b/pkg/archive/testdata/archive/emmy/1404/397/1609300556/data.json.gz differ diff --git a/pkg/archive/testdata/archive/emmy/1404/397/1609300556/meta.json b/pkg/archive/testdata/archive/emmy/1404/397/1609300556/meta.json new file mode 100644 index 0000000..e1fff10 --- /dev/null +++ b/pkg/archive/testdata/archive/emmy/1404/397/1609300556/meta.json @@ -0,0 +1,194 @@ +{ + "stopTime": 1609387081, + "resources": [ + { + "hostname": "e0151" + }, + { + "hostname": "e0152" + }, + { + "hostname": "e0153" + }, + { + "hostname": "e0232" + }, + { + "hostname": "e0303" + }, + { + "hostname": "e0314" + }, + { + "hostname": "e0344" + }, + { + "hostname": "e0345" + }, + { + "hostname": "e0348" + }, + { + "hostname": "e0507" + }, + { + "hostname": "e0518" + }, + { + "hostname": "e0520" + }, + { + "hostname": "e0522" + }, + { + "hostname": "e0526" + }, + { + "hostname": "e0527" + }, + { + "hostname": "e0528" + }, + { + "hostname": "e0530" + }, + { + "hostname": "e0551" + }, + { + "hostname": "e0604" + }, + { + "hostname": "e0613" + }, + { + "hostname": "e0634" + }, + { + "hostname": "e0639" + }, + { + "hostname": "e0640" + }, + { + "hostname": "e0651" + }, + { + "hostname": "e0653" + }, + { + "hostname": "e0701" + }, + { + "hostname": "e0704" + }, + { + "hostname": "e0751" + }, + { + "hostname": "e0809" + }, + { + "hostname": "e0814" + }, + { + "hostname": "e0819" + }, + { + "hostname": "e0908" + } + ], + "walltime": 10, + "cluster": "emmy", + "subCluster": "haswell", + "jobState": "completed", + "statistics": { + "clock": { + "max": 2634.9, + "unit": { + "base": "Hz", + "prefix": "M" + }, + "min": 0, + "avg": 2597.8 + }, + "cpu_load": { + "max": 27.41, + "min": 0, + "avg": 18.39, + "unit": { + "base": "load" + } + }, + "mem_bw": { + "min": 0, + "avg": 63.23, + "unit": { + "base": "B/s", + "prefix": "G" + }, + "max": 75.06 + }, + "ipc": { + "min": 0.0, + "avg": 1.53846153846154, + "unit": { + "base": "IPC" + }, + "max": 0.490196078431373 + }, + "rapl_power": { + "min": 0, + "avg": 227.32, + "unit": { + "base": "W" + }, + "max": 256.22 + }, + "mem_used": { + "min": 1.5, + "avg": 27.77, + "unit": { + "base": "B", + "prefix": "G" + }, + "max": 37.43 + }, + "flops_sp": { + "unit": { + "base": "F/s", + "prefix": "G" + }, + "max": 413.21, + "min": 0, + "avg": 224.41 + }, + "flops_dp": { + "max": 5.72, + "unit": { + "base": "F/s", + "prefix": "G" + }, + "min": 0, + "avg": 0 + }, + "flops_any": { + "min": 0, + "avg": 224.42, + "max": 413.21, + "unit": { + "base": "F/s", + "prefix": "G" + } + } + }, + "exclusive": 1, + "jobId": 1404397, + "tags": [], + "partition": "work", + "project": "no project", + "user": "emmyUser6", + "startTime": 1609300556, + "duration": 86525, + "numNodes": 32 +} diff --git a/pkg/archive/testdata/archive/emmy/cluster.json b/pkg/archive/testdata/archive/emmy/cluster.json new file mode 100644 index 0000000..e0105ec --- /dev/null +++ b/pkg/archive/testdata/archive/emmy/cluster.json @@ -0,0 +1,974 @@ +{ + "name": "emmy", + "subClusters": [ + { + "name": "haswell", + "processorType": "Intel Xeon E3-1240 v3", + "socketsPerNode": 1, + "coresPerSocket": 4, + "threadsPerCore": 1, + "flopRateScalar": { + "unit": { + "prefix": "G", + "base": "F/s" + }, + "value": 14 + }, + "flopRateSimd": { + "unit": { + "prefix": "G", + "base": "F/s" + }, + "value": 112 + }, + "memoryBandwidth": { + "unit": { + "prefix": "G", + "base": "B/s" + }, + "value": 24 + }, + "nodes": "w11[27-45,49-63,69-72]", + "topology": { + "node": [ + 0, + 1, + 2, + 3 + ], + "socket": [ + [ + 0, + 1, + 2, + 3 + ] + ], + "memoryDomain": [ + [ + 0, + 1, + 2, + 3 + ] + ], + "core": [ + [ + 0 + ], + [ + 1 + ], + [ + 2 + ], + [ + 3 + ] + ] + } + }, + { + "name": "skylake", + "processorType": "Intel Xeon E3-1240 v5 ", + "socketsPerNode": 1, + "coresPerSocket": 4, + "threadsPerCore": 1, + "flopRateScalar": { + "unit": { + "prefix": "G", + "base": "F/s" + }, + "value": 14 + }, + "flopRateSimd": { + "unit": { + "prefix": "G", + "base": "F/s" + }, + "value": 112 + }, + "memoryBandwidth": { + "unit": { + "prefix": "G", + "base": "B/s" + }, + "value": 64 + }, + "nodes": "w12[01-08],w13[01-31,33-56]", + "topology": { + "node": [ + 0, + 1, + 2, + 3 + ], + "socket": [ + [ + 0, + 1, + 2, + 3 + ] + ], + "memoryDomain": [ + [ + 0, + 1, + 2, + 3 + ] + ], + "core": [ + [ + 0 + ], + [ + 1 + ], + [ + 2 + ], + [ + 3 + ] + ] + } + }, + { + "name": "kabylake", + "processorType": "Intel Xeon E3-1240 v6", + "socketsPerNode": 1, + "coresPerSocket": 4, + "threadsPerCore": 1, + "flopRateScalar": { + "unit": { + "prefix": "G", + "base": "F/s" + }, + "value": 14 + }, + "flopRateSimd": { + "unit": { + "prefix": "G", + "base": "F/s" + }, + "value": 112 + }, + "memoryBandwidth": { + "unit": { + "prefix": "G", + "base": "B/s" + }, + "value": 24 + }, + "nodes": "w14[01-56],w15[01-05,07-56]", + "topology": { + "node": [ + 0, + 1, + 2, + 3 + ], + "socket": [ + [ + 0, + 1, + 2, + 3 + ] + ], + "memoryDomain": [ + [ + 0, + 1, + 2, + 3 + ] + ], + "core": [ + [ + 0 + ], + [ + 1 + ], + [ + 2 + ], + [ + 3 + ] + ] + } + }, + { + "name": "icelake", + "processorType": "Intel Xeon Gold 6326", + "socketsPerNode": 2, + "coresPerSocket": 16, + "threadsPerCore": 1, + "flopRateScalar": { + "unit": { + "prefix": "G", + "base": "F/s" + }, + "value": 432 + }, + "flopRateSimd": { + "unit": { + "prefix": "G", + "base": "F/s" + }, + "value": 9216 + }, + "memoryBandwidth": { + "unit": { + "prefix": "G", + "base": "B/s" + }, + "value": 350 + }, + "nodes": "w22[01-35],w23[01-35]", + "topology": { + "node": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71 + ], + "socket": [ + [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35 + ], + [ + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71 + ] + ], + "memoryDomain": [ + [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17 + ], + [ + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35 + ], + [ + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53 + ], + [ + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71 + ] + ], + "core": [ + [ + 0 + ], + [ + 1 + ], + [ + 2 + ], + [ + 3 + ], + [ + 4 + ], + [ + 5 + ], + [ + 6 + ], + [ + 7 + ], + [ + 8 + ], + [ + 9 + ], + [ + 10 + ], + [ + 11 + ], + [ + 12 + ], + [ + 13 + ], + [ + 14 + ], + [ + 15 + ], + [ + 16 + ], + [ + 17 + ], + [ + 18 + ], + [ + 19 + ], + [ + 20 + ], + [ + 21 + ], + [ + 22 + ], + [ + 23 + ], + [ + 24 + ], + [ + 25 + ], + [ + 26 + ], + [ + 27 + ], + [ + 28 + ], + [ + 29 + ], + [ + 30 + ], + [ + 31 + ], + [ + 32 + ], + [ + 33 + ], + [ + 34 + ], + [ + 35 + ], + [ + 36 + ], + [ + 37 + ], + [ + 38 + ], + [ + 39 + ], + [ + 40 + ], + [ + 41 + ], + [ + 42 + ], + [ + 43 + ], + [ + 44 + ], + [ + 45 + ], + [ + 46 + ], + [ + 47 + ], + [ + 48 + ], + [ + 49 + ], + [ + 50 + ], + [ + 51 + ], + [ + 52 + ], + [ + 53 + ], + [ + 54 + ], + [ + 55 + ], + [ + 56 + ], + [ + 57 + ], + [ + 58 + ], + [ + 59 + ], + [ + 60 + ], + [ + 61 + ], + [ + 62 + ], + [ + 63 + ], + [ + 64 + ], + [ + 65 + ], + [ + 66 + ], + [ + 67 + ], + [ + 68 + ], + [ + 69 + ], + [ + 70 + ], + [ + 71 + ] + ] + } + } + ], + "metricConfig": [ + { + "name": "cpu_load", + "scope": "node", + "unit": { + "base": "" + }, + "aggregation": "avg", + "timestep": 60, + "peak": 4, + "normal": 4, + "caution": 4, + "alert": 1, + "subClusters": [ + { + "name": "icelake", + "peak": 32, + "normal": 32, + "caution": 16, + "alert": 1 + } + ] + }, + { + "name": "cpu_user", + "scope": "hwthread", + "unit": { + "base": "" + }, + "aggregation": "avg", + "timestep": 60, + "peak": 100, + "normal": 50, + "caution": 20, + "alert": 10, + "subClusters": [ + { + "name": "haswell", + "remove": true + }, + { + "name": "skylake", + "remove": true + } + ] + }, + { + "name": "ipc", + "scope": "hwthread", + "unit": { + "base": "IPC" + }, + "aggregation": "avg", + "timestep": 60, + "peak": 4, + "normal": 2, + "caution": 1, + "alert": 0.25 + }, + { + "name": "mem_used", + "scope": "node", + "unit": { + "prefix": "G", + "base": "B" + }, + "aggregation": "sum", + "timestep": 60, + "peak": 32, + "normal": 16, + "caution": 28, + "alert": 30, + "subClusters": [ + { + "name": "icelake", + "peak": 256, + "normal": 128, + "caution": 245, + "alert": 255 + } + ] + }, + { + "name": "flops_any", + "scope": "hwthread", + "unit": { + "prefix": "G", + "base": "F/s" + }, + "aggregation": "sum", + "timestep": 60, + "peak": 112, + "normal": 50, + "caution": 20, + "alert": 10, + "subClusters": [ + { + "name": "icelake", + "peak": 9216, + "normal": 432, + "caution": 100, + "alert": 50 + } + ] + }, + { + "name": "flops_dp", + "scope": "hwthread", + "unit": { + "prefix": "G", + "base": "F/s" + }, + "aggregation": "sum", + "timestep": 60, + "peak": 56, + "normal": 30, + "caution": 15, + "alert": 5, + "subClusters": [ + { + "name": "icelake", + "peak": 4108, + "normal": 220, + "caution": 60, + "alert": 30 + } + ] + }, + { + "name": "flops_sp", + "scope": "hwthread", + "unit": { + "prefix": "G", + "base": "F/s" + }, + "aggregation": "sum", + "timestep": 60, + "peak": 112, + "normal": 50, + "caution": 20, + "alert": 10, + "subClusters": [ + { + "name": "icelake", + "peak": 9216, + "normal": 432, + "caution": 100, + "alert": 50 + } + ] + }, + { + "name": "mem_bw", + "scope": "socket", + "unit": { + "prefix": "G", + "base": "B/s" + }, + "aggregation": "sum", + "timestep": 60, + "peak": 24, + "normal": 10, + "caution": 5, + "alert": 2, + "subClusters": [ + { + "name": "icelake", + "peak": 350, + "normal": 100, + "caution": 50, + "alert": 25 + } + ] + }, + { + "name": "clock", + "scope": "hwthread", + "unit": { + "prefix": "M", + "base": "Hz" + }, + "aggregation": "avg", + "timestep": 60, + "peak": 2900, + "normal": 2900, + "caution": 1500, + "alert": 1200, + "subClusters": [ + { + "name": "haswell", + "peak": 3500, + "normal": 3500, + "caution": 1500, + "alert": 1200 + }, + { + "name": "skylake", + "peak": 3500, + "normal": 3500, + "caution": 1500, + "alert": 1200 + }, + { + "name": "kabylake", + "peak": 3700, + "normal": 3700, + "caution": 1500, + "alert": 1200 + } + ] + }, + { + "name": "vectorization_ratio", + "scope": "hwthread", + "unit": { + "base": "" + }, + "aggregation": "avg", + "timestep": 60, + "peak": 100, + "normal": 60, + "caution": 40, + "alert": 10 + }, + { + "name": "nfs4_read", + "scope": "node", + "unit": { + "prefix": "M", + "base": "B/s" + }, + "aggregation": "sum", + "timestep": 60, + "peak": 6, + "normal": 4, + "caution": 2, + "alert": 1, + "subClusters": [ + { + "name": "haswell", + "remove": true + } + ] + }, + { + "name": "nfs4_write", + "scope": "node", + "unit": { + "prefix": "M", + "base": "B/s" + }, + "aggregation": "sum", + "timestep": 60, + "peak": 6, + "normal": 4, + "caution": 2, + "alert": 1, + "subClusters": [ + { + "name": "haswell", + "remove": true + } + ] + }, + { + "name": "nfs4_total", + "scope": "node", + "unit": { + "prefix": "M", + "base": "B/s" + }, + "aggregation": "sum", + "timestep": 60, + "peak": 6, + "normal": 4, + "caution": 2, + "alert": 1, + "subClusters": [ + { + "name": "haswell", + "remove": true + } + ] + } + ] +} diff --git a/pkg/archive/testdata/archive/version.txt b/pkg/archive/testdata/archive/version.txt new file mode 100644 index 0000000..d00491f --- /dev/null +++ b/pkg/archive/testdata/archive/version.txt @@ -0,0 +1 @@ +1