ClusterCockpit/cc-backend
1
0
Fork 0
mirror of https://github.com/ClusterCockpit/cc-backend synced 2024-12-24 12:29:05 +01:00
Code Issues Packages Releases Activity

Fix archive test

Browse Source 
Add testdata
This commit is contained in:
Jan Eitzinger 2023-05-04 18:54:34 +02:00
parent 4f7a638619
commit 0a43c4a146
7 changed files with 1374 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
pkg/archive/fsBackend_test.go
View File

@ -20,7 +20,7 @@ func init() {
func TestInitEmptyPath(t *testing.T) { func TestInitEmptyPath(t *testing.T) {
var fsa FsArchive var fsa FsArchive
_, err := fsa.Init(json.RawMessage("{\"kind\":\"../../test/archive\"}")) _, err := fsa.Init(json.RawMessage("{\"kind\":\"testdata/archive\"}"))
if err == nil { if err == nil {
t.Fatal(err) t.Fatal(err)
} }
@ -28,14 +28,14 @@ func TestInitEmptyPath(t *testing.T) {
func TestInitNoJson(t *testing.T) { func TestInitNoJson(t *testing.T) {
var fsa FsArchive var fsa FsArchive
_, err := fsa.Init(json.RawMessage("\"path\":\"../../test/archive\"}")) _, err := fsa.Init(json.RawMessage("\"path\":\"testdata/archive\"}"))
if err == nil { if err == nil {
t.Fatal(err) t.Fatal(err)
} }
} }
func TestInitNotExists(t *testing.T) { func TestInitNotExists(t *testing.T) {
var fsa FsArchive var fsa FsArchive
_, err := fsa.Init(json.RawMessage("{\"path\":\"../../test/job-archive\"}")) _, err := fsa.Init(json.RawMessage("{\"path\":\"testdata/job-archive\"}"))
if err == nil { if err == nil {
t.Fatal(err) t.Fatal(err)
} }
@ -43,11 +43,11 @@ func TestInitNotExists(t *testing.T) {
func TestInit(t *testing.T) { func TestInit(t *testing.T) {
var fsa FsArchive var fsa FsArchive
version, err := fsa.Init(json.RawMessage("{\"path\":\"../../test/archive\"}")) version, err := fsa.Init(json.RawMessage("{\"path\":\"testdata/archive\"}"))
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
if fsa.path != "../../test/archive" { if fsa.path != "testdata/archive" {
t.Fail() t.Fail()
} }
if version != 1 { if version != 1 {
@ -60,12 +60,12 @@ func TestInit(t *testing.T) {
func TestLoadJobMetaInternal(t *testing.T) { func TestLoadJobMetaInternal(t *testing.T) {
var fsa FsArchive var fsa FsArchive
_, err := fsa.Init(json.RawMessage("{\"path\":\"../../test/archive\"}")) _, err := fsa.Init(json.RawMessage("{\"path\":\"testdata/archive\"}"))
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
job, err := loadJobMeta("../../test/archive/emmy/1404/397/1609300556/meta.json") job, err := loadJobMeta("testdata/archive/emmy/1404/397/1609300556/meta.json")
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -83,7 +83,7 @@ func TestLoadJobMetaInternal(t *testing.T) {
func TestLoadJobMeta(t *testing.T) { func TestLoadJobMeta(t *testing.T) {
var fsa FsArchive var fsa FsArchive
_, err := fsa.Init(json.RawMessage("{\"path\":\"../../test/archive\"}")) _, err := fsa.Init(json.RawMessage("{\"path\":\"testdata/archive\"}"))
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -111,7 +111,7 @@ func TestLoadJobMeta(t *testing.T) {
func TestLoadJobData(t *testing.T) { func TestLoadJobData(t *testing.T) {
var fsa FsArchive var fsa FsArchive
_, err := fsa.Init(json.RawMessage("{\"path\":\"../../test/archive\"}")) _, err := fsa.Init(json.RawMessage("{\"path\":\"testdata/archive\"}"))
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -137,7 +137,7 @@ func TestLoadJobData(t *testing.T) {
func TestLoadCluster(t *testing.T) { func TestLoadCluster(t *testing.T) {
var fsa FsArchive var fsa FsArchive
_, err := fsa.Init(json.RawMessage("{\"path\":\"../../test/archive\"}")) _, err := fsa.Init(json.RawMessage("{\"path\":\"testdata/archive\"}"))
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -154,7 +154,7 @@ func TestLoadCluster(t *testing.T) {
func TestIter(t *testing.T) { func TestIter(t *testing.T) {
var fsa FsArchive var fsa FsArchive
_, err := fsa.Init(json.RawMessage("{\"path\":\"../../test/archive\"}")) _, err := fsa.Init(json.RawMessage("{\"path\":\"testdata/archive\"}"))
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }

BIN
pkg/archive/testdata/archive/emmy/1403/244/1608923076/data.json.gz vendored Normal file
View File

Binary file not shown.

194
pkg/archive/testdata/archive/emmy/1403/244/1608923076/meta.json vendored Normal file
View File

@ -0,0 +1,194 @@
{
"exclusive": 1,
"jobId": 1403244,
"statistics": {
"mem_bw": {
"avg": 63.57,
"min": 0,
"unit": {
"base": "B/s",
"prefix": "G"
},
"max": 74.5
},
"rapl_power": {
"avg": 228.07,
"min": 0,
"unit": {
"base": "W"
},
"max": 258.56
},
"ipc": {
"unit": {
"base": "IPC"
},
"max": 0.510204081632653,
"avg": 1.53846153846154,
"min": 0.0
},
"clock": {
"min": 1380.32,
"avg": 2599.39,
"unit": {
"base": "Hz",
"prefix": "M"
},
"max": 2634.46
},
"cpu_load": {
"avg": 18.4,
"min": 0,
"max": 23.58,
"unit": {
"base": "load"
}
},
"flops_any": {
"max": 404.62,
"unit": {
"base": "F/s",
"prefix": "G"
},
"avg": 225.59,
"min": 0
},
"flops_dp": {
"max": 0.24,
"unit": {
"base": "F/s",
"prefix": "G"
},
"min": 0,
"avg": 0
},
"mem_used": {
"min": 1.55,
"avg": 27.84,
"unit": {
"base": "B",
"prefix": "G"
},
"max": 37.5
},
"flops_sp": {
"min": 0,
"avg": 225.59,
"max": 404.62,
"unit": {
"base": "F/s",
"prefix": "G"
}
}
},
"resources": [
{
"hostname": "e0102"
},
{
"hostname": "e0103"
},
{
"hostname": "e0105"
},
{
"hostname": "e0106"
},
{
"hostname": "e0107"
},
{
"hostname": "e0108"
},
{
"hostname": "e0114"
},
{
"hostname": "e0320"
},
{
"hostname": "e0321"
},
{
"hostname": "e0325"
},
{
"hostname": "e0404"
},
{
"hostname": "e0415"
},
{
"hostname": "e0433"
},
{
"hostname": "e0437"
},
{
"hostname": "e0439"
},
{
"hostname": "e0501"
},
{
"hostname": "e0503"
},
{
"hostname": "e0505"
},
{
"hostname": "e0506"
},
{
"hostname": "e0512"
},
{
"hostname": "e0513"
},
{
"hostname": "e0514"
},
{
"hostname": "e0653"
},
{
"hostname": "e0701"
},
{
"hostname": "e0716"
},
{
"hostname": "e0727"
},
{
"hostname": "e0728"
},
{
"hostname": "e0925"
},
{
"hostname": "e0926"
},
{
"hostname": "e0929"
},
{
"hostname": "e0934"
},
{
"hostname": "e0951"
}
],
"walltime": 10,
"jobState": "completed",
"cluster": "emmy",
"subCluster": "haswell",
"stopTime": 1609009562,
"user": "emmyUser6",
"startTime": 1608923076,
"partition": "work",
"tags": [],
"project": "no project",
"numNodes": 32,
"duration": 86486
}

BIN
pkg/archive/testdata/archive/emmy/1404/397/1609300556/data.json.gz vendored Normal file
View File

Binary file not shown.

194
pkg/archive/testdata/archive/emmy/1404/397/1609300556/meta.json vendored Normal file
View File

@ -0,0 +1,194 @@
{
"stopTime": 1609387081,
"resources": [
{
"hostname": "e0151"
},
{
"hostname": "e0152"
},
{
"hostname": "e0153"
},
{
"hostname": "e0232"
},
{
"hostname": "e0303"
},
{
"hostname": "e0314"
},
{
"hostname": "e0344"
},
{
"hostname": "e0345"
},
{
"hostname": "e0348"
},
{
"hostname": "e0507"
},
{
"hostname": "e0518"
},
{
"hostname": "e0520"
},
{
"hostname": "e0522"
},
{
"hostname": "e0526"
},
{
"hostname": "e0527"
},
{
"hostname": "e0528"
},
{
"hostname": "e0530"
},
{
"hostname": "e0551"
},
{
"hostname": "e0604"
},
{
"hostname": "e0613"
},
{
"hostname": "e0634"
},
{
"hostname": "e0639"
},
{
"hostname": "e0640"
},
{
"hostname": "e0651"
},
{
"hostname": "e0653"
},
{
"hostname": "e0701"
},
{
"hostname": "e0704"
},
{
"hostname": "e0751"
},
{
"hostname": "e0809"
},
{
"hostname": "e0814"
},
{
"hostname": "e0819"
},
{
"hostname": "e0908"
}
],
"walltime": 10,
"cluster": "emmy",
"subCluster": "haswell",
"jobState": "completed",
"statistics": {
"clock": {
"max": 2634.9,
"unit": {
"base": "Hz",
"prefix": "M"
},
"min": 0,
"avg": 2597.8
},
"cpu_load": {
"max": 27.41,
"min": 0,
"avg": 18.39,
"unit": {
"base": "load"
}
},
"mem_bw": {
"min": 0,
"avg": 63.23,
"unit": {
"base": "B/s",
"prefix": "G"
},
"max": 75.06
},
"ipc": {
"min": 0.0,
"avg": 1.53846153846154,
"unit": {
"base": "IPC"
},
"max": 0.490196078431373
},
"rapl_power": {
"min": 0,
"avg": 227.32,
"unit": {
"base": "W"
},
"max": 256.22
},
"mem_used": {
"min": 1.5,
"avg": 27.77,
"unit": {
"base": "B",
"prefix": "G"
},
"max": 37.43
},
"flops_sp": {
"unit": {
"base": "F/s",
"prefix": "G"
},
"max": 413.21,
"min": 0,
"avg": 224.41
},
"flops_dp": {
"max": 5.72,
"unit": {
"base": "F/s",
"prefix": "G"
},
"min": 0,
"avg": 0
},
"flops_any": {
"min": 0,
"avg": 224.42,
"max": 413.21,
"unit": {
"base": "F/s",
"prefix": "G"
}
}
},
"exclusive": 1,
"jobId": 1404397,
"tags": [],
"partition": "work",
"project": "no project",
"user": "emmyUser6",
"startTime": 1609300556,
"duration": 86525,
"numNodes": 32
}

974
pkg/archive/testdata/archive/emmy/cluster.json vendored Normal file
View File

@ -0,0 +1,974 @@
{
"name": "emmy",
"subClusters": [
{
"name": "haswell",
"processorType": "Intel Xeon E3-1240 v3",
"socketsPerNode": 1,
"coresPerSocket": 4,
"threadsPerCore": 1,
"flopRateScalar": {
"unit": {
"prefix": "G",
"base": "F/s"
},
"value": 14
},
"flopRateSimd": {
"unit": {
"prefix": "G",
"base": "F/s"
},
"value": 112
},
"memoryBandwidth": {
"unit": {
"prefix": "G",
"base": "B/s"
},
"value": 24
},
"nodes": "w11[27-45,49-63,69-72]",
"topology": {
"node": [
0,
1,
2,
3
],
"socket": [
[
0,
1,
2,
3
]
],
"memoryDomain": [
[
0,
1,
2,
3
]
],
"core": [
[
0
],
[
1
],
[
2
],
[
3
]
]
}
},
{
"name": "skylake",
"processorType": "Intel Xeon E3-1240 v5 ",
"socketsPerNode": 1,
"coresPerSocket": 4,
"threadsPerCore": 1,
"flopRateScalar": {
"unit": {
"prefix": "G",
"base": "F/s"
},
"value": 14
},
"flopRateSimd": {
"unit": {
"prefix": "G",
"base": "F/s"
},
"value": 112
},
"memoryBandwidth": {
"unit": {
"prefix": "G",
"base": "B/s"
},
"value": 64
},
"nodes": "w12[01-08],w13[01-31,33-56]",
"topology": {
"node": [
0,
1,
2,
3
],
"socket": [
[
0,
1,
2,
3
]
],
"memoryDomain": [
[
0,
1,
2,
3
]
],
"core": [
[
0
],
[
1
],
[
2
],
[
3
]
]
}
},
{
"name": "kabylake",
"processorType": "Intel Xeon E3-1240 v6",
"socketsPerNode": 1,
"coresPerSocket": 4,
"threadsPerCore": 1,
"flopRateScalar": {
"unit": {
"prefix": "G",
"base": "F/s"
},
"value": 14
},
"flopRateSimd": {
"unit": {
"prefix": "G",
"base": "F/s"
},
"value": 112
},
"memoryBandwidth": {
"unit": {
"prefix": "G",
"base": "B/s"
},
"value": 24
},
"nodes": "w14[01-56],w15[01-05,07-56]",
"topology": {
"node": [
0,
1,
2,
3
],
"socket": [
[
0,
1,
2,
3
]
],
"memoryDomain": [
[
0,
1,
2,
3
]
],
"core": [
[
0
],
[
1
],
[
2
],
[
3
]
]
}
},
{
"name": "icelake",
"processorType": "Intel Xeon Gold 6326",
"socketsPerNode": 2,
"coresPerSocket": 16,
"threadsPerCore": 1,
"flopRateScalar": {
"unit": {
"prefix": "G",
"base": "F/s"
},
"value": 432
},
"flopRateSimd": {
"unit": {
"prefix": "G",
"base": "F/s"
},
"value": 9216
},
"memoryBandwidth": {
"unit": {
"prefix": "G",
"base": "B/s"
},
"value": 350
},
"nodes": "w22[01-35],w23[01-35]",
"topology": {
"node": [
0,
1,
2,
3,
4,
5,
6,
7,
8,
9,
10,
11,
12,
13,
14,
15,
16,
17,
18,
19,
20,
21,
22,
23,
24,
25,
26,
27,
28,
29,
30,
31,
32,
33,
34,
35,
36,
37,
38,
39,
40,
41,
42,
43,
44,
45,
46,
47,
48,
49,
50,
51,
52,
53,
54,
55,
56,
57,
58,
59,
60,
61,
62,
63,
64,
65,
66,
67,
68,
69,
70,
71
],
"socket": [
[
0,
1,
2,
3,
4,
5,
6,
7,
8,
9,
10,
11,
12,
13,
14,
15,
16,
17,
18,
19,
20,
21,
22,
23,
24,
25,
26,
27,
28,
29,
30,
31,
32,
33,
34,
35
],
[
36,
37,
38,
39,
40,
41,
42,
43,
44,
45,
46,
47,
48,
49,
50,
51,
52,
53,
54,
55,
56,
57,
58,
59,
60,
61,
62,
63,
64,
65,
66,
67,
68,
69,
70,
71
]
],
"memoryDomain": [
[
0,
1,
2,
3,
4,
5,
6,
7,
8,
9,
10,
11,
12,
13,
14,
15,
16,
17
],
[
18,
19,
20,
21,
22,
23,
24,
25,
26,
27,
28,
29,
30,
31,
32,
33,
34,
35
],
[
36,
37,
38,
39,
40,
41,
42,
43,
44,
45,
46,
47,
48,
49,
50,
51,
52,
53
],
[
54,
55,
56,
57,
58,
59,
60,
61,
62,
63,
64,
65,
66,
67,
68,
69,
70,
71
]
],
"core": [
[
0
],
[
1
],
[
2
],
[
3
],
[
4
],
[
5
],
[
6
],
[
7
],
[
8
],
[
9
],
[
10
],
[
11
],
[
12
],
[
13
],
[
14
],
[
15
],
[
16
],
[
17
],
[
18
],
[
19
],
[
20
],
[
21
],
[
22
],
[
23
],
[
24
],
[
25
],
[
26
],
[
27
],
[
28
],
[
29
],
[
30
],
[
31
],
[
32
],
[
33
],
[
34
],
[
35
],
[
36
],
[
37
],
[
38
],
[
39
],
[
40
],
[
41
],
[
42
],
[
43
],
[
44
],
[
45
],
[
46
],
[
47
],
[
48
],
[
49
],
[
50
],
[
51
],
[
52
],
[
53
],
[
54
],
[
55
],
[
56
],
[
57
],
[
58
],
[
59
],
[
60
],
[
61
],
[
62
],
[
63
],
[
64
],
[
65
],
[
66
],
[
67
],
[
68
],
[
69
],
[
70
],
[
71
]
]
}
}
],
"metricConfig": [
{
"name": "cpu_load",
"scope": "node",
"unit": {
"base": ""
},
"aggregation": "avg",
"timestep": 60,
"peak": 4,
"normal": 4,
"caution": 4,
"alert": 1,
"subClusters": [
{
"name": "icelake",
"peak": 32,
"normal": 32,
"caution": 16,
"alert": 1
}
]
},
{
"name": "cpu_user",
"scope": "hwthread",
"unit": {
"base": ""
},
"aggregation": "avg",
"timestep": 60,
"peak": 100,
"normal": 50,
"caution": 20,
"alert": 10,
"subClusters": [
{
"name": "haswell",
"remove": true
},
{
"name": "skylake",
"remove": true
}
]
},
{
"name": "ipc",
"scope": "hwthread",
"unit": {
"base": "IPC"
},
"aggregation": "avg",
"timestep": 60,
"peak": 4,
"normal": 2,
"caution": 1,
"alert": 0.25
},
{
"name": "mem_used",
"scope": "node",
"unit": {
"prefix": "G",
"base": "B"
},
"aggregation": "sum",
"timestep": 60,
"peak": 32,
"normal": 16,
"caution": 28,
"alert": 30,
"subClusters": [
{
"name": "icelake",
"peak": 256,
"normal": 128,
"caution": 245,
"alert": 255
}
]
},
{
"name": "flops_any",
"scope": "hwthread",
"unit": {
"prefix": "G",
"base": "F/s"
},
"aggregation": "sum",
"timestep": 60,
"peak": 112,
"normal": 50,
"caution": 20,
"alert": 10,
"subClusters": [
{
"name": "icelake",
"peak": 9216,
"normal": 432,
"caution": 100,
"alert": 50
}
]
},
{
"name": "flops_dp",
"scope": "hwthread",
"unit": {
"prefix": "G",
"base": "F/s"
},
"aggregation": "sum",
"timestep": 60,
"peak": 56,
"normal": 30,
"caution": 15,
"alert": 5,
"subClusters": [
{
"name": "icelake",
"peak": 4108,
"normal": 220,
"caution": 60,
"alert": 30
}
]
},
{
"name": "flops_sp",
"scope": "hwthread",
"unit": {
"prefix": "G",
"base": "F/s"
},
"aggregation": "sum",
"timestep": 60,
"peak": 112,
"normal": 50,
"caution": 20,
"alert": 10,
"subClusters": [
{
"name": "icelake",
"peak": 9216,
"normal": 432,
"caution": 100,
"alert": 50
}
]
},
{
"name": "mem_bw",
"scope": "socket",
"unit": {
"prefix": "G",
"base": "B/s"
},
"aggregation": "sum",
"timestep": 60,
"peak": 24,
"normal": 10,
"caution": 5,
"alert": 2,
"subClusters": [
{
"name": "icelake",
"peak": 350,
"normal": 100,
"caution": 50,
"alert": 25
}
]
},
{
"name": "clock",
"scope": "hwthread",
"unit": {
"prefix": "M",
"base": "Hz"
},
"aggregation": "avg",
"timestep": 60,
"peak": 2900,
"normal": 2900,
"caution": 1500,
"alert": 1200,
"subClusters": [
{
"name": "haswell",
"peak": 3500,
"normal": 3500,
"caution": 1500,
"alert": 1200
},
{
"name": "skylake",
"peak": 3500,
"normal": 3500,
"caution": 1500,
"alert": 1200
},
{
"name": "kabylake",
"peak": 3700,
"normal": 3700,
"caution": 1500,
"alert": 1200
}
]
},
{
"name": "vectorization_ratio",
"scope": "hwthread",
"unit": {
"base": ""
},
"aggregation": "avg",
"timestep": 60,
"peak": 100,
"normal": 60,
"caution": 40,
"alert": 10
},
{
"name": "nfs4_read",
"scope": "node",
"unit": {
"prefix": "M",
"base": "B/s"
},
"aggregation": "sum",
"timestep": 60,
"peak": 6,
"normal": 4,
"caution": 2,
"alert": 1,
"subClusters": [
{
"name": "haswell",
"remove": true
}
]
},
{
"name": "nfs4_write",
"scope": "node",
"unit": {
"prefix": "M",
"base": "B/s"
},
"aggregation": "sum",
"timestep": 60,
"peak": 6,
"normal": 4,
"caution": 2,
"alert": 1,
"subClusters": [
{
"name": "haswell",
"remove": true
}
]
},
{
"name": "nfs4_total",
"scope": "node",
"unit": {
"prefix": "M",
"base": "B/s"
},
"aggregation": "sum",
"timestep": 60,
"peak": 6,
"normal": 4,
"caution": 2,
"alert": 1,
"subClusters": [
{
"name": "haswell",
"remove": true
}
]
}
]
}

1
pkg/archive/testdata/archive/version.txt vendored Normal file
View File

@ -0,0 +1 @@
1