ClusterCockpit/cc-specifications
1
0
Fork 0
mirror of https://github.com/ClusterCockpit/cc-specifications.git synced 2024-09-20 03:17:27 +02:00
Code Issues Packages Projects Releases Activity

Add filesystems array. Fix Bugs.

Browse Source
This commit is contained in:
Jan Eitzinger 2022-08-25 17:33:18 +02:00
parent 5ea38d5a6c
commit 8aefd3e34a
5 changed files with 323 additions and 376 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
datastructures/cluster.schema.json
View File

@ -69,6 +69,7 @@
"alert"
]
}
}
},
"required": [
"name",
@ -77,7 +78,6 @@
"timestep"
]
}
}
},
"subClusters": {
"description": "Array of cluster hardware partitions",
@ -202,12 +202,14 @@
"model"
]
}
}
},
"required":[
"node",
"socket",
"memoryDomain"
]
}
},
"required":[
"name",
@ -221,13 +223,11 @@
"memoryBandwidth"
]
}
}
},
"required":[
"name",
"metricConfig",
"subClusters"
]
}
}
}
}

185
datastructures/job-data.schema.json
View File

@ -36,9 +36,7 @@
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json"
}
},
"required": [
"node"
]
"minProperties": 1
},
"mem_bw": {
"description": "Main memory bandwidth",
@ -53,9 +51,7 @@
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json"
}
},
"required": [
"node"
]
"minProperties": 1
},
"net_bw": {
"description": "Total fast interconnect network bandwidth",
@ -69,18 +65,6 @@
"node"
]
},
"file_bw": {
"description": "Total file IO bandwidth",
"type": "object",
"properties": {
"node": {
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json"
}
},
"required": [
"node"
]
},
"ipc": {
"description": "Instructions executed per cycle",
"properties": {
@ -100,9 +84,7 @@
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json"
}
},
"required": [
"node"
]
"minProperties": 1
},
"cpu_used": {
"description": "CPU active core utilization",
@ -123,27 +105,13 @@
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json"
}
},
"required": [
"node"
]
"minProperties": 1
},
"cpu_load": {
"description": "CPU requested core utilization (load 1m)",
"properties": {
"node": {
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json"
},
"socket": {
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json"
},
"memoryDomain": {
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json"
},
"core": {
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json"
},
"hwthread": {
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json"
}
},
"required": [
@ -169,9 +137,7 @@
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json"
}
},
"required": [
"node"
]
"minProperties": 1
},
"flops_sp": {
"description": "Single precision flops rate",
@ -192,9 +158,7 @@
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json"
}
},
"required": [
"node"
]
"minProperties": 1
},
"vectorization_ratio": {
"description": "Fraction of arithmetic instructions using SIMD instructions",
@ -215,9 +179,7 @@
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json"
}
},
"required": [
"node"
]
"minProperties": 1
},
"cpu_power": {
"description": "CPU power consumption",
@ -229,9 +191,7 @@
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json"
}
},
"required": [
"node"
]
"minProperties": 1
},
"mem_power": {
"description": "Memory power consumption",
@ -243,9 +203,7 @@
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json"
}
},
"required": [
"node"
]
"minProperties": 1
},
"acc_utilization": {
"description": "GPU utilization",
@ -255,7 +213,7 @@
}
},
"required": [
"node"
"accelerator"
]
},
"acc_mem_used": {
@ -266,7 +224,7 @@
}
},
"required": [
"node"
"accelerator"
]
},
"acc_power": {
@ -277,7 +235,7 @@
}
},
"required": [
"node"
"accelerator"
]
},
"clock": {
@ -299,9 +257,7 @@
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json"
}
},
"required": [
"node"
]
"minProperties": 1
},
"eth_read_bw": {
"description": "Ethernet read bandwidth",
@ -325,8 +281,29 @@
"node"
]
},
"pfs_read_bw": {
"description": "Parallel file system read bandwidth",
"filesystems": {
"description": "Array of filesystems",
"type": "array",
"items": {
"type": "object",
"properties": {
"name": {
"type": "string"
},
"type": {
"type": "string",
"enum": [
"nfs",
"lustre",
"gpfs",
"nvme",
"ssd",
"hdd",
"beegfs"
]
},
"read_bw": {
"description": "File system read bandwidth",
"properties": {
"node": {
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json"
@ -336,8 +313,8 @@
"node"
]
},
"pfs_write_bw": {
"description": "Parallel file system write bandwidth",
"write_bw": {
"description": "File system write bandwidth",
"properties": {
"node": {
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json"
@ -347,8 +324,8 @@
"node"
]
},
"pfs_read_req": {
"description": "Parallel file system read requests",
"read_req": {
"description": "File system read requests",
"properties": {
"node": {
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json"
@ -358,8 +335,8 @@
"node"
]
},
"pfs_write_req": {
"description": "Parallel file system write requests",
"write_req": {
"description": "File system write requests",
"properties": {
"node": {
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json"
@ -369,8 +346,8 @@
"node"
]
},
"pfs_inodes": {
"description": "Parallel file system inodes used",
"inodes": {
"description": "File system write requests",
"properties": {
"node": {
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json"
@ -380,8 +357,8 @@
"node"
]
},
"pfs_accesses": {
"description": "Parallel file system open and close",
"accesses": {
"description": "File system open and close",
"properties": {
"node": {
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json"
@ -391,8 +368,8 @@
"node"
]
},
"pfs_fsync": {
"description": "Parallel file system fsync",
"fsync": {
"description": "File system fsync",
"properties": {
"node": {
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json"
@ -402,8 +379,8 @@
"node"
]
},
"pfs_create": {
"description": "Parallel file system create",
"create": {
"description": "File system create",
"properties": {
"node": {
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json"
@ -413,8 +390,8 @@
"node"
]
},
"pfs_open": {
"description": "Parallel file system open",
"open": {
"description": "File system open",
"properties": {
"node": {
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json"
@ -424,8 +401,8 @@
"node"
]
},
"pfs_close": {
"description": "Parallel file system close",
"close": {
"description": "File system close",
"properties": {
"node": {
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json"
@ -435,8 +412,8 @@
"node"
]
},
"pfs_seek": {
"description": "Parallel file system seek",
"seek": {
"description": "File system seek",
"properties": {
"node": {
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json"
@ -445,51 +422,19 @@
"required": [
"node"
]
},
"fs_read_bw": {
"description": "Local file system read bandwidth",
"properties": {
"node": {
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json"
}
},
"required": [
"node"
"name",
"type",
"read_bw",
"write_bw",
"inodes"
]
},
"fs_write_bw": {
"description": "Local file system write bandwidth",
"properties": {
"node": {
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json"
"minItems": 1
}
},
"required": [
"node"
]
},
"fs_inodes": {
"description": "Local file system inodes used",
"properties": {
"node": {
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json"
}
},
"required": [
"node"
]
},
"fs_accesses": {
"description": "Local file system open and close",
"properties": {
"node": {
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json"
}
},
"required": [
"node"
]
},
"ic_rcv_packets": {
"description": "Network interconnect read packets",
"properties": {
@ -534,13 +479,13 @@
"node"
]
}
},
"required": [
},
"required": [
"cpu_used",
"mem_used",
"flops_any",
"mem_bw",
"net_bw",
"file_bw"
]
]
}

142
datastructures/job-meta.schema.json
View File

@ -238,66 +238,6 @@
"description": "Ethernet write bandwidth",
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-statistics.schema.json"
},
"pfs_read_bw": {
"description": "Parallel file system read bandwidth",
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-statistics.schema.json"
},
"pfs_write_bw": {
"description": "Parallel file system write bandwidth",
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-statistics.schema.json"
},
"pfs_read_req": {
"description": "Parallel file system read requests",
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-statistics.schema.json"
},
"pfs_write_req": {
"description": "Parallel file system write requests",
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-statistics.schema.json"
},
"pfs_inodes": {
"description": "Parallel file system inodes used",
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-statistics.schema.json"
},
"pfs_accesses": {
"description": "Parallel file system open and close",
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-statistics.schema.json"
},
"pfs_fsync": {
"description": "Parallel file system fsync",
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-statistics.schema.json"
},
"pfs_create": {
"description": "Parallel file system create",
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-statistics.schema.json"
},
"pfs_open": {
"description": "Parallel file system open",
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-statistics.schema.json"
},
"pfs_close": {
"description": "Parallel file system close",
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-statistics.schema.json"
},
"pfs_seek": {
"description": "Parallel file system seek",
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-statistics.schema.json"
},
"fs_read_bw": {
"description": "Local file system read bandwidth",
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-statistics.schema.json"
},
"fs_write_bw": {
"description": "Local file system write bandwidth",
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-statistics.schema.json"
},
"fs_inodes": {
"description": "Local file system inodes used",
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-statistics.schema.json"
},
"fs_accesses": {
"description": "Local file system open and close",
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-statistics.schema.json"
},
"ic_rcv_packets": {
"description": "Network interconnect read packets",
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-statistics.schema.json"
@ -313,15 +253,89 @@
"ic_write_bw": {
"description": "Network interconnect write bandwidth",
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-statistics.schema.json"
},
"filesystems": {
"description": "Array of filesystems",
"type": "array",
"items": {
"type": "object",
"properties": {
"name": {
"type": "string"
},
"type": {
"type": "string",
"enum": [
"nfs",
"lustre",
"gpfs",
"nvme",
"ssd",
"hdd",
"beegfs"
]
},
"read_bw": {
"description": "File system read bandwidth",
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-statistics.schema.json"
},
"write_bw": {
"description": "File system write bandwidth",
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-statistics.schema.json"
},
"read_req": {
"description": "File system read requests",
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-statistics.schema.json"
},
"write_req": {
"description": "File system write requests",
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-statistics.schema.json"
},
"inodes": {
"description": "File system write requests",
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-statistics.schema.json"
},
"accesses": {
"description": "File system open and close",
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-statistics.schema.json"
},
"fsync": {
"description": "File system fsync",
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-statistics.schema.json"
},
"create": {
"description": "File system create",
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-statistics.schema.json"
},
"open": {
"description": "File system open",
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-statistics.schema.json"
},
"close": {
"description": "File system close",
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-statistics.schema.json"
},
"seek": {
"description": "File system seek",
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-statistics.schema.json"
}
},
"required": [
"name",
"type",
"read_bw",
"write_bw",
"inodes"
]
},
"minItems": 1
}
},
"required": [
"cpu_used",
"mem_used",
"flops_any",
"mem_bw",
"net_bw",
"file_bw"
"mem_bw"
]
}
},
@ -330,12 +344,10 @@
"user",
"project",
"cluster",
"subCluster",
"partition",
"numNodes",
"exclusive",
"startTime",
"stopTime",
"jobState",
"duration",
"resources",

14
datastructures/job-metric-data.schema.json
View File

@ -9,18 +9,6 @@
"description": "Metric unit",
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/unit.schema.json"
},
"scope": {
"type": "string",
"enum": [
"node",
"hwthread",
"core",
"memoryDomain",
"die",
"socket",
"accelerator"
]
},
"timestep": {
"description": "Measurement interval in seconds",
"type": "integer"
@ -214,6 +202,7 @@
},
"required": [
"hostname",
"statistics",
"data"
]
}
@ -221,7 +210,6 @@
},
"required": [
"unit",
"scope",
"timestep",
"series"
]

2
datastructures/unit.schema.json
View File

@ -15,6 +15,8 @@
"F/s",
"CPI",
"IPC",
"cpu_used",
"cpu_load",
"Hz"
]
},