ClusterCockpit/cc-specifications
1
0
Fork 0
mirror of https://github.com/ClusterCockpit/cc-specifications.git synced 2024-12-26 13:29:05 +01:00
Code Issues Packages Projects Releases Activity

Add filesystems array. Fix Bugs.

Browse Source
This commit is contained in:
Jan Eitzinger 2022-08-25 17:33:18 +02:00
parent 5ea38d5a6c
commit 8aefd3e34a
5 changed files with 323 additions and 376 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
datastructures/cluster.schema.json
View File

@ -69,6 +69,7 @@
"alert" "alert"
] ]
} }
}
}, },
"required": [ "required": [
"name", "name",
@ -77,7 +78,6 @@
"timestep" "timestep"
] ]
} }
}
}, },
"subClusters": { "subClusters": {
"description": "Array of cluster hardware partitions", "description": "Array of cluster hardware partitions",
@ -202,12 +202,14 @@
"model" "model"
] ]
} }
}
}, },
"required":[ "required":[
"node", "node",
"socket", "socket",
"memoryDomain" "memoryDomain"
] ]
}
}, },
"required":[ "required":[
"name", "name",
@ -221,13 +223,11 @@
"memoryBandwidth" "memoryBandwidth"
] ]
} }
}
}, },
"required":[ "required":[
"name", "name",
"metricConfig", "metricConfig",
"subClusters" "subClusters"
] ]
}
}
}
} }

185
datastructures/job-data.schema.json
View File

@ -36,9 +36,7 @@
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json" "$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json"
} }
}, },
"required": [ "minProperties": 1
"node"
]
}, },
"mem_bw": { "mem_bw": {
"description": "Main memory bandwidth", "description": "Main memory bandwidth",
@ -53,9 +51,7 @@
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json" "$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json"
} }
}, },
"required": [ "minProperties": 1
"node"
]
}, },
"net_bw": { "net_bw": {
"description": "Total fast interconnect network bandwidth", "description": "Total fast interconnect network bandwidth",
@ -69,18 +65,6 @@
"node" "node"
] ]
}, },
"file_bw": {
"description": "Total file IO bandwidth",
"type": "object",
"properties": {
"node": {
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json"
}
},
"required": [
"node"
]
},
"ipc": { "ipc": {
"description": "Instructions executed per cycle", "description": "Instructions executed per cycle",
"properties": { "properties": {
@ -100,9 +84,7 @@
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json" "$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json"
} }
}, },
"required": [ "minProperties": 1
"node"
]
}, },
"cpu_used": { "cpu_used": {
"description": "CPU active core utilization", "description": "CPU active core utilization",
@ -123,27 +105,13 @@
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json" "$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json"
} }
}, },
"required": [ "minProperties": 1
"node"
]
}, },
"cpu_load": { "cpu_load": {
"description": "CPU requested core utilization (load 1m)", "description": "CPU requested core utilization (load 1m)",
"properties": { "properties": {
"node": { "node": {
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json" "$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json"
},
"socket": {
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json"
},
"memoryDomain": {
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json"
},
"core": {
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json"
},
"hwthread": {
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json"
} }
}, },
"required": [ "required": [
@ -169,9 +137,7 @@
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json" "$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json"
} }
}, },
"required": [ "minProperties": 1
"node"
]
}, },
"flops_sp": { "flops_sp": {
"description": "Single precision flops rate", "description": "Single precision flops rate",
@ -192,9 +158,7 @@
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json" "$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json"
} }
}, },
"required": [ "minProperties": 1
"node"
]
}, },
"vectorization_ratio": { "vectorization_ratio": {
"description": "Fraction of arithmetic instructions using SIMD instructions", "description": "Fraction of arithmetic instructions using SIMD instructions",
@ -215,9 +179,7 @@
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json" "$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json"
} }
}, },
"required": [ "minProperties": 1
"node"
]
}, },
"cpu_power": { "cpu_power": {
"description": "CPU power consumption", "description": "CPU power consumption",
@ -229,9 +191,7 @@
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json" "$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json"
} }
}, },
"required": [ "minProperties": 1
"node"
]
}, },
"mem_power": { "mem_power": {
"description": "Memory power consumption", "description": "Memory power consumption",
@ -243,9 +203,7 @@
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json" "$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json"
} }
}, },
"required": [ "minProperties": 1
"node"
]
}, },
"acc_utilization": { "acc_utilization": {
"description": "GPU utilization", "description": "GPU utilization",
@ -255,7 +213,7 @@
} }
}, },
"required": [ "required": [
"node" "accelerator"
] ]
}, },
"acc_mem_used": { "acc_mem_used": {
@ -266,7 +224,7 @@
} }
}, },
"required": [ "required": [
"node" "accelerator"
] ]
}, },
"acc_power": { "acc_power": {
@ -277,7 +235,7 @@
} }
}, },
"required": [ "required": [
"node" "accelerator"
] ]
}, },
"clock": { "clock": {
@ -299,9 +257,7 @@
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json" "$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json"
} }
}, },
"required": [ "minProperties": 1
"node"
]
}, },
"eth_read_bw": { "eth_read_bw": {
"description": "Ethernet read bandwidth", "description": "Ethernet read bandwidth",
@ -325,8 +281,29 @@
"node" "node"
] ]
}, },
"pfs_read_bw": { "filesystems": {
"description": "Parallel file system read bandwidth", "description": "Array of filesystems",
"type": "array",
"items": {
"type": "object",
"properties": {
"name": {
"type": "string"
},
"type": {
"type": "string",
"enum": [
"nfs",
"lustre",
"gpfs",
"nvme",
"ssd",
"hdd",
"beegfs"
]
},
"read_bw": {
"description": "File system read bandwidth",
"properties": { "properties": {
"node": { "node": {
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json" "$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json"
@ -336,8 +313,8 @@
"node" "node"
] ]
}, },
"pfs_write_bw": { "write_bw": {
"description": "Parallel file system write bandwidth", "description": "File system write bandwidth",
"properties": { "properties": {
"node": { "node": {
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json" "$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json"
@ -347,8 +324,8 @@
"node" "node"
] ]
}, },
"pfs_read_req": { "read_req": {
"description": "Parallel file system read requests", "description": "File system read requests",
"properties": { "properties": {
"node": { "node": {
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json" "$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json"
@ -358,8 +335,8 @@
"node" "node"
] ]
}, },
"pfs_write_req": { "write_req": {
"description": "Parallel file system write requests", "description": "File system write requests",
"properties": { "properties": {
"node": { "node": {
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json" "$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json"
@ -369,8 +346,8 @@
"node" "node"
] ]
}, },
"pfs_inodes": { "inodes": {
"description": "Parallel file system inodes used", "description": "File system write requests",
"properties": { "properties": {
"node": { "node": {
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json" "$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json"
@ -380,8 +357,8 @@
"node" "node"
] ]
}, },
"pfs_accesses": { "accesses": {
"description": "Parallel file system open and close", "description": "File system open and close",
"properties": { "properties": {
"node": { "node": {
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json" "$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json"
@ -391,8 +368,8 @@
"node" "node"
] ]
}, },
"pfs_fsync": { "fsync": {
"description": "Parallel file system fsync", "description": "File system fsync",
"properties": { "properties": {
"node": { "node": {
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json" "$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json"
@ -402,8 +379,8 @@
"node" "node"
] ]
}, },
"pfs_create": { "create": {
"description": "Parallel file system create", "description": "File system create",
"properties": { "properties": {
"node": { "node": {
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json" "$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json"
@ -413,8 +390,8 @@
"node" "node"
] ]
}, },
"pfs_open": { "open": {
"description": "Parallel file system open", "description": "File system open",
"properties": { "properties": {
"node": { "node": {
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json" "$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json"
@ -424,8 +401,8 @@
"node" "node"
] ]
}, },
"pfs_close": { "close": {
"description": "Parallel file system close", "description": "File system close",
"properties": { "properties": {
"node": { "node": {
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json" "$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json"
@ -435,8 +412,8 @@
"node" "node"
] ]
}, },
"pfs_seek": { "seek": {
"description": "Parallel file system seek", "description": "File system seek",
"properties": { "properties": {
"node": { "node": {
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json" "$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json"
@ -445,51 +422,19 @@
"required": [ "required": [
"node" "node"
] ]
},
"fs_read_bw": {
"description": "Local file system read bandwidth",
"properties": {
"node": {
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json"
} }
}, },
"required": [ "required": [
"node" "name",
"type",
"read_bw",
"write_bw",
"inodes"
] ]
}, },
"fs_write_bw": { "minItems": 1
"description": "Local file system write bandwidth",
"properties": {
"node": {
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json"
} }
}, },
"required": [
"node"
]
},
"fs_inodes": {
"description": "Local file system inodes used",
"properties": {
"node": {
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json"
}
},
"required": [
"node"
]
},
"fs_accesses": {
"description": "Local file system open and close",
"properties": {
"node": {
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json"
}
},
"required": [
"node"
]
},
"ic_rcv_packets": { "ic_rcv_packets": {
"description": "Network interconnect read packets", "description": "Network interconnect read packets",
"properties": { "properties": {
@ -534,13 +479,13 @@
"node" "node"
] ]
} }
}, },
"required": [ "required": [
"cpu_used", "cpu_used",
"mem_used", "mem_used",
"flops_any", "flops_any",
"mem_bw", "mem_bw",
"net_bw", "net_bw",
"file_bw" "file_bw"
] ]
} }

142
datastructures/job-meta.schema.json
View File

@ -238,66 +238,6 @@
"description": "Ethernet write bandwidth", "description": "Ethernet write bandwidth",
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-statistics.schema.json" "$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-statistics.schema.json"
}, },
"pfs_read_bw": {
"description": "Parallel file system read bandwidth",
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-statistics.schema.json"
},
"pfs_write_bw": {
"description": "Parallel file system write bandwidth",
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-statistics.schema.json"
},
"pfs_read_req": {
"description": "Parallel file system read requests",
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-statistics.schema.json"
},
"pfs_write_req": {
"description": "Parallel file system write requests",
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-statistics.schema.json"
},
"pfs_inodes": {
"description": "Parallel file system inodes used",
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-statistics.schema.json"
},
"pfs_accesses": {
"description": "Parallel file system open and close",
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-statistics.schema.json"
},
"pfs_fsync": {
"description": "Parallel file system fsync",
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-statistics.schema.json"
},
"pfs_create": {
"description": "Parallel file system create",
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-statistics.schema.json"
},
"pfs_open": {
"description": "Parallel file system open",
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-statistics.schema.json"
},
"pfs_close": {
"description": "Parallel file system close",
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-statistics.schema.json"
},
"pfs_seek": {
"description": "Parallel file system seek",
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-statistics.schema.json"
},
"fs_read_bw": {
"description": "Local file system read bandwidth",
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-statistics.schema.json"
},
"fs_write_bw": {
"description": "Local file system write bandwidth",
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-statistics.schema.json"
},
"fs_inodes": {
"description": "Local file system inodes used",
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-statistics.schema.json"
},
"fs_accesses": {
"description": "Local file system open and close",
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-statistics.schema.json"
},
"ic_rcv_packets": { "ic_rcv_packets": {
"description": "Network interconnect read packets", "description": "Network interconnect read packets",
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-statistics.schema.json" "$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-statistics.schema.json"
@ -313,15 +253,89 @@
"ic_write_bw": { "ic_write_bw": {
"description": "Network interconnect write bandwidth", "description": "Network interconnect write bandwidth",
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-statistics.schema.json" "$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-statistics.schema.json"
},
"filesystems": {
"description": "Array of filesystems",
"type": "array",
"items": {
"type": "object",
"properties": {
"name": {
"type": "string"
},
"type": {
"type": "string",
"enum": [
"nfs",
"lustre",
"gpfs",
"nvme",
"ssd",
"hdd",
"beegfs"
]
},
"read_bw": {
"description": "File system read bandwidth",
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-statistics.schema.json"
},
"write_bw": {
"description": "File system write bandwidth",
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-statistics.schema.json"
},
"read_req": {
"description": "File system read requests",
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-statistics.schema.json"
},
"write_req": {
"description": "File system write requests",
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-statistics.schema.json"
},
"inodes": {
"description": "File system write requests",
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-statistics.schema.json"
},
"accesses": {
"description": "File system open and close",
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-statistics.schema.json"
},
"fsync": {
"description": "File system fsync",
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-statistics.schema.json"
},
"create": {
"description": "File system create",
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-statistics.schema.json"
},
"open": {
"description": "File system open",
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-statistics.schema.json"
},
"close": {
"description": "File system close",
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-statistics.schema.json"
},
"seek": {
"description": "File system seek",
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-statistics.schema.json"
}
},
"required": [
"name",
"type",
"read_bw",
"write_bw",
"inodes"
]
},
"minItems": 1
} }
}, },
"required": [ "required": [
"cpu_used", "cpu_used",
"mem_used", "mem_used",
"flops_any", "flops_any",
"mem_bw", "mem_bw"
"net_bw",
"file_bw"
] ]
} }
}, },
@ -330,12 +344,10 @@
"user", "user",
"project", "project",
"cluster", "cluster",
"subCluster",
"partition", "partition",
"numNodes", "numNodes",
"exclusive", "exclusive",
"startTime", "startTime",
"stopTime",
"jobState", "jobState",
"duration", "duration",
"resources", "resources",

14
datastructures/job-metric-data.schema.json
View File

@ -9,18 +9,6 @@
"description": "Metric unit", "description": "Metric unit",
"$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/unit.schema.json" "$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/unit.schema.json"
}, },
"scope": {
"type": "string",
"enum": [
"node",
"hwthread",
"core",
"memoryDomain",
"die",
"socket",
"accelerator"
]
},
"timestep": { "timestep": {
"description": "Measurement interval in seconds", "description": "Measurement interval in seconds",
"type": "integer" "type": "integer"
@ -214,6 +202,7 @@
}, },
"required": [ "required": [
"hostname", "hostname",
"statistics",
"data" "data"
] ]
} }
@ -221,7 +210,6 @@
}, },
"required": [ "required": [
"unit", "unit",
"scope",
"timestep", "timestep",
"series" "series"
] ]

2
datastructures/unit.schema.json
View File

@ -15,6 +15,8 @@
"F/s", "F/s",
"CPI", "CPI",
"IPC", "IPC",
"cpu_used",
"cpu_load",
"Hz" "Hz"
] ]
}, },