From 8aefd3e34ac272ebb8bf261289b4f7b33d75e1c3 Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Thu, 25 Aug 2022 17:33:18 +0200 Subject: [PATCH] Add filesystems array. Fix Bugs. --- datastructures/cluster.schema.json | 52 +-- datastructures/job-data.schema.json | 489 +++++++++------------ datastructures/job-meta.schema.json | 142 +++--- datastructures/job-metric-data.schema.json | 14 +- datastructures/unit.schema.json | 2 + 5 files changed, 323 insertions(+), 376 deletions(-) diff --git a/datastructures/cluster.schema.json b/datastructures/cluster.schema.json index 04bf792..0291b65 100644 --- a/datastructures/cluster.schema.json +++ b/datastructures/cluster.schema.json @@ -69,14 +69,14 @@ "alert" ] } - }, - "required": [ - "name", - "unit", - "scope", - "timestep" - ] - } + } + }, + "required": [ + "name", + "unit", + "scope", + "timestep" + ] } }, "subClusters": { @@ -202,32 +202,32 @@ "model" ] } - }, - "required":[ - "node", - "socket", - "memoryDomain" - ] + } }, "required":[ - "name", - "topology", - "processorType", - "socketsPerNode", - "coresPerSocket", - "threadsPerCore", - "flopRateScalar", - "flopRateSimd", - "memoryBandwidth" + "node", + "socket", + "memoryDomain" ] } }, "required":[ "name", - "metricConfig", - "subClusters" + "topology", + "processorType", + "socketsPerNode", + "coresPerSocket", + "threadsPerCore", + "flopRateScalar", + "flopRateSimd", + "memoryBandwidth" ] } } - } + }, + "required":[ + "name", + "metricConfig", + "subClusters" + ] } diff --git a/datastructures/job-data.schema.json b/datastructures/job-data.schema.json index e187fd7..71803df 100644 --- a/datastructures/job-data.schema.json +++ b/datastructures/job-data.schema.json @@ -36,9 +36,7 @@ "$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json" } }, - "required": [ - "node" - ] + "minProperties": 1 }, "mem_bw": { "description": "Main memory bandwidth", @@ -53,9 +51,7 @@ "$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json" } }, - "required": [ - "node" - ] + "minProperties": 1 }, "net_bw": { "description": "Total fast interconnect network bandwidth", @@ -69,18 +65,6 @@ "node" ] }, - "file_bw": { - "description": "Total file IO bandwidth", - "type": "object", - "properties": { - "node": { - "$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json" - } - }, - "required": [ - "node" - ] - }, "ipc": { "description": "Instructions executed per cycle", "properties": { @@ -100,9 +84,7 @@ "$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json" } }, - "required": [ - "node" - ] + "minProperties": 1 }, "cpu_used": { "description": "CPU active core utilization", @@ -123,27 +105,13 @@ "$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json" } }, - "required": [ - "node" - ] + "minProperties": 1 }, "cpu_load": { "description": "CPU requested core utilization (load 1m)", "properties": { "node": { "$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json" - }, - "socket": { - "$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json" - }, - "memoryDomain": { - "$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json" - }, - "core": { - "$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json" - }, - "hwthread": { - "$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json" } }, "required": [ @@ -169,9 +137,7 @@ "$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json" } }, - "required": [ - "node" - ] + "minProperties": 1 }, "flops_sp": { "description": "Single precision flops rate", @@ -192,9 +158,7 @@ "$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json" } }, - "required": [ - "node" - ] + "minProperties": 1 }, "vectorization_ratio": { "description": "Fraction of arithmetic instructions using SIMD instructions", @@ -215,9 +179,7 @@ "$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json" } }, - "required": [ - "node" - ] + "minProperties": 1 }, "cpu_power": { "description": "CPU power consumption", @@ -229,9 +191,7 @@ "$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json" } }, - "required": [ - "node" - ] + "minProperties": 1 }, "mem_power": { "description": "Memory power consumption", @@ -243,9 +203,7 @@ "$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json" } }, - "required": [ - "node" - ] + "minProperties": 1 }, "acc_utilization": { "description": "GPU utilization", @@ -255,7 +213,7 @@ } }, "required": [ - "node" + "accelerator" ] }, "acc_mem_used": { @@ -266,7 +224,7 @@ } }, "required": [ - "node" + "accelerator" ] }, "acc_power": { @@ -277,7 +235,7 @@ } }, "required": [ - "node" + "accelerator" ] }, "clock": { @@ -299,9 +257,7 @@ "$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json" } }, - "required": [ - "node" - ] + "minProperties": 1 }, "eth_read_bw": { "description": "Ethernet read bandwidth", @@ -325,222 +281,211 @@ "node" ] }, - "pfs_read_bw": { - "description": "Parallel file system read bandwidth", - "properties": { - "node": { - "$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json" - } + "filesystems": { + "description": "Array of filesystems", + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "type": { + "type": "string", + "enum": [ + "nfs", + "lustre", + "gpfs", + "nvme", + "ssd", + "hdd", + "beegfs" + ] + }, + "read_bw": { + "description": "File system read bandwidth", + "properties": { + "node": { + "$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json" + } + }, + "required": [ + "node" + ] + }, + "write_bw": { + "description": "File system write bandwidth", + "properties": { + "node": { + "$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json" + } + }, + "required": [ + "node" + ] + }, + "read_req": { + "description": "File system read requests", + "properties": { + "node": { + "$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json" + } + }, + "required": [ + "node" + ] + }, + "write_req": { + "description": "File system write requests", + "properties": { + "node": { + "$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json" + } + }, + "required": [ + "node" + ] + }, + "inodes": { + "description": "File system write requests", + "properties": { + "node": { + "$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json" + } + }, + "required": [ + "node" + ] + }, + "accesses": { + "description": "File system open and close", + "properties": { + "node": { + "$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json" + } + }, + "required": [ + "node" + ] + }, + "fsync": { + "description": "File system fsync", + "properties": { + "node": { + "$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json" + } + }, + "required": [ + "node" + ] + }, + "create": { + "description": "File system create", + "properties": { + "node": { + "$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json" + } + }, + "required": [ + "node" + ] + }, + "open": { + "description": "File system open", + "properties": { + "node": { + "$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json" + } + }, + "required": [ + "node" + ] + }, + "close": { + "description": "File system close", + "properties": { + "node": { + "$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json" + } + }, + "required": [ + "node" + ] + }, + "seek": { + "description": "File system seek", + "properties": { + "node": { + "$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json" + } + }, + "required": [ + "node" + ] + } + }, + "required": [ + "name", + "type", + "read_bw", + "write_bw", + "inodes" + ] }, - "required": [ - "node" - ] - }, - "pfs_write_bw": { - "description": "Parallel file system write bandwidth", - "properties": { - "node": { - "$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json" - } - }, - "required": [ - "node" - ] - }, - "pfs_read_req": { - "description": "Parallel file system read requests", - "properties": { - "node": { - "$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json" - } - }, - "required": [ - "node" - ] - }, - "pfs_write_req": { - "description": "Parallel file system write requests", - "properties": { - "node": { - "$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json" - } - }, - "required": [ - "node" - ] - }, - "pfs_inodes": { - "description": "Parallel file system inodes used", - "properties": { - "node": { - "$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json" - } - }, - "required": [ - "node" - ] - }, - "pfs_accesses": { - "description": "Parallel file system open and close", - "properties": { - "node": { - "$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json" - } - }, - "required": [ - "node" - ] - }, - "pfs_fsync": { - "description": "Parallel file system fsync", - "properties": { - "node": { - "$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json" - } - }, - "required": [ - "node" - ] - }, - "pfs_create": { - "description": "Parallel file system create", - "properties": { - "node": { - "$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json" - } - }, - "required": [ - "node" - ] - }, - "pfs_open": { - "description": "Parallel file system open", - "properties": { - "node": { - "$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json" - } - }, - "required": [ - "node" - ] - }, - "pfs_close": { - "description": "Parallel file system close", - "properties": { - "node": { - "$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json" - } - }, - "required": [ - "node" - ] - }, - "pfs_seek": { - "description": "Parallel file system seek", - "properties": { - "node": { - "$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json" - } - }, - "required": [ - "node" - ] - }, - "fs_read_bw": { - "description": "Local file system read bandwidth", - "properties": { - "node": { - "$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json" - } - }, - "required": [ - "node" - ] - }, - "fs_write_bw": { - "description": "Local file system write bandwidth", - "properties": { - "node": { - "$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json" - } - }, - "required": [ - "node" - ] - }, - "fs_inodes": { - "description": "Local file system inodes used", - "properties": { - "node": { - "$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json" - } - }, - "required": [ - "node" - ] - }, - "fs_accesses": { - "description": "Local file system open and close", - "properties": { - "node": { - "$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json" - } - }, - "required": [ - "node" - ] - }, - "ic_rcv_packets": { - "description": "Network interconnect read packets", - "properties": { - "node": { - "$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json" - } - }, - "required": [ - "node" - ] - }, - "ic_send_packets": { - "description": "Network interconnect send packet", - "properties": { - "node": { - "$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json" - } - }, - "required": [ - "node" - ] - }, - "ic_read_bw": { - "description": "Network interconnect read bandwidth", - "properties": { - "node": { - "$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json" - } - }, - "required": [ - "node" - ] - }, - "ic_write_bw": { - "description": "Network interconnect write bandwidth", - "properties": { - "node": { - "$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json" - } - }, - "required": [ - "node" - ] + "minItems": 1 } }, - "required": [ - "cpu_used", - "mem_used", - "flops_any", - "mem_bw", - "net_bw", - "file_bw" - ] + "ic_rcv_packets": { + "description": "Network interconnect read packets", + "properties": { + "node": { + "$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json" + } + }, + "required": [ + "node" + ] + }, + "ic_send_packets": { + "description": "Network interconnect send packet", + "properties": { + "node": { + "$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json" + } + }, + "required": [ + "node" + ] + }, + "ic_read_bw": { + "description": "Network interconnect read bandwidth", + "properties": { + "node": { + "$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json" + } + }, + "required": [ + "node" + ] + }, + "ic_write_bw": { + "description": "Network interconnect write bandwidth", + "properties": { + "node": { + "$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-data.schema.json" + } + }, + "required": [ + "node" + ] + } +}, +"required": [ + "cpu_used", + "mem_used", + "flops_any", + "mem_bw", + "net_bw", + "file_bw" +] } diff --git a/datastructures/job-meta.schema.json b/datastructures/job-meta.schema.json index b46990d..a77e340 100644 --- a/datastructures/job-meta.schema.json +++ b/datastructures/job-meta.schema.json @@ -238,66 +238,6 @@ "description": "Ethernet write bandwidth", "$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-statistics.schema.json" }, - "pfs_read_bw": { - "description": "Parallel file system read bandwidth", - "$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-statistics.schema.json" - }, - "pfs_write_bw": { - "description": "Parallel file system write bandwidth", - "$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-statistics.schema.json" - }, - "pfs_read_req": { - "description": "Parallel file system read requests", - "$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-statistics.schema.json" - }, - "pfs_write_req": { - "description": "Parallel file system write requests", - "$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-statistics.schema.json" - }, - "pfs_inodes": { - "description": "Parallel file system inodes used", - "$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-statistics.schema.json" - }, - "pfs_accesses": { - "description": "Parallel file system open and close", - "$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-statistics.schema.json" - }, - "pfs_fsync": { - "description": "Parallel file system fsync", - "$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-statistics.schema.json" - }, - "pfs_create": { - "description": "Parallel file system create", - "$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-statistics.schema.json" - }, - "pfs_open": { - "description": "Parallel file system open", - "$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-statistics.schema.json" - }, - "pfs_close": { - "description": "Parallel file system close", - "$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-statistics.schema.json" - }, - "pfs_seek": { - "description": "Parallel file system seek", - "$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-statistics.schema.json" - }, - "fs_read_bw": { - "description": "Local file system read bandwidth", - "$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-statistics.schema.json" - }, - "fs_write_bw": { - "description": "Local file system write bandwidth", - "$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-statistics.schema.json" - }, - "fs_inodes": { - "description": "Local file system inodes used", - "$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-statistics.schema.json" - }, - "fs_accesses": { - "description": "Local file system open and close", - "$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-statistics.schema.json" - }, "ic_rcv_packets": { "description": "Network interconnect read packets", "$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-statistics.schema.json" @@ -313,15 +253,89 @@ "ic_write_bw": { "description": "Network interconnect write bandwidth", "$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-statistics.schema.json" + }, + "filesystems": { + "description": "Array of filesystems", + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "type": { + "type": "string", + "enum": [ + "nfs", + "lustre", + "gpfs", + "nvme", + "ssd", + "hdd", + "beegfs" + ] + }, + "read_bw": { + "description": "File system read bandwidth", + "$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-statistics.schema.json" + }, + "write_bw": { + "description": "File system write bandwidth", + "$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-statistics.schema.json" + }, + "read_req": { + "description": "File system read requests", + "$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-statistics.schema.json" + }, + "write_req": { + "description": "File system write requests", + "$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-statistics.schema.json" + }, + "inodes": { + "description": "File system write requests", + "$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-statistics.schema.json" + }, + "accesses": { + "description": "File system open and close", + "$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-statistics.schema.json" + }, + "fsync": { + "description": "File system fsync", + "$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-statistics.schema.json" + }, + "create": { + "description": "File system create", + "$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-statistics.schema.json" + }, + "open": { + "description": "File system open", + "$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-statistics.schema.json" + }, + "close": { + "description": "File system close", + "$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-statistics.schema.json" + }, + "seek": { + "description": "File system seek", + "$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/job-metric-statistics.schema.json" + } + }, + "required": [ + "name", + "type", + "read_bw", + "write_bw", + "inodes" + ] + }, + "minItems": 1 } }, "required": [ "cpu_used", "mem_used", "flops_any", - "mem_bw", - "net_bw", - "file_bw" + "mem_bw" ] } }, @@ -330,12 +344,10 @@ "user", "project", "cluster", - "subCluster", "partition", "numNodes", "exclusive", "startTime", - "stopTime", "jobState", "duration", "resources", diff --git a/datastructures/job-metric-data.schema.json b/datastructures/job-metric-data.schema.json index 9c88896..8200864 100644 --- a/datastructures/job-metric-data.schema.json +++ b/datastructures/job-metric-data.schema.json @@ -9,18 +9,6 @@ "description": "Metric unit", "$ref": "https://raw.githubusercontent.com/ClusterCockpit/cc-specifications/master/datastructures/unit.schema.json" }, - "scope": { - "type": "string", - "enum": [ - "node", - "hwthread", - "core", - "memoryDomain", - "die", - "socket", - "accelerator" - ] - }, "timestep": { "description": "Measurement interval in seconds", "type": "integer" @@ -214,6 +202,7 @@ }, "required": [ "hostname", + "statistics", "data" ] } @@ -221,7 +210,6 @@ }, "required": [ "unit", - "scope", "timestep", "series" ] diff --git a/datastructures/unit.schema.json b/datastructures/unit.schema.json index c9e893f..1628197 100644 --- a/datastructures/unit.schema.json +++ b/datastructures/unit.schema.json @@ -15,6 +15,8 @@ "F/s", "CPI", "IPC", + "cpu_used", + "cpu_load", "Hz" ] },