73 lines
2.5 KiB
Python
Executable File
73 lines
2.5 KiB
Python
Executable File
import pandas as pd
|
|
import os
|
|
|
|
data_markers = {
|
|
"block_separator": "#----------------------------------------------------------------",
|
|
"benchmark_type": "# Benchmarking",
|
|
"processes_num": "# #processes = ",
|
|
"min_bytelen": "# Minimum message length in bytes",
|
|
"max_bytelen": "# Maximum message length in bytes",
|
|
"mpi_datatype": "# MPI_Datatype :",
|
|
"mpi_red_datatype": "# MPI_Datatype for reductions :",
|
|
"mpi_red_op": "# MPI_Op",
|
|
"end_of_table": "# All processes entering MPI_Finalize",
|
|
}
|
|
|
|
column_names = [
|
|
"benchmark_type",
|
|
"proc_num",
|
|
"msg_size_bytes",
|
|
"repetitions",
|
|
"t_min_usec",
|
|
"t_max_usec",
|
|
"t_avg_usec",
|
|
"mpi_datatype",
|
|
"mpi_red_datatype",
|
|
"mpi_red_op",
|
|
]
|
|
|
|
data = list()
|
|
for file in os.listdir("data/"):
|
|
with open("data/"+file, 'r') as f:
|
|
lines = f.readlines()
|
|
past_preheader = False
|
|
in_header = False
|
|
in_body = False
|
|
btype = None
|
|
proc_num = None
|
|
mpi_datatype = None
|
|
mpi_red_datatype = None
|
|
mpi_red_op = None
|
|
|
|
for line in lines:
|
|
if data_markers["block_separator"] in line:
|
|
if in_header and not past_preheader:
|
|
past_preheader = True
|
|
elif in_header and past_preheader:
|
|
in_body = True
|
|
in_header = not in_header
|
|
continue
|
|
if not in_header and not in_body and past_preheader:
|
|
if data_markers["mpi_datatype"] in line:
|
|
mpi_datatype = line.split()[-1]
|
|
elif data_markers["mpi_red_datatype"] in line:
|
|
mpi_red_datatype = line.split()[-1]
|
|
elif data_markers["mpi_red_op"] in line:
|
|
mpi_red_op = line.split()[-1]
|
|
|
|
if past_preheader and in_header:
|
|
if data_markers["benchmark_type"] in line:
|
|
btype = line.split()[2]
|
|
if data_markers["processes_num"] in line:
|
|
proc_num = int(line.split()[3])
|
|
if in_body:
|
|
if "#" in line or "".join(line.split()) == "":
|
|
continue
|
|
if data_markers["end_of_table"] in line:
|
|
break
|
|
data.append([btype, proc_num]+[int(s) if s.isdigit()
|
|
else float(s) for s in line.split()] + [mpi_datatype, mpi_red_datatype, mpi_red_op])
|
|
|
|
df = pd.DataFrame(data, columns=column_names)
|
|
df.to_csv("data.csv", index=False)
|