104 lines
3.6 KiB
Python
Executable File
104 lines
3.6 KiB
Python
Executable File
from venv import create
|
|
import pandas as pd
|
|
import os
|
|
|
|
data_markers = {
|
|
"block_separator": "#----------------------------------------------------------------",
|
|
"benchmark_type": "# Benchmarking",
|
|
"processes_num": "# #processes = ",
|
|
"min_bytelen": "# Minimum message length in bytes",
|
|
"max_bytelen": "# Maximum message length in bytes",
|
|
"mpi_datatype": "# MPI_Datatype :",
|
|
"mpi_red_datatype": "# MPI_Datatype for reductions :",
|
|
"mpi_red_op": "# MPI_Op",
|
|
"end_of_table": "# All processes entering MPI_Finalize",
|
|
"creation_time": "# CREATION_TIME :",
|
|
"n_nodes": "# N_NODES :",
|
|
"off_mem_flag": "# OFF_MEM_FLAG :"
|
|
}
|
|
|
|
column_names = [
|
|
"benchmark_type",
|
|
"proc_num",
|
|
"msg_size_bytes",
|
|
"repetitions",
|
|
"t_min_usec",
|
|
"t_max_usec",
|
|
"t_avg_usec",
|
|
"mpi_datatype",
|
|
"mpi_red_datatype",
|
|
"mpi_red_op",
|
|
"creation_time",
|
|
"n_nodes",
|
|
"off_mem_flag",
|
|
]
|
|
|
|
data = list()
|
|
|
|
for file in os.listdir("data/"):
|
|
with open("data/"+file, 'r') as f:
|
|
lines = f.readlines()
|
|
|
|
past_preheader = False
|
|
in_header = False
|
|
in_body = False
|
|
|
|
btype = "NA"
|
|
proc_num = "NA"
|
|
mpi_datatype = "NA"
|
|
mpi_red_datatype = "NA"
|
|
mpi_red_op = "NA"
|
|
creation_time = "NA"
|
|
n_nodes = "NA"
|
|
off_mem_flag = "NA"
|
|
|
|
for line in lines:
|
|
if data_markers["block_separator"] in line:
|
|
if in_header and not past_preheader:
|
|
past_preheader = True
|
|
elif in_header and past_preheader:
|
|
in_body = True
|
|
in_header = not in_header
|
|
continue
|
|
if not in_header and not in_body and past_preheader:
|
|
if data_markers["mpi_datatype"] in line:
|
|
mpi_datatype = line.split()[-1]
|
|
elif data_markers["mpi_red_datatype"] in line:
|
|
mpi_red_datatype = line.split()[-1]
|
|
elif data_markers["mpi_red_op"] in line:
|
|
mpi_red_op = line.split()[-1]
|
|
|
|
if not in_header and not in_body and not past_preheader:
|
|
if data_markers["n_nodes"] in line:
|
|
n_nodes = line.split()[-1]
|
|
if data_markers["creation_time"] in line:
|
|
creation_time = line.split()[-1]
|
|
if data_markers["off_mem_flag"] in line:
|
|
off_mem_flag = line.split(":")[-1].strip()
|
|
if off_mem_flag == "": off_mem_flag = "NA"
|
|
else: off_mem_flag = off_mem_flag.replace("-off_cache","")
|
|
|
|
if past_preheader and in_header:
|
|
if data_markers["benchmark_type"] in line:
|
|
btype = line.split()[2]
|
|
if data_markers["processes_num"] in line:
|
|
proc_num = int(line.split()[3])
|
|
if in_body:
|
|
if "#" in line or "".join(line.split()) == "":
|
|
continue
|
|
if data_markers["end_of_table"] in line:
|
|
break
|
|
data.append([btype, proc_num]+[int(s) if s.isdigit()
|
|
else float(s) for s in line.split()] +
|
|
[
|
|
mpi_datatype,
|
|
mpi_red_datatype,
|
|
mpi_red_op,
|
|
creation_time,
|
|
n_nodes,
|
|
off_mem_flag,
|
|
])
|
|
|
|
df = pd.DataFrame(data, columns=column_names)
|
|
df.to_csv("data.csv", index=False)
|