initial_commit

2025-05-28 19:46:24 +02:00
commit 8b80f1fd28
6 changed files with 261 additions and 0 deletions
--- a/postprocess_data.py
+++ b/postprocess_data.py
@@ -0,0 +1,72 @@
+import pandas as pd
+import os
+
+data_markers = {
+    "block_separator": "#----------------------------------------------------------------",
+    "benchmark_type": "# Benchmarking",
+    "processes_num": "# #processes = ",
+    "min_bytelen": "# Minimum message length in bytes",
+    "max_bytelen": "# Maximum message length in bytes",
+    "mpi_datatype": "# MPI_Datatype                   :",
+    "mpi_red_datatype": "# MPI_Datatype for reductions    :",
+    "mpi_red_op": "# MPI_Op",
+    "end_of_table": "# All processes entering MPI_Finalize",
+}
+
+column_names = [
+    "benchmark_type",
+    "proc_num",
+    "msg_size_bytes",
+    "repetitions",
+    "t_min_usec",
+    "t_max_usec",
+    "t_avg_usec",
+    "mpi_datatype",
+    "mpi_red_datatype",
+    "mpi_red_op",
+]
+
+data = list()
+for file in os.listdir("data/"):
+    with open("data/"+file, 'r') as f:
+        lines = f.readlines()
+        past_preheader = False
+        in_header = False
+        in_body = False
+        btype = None
+        proc_num = None
+        mpi_datatype = None
+        mpi_red_datatype = None
+        mpi_red_op = None
+
+        for line in lines:
+            if data_markers["block_separator"] in line:
+                if in_header and not past_preheader:
+                    past_preheader = True
+                elif in_header and past_preheader:
+                    in_body = True
+                in_header = not in_header
+                continue
+            if not in_header and not in_body and past_preheader:
+                if data_markers["mpi_datatype"] in line:
+                    mpi_datatype = line.split()[-1]
+                elif data_markers["mpi_red_datatype"] in line:
+                    mpi_red_datatype = line.split()[-1]
+                elif data_markers["mpi_red_op"] in line:
+                    mpi_red_op = line.split()[-1]
+
+            if past_preheader and in_header:
+                if data_markers["benchmark_type"] in line:
+                    btype = line.split()[2]
+                if data_markers["processes_num"] in line:
+                    proc_num = int(line.split()[3])
+            if in_body:
+                if "#" in line or "".join(line.split()) == "":
+                    continue
+                if data_markers["end_of_table"] in line:
+                    break
+                data.append([btype, proc_num]+[int(s) if s.isdigit()
+                            else float(s) for s in line.split()] + [mpi_datatype, mpi_red_datatype, mpi_red_op])
+
+df = pd.DataFrame(data, columns=column_names)
+df.to_csv("data.csv", index=False)