Added some Metadata

launch_bench_singlenode.py

@@ -15,19 +15,20 @@ def write_batch(batch_fpath: str, batch_content: str):
         _ = handle.write(batch_content)
 
 
-collectives = ["Reduce",
+collectives = [
-               "Reduce_scatter",
+    "Reduce",
-               "Allreduce",
+    "Reduce_scatter",
-               "Allgather",
+    "Allreduce",
-               "Allgatherv",
+    "Allgather",
-               "Scatter",
+    "Allgatherv",
-               "Scatterv",
+    "Scatter",
-               "Gather",
+    "Scatterv",
-               "Gatherv",
+    "Gather",
-               "Alltoall",
+    "Gatherv",
-               "Bcast",
+    "Alltoall",
-               # "Barrier"
+    "Bcast",
-               ]
+    # "Barrier"
+]
 
 procnt = [
     18,
@@ -35,18 +36,21 @@ procnt = [
     54,
     72
 ]
+
 mpi1_bin = "/home/hpc/ihpc/ihpc136h/workspace/prototyping/bin/IMB-MPI1"
 slurm_template = load_template("templates/bench.template")
 
-template_parameter = {"time_stamp": datetime.now().strftime("%y_%m_%d_%H-%M-%S"),
+template_parameter = {
-                      "job_name": "",
+    "time_stamp": datetime.now().strftime("%y_%m_%d_%H-%M-%S"),
-                      "output_dir": os.getcwd()+"/output/",
+    "job_name": "",
-                      "err_dir": os.getcwd()+"/error/",
+    "output_dir": os.getcwd()+"/output/",
-                      "data_dir": os.getcwd()+"/data/",
+    "err_dir": os.getcwd()+"/error/",
-                      "n_procs": 18,
+    "data_dir": os.getcwd()+"/data/",
-                      "off_mem_flag": "",
+    "n_procs": 18,
-                      "bin": mpi1_bin
+    "off_mem_flag": "-off_cache 50",
-                      }
+    "bin": mpi1_bin,
+    "n_nodes": 1
+}
 
 output_dir = os.getcwd()+"/output/"
 err_dir = os.getcwd()+"/error/"
@@ -76,4 +80,4 @@ for n_procs in procnt:
         log += "\tSTDOUT:" + result.stdout + "\n"
         log += "\tSTDERR:" + result.stderr + "\n"
 print(log)
-_ = subprocess.run(["./clean.sh"])
+# _ = subprocess.run(["./clean.sh"])

postprocess_data.py

@@ -1,3 +1,4 @@
+from venv import create
 import pandas as pd
 import os
 
@@ -11,6 +12,9 @@ data_markers = {
     "mpi_red_datatype": "# MPI_Datatype for reductions    :",
     "mpi_red_op": "# MPI_Op",
     "end_of_table": "# All processes entering MPI_Finalize",
+    "creation_time": "# CREATION_TIME :",
+    "n_nodes": "# N_NODES       :",
+    "off_mem_flag": "# OFF_MEM_FLAG  :"
 }
 
 column_names = [
@@ -24,20 +28,29 @@ column_names = [
     "mpi_datatype",
     "mpi_red_datatype",
     "mpi_red_op",
+    "creation_time",
+    "n_nodes",
+    "off_mem_flag",
 ]
 
 data = list()
+
 for file in os.listdir("data/"):
     with open("data/"+file, 'r') as f:
         lines = f.readlines()
+
         past_preheader = False
         in_header = False
         in_body = False
-        btype = None
+
-        proc_num = None
+        btype = "NA"
-        mpi_datatype = None
+        proc_num = "NA"
-        mpi_red_datatype = None
+        mpi_datatype = "NA"
-        mpi_red_op = None
+        mpi_red_datatype = "NA"
+        mpi_red_op = "NA"
+        creation_time = "NA"
+        n_nodes = "NA"
+        off_mem_flag = "NA"
 
         for line in lines:
             if data_markers["block_separator"] in line:
@@ -55,6 +68,16 @@ for file in os.listdir("data/"):
                 elif data_markers["mpi_red_op"] in line:
                     mpi_red_op = line.split()[-1]
 
+            if not in_header and not in_body and not past_preheader:
+                if data_markers["n_nodes"] in line:
+                    n_nodes = line.split()[-1]
+                if data_markers["creation_time"] in line:
+                    creation_time = line.split()[-1]
+                if data_markers["off_mem_flag"] in line:
+                    off_mem_flag = line.split(":")[-1].strip()
+                    if off_mem_flag == "": off_mem_flag = "NA"
+                    else: off_mem_flag = off_mem_flag.replace("-off_cache","")
+
             if past_preheader and in_header:
                 if data_markers["benchmark_type"] in line:
                     btype = line.split()[2]
@@ -66,7 +89,15 @@ for file in os.listdir("data/"):
                 if data_markers["end_of_table"] in line:
                     break
                 data.append([btype, proc_num]+[int(s) if s.isdigit()
-                            else float(s) for s in line.split()] + [mpi_datatype, mpi_red_datatype, mpi_red_op])
+                            else float(s) for s in line.split()] +
+                            [
+                                mpi_datatype,
+                                mpi_red_datatype,
+                                mpi_red_op,
+                                creation_time,
+                                n_nodes,
+                                off_mem_flag,
+                ])
 
 df = pd.DataFrame(data, columns=column_names)
 df.to_csv("data.csv", index=False)

templates/bench.template

@@ -2,7 +2,7 @@
 #SBATCH --job-name={job_name}_{n_procs}
 #SBATCH --output={output_dir}{job_name}_{n_procs}.out
 #SBATCH --error={err_dir}{job_name}_{n_procs}.err
-#SBATCH --nodes=1
+#SBATCH --nodes={n_nodes}
 #SBATCH --time=00:10:00
 #SBATCH --export=NONE
 
@@ -12,7 +12,13 @@ module load intel intelmpi likwid
 
 unset I_MPI_PMI_LIBRARY
 export  LIKWID_SILENT=1
-echo CREATION_TIME {time_stamp}
 
-srun --cpu-freq=2000000-2000000:performance  ./likwid-mpirun -np {n_procs} -mpi intelmpi -omp intel -nperdomain M:18 {bin} {job_name} -npmin {n_procs} {off_mem_flag}  > {data_dir}/{job_name}_{n_procs}.dat
+OUTPUT_FILENAME="{data_dir}/{job_name}_$SLURM_JOB_ID.dat"
+
+echo  "# CREATION_TIME : {time_stamp}" > $OUTPUT_FILENAME
+echo  "# N_NODES       : {n_nodes}" >> $OUTPUT_FILENAME
+echo  "# OFF_MEM_FLAG  : {off_mem_flag}">> $OUTPUT_FILENAME
+
+srun --cpu-freq=2000000-2000000:performance  ./likwid-mpirun -np {n_procs} -mpi intelmpi -omp intel -nperdomain M:18 {bin} {job_name} -npmin {n_procs} {off_mem_flag}  >> $OUTPUT_FILENAME 
+