new data: runs with collectives pinned to different available algorithms

.gitignore

@@ -1 +0,0 @@
-**/.DS_Store

launch_alg_bench.py

@@ -1,206 +0,0 @@
-import os
-import subprocess
-from datetime import datetime
-
-################ HELPER FUNCTIONS ################
-
-
-def load_template(template_path: str):
-    output_template = ""
-    with open(template_path, "r") as handle:
-        output_template = handle.read()
-    return output_template
-
-
-def write_batch(batch_fpath: str, batch_content: str):
-    with open(batch_fpath, "w") as handle:
-        _ = handle.write(batch_content)
-
-
-################### SETUP DIRS ###################
-output_dir = os.getcwd()+"/output/"
-err_dir = os.getcwd()+"/error/"
-batch_files_dir = os.getcwd()+"/batchs/"
-data_dir = os.getcwd()+"/data/"
-
-if os.path.isdir(output_dir) == False:
-    os.mkdir(output_dir)
-if os.path.isdir(err_dir) == False:
-    os.mkdir(err_dir)
-if os.path.isdir(data_dir) == False:
-    os.mkdir(data_dir)
-if os.path.isdir(batch_files_dir) == False:
-    os.mkdir(batch_files_dir)
-
-################ GLOBAL DEFAULTS #################
-mpi1_bin = "/home/hpc/ihpc/ihpc136h/workspace/mpi-benchmark-tool/bin/IMB-MPI1"
-default_parameter = {
-    "time_stamp": datetime.now().strftime("%y_%m_%d_%H-%M-%S"),
-    "job_name": "",
-    "output_dir": os.getcwd()+"/output/",
-    "err_dir": os.getcwd()+"/error/",
-    "data_dir": os.getcwd()+"/data/",
-    "n_procs": 18,
-    "off_cache_flag": "",
-    "bin": mpi1_bin,
-    "n_nodes": 1
-}
-
-algs_dic = [{'name': "Allgather",
-             'flag': "I_MPI_ADJUST_ALLGATHER",
-             'algs': [
-                 "Recursive doubling ",
-                 "Bruck`s ",
-                 "Ring ",
-                 "Topology aware Gatherv + Bcast ",
-                 "Knomial ",
-             ]},
-            {'name': "Allreduce",
-             'flag': "I_MPI_ADJUST_ALLREDUCE",
-             'algs': [
-                 "Recursive doubling ",
-                 "Rabenseifner`s ",
-                 "Reduce + Bcast ",
-                 "Topology aware Reduce + Bcast ",
-                 "Binomial gather + scatter ",
-                 "Topology aware binominal gather + scatter ",
-                 "Shumilin`s ring ",
-                 "Ring ",
-                 "Knomial ",
-                 "Topology aware SHM-based flat ",
-                 "Topology aware SHM-based Knomial ",
-                 "Topology aware SHM-based Knary ",
-             ]},
-
-            {'name': "Alltoall",
-             'flag': "I_MPI_ADJUST_ALLTOALL",
-             'algs': [
-                 "Bruck`s ",
-                 "Isend/Irecv + waitall ",
-                 "Pair wise exchange ",
-                 "Plum`s ",
-             ]},
-            {'name': "Barrier",
-             'flag': "I_MPI_ADJUST_BARRIER",
-             'algs': [
-                 "Dissemination ",
-                 "Recursive doubling ",
-                 "Topology aware dissemination ",
-                 "Topology aware recursive doubling ",
-                 "Binominal gather + scatter ",
-                 "Topology aware binominal gather + scatter ",
-                 "Topology aware SHM-based flat ",
-                 "Topology aware SHM-based Knomial ",
-                 "Topology aware SHM-based Knary ",
-             ]},
-            {'name': "Bcast",
-             'flag': "I_MPI_ADJUST_BCAST",
-             'algs': [
-                 "Binomial ",
-                 "Recursive doubling ",
-                 "Ring ",
-                 "Topology aware binomial ",
-                 "Topology aware recursive doubling ",
-                 "Topology aware ring ",
-                 "Shumilin`s ",
-                 "Knomial ",
-                 "Topology aware SHM-based flat ",
-                 "Topology aware SHM-based Knomial ",
-                 "Topology aware SHM-based Knary ",
-                 "NUMA aware SHM-based (SSE4.2) ",
-                 "NUMA aware SHM-based (AVX2) ",
-                 "NUMA aware SHM-based (AVX512) ",
-             ]},
-
-            {'name': "Gather",
-             'flag': "I_MPI_ADJUST_GATHER",
-             'algs': [
-                 "Binomial ",
-                 "Topology aware binomial ",
-                 "Shumilin`s ",
-                 "Binomial with segmentation ",
-             ]},
-
-            {'name': "Reduce_scatter",
-             'flag': "I_MPI_ADJUST_REDUCE_SCATTER",
-             'algs': [
-                 "Recursive halving ",
-                 "Pair wise exchange ",
-                 "Recursive doubling ",
-                 "Reduce + Scatterv ",
-                 "Topology aware Reduce + Scatterv ",
-             ]},
-
-            {'name': "Reduce",
-             'flag': "I_MPI_ADJUST_REDUCE",
-             'algs': [
-                 "Shumilin`s ",
-                 "Binomial ",
-                 "Topology aware Shumilin`s ",
-                 "Topology aware binomial ",
-                 "Rabenseifner`s ",
-                 "Topology aware Rabenseifner`s ",
-                 "Knomial ",
-                 "Topology aware SHM-based flat ",
-                 "Topology aware SHM-based Knomial ",
-                 "Topology aware SHM-based Knary ",
-                 "Topology aware SHM-based binomial ",
-             ]},
-
-            {'name': "Scatter",
-             'flag': "I_MPI_ADJUST_SCATTER",
-             'algs': [
-                 "Binomial ",
-                 "Topology aware binomial ",
-                 "Shumilin`s ",
-             ]},
-            ]
-log = ""
-
-############## MULTIPLE-NODE LAUNCH ##############
-off_cache_flags = [
-    "-off_cache -1",
-    "-off_cache 50",
-    ""
-]
-
-ndcnt = [
-    2,
-    3,
-    4,
-    5,
-    6,
-    7,
-    8,
-    9,
-    10
-]
-
-proc_per_node = 72
-multiple_node_parameter = dict(default_parameter)
-multiple_node_template = load_template("./templates/multinode_algs.template")
-
-for flag in off_cache_flags:
-    multiple_node_parameter["off_cache_flag"] = flag
-    for n_nodes in ndcnt:
-        n_procs = n_nodes*proc_per_node
-        multiple_node_parameter["n_procs"] = int(n_procs)
-        multiple_node_parameter["n_nodes"] = n_nodes
-        for alg_conf in algs_dic:
-            collective = alg_conf['name']
-            multiple_node_parameter["job_name"] = collective
-            multiple_node_parameter["alg_flag"] = alg_conf['flag']
-            algs = alg_conf["algs"]
-            for idx, alg in enumerate(algs):
-                multiple_node_parameter["alg_name"] = alg
-                multiple_node_parameter["alg_idx"] = idx
-                batch_file = os.path.join(batch_files_dir,
-                                          f"{collective}_{alg.strip().replace('`','').replace(' ','_').replace('/','_')}.sh")
-                write_batch(batch_file,
-                            multiple_node_template.format(**multiple_node_parameter))
-                result = subprocess.run(["sbatch", batch_file],
-                                        capture_output=True, text=True)
-                log += f"#{collective} {n_procs}" + "\n"
-                log += "\tSTDOUT:" + result.stdout + "\n"
-                log += "\tSTDERR:" + result.stderr + "\n"
-print(log)

postprocess_data_algs.py

@@ -1,112 +0,0 @@
-from venv import create
-import pandas as pd
-import os
-
-data_markers = {
-    "block_separator": "#----------------------------------------------------------------",
-    "benchmark_type": "# Benchmarking",
-    "processes_num": "# #processes = ",
-    "min_bytelen": "# Minimum message length in bytes",
-    "max_bytelen": "# Maximum message length in bytes",
-    "mpi_datatype": "# MPI_Datatype                   :",
-    "mpi_red_datatype": "# MPI_Datatype for reductions    :",
-    "mpi_red_op": "# MPI_Op",
-    "end_of_table": "# All processes entering MPI_Finalize",
-    "creation_time": "# CREATION_TIME :",
-    "n_nodes": "# N_NODES       :",
-    "off_cache_flag": "# OFF_CACHE_FLAG  :",
-    "algorithm":"# ALGORITHM  :"
-}
-
-column_names = [
-    "benchmark_type",
-    "proc_num",
-    "msg_size_bytes",
-    "repetitions",
-    "t_min_usec",
-    "t_max_usec",
-    "t_avg_usec",
-    "mpi_datatype",
-    "mpi_red_datatype",
-    "mpi_red_op",
-    "creation_time",
-    "n_nodes",
-    "off_cache_flag",
-    "algorithm"
-]
-
-data = list()
-
-for file in os.listdir("data/"):
-    with open("data/"+file, 'r') as f:
-        lines = f.readlines()
-
-        past_preheader = False
-        in_header = False
-        in_body = False
-
-        btype = "NA"
-        proc_num = "NA"
-        mpi_datatype = "NA"
-        mpi_red_datatype = "NA"
-        mpi_red_op = "NA"
-        creation_time = "NA"
-        n_nodes = "NA"
-        off_cache_flag = "NA"
-        algorithm = "NA"
-
-        for line in lines:
-            if data_markers["block_separator"] in line:
-                if in_header and not past_preheader:
-                    past_preheader = True
-                elif in_header and past_preheader:
-                    in_body = True
-                in_header = not in_header
-                continue
-            if not in_header and not in_body and past_preheader:
-                if data_markers["mpi_datatype"] in line:
-                    mpi_datatype = line.split()[-1]
-                elif data_markers["mpi_red_datatype"] in line:
-                    mpi_red_datatype = line.split()[-1]
-                elif data_markers["mpi_red_op"] in line:
-                    mpi_red_op = line.split()[-1]
-
-            if not in_header and not in_body and not past_preheader:
-                if data_markers["n_nodes"] in line:
-                    n_nodes = line.split()[-1]
-                if data_markers["creation_time"] in line:
-                    creation_time = line.split()[-1]
-                if data_markers["off_cache_flag"] in line:
-                    off_cache_flag = line.split(":")[-1].strip()
-                    if off_cache_flag == "": off_cache_flag = "NA"
-                    else: off_cache_flag = off_cache_flag.replace("-off_cache","")
-                if data_markers["algorithm"] in line:
-                    algorithm = line.split(":")[-1].strip()
-
-            if past_preheader and in_header:
-                if data_markers["benchmark_type"] in line:
-                    btype = line.split()[2]
-                if data_markers["processes_num"] in line:
-                    proc_num = int(line.split()[3])
-
-            if in_body:
-                if "#" in line or "".join(line.split()) == "":
-                    continue
-                if data_markers["end_of_table"] in line:
-                    break
-                if("int-overflow" in line) : continue 
-                if("out-of-mem" in line) : continue 
-                data.append([btype, proc_num]+[int(s) if s.isdigit()
-                            else float(s) for s in line.split()] +
-                            [
-                                mpi_datatype,
-                                mpi_red_datatype,
-                                mpi_red_op,
-                                creation_time,
-                                n_nodes,
-                                off_cache_flag,
-                                algorithm
-                ])
-
-df = pd.DataFrame(data, columns=column_names)
-df.to_csv("data.csv", index=False)

results-and-plotting/python/scripts/plot_alg.py

@@ -1,104 +0,0 @@
-from matplotlib.ticker import FuncFormatter
-import pandas as pd
-import matplotlib.pyplot as plt
-import numpy as np
-from matplotlib.lines import Line2D
-from enum import Enum
-
-
-class columns ():
-    benchmark_type = 'benchmark_type'
-    proc_num = 'proc_num'
-    msg_size_bytes = 'msg_size_bytes'
-    repetitions = 'repetitions'
-    t_min_usec = 't_min_usec'
-    t_max_usec = 't_max_usec'
-    t_avg_usec = 't_avg_usec'
-    mpi_datatype = 'mpi_datatype'
-    mpi_red_datatype = 'mpi_red_datatype'
-    mpi_red_op = 'mpi_red_op'
-    creation_time = 'creation_time'
-    n_nodes = 'n_nodes'
-    off_cache_flag = 'off_cache_flag'
-    algorithm = 'algorithm'
-
-
-class collectives(Enum):
-    Bcast = 'Bcast'
-    Reduce = 'Reduce'
-    Allreduce = 'Allreduce'
-    Alltoall = 'Alltoall'
-    Scatter = 'Scatter'
-    Reduce_scatter = 'Reduce_scatter'
-    Allgather = 'Allgather'
-    Gather = 'Gather'
-
-
-data_file = "./data/data_04_11_25_algs.csv"
-
-df_multinode = pd.read_csv(data_file, delimiter=',')
-df_multinode.fillna(0, inplace=True)
-df_multinode = df_multinode[df_multinode[columns.off_cache_flag] == 50]
-for c in collectives:
-
-    df_single = df_multinode[df_multinode[columns.benchmark_type]
-                             == c.value]
-    df_single = df_single[df_single[columns.msg_size_bytes] > 1000]
-    df_single = df_single[[columns.proc_num, columns.msg_size_bytes,
-                           columns.t_avg_usec, columns.algorithm]]
-
-    df_gather_best = df_single.loc[  # pyright: ignore
-        df_single.groupby([columns.msg_size_bytes, columns.proc_num])[  # pyright: ignore
-            columns.t_avg_usec].idxmin()
-    ].reset_index(drop=True)
-
-    df_gather_best = df_gather_best[df_gather_best[columns.msg_size_bytes] > 2**17]
-
-    pivot_best = df_gather_best.pivot(index=columns.msg_size_bytes,  # pyright: ignore
-                                      columns=columns.proc_num, values=columns.t_avg_usec)
-
-    X = pivot_best.columns.values   # proc_num
-    Y = pivot_best.index.values     # msg_size_bytes
-    X, Y = np.meshgrid(X, Y)   # pyright: ignore
-    Z = pivot_best.values
-
-    alg_pivot = df_gather_best.pivot(
-        index=columns.msg_size_bytes,
-        columns=columns.proc_num,
-        values=columns.algorithm
-    )
-
-    algorithms = alg_pivot.values.flatten()
-    unique_algs = sorted(pd.unique(algorithms[~pd.isna(algorithms)]))
-    color_map = {alg: i for i, alg in enumerate(unique_algs)}
-    color_values = np.array([color_map.get(a, np.nan) for a in algorithms])
-
-    fig = plt.figure(figsize=(16, 9))
-    ax = fig.add_subplot(111, projection='3d')
-    surf = ax.plot_wireframe(X, Y, Z,
-                             color='black', linewidths=1)
-    surf_points = ax.scatter(X,
-                             Y, Z, c=color_values, cmap='viridis', s=20, depthshade=False)  # pyright: ignore
-
-    handles = [
-        Line2D([0], [0],
-               marker='o', color='w',
-               label=alg,
-               markerfacecolor=plt.cm.viridis(
-                   color_map[alg] / max(len(unique_algs)-1, 1)),
-               markersize=8)
-        for alg in unique_algs
-    ]
-
-    ax.legend(handles=handles, title="Algorithm", loc='upper right')
-    ax.set_xlabel("Process Count")
-    ax.set_ylabel("Message Size [B]")
-    ax.set_zlabel("Average Time [μs]")
-    ax.set_title(f"{c.value}")
-    ax.set_xticks(pivot_best.columns.values)  # pyright: ignore
-    ax.set_xticklabels(pivot_best.columns.values)
-    ax.set_yticks(Y[:, 0])
-    ymin, ymax = ax.get_ylim()
-    ax.set_ylim(ymin*0.8, ymax)   # 30% more space at top
-    ax.set_yticklabels([f"$2^{{{int(np.log2(v))}}}$" for v in Y[:, 0]])
-    plt.savefig(f"./plots/{c.value.lower()}_algcomp.png")

results-and-plotting/python/scripts/plot_alg_log.py

@@ -1,127 +0,0 @@
-from matplotlib.ticker import FuncFormatter
-import pandas as pd
-import matplotlib.pyplot as plt
-import numpy as np
-from matplotlib.lines import Line2D
-from enum import Enum
-
-
-class columns ():
-    benchmark_type = 'benchmark_type'
-    proc_num = 'proc_num'
-    msg_size_bytes = 'msg_size_bytes'
-    repetitions = 'repetitions'
-    t_min_usec = 't_min_usec'
-    t_max_usec = 't_max_usec'
-    t_avg_usec = 't_avg_usec'
-    mpi_datatype = 'mpi_datatype'
-    mpi_red_datatype = 'mpi_red_datatype'
-    mpi_red_op = 'mpi_red_op'
-    creation_time = 'creation_time'
-    n_nodes = 'n_nodes'
-    off_cache_flag = 'off_cache_flag'
-    algorithm = 'algorithm'
-
-
-class collectives(Enum):
-    Bcast = 'Bcast'
-    Reduce = 'Reduce'
-    Allreduce = 'Allreduce'
-    Alltoall = 'Alltoall'
-    Scatter = 'Scatter'
-    Reduce_scatter = 'Reduce_scatter'
-    Allgather = 'Allgather'
-    Gather = 'Gather'
-
-
-def log_notation(val, pos):
-    return f" $1e{int(val)}$" if val != 0 else "1"
-
-
-def log2_notation(val, pos):
-    return "$2^{"+str(int(val))+"}$" if val != 0 else "1"
-
-
-data_file = "./data/data_04_11_25_algs.csv"
-
-df_multinode = pd.read_csv(data_file, delimiter=',')
-df_multinode.fillna(0, inplace=True)
-df_multinode = df_multinode[df_multinode[columns.off_cache_flag] == 50]
-for c in collectives:
-
-    df_single = df_multinode[df_multinode[columns.benchmark_type]
-                             == c.value]
-    df_single = df_single[df_single[columns.msg_size_bytes] > 1000]
-    df_single = df_single[[columns.proc_num, columns.msg_size_bytes,
-                           columns.t_avg_usec, columns.algorithm]]
-
-    df_gather_best = df_single.loc[  # pyright: ignore
-        df_single.groupby([columns.msg_size_bytes, columns.proc_num])[  # pyright: ignore
-            columns.t_avg_usec].idxmin()
-    ].reset_index(drop=True)
-
-    df_gather_worst = df_single.loc[  # pyright: ignore
-        df_single.groupby([columns.msg_size_bytes, columns.proc_num])[  # pyright: ignore
-            columns.t_avg_usec].idxmax()
-    ].reset_index(drop=True)
-    # df_gather_select = df_gather_select[df_gather_select[columns.msg_size_bytes] > 2**17]
-
-    pivot_best = df_gather_best.pivot(index=columns.msg_size_bytes,  # pyright: ignore
-                                      columns=columns.proc_num, values=columns.t_avg_usec)
-    pivot_worst = df_gather_worst.pivot(index=columns.msg_size_bytes,  # pyright: ignore
-                                        columns=columns.proc_num, values=columns.t_avg_usec)
-
-    X = pivot_best.columns.values   # proc_num
-    Y = pivot_best.index.values     # msg_size_bytes
-    X, Y = np.meshgrid(X, Y)   # pyright: ignore
-    Z = pivot_best.values
-
-    X_w = pivot_worst.columns.values   # proc_num
-    Y_w = pivot_worst.index.values     # msg_size_bytes
-    X_w, Y_w = np.meshgrid(X_w, Y_w)   # pyright: ignore
-    Z_w = pivot_worst.values
-
-    alg_pivot = df_gather_best.pivot(
-        index=columns.msg_size_bytes,
-        columns=columns.proc_num,
-        values=columns.algorithm
-    )
-
-    algorithms = alg_pivot.values.flatten()
-    unique_algs = sorted(pd.unique(algorithms[~pd.isna(algorithms)]))
-    color_map = {alg: i for i, alg in enumerate(unique_algs)}
-    color_values = np.array([color_map.get(a, np.nan) for a in algorithms])
-
-    fig = plt.figure(figsize=(16, 9))
-    ax = fig.add_subplot(111, projection='3d')
-    surf = ax.plot_wireframe(X, np.log2(Y), np.log(Z),
-                             color='black', linewidths=1)
-    surf = ax.plot_wireframe(X_w, np.log2(
-        Y_w), np.log(Z_w), color='gray', linewidths=0.3)
-
-    surf_points = ax.scatter(X, np.log2(
-        Y), np.log(Z), c=color_values, cmap='viridis', s=20, depthshade=False)  # pyright: ignore
-
-    surf_points = ax.scatter(X_w, np.log2(
-        Y_w), np.log(Z_w), c='gray', alpha=0.2,  s=20, depthshade=False)  # pyright: ignore
-
-    handles = [
-        Line2D([0], [0],
-               marker='o', color='w',
-               label=alg,
-               markerfacecolor=plt.cm.viridis(
-                   color_map[alg] / max(len(unique_algs)-1, 1)),
-               markersize=8)
-        for alg in unique_algs
-    ]
-
-    ax.legend(handles=handles, title="Algorithm", loc='upper right')
-    ax.set_xlabel("Process Count")
-    ax.set_ylabel("Message Size [B] (log2)")
-    ax.set_zlabel("Average Time [μs] (log10)")
-    ax.set_title(f"{c.value}")
-    ax.set_xticks(pivot_best.columns.values)  # pyright: ignore
-    ax.set_xticklabels(pivot_best.columns.values)
-    ax.yaxis.set_major_formatter(FuncFormatter(log2_notation))
-    ax.zaxis.set_major_formatter(FuncFormatter(log_notation))
-    plt.savefig(f"./plots/{c.value.lower()}_algcomp_log.png")

templates/multinode_algs.template

@@ -1,12 +1,11 @@
 #!/bin/bash -l
-#SBATCH --job-name={job_name}_{n_procs}_{alg_idx}
+#SBATCH --job-name={job_name}_{n_procs}
 #SBATCH --output={output_dir}{job_name}_{n_procs}.out
 #SBATCH --error={err_dir}{job_name}_{n_procs}.err
 #SBATCH --nodes={n_nodes}
 #SBATCH --nodelist=f01[01-64]
 #SBATCH --time=00:30:00
 #SBATCH --export=NONE
-
 # SwitchName=fswibl01 Level=0 LinkSpeed=1 Nodes=f01[01-64]
 # SwitchName=fswibl02 Level=0 LinkSpeed=1 Nodes=f02[01-64]
 # SwitchName=fswibl03 Level=0 LinkSpeed=1 Nodes=f03[01-64]
@@ -20,19 +19,17 @@
 # SwitchName=fswibl11 Level=0 LinkSpeed=1 Nodes=f08[01-64]
 # SwitchName=fswibl12 Level=0 LinkSpeed=1 Nodes=f09[01-64]
 # SwitchName=fswibl13 Level=0 LinkSpeed=1 Nodes=f10[01-64]
-
 unset SLURM_EXPORT_ENV
 
 module load intel intelmpi 
 
-export {alg_flag}={alg_idx}
+export I_MPI_ADJUST_{capital_jobname}={algnumber}
 
 OUTPUT_FILENAME="{data_dir}/{job_name}_$SLURM_JOB_ID.dat"
 
 echo  "# CREATION_TIME : {time_stamp}" > $OUTPUT_FILENAME
 echo  "# N_NODES       : {n_nodes}" >> $OUTPUT_FILENAME
 echo  "# OFF_CACHE_FLAG  : {off_cache_flag}">> $OUTPUT_FILENAME
-echo  "# ALGORITHM  : {alg_name}">> $OUTPUT_FILENAME
 
 srun --cpu-freq=2000000-2000000:performance -N {n_nodes} -n{n_procs} {bin} {job_name} -npmin {n_procs} {off_cache_flag} -mem 2 -time 60 >> $OUTPUT_FILENAME