Ignore Mac Trash

.gitignore

@@ -1,11 +1 @@
-# Ignore everything
-*
-
-# But not these!
-!.gitignore
-!README.md
-!*.py
-!*.template
-# Optional: Keep subdirectories and their Python files
-
-!*/
+**/.DS_Store

launch_alg_bench.py

@@ -3,6 +3,8 @@ import subprocess
 from datetime import datetime
 
 ################ HELPER FUNCTIONS ################
+
+
 def load_template(template_path: str):
     output_template = ""
     with open(template_path, "r") as handle:
@@ -14,6 +16,7 @@ def write_batch(batch_fpath: str, batch_content: str):
     with open(batch_fpath, "w") as handle:
         _ = handle.write(batch_content)
 
+
 ################### SETUP DIRS ###################
 output_dir = os.getcwd()+"/output/"
 err_dir = os.getcwd()+"/error/"

results-and-plotting/python/notebooks/bcast_analysis.ipynb

@@ -0,0 +1,175 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "da7c16b4",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pandas as pd\n",
+    "import numpy as np\n",
+    "import matplotlib.pyplot as plt\n",
+    "import seaborn as sns\n",
+    "from scipy.optimize import curve_fit\n",
+    "from matplotlib.cm import get_cmap"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "47341b1d",
+   "metadata": {},
+   "source": [
+    "# Alltoall "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "1cc39aab",
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "",
+     "evalue": "",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[1;31mnotebook controller is DISPOSED. \n",
+      "\u001b[1;31mView Jupyter <a href='command:jupyter.viewOutput'>log</a> for further details."
+     ]
+    }
+   ],
+   "source": [
+    "df_multinode = pd.read_csv(\"../data/data-multi-defand100cflag.csv\",delimiter = \",\")\n",
+    "df_multinode['benchmark_type'].unique()\n",
+    "df_gather = df_multinode[df_multinode[\"benchmark_type\"]==\"Bcast\"][df_multinode['msg_size_bytes']>1024][df_multinode['off_cache_flag']==-1]\n",
+    "df_gather.columns.tolist()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "4336d3c6",
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "",
+     "evalue": "",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[1;31mnotebook controller is DISPOSED. \n",
+      "\u001b[1;31mView Jupyter <a href='command:jupyter.viewOutput'>log</a> for further details."
+     ]
+    }
+   ],
+   "source": [
+    "def model(proc_num, alpha, beta, msg_size):\n",
+    "    return (alpha * msg_size * (proc_num - 72) * 72) / (12.5 * 1e3) + 1e6*beta\n",
+    "\n",
+    "results = []\n",
+    "msg_sizes = sorted(df_gather['msg_size_bytes'].unique())\n",
+    "n_rows = int(np.ceil(len(msg_sizes) / 3))\n",
+    "n_cols = min(len(msg_sizes), 3)\n",
+    "fig, axes = plt.subplots(n_rows, n_cols, figsize=(5*n_cols, 4*n_rows), squeeze=False)\n",
+    "cmap = get_cmap('tab10')\n",
+    "\n",
+    "for idx, (msg_size, group) in enumerate(df_gather.groupby('msg_size_bytes')):\n",
+    "    x = group['proc_num'].values.copy()\n",
+    "    y = group['t_avg_usec'].values.copy()\n",
+    "    sorted_indices = np.argsort(x)\n",
+    "    x = x[sorted_indices]\n",
+    "    y = y[sorted_indices]\n",
+    "    fit_func = lambda proc_num, alpha, beta: model(proc_num, alpha, beta, msg_size)\n",
+    "    popt, _ = curve_fit(fit_func, x, y, bounds=([1, 0], [np.inf, np.inf]))\n",
+    "    alpha, beta = popt\n",
+    "    results.append({'msg_size_bytes': msg_size, 'alpha': alpha, 'beta': beta})\n",
+    "\n",
+    "    x_fit = np.linspace(min(x), max(x), 100)\n",
+    "    y_fit = fit_func(x_fit, alpha, beta)\n",
+    "    y_speed = model(x_fit,1,0,msg_size)\n",
+    "    row, col = divmod(idx, n_cols)\n",
+    "    ax = axes[row][col]\n",
+    "\n",
+    "    color = cmap(idx % 10)\n",
+    "    # ax.scatter(x, y/1e6, label='Data', color=color)\n",
+    "    ax.plot(x, y/1e6, label='Data', color=color)\n",
+    "    # ax.plot(x_fit, y_fit/1e6, linestyle='--', color=color, alpha=0.5, label='Fit')\n",
+    "    # ax.plot(x_fit, y_speed/1e6, linestyle='--', color='red', alpha=0.1, label='Fit')\n",
+    "    ax.set_title(f'msg_size: {msg_size} bytes')\n",
+    "    ax.set_xlabel('num. proc.')\n",
+    "    ax.set_ylabel('Average Time [s]')\n",
+    "    ax.set_xticks(x)\n",
+    "    ax.grid(True)\n",
+    "    max_data =(x[-1]-72)*72*msg_size\n",
+    "    min_data =(x[0]-72)*72*msg_size\n",
+    "\n",
+    "    textstr = \"\"\n",
+    "    # if(max_data > 1e9):\n",
+    "    #     textstr+=f\"max data = {max_data/1e9:0.2f}GB\\n\" \n",
+    "    # else:\n",
+    "    #     textstr+=f\"max data = {max_data/1e6:0.2f}MB\\n\" \n",
+    "\n",
+    "    # if(min_data > 1e9):\n",
+    "    #     textstr+=f\"min data = {min_data/1e9:0.2f}GB\\n\" \n",
+    "    # else:\n",
+    "    #     textstr+=f\"min data = {min_data/1e6:0.2f}MB\\n\" \n",
+    "    # textstr += r\"$\\alpha$\" +f\"= {alpha:.3e}\\n\"+r\"$b_{eff}=$\"+f\"{12.5/alpha:0.3f}Gbps\\n\"+\\\n",
+    "    #            r\"$\\beta$\"+f\"= {beta:.3e} s\"\n",
+    "    # ax.text(0.95, 0.05, textstr, transform=ax.transAxes,\n",
+    "    #         fontsize=10, verticalalignment='bottom',\n",
+    "    #         horizontalalignment='right',\n",
+    "    #         bbox=dict(boxstyle='round', facecolor='white', alpha=0.8))\n",
+    "\n",
+    "fig.suptitle('Alltoall Time Fit per Message Size\\nDots = Data Points | Dashed Lines = Fits\\n off_mem=-1', fontsize=14)\n",
+    "fig.tight_layout(rect=[0, 0.03, 1, 0.95])\n",
+    "# plt.savefig(\"plots/alltoall_analysis.png\",dpi=300)\n",
+    "plt.show()\n",
+    "\n",
+    "fit_results = pd.DataFrame(results)\n",
+    "fit_results['inv_alpha'] = 1 / fit_results['alpha']\n",
+    "print(fit_results)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "ce632d6f",
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "",
+     "evalue": "",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[1;31mnotebook controller is DISPOSED. \n",
+      "\u001b[1;31mView Jupyter <a href='command:jupyter.viewOutput'>log</a> for further details."
+     ]
+    }
+   ],
+   "source": [
+    "df_gather[df_gather['msg_size_bytes']==1048576]"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "data",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.13.2"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

results-and-plotting/python/scripts/plot_alg.py

@@ -0,0 +1,104 @@
+from matplotlib.ticker import FuncFormatter
+import pandas as pd
+import matplotlib.pyplot as plt
+import numpy as np
+from matplotlib.lines import Line2D
+from enum import Enum
+
+
+class columns ():
+    benchmark_type = 'benchmark_type'
+    proc_num = 'proc_num'
+    msg_size_bytes = 'msg_size_bytes'
+    repetitions = 'repetitions'
+    t_min_usec = 't_min_usec'
+    t_max_usec = 't_max_usec'
+    t_avg_usec = 't_avg_usec'
+    mpi_datatype = 'mpi_datatype'
+    mpi_red_datatype = 'mpi_red_datatype'
+    mpi_red_op = 'mpi_red_op'
+    creation_time = 'creation_time'
+    n_nodes = 'n_nodes'
+    off_cache_flag = 'off_cache_flag'
+    algorithm = 'algorithm'
+
+
+class collectives(Enum):
+    Bcast = 'Bcast'
+    Reduce = 'Reduce'
+    Allreduce = 'Allreduce'
+    Alltoall = 'Alltoall'
+    Scatter = 'Scatter'
+    Reduce_scatter = 'Reduce_scatter'
+    Allgather = 'Allgather'
+    Gather = 'Gather'
+
+
+data_file = "./data/data_04_11_25_algs.csv"
+
+df_multinode = pd.read_csv(data_file, delimiter=',')
+df_multinode.fillna(0, inplace=True)
+df_multinode = df_multinode[df_multinode[columns.off_cache_flag] == 50]
+for c in collectives:
+
+    df_single = df_multinode[df_multinode[columns.benchmark_type]
+                             == c.value]
+    df_single = df_single[df_single[columns.msg_size_bytes] > 1000]
+    df_single = df_single[[columns.proc_num, columns.msg_size_bytes,
+                           columns.t_avg_usec, columns.algorithm]]
+
+    df_gather_best = df_single.loc[  # pyright: ignore
+        df_single.groupby([columns.msg_size_bytes, columns.proc_num])[  # pyright: ignore
+            columns.t_avg_usec].idxmin()
+    ].reset_index(drop=True)
+
+    df_gather_best = df_gather_best[df_gather_best[columns.msg_size_bytes] > 2**17]
+
+    pivot_best = df_gather_best.pivot(index=columns.msg_size_bytes,  # pyright: ignore
+                                      columns=columns.proc_num, values=columns.t_avg_usec)
+
+    X = pivot_best.columns.values   # proc_num
+    Y = pivot_best.index.values     # msg_size_bytes
+    X, Y = np.meshgrid(X, Y)   # pyright: ignore
+    Z = pivot_best.values
+
+    alg_pivot = df_gather_best.pivot(
+        index=columns.msg_size_bytes,
+        columns=columns.proc_num,
+        values=columns.algorithm
+    )
+
+    algorithms = alg_pivot.values.flatten()
+    unique_algs = sorted(pd.unique(algorithms[~pd.isna(algorithms)]))
+    color_map = {alg: i for i, alg in enumerate(unique_algs)}
+    color_values = np.array([color_map.get(a, np.nan) for a in algorithms])
+
+    fig = plt.figure(figsize=(16, 9))
+    ax = fig.add_subplot(111, projection='3d')
+    surf = ax.plot_wireframe(X, Y, Z,
+                             color='black', linewidths=1)
+    surf_points = ax.scatter(X,
+                             Y, Z, c=color_values, cmap='viridis', s=20, depthshade=False)  # pyright: ignore
+
+    handles = [
+        Line2D([0], [0],
+               marker='o', color='w',
+               label=alg,
+               markerfacecolor=plt.cm.viridis(
+                   color_map[alg] / max(len(unique_algs)-1, 1)),
+               markersize=8)
+        for alg in unique_algs
+    ]
+
+    ax.legend(handles=handles, title="Algorithm", loc='upper right')
+    ax.set_xlabel("Process Count")
+    ax.set_ylabel("Message Size [B]")
+    ax.set_zlabel("Average Time [μs]")
+    ax.set_title(f"{c.value}")
+    ax.set_xticks(pivot_best.columns.values)  # pyright: ignore
+    ax.set_xticklabels(pivot_best.columns.values)
+    ax.set_yticks(Y[:, 0])
+    ymin, ymax = ax.get_ylim()
+    ax.set_ylim(ymin*0.8, ymax)   # 30% more space at top
+    ax.set_yticklabels([f"$2^{{{int(np.log2(v))}}}$" for v in Y[:, 0]])
+    plt.savefig(f"./plots/{c.value.lower()}_algcomp.png")

results-and-plotting/python/scripts/plot_alg_log.py

@@ -0,0 +1,127 @@
+from matplotlib.ticker import FuncFormatter
+import pandas as pd
+import matplotlib.pyplot as plt
+import numpy as np
+from matplotlib.lines import Line2D
+from enum import Enum
+
+
+class columns ():
+    benchmark_type = 'benchmark_type'
+    proc_num = 'proc_num'
+    msg_size_bytes = 'msg_size_bytes'
+    repetitions = 'repetitions'
+    t_min_usec = 't_min_usec'
+    t_max_usec = 't_max_usec'
+    t_avg_usec = 't_avg_usec'
+    mpi_datatype = 'mpi_datatype'
+    mpi_red_datatype = 'mpi_red_datatype'
+    mpi_red_op = 'mpi_red_op'
+    creation_time = 'creation_time'
+    n_nodes = 'n_nodes'
+    off_cache_flag = 'off_cache_flag'
+    algorithm = 'algorithm'
+
+
+class collectives(Enum):
+    Bcast = 'Bcast'
+    Reduce = 'Reduce'
+    Allreduce = 'Allreduce'
+    Alltoall = 'Alltoall'
+    Scatter = 'Scatter'
+    Reduce_scatter = 'Reduce_scatter'
+    Allgather = 'Allgather'
+    Gather = 'Gather'
+
+
+def log_notation(val, pos):
+    return f" $1e{int(val)}$" if val != 0 else "1"
+
+
+def log2_notation(val, pos):
+    return "$2^{"+str(int(val))+"}$" if val != 0 else "1"
+
+
+data_file = "./data/data_04_11_25_algs.csv"
+
+df_multinode = pd.read_csv(data_file, delimiter=',')
+df_multinode.fillna(0, inplace=True)
+df_multinode = df_multinode[df_multinode[columns.off_cache_flag] == 50]
+for c in collectives:
+
+    df_single = df_multinode[df_multinode[columns.benchmark_type]
+                             == c.value]
+    df_single = df_single[df_single[columns.msg_size_bytes] > 1000]
+    df_single = df_single[[columns.proc_num, columns.msg_size_bytes,
+                           columns.t_avg_usec, columns.algorithm]]
+
+    df_gather_best = df_single.loc[  # pyright: ignore
+        df_single.groupby([columns.msg_size_bytes, columns.proc_num])[  # pyright: ignore
+            columns.t_avg_usec].idxmin()
+    ].reset_index(drop=True)
+
+    df_gather_worst = df_single.loc[  # pyright: ignore
+        df_single.groupby([columns.msg_size_bytes, columns.proc_num])[  # pyright: ignore
+            columns.t_avg_usec].idxmax()
+    ].reset_index(drop=True)
+    # df_gather_select = df_gather_select[df_gather_select[columns.msg_size_bytes] > 2**17]
+
+    pivot_best = df_gather_best.pivot(index=columns.msg_size_bytes,  # pyright: ignore
+                                      columns=columns.proc_num, values=columns.t_avg_usec)
+    pivot_worst = df_gather_worst.pivot(index=columns.msg_size_bytes,  # pyright: ignore
+                                        columns=columns.proc_num, values=columns.t_avg_usec)
+
+    X = pivot_best.columns.values   # proc_num
+    Y = pivot_best.index.values     # msg_size_bytes
+    X, Y = np.meshgrid(X, Y)   # pyright: ignore
+    Z = pivot_best.values
+
+    X_w = pivot_worst.columns.values   # proc_num
+    Y_w = pivot_worst.index.values     # msg_size_bytes
+    X_w, Y_w = np.meshgrid(X_w, Y_w)   # pyright: ignore
+    Z_w = pivot_worst.values
+
+    alg_pivot = df_gather_best.pivot(
+        index=columns.msg_size_bytes,
+        columns=columns.proc_num,
+        values=columns.algorithm
+    )
+
+    algorithms = alg_pivot.values.flatten()
+    unique_algs = sorted(pd.unique(algorithms[~pd.isna(algorithms)]))
+    color_map = {alg: i for i, alg in enumerate(unique_algs)}
+    color_values = np.array([color_map.get(a, np.nan) for a in algorithms])
+
+    fig = plt.figure(figsize=(16, 9))
+    ax = fig.add_subplot(111, projection='3d')
+    surf = ax.plot_wireframe(X, np.log2(Y), np.log(Z),
+                             color='black', linewidths=1)
+    surf = ax.plot_wireframe(X_w, np.log2(
+        Y_w), np.log(Z_w), color='gray', linewidths=0.3)
+
+    surf_points = ax.scatter(X, np.log2(
+        Y), np.log(Z), c=color_values, cmap='viridis', s=20, depthshade=False)  # pyright: ignore
+
+    surf_points = ax.scatter(X_w, np.log2(
+        Y_w), np.log(Z_w), c='gray', alpha=0.2,  s=20, depthshade=False)  # pyright: ignore
+
+    handles = [
+        Line2D([0], [0],
+               marker='o', color='w',
+               label=alg,
+               markerfacecolor=plt.cm.viridis(
+                   color_map[alg] / max(len(unique_algs)-1, 1)),
+               markersize=8)
+        for alg in unique_algs
+    ]
+
+    ax.legend(handles=handles, title="Algorithm", loc='upper right')
+    ax.set_xlabel("Process Count")
+    ax.set_ylabel("Message Size [B] (log2)")
+    ax.set_zlabel("Average Time [μs] (log10)")
+    ax.set_title(f"{c.value}")
+    ax.set_xticks(pivot_best.columns.values)  # pyright: ignore
+    ax.set_xticklabels(pivot_best.columns.values)
+    ax.yaxis.set_major_formatter(FuncFormatter(log2_notation))
+    ax.zaxis.set_major_formatter(FuncFormatter(log_notation))
+    plt.savefig(f"./plots/{c.value.lower()}_algcomp_log.png")

results-and-plotting/python/scripts/plot_all.py

@@ -0,0 +1,69 @@
+import pandas as pd
+import numpy as np
+import matplotlib.pyplot as plt
+import seaborn as sns
+from scipy.optimize import curve_fit
+import matplotlib.cm as cm
+
+
+def max_transfer_size(msg_size, np_procs, benchmark_type):
+    if benchmark_type == 'Allgather':
+        return (np_procs-72)*msg_size
+    elif benchmark_type == 'Scatter':
+        return (np_procs-72)*msg_size  # ?
+    elif benchmark_type == 'Alltoall':
+        return 72*(np_procs-72)*msg_size
+    elif benchmark_type == 'Bcast':
+        return msg_size
+    elif benchmark_type == 'Gather':
+        return (np_procs)*msg_size  # ?
+    elif benchmark_type == 'Reduce_scatter':
+        return 0.25*(np_procs-72)*(1/72)*msg_size  # ?
+    elif benchmark_type == 'Allreduce':
+        return 0.25*(np_procs-72)*(1/72)*msg_size
+    elif benchmark_type == 'Reduce':
+        return 0.25*(np_procs-72)*(1/72)*msg_size
+
+
+data_file = "data/data-multi-defand100cflag.csv"
+df_multinode = pd.read_csv(data_file, delimiter=',')
+df_multinode_offdef = df_multinode[df_multinode['off_cache_flag'] == 100]
+benchmarks = df_multinode_offdef['benchmark_type'].unique().tolist()
+benchmarks = [x for x in benchmarks if x[-1] != 'v']
+print(benchmarks)
+df_multinode_offdef = df_multinode_offdef[df_multinode_offdef['benchmark_type'].isin(
+    benchmarks)][df_multinode_offdef['msg_size_bytes'] > 1000]
+
+df_multinode_offdef["max_transfer"] = df_multinode_offdef.apply(
+    lambda row: max_transfer_size(
+        msg_size=row["msg_size_bytes"],
+        np_procs=row["proc_num"],
+        benchmark_type=row["benchmark_type"]
+    ),
+    axis=1
+)
+
+df_multinode_offdef["bytes/usec"] = df_multinode_offdef["max_transfer"] / \
+    df_multinode_offdef["t_avg_usec"]
+df_multinode_offdef = df_multinode_offdef[df_multinode_offdef['benchmark_type']!='Allgather'][df_multinode_offdef['benchmark_type']!='Alltoall']
+df_multinode_offdef = df_multinode_offdef[['benchmark_type','msg_size_bytes','t_avg_usec','proc_num']]
+
+plt.figure(figsize=(16, 9))
+sns.barplot(
+    data=df_multinode_offdef,
+    x="benchmark_type",
+    y="t_avg_usec",
+    dodge=True,
+    hue=df_multinode_offdef["msg_size_bytes"].astype(str),
+)
+# plt.yscale("log")
+plt.title("Average Time (usec) per Benchmark Type and Message Size")
+plt.ylabel("Average Time (usec)")
+plt.xlabel("Benchmark Type")
+plt.xticks(rotation=45)
+plt.legend(title="Message Size (bytes)")
+plt.tight_layout()
+# plt.show()
+plt.savefig("./plots/benchmark_avg_time_barplot.png", dpi=300)
+plt.close()
+

results-and-plotting/python/scripts/plot_fast_group.py

@@ -0,0 +1,64 @@
+import pandas as pd
+import matplotlib.pyplot as plt
+import seaborn as sns
+import numpy as np
+from mpl_toolkits.mplot3d import Axes3D
+
+data_file = "data/data-multi-defand100cflag.csv"
+df_multinode = pd.read_csv(data_file, delimiter=',')
+df_multinode_offdef = df_multinode[df_multinode['off_cache_flag'] == 100]
+df_multinode_offdef = df_multinode_offdef[['benchmark_type','msg_size_bytes','t_avg_usec','proc_num']]
+
+benchmarks = df_multinode_offdef['benchmark_type'].unique().tolist()
+benchmarks = [x for x in benchmarks if x[-1] != 'v']
+df_multinode_offdef = df_multinode_offdef[df_multinode_offdef['benchmark_type'].isin(
+    benchmarks)][df_multinode_offdef['msg_size_bytes'] > 1000]
+fast_benchmarks = ["Allreduce","Bcast","Reduce","Reduce_scatter"]
+df_multinode_offdef = df_multinode_offdef[df_multinode_offdef["benchmark_type"].isin(fast_benchmarks)]
+
+plt.figure(figsize=(16, 9))
+sns.barplot(
+    data=df_multinode_offdef,
+    x="benchmark_type",
+    y="t_avg_usec",
+    dodge=True,
+    hue=df_multinode_offdef["msg_size_bytes"].astype(str),
+)
+
+plt.ylim(0)
+plt.title("Average Time (usec) per Benchmark Type and Message Size")
+plt.ylabel("Average Time (usec)")
+plt.xlabel("Benchmark Type")
+plt.xticks(rotation=45)
+plt.legend(title="Message Size (bytes)")
+plt.tight_layout()
+plt.savefig("./plots/fbenchmarks_avg_time_barplot.png", dpi=300)
+plt.close()
+
+df_allreduce= df_multinode_offdef[df_multinode_offdef["benchmark_type"]=="Allreduce"]
+df_allreduce = df_allreduce[['msg_size_bytes','t_avg_usec','proc_num']]
+df_allreduce = df_allreduce[df_allreduce['msg_size_bytes']>2**17]
+pivot = df_allreduce.pivot(index="msg_size_bytes", columns="proc_num", values="t_avg_usec")
+X = pivot.columns.values   # proc_num
+Y = pivot.index.values     # msg_size_bytes
+X, Y = np.meshgrid(X, Y)
+Z = pivot.values
+
+fig = plt.figure(figsize=(16, 9))
+ax = fig.add_subplot(111, projection='3d')
+surf = ax.plot_surface(X, Y, Z, cmap="viridis", edgecolor='k')
+cbar = fig.colorbar(surf, ax=ax, shrink=0.6, pad=0.01, location='left')
+cbar.set_label("Average Time (μs)")
+ax.set_xlabel("Process Count")
+ax.set_ylabel("Message Size (B)")
+ax.set_zlabel("Average Time (μs)")
+ax.set_title("Allreduce")
+ax.set_xticks(pivot.columns.values)           # use the actual process count values
+ax.set_xticklabels(pivot.columns.values)
+ax.set_yticks(Y[:, 0])
+ymin, ymax = ax.get_ylim()
+ax.set_ylim(ymin*0.8, ymax)   # 30% more space at top
+ax.set_yticklabels([f"$2^{{{int(np.log2(v))}}}$" for v in Y[:, 0]])
+plt.tight_layout()
+plt.savefig("./plots/allreduce_surface.png", dpi=300)
+plt.close()

results-and-plotting/python/scripts/plot_mid_group.py

@@ -0,0 +1,66 @@
+import pandas as pd
+import matplotlib.pyplot as plt
+import seaborn as sns
+import numpy as np
+from mpl_toolkits.mplot3d import Axes3D
+
+data_file = "data/data-multi-defand100cflag.csv"
+df_multinode = pd.read_csv(data_file, delimiter=',')
+df_multinode_offdef = df_multinode[df_multinode['off_cache_flag'] == 100]
+df_multinode_offdef = df_multinode_offdef[['benchmark_type','msg_size_bytes','t_avg_usec','proc_num']]
+
+benchmarks = df_multinode_offdef['benchmark_type'].unique().tolist()
+benchmarks = [x for x in benchmarks if x[-1] != 'v']
+df_multinode_offdef = df_multinode_offdef[df_multinode_offdef['benchmark_type'].isin(
+    benchmarks)][df_multinode_offdef['msg_size_bytes'] > 1000]
+# fast_benchmarks = ["Allreduce","Bcast","Reduce","Reduce_scatter"]
+
+medium_benchmarks = ["Gather","Scatter"]
+df_multinode_offdef = df_multinode_offdef[df_multinode_offdef["benchmark_type"].isin(medium_benchmarks)]
+
+plt.figure(figsize=(16, 9))
+sns.barplot(
+    data=df_multinode_offdef,
+    x="benchmark_type",
+    y="t_avg_usec",
+    dodge=True,
+    hue=df_multinode_offdef["msg_size_bytes"].astype(str),
+)
+
+plt.ylim(0)
+plt.title("Average Time (usec) per Benchmark Type and Message Size")
+plt.ylabel("Average Time (usec)")
+plt.xlabel("Benchmark Type")
+plt.xticks(rotation=45)
+plt.legend(title="Message Size (bytes)")
+plt.tight_layout()
+plt.savefig("./plots/mbenchmarks_avg_time_barplot.png", dpi=300)
+plt.close()
+
+df_gather = df_multinode_offdef[df_multinode_offdef['benchmark_type']=='Gather']
+df_gather = df_gather[['msg_size_bytes','t_avg_usec','proc_num']]
+df_gather = df_gather[df_gather['msg_size_bytes']>2**17]
+pivot = df_gather.pivot(index="msg_size_bytes", columns="proc_num", values="t_avg_usec")
+X = pivot.columns.values   # proc_num
+Y = pivot.index.values     # msg_size_bytes
+X, Y = np.meshgrid(X, Y)
+Z = pivot.values
+
+fig = plt.figure(figsize=(16, 9))
+ax = fig.add_subplot(111, projection='3d')
+surf = ax.plot_surface(X, Y, Z, cmap="viridis", edgecolor='k')
+cbar = fig.colorbar(surf, ax=ax, shrink=0.6, pad=0.01, location='left')
+cbar.set_label("Average Time (μs)")
+ax.set_xlabel("Process Count")
+ax.set_ylabel("Message Size (B)")
+ax.set_zlabel("Average Time (μs)")
+ax.set_title("Gather")
+ax.set_xticks(pivot.columns.values)           # use the actual process count values
+ax.set_xticklabels(pivot.columns.values)
+ax.set_yticks(Y[:, 0])
+ymin, ymax = ax.get_ylim()
+ax.set_ylim(ymin*0.8, ymax)   # 30% more space at top
+ax.set_yticklabels([f"$2^{{{int(np.log2(v))}}}$" for v in Y[:, 0]])
+plt.tight_layout()
+plt.savefig("./plots/gather_surface.png", dpi=300)
+plt.close()

results-and-plotting/python/scripts/plot_slow_group.py

@@ -0,0 +1,93 @@
+import pandas as pd
+import matplotlib.pyplot as plt
+import seaborn as sns
+import numpy as np
+from mpl_toolkits.mplot3d import Axes3D
+
+data_file = "data/data-multi-defand100cflag.csv"
+df_multinode = pd.read_csv(data_file, delimiter=',')
+df_multinode_offdef = df_multinode[df_multinode['off_cache_flag'] == 100]
+df_multinode_offdef = df_multinode_offdef[['benchmark_type','msg_size_bytes','t_avg_usec','proc_num']]
+
+benchmarks = df_multinode_offdef['benchmark_type'].unique().tolist()
+benchmarks = [x for x in benchmarks if x[-1] != 'v']
+df_multinode_offdef = df_multinode_offdef[df_multinode_offdef['benchmark_type'].isin(
+    benchmarks)][df_multinode_offdef['msg_size_bytes'] > 1000]
+
+slow_benchmarks = ["Alltoall","Allgather"]
+df_multinode_offdef = df_multinode_offdef[df_multinode_offdef["benchmark_type"].isin(slow_benchmarks)]
+
+plt.figure(figsize=(16, 9))
+sns.barplot(
+    data=df_multinode_offdef,
+    x="benchmark_type",
+    y="t_avg_usec",
+    dodge=True,
+    hue=df_multinode_offdef["msg_size_bytes"].astype(str),
+)
+
+plt.ylim(0)
+plt.title("Average Time (usec) per Benchmark Type and Message Size")
+plt.ylabel("Average Time (usec)")
+plt.xlabel("Benchmark Type")
+plt.xticks(rotation=45)
+plt.legend(title="Message Size (bytes)")
+plt.tight_layout()
+plt.savefig("./plots/sbenchmarks_avg_time_barplot.png", dpi=300)
+plt.close()
+
+df_alltoall = df_multinode_offdef[df_multinode_offdef['benchmark_type']=='Alltoall']
+df_alltoall = df_alltoall[['msg_size_bytes','t_avg_usec','proc_num']]
+df_alltoall = df_alltoall[df_alltoall['msg_size_bytes']>2**17]
+pivot = df_alltoall.pivot(index="msg_size_bytes", columns="proc_num", values="t_avg_usec")
+X = pivot.columns.values   # proc_num
+Y = pivot.index.values     # msg_size_bytes
+X, Y = np.meshgrid(X, Y)
+Z = pivot.values
+
+fig = plt.figure(figsize=(16, 9))
+ax = fig.add_subplot(111, projection='3d')
+surf = ax.plot_surface(X, Y, Z, cmap="viridis", edgecolor='k')
+cbar = fig.colorbar(surf, ax=ax, shrink=0.6, pad=0.01, location='left')
+cbar.set_label("Average Time (μs)")
+ax.set_xlabel("Process Count")
+ax.set_ylabel("Message Size (B)")
+ax.set_zlabel("Average Time (μs)")
+ax.set_title("Alltoall")
+ax.set_xticks(pivot.columns.values)           # use the actual process count values
+ax.set_xticklabels(pivot.columns.values)
+ax.set_yticks(Y[:, 0])
+ymin, ymax = ax.get_ylim()
+ax.set_ylim(ymin*0.8, ymax)   # 30% more space at top
+ax.set_yticklabels([f"$2^{{{int(np.log2(v))}}}$" for v in Y[:, 0]])
+plt.tight_layout()
+plt.savefig("./plots/alltoall_surface.png", dpi=300)
+plt.close()
+
+df_allgather = df_multinode_offdef[df_multinode_offdef['benchmark_type']=='Allgather']
+df_allgather = df_allgather[['msg_size_bytes','t_avg_usec','proc_num']]
+df_allgather = df_allgather[df_allgather['msg_size_bytes']>2**17]
+pivot = df_allgather.pivot(index="msg_size_bytes", columns="proc_num", values="t_avg_usec")
+X = pivot.columns.values   # proc_num
+Y = pivot.index.values     # msg_size_bytes
+X, Y = np.meshgrid(X, Y)
+Z = pivot.values
+
+fig = plt.figure(figsize=(16, 9))
+ax = fig.add_subplot(111, projection='3d')
+surf = ax.plot_surface(X, Y, Z, cmap="viridis", edgecolor='k')
+cbar = fig.colorbar(surf, ax=ax, shrink=0.6, pad=0.01, location='left')
+cbar.set_label("Average Time (μs)")
+ax.set_xlabel("Process Count")
+ax.set_ylabel("Message Size (B)")
+ax.set_zlabel("Average Time (μs)")
+ax.set_title("Allgather")
+ax.set_xticks(pivot.columns.values)           # use the actual process count values
+ax.set_xticklabels(pivot.columns.values)
+ax.set_yticks(Y[:, 0])
+ymin, ymax = ax.get_ylim()
+ax.set_ylim(ymin*0.8, ymax)   # 30% more space at top
+ax.set_yticklabels([f"$2^{{{int(np.log2(v))}}}$" for v in Y[:, 0]])
+plt.tight_layout()
+plt.savefig("./plots/allgather_surface.png", dpi=300)
+plt.close()