Commit 3d5ede86 authored by Sudheer Chunduri's avatar Sudheer Chunduri
Browse files

Merge branch 'master' into 'master'

Add initial gpcnet artifact

See merge request networkbench/routing_aries!1
parents fc66dc67 910f913b
From e615c0726d15caa5e9aa6aa628df42464759a139 Mon Sep 17 00:00:00 2001
From: Justs Zarins <jzarins@thetalogin4.tmi.alcf.anl.gov>
Date: Fri, 24 Jan 2020 15:29:48 +0000
Subject: [PATCH] Add verbose to Makefile and remove .rec file creation in
verbose mode
---
Makefile | 2 +-
utils.c | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/Makefile b/Makefile
index 72b6891..fc01388 100644
--- a/Makefile
+++ b/Makefile
@@ -1,7 +1,7 @@
CFLAGS = -I .
LIBS = -lm
CC = cc
-FLAGS =
+FLAGS = -DVERBOSE
%.o: %.c $(DEPS)
$(CC) -c -o $@ $< $(CFLAGS) $(FLAGS)
diff --git a/utils.c b/utils.c
index 69c36c2..1e9e155 100644
--- a/utils.c
+++ b/utils.c
@@ -753,7 +753,7 @@ int summarize_pairs_performance(CommConfig_t *config, MPI_Comm comm, char *lnode
char *all_perf_strs;
struct timespec timestmp;
-#ifndef VERBOSE
+#ifdef VERBOSE
return 0;
#else
--
2.12.3
import numpy as np
import matplotlib as mpl
import matplotlib.font_manager as fm
from matplotlib import pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
import seaborn as sns
import itertools, numpy
import glob
import os
import re
import time
import re
from numpy import genfromtxt
import pandas as pd
#from IPython.display import set_matplotlib_formats
#set_matplotlib_formats('pdf', 'png')
#set_matplotlib_formats('png')
#from palettable.cmocean.diverging import Delta_20
#cmap=Delta_20.hex_colors
plt.rcParams['font.style'] = 'normal'
plt.rcParams['font.serif'] = 'Times'
#matplotlib.rcParams.update({'font.size': 14})
#from sklearn.model_selection import train_test_split
#from scipy.stats import linregress
#from scipy.stats import skew,pearsonr, kurtosis, kurtosistest, iqr, ks_2samp, ttest_ind, gaussian_kde
#from scipy.stats import bayes_mvs
#from statsmodels import robust
mpl.rcParams['ps.useafm'] = True
#mpl.rcParams['pdf.use14corefonts'] = True
mpl.rcParams['text.usetex'] = True
mpl.rc('text', usetex=True)
mpl.rcParams['text.latex.unicode']=True
mpl.use( "cairo", warn=False, force=True)
print(mpl.pyplot.get_backend())
mpl.rcParams['text.latex.preamble'] = [
r'\usepackage{siunitx}', # i need upright \micro symbols, but you need...
r'\sisetup{detect-all}', # ...this to force siunitx to actually use your fonts
r'\usepackage[T1]{fontenc}',
r'\usepackage[default]{gillius}', # set the normal font here
r'\usepackage{sansmath}', # load up the sansmath so that math -> helvet
r'\sansmath' # <- tricky! -- gotta actually tell tex to use!
]
#mpl.rcParams["axes.spines.right"] = False
#mpl.rcParams["axes.spines.top"] = False
def init_plotting():
plt.rcParams['figure.figsize'] = (28, 7)
plt.rcParams['font.size'] = 48
print(plt.rcParams['font.family'] )
plt.rcParams['axes.labelsize'] = plt.rcParams['font.size']
plt.rcParams['axes.titlesize'] = plt.rcParams['font.size']
plt.rcParams['legend.fontsize'] = 1.0*plt.rcParams['font.size']
plt.rcParams['xtick.labelsize'] = plt.rcParams['font.size']
plt.rcParams['ytick.labelsize'] = plt.rcParams['font.size']
# print(plt.rcParams['savefig.dpi'])
# plt.rcParams['savefig.dpi'] = 2*plt.rcParams['savefig.dpi']
plt.rcParams['xtick.minor.visible']=False
plt.rcParams['ytick.minor.visible']=False
plt.rcParams['xtick.major.size'] = 6
plt.rcParams['xtick.minor.size'] = 3
plt.rcParams['xtick.major.width'] = 2
plt.rcParams['xtick.minor.width'] = 2
plt.rcParams['ytick.major.size'] = 6
plt.rcParams['ytick.minor.size'] = 3
plt.rcParams['ytick.major.width'] = 2
plt.rcParams['ytick.minor.width'] = 2
plt.rcParams['xtick.major.pad']='8'
plt.rcParams['ytick.major.pad']='8'
plt.rcParams['ytick.color'] = "#808080"
plt.rcParams['xtick.color'] = "#808080"
plt.rcParams['xtick.direction'] = 'in'
plt.rcParams['ytick.direction'] = 'in'
#ax.tick_params(direction='out', length=6, width=2, colors='r',
# grid_color='r', grid_alpha=0.5)
plt.rcParams['legend.frameon'] = False
plt.rcParams['legend.loc'] = 'best'
plt.rcParams['axes.linewidth'] = 3
plt.gca().spines['bottom'].set_color('#808080')
plt.gca().spines['left'].set_color('#808080')
## to hide the spines
#plt.gca().spines['top'].set_visible(False)
#plt.gca().spines['right'].set_visible(False)
plt.gca().spines['top'].set_color('#808080')
plt.gca().spines['right'].set_color('#808080')
plt.gca().spines['top'].set_linewidth(.8)
plt.gca().spines['right'].set_linewidth(.8)
plt.grid(True,linestyle='-', linewidth=0.01)
plt.gca().legend(ncol=4)
plt.gca().legend(prop={'family': 'monospace'})
def hide_spines():
"""Hides the top and rightmost axis spines from view for all active
figures and their respective axes."""
# Retrieve a list of all current figures.
figures = [x for x in mpl._pylab_helpers.Gcf.get_all_fig_managers()]
for figure in figures:
# Get all Axis instances related to the figure.
for ax in figure.canvas.figure.get_axes():
ax.spines['left'].set_color("#808080")
ax.spines['bottom'].set_color("#808080")
ax.spines['top'].set_color("#808080")
ax.spines['right'].set_color("#808080")
#ax.spines['right'].set_visible('False')
#ax.spines['top'].set_visible('False')
ax.xaxis.set_ticks_position('bottom')
ax.yaxis.set_ticks_position('left')
ax.spines['top'].set_linewidth(0.8)
#ax.spines['top'].set_linestyle(':')
ax.spines['right'].set_linewidth(0.8)
#ax.spines['right'].set_linestyle(':')
#plt.gca().spines['bottom'].set_color('gray')
#plt.gca().spines['left'].set_color('gray')
#plt.grid(True,linestyle='-', linewidth=0.01)
#plt.gca().legend(ncol=4)
#plt.gca().legend(prop={'family': 'monospace'})
init_plotting()
#hide_spines()
1) Use
bash get_code.sh
to get code and apply modifications.
2) cd gpcnet
3) Modify Makefile for your compilation environment and run
make all
4) Modify submit.cobalt to suit your submission system and change "benchmark" variable to point to "network_load_test" in your GPCNeT directory.
5) qsub submit.cobalt
6) python3 gather_data.py ./Probe_ADAPTIVE_*__nnodes_512/summary.out
7) To generate plot:
python3 analyse_data.py dataset.pickle
From 3c5c4f4b719c208ba89206aaf6c054571cba3e0e Mon Sep 17 00:00:00 2001
From: Justs Zarins <jzarins@thetalogin4.tmi.alcf.anl.gov>
Date: Fri, 24 Jan 2020 15:56:52 +0000
Subject: [PATCH] Set static seed; add another net test
---
network_load_test.c | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/network_load_test.c b/network_load_test.c
index 8f44a76..99b0596 100644
--- a/network_load_test.c
+++ b/network_load_test.c
@@ -22,7 +22,7 @@
#define VERSION 1.1
-#define NUM_NETWORK_TESTS 3
+#define NUM_NETWORK_TESTS 4
#define NUM_CONGESTOR_TESTS 4
/* loop counts for the various tests */
@@ -536,7 +536,7 @@ int build_subcomms(int ncongestor_tests, CommConfig_t *config, CommNodes_t *node
int seed = (int)now;
/* to use a fixed seed for debug purposes, uncomment the following line */
- //seed = RSEED;
+ seed = RSEED;
shuffle(allnodes, nodes->nnodes, seed, 0);
/* pull out the different node types */
@@ -642,6 +642,7 @@ int main(int argc, char* argv[])
network_tests_list[0] = P2P_LATENCY;
network_tests_list[1] = P2P_NEIGHBORS;
network_tests_list[2] = ALLREDUCE_LATENCY;
+ network_tests_list[3] = P2P_BANDWIDTH;
congestor_tests_list[0] = A2A_CONGESTOR;
congestor_tests_list[1] = P2P_INCAST_CONGESTOR;
--
2.12.3
import numpy as np
import pandas as pd
import sys
import os
import glob
import pickle
import re
import datetime
import Common_lib as cl
import seaborn as sns
import matplotlib.pyplot as plt
def save_dataframe(df, filename):
with open(filename, "wb") as f:
print("Saving data to", filename)
pickle.dump(df, f)
def load_dataframe(filename):
df = None
with open(filename, "rb") as f:
print("Loading data from", filename)
df = pickle.load(f)
return df
def get_zscore_hists(df_rows, sigma_cutoff=3, return_z_normalised_hists=False):
all_binv = df_rows[0].binv
all_binc = df_rows[0].binc
if len(df_rows) > 1:
all_binv = np.append(df_rows[0].binv, [r.binv for r in df_rows[1:]])
all_binc = np.append(df_rows[0].binc, [r.binc for r in df_rows[1:]])
mean = calc_mean(all_binc, all_binv)
stddev = calc_stddev(all_binc, all_binv)
zbinvs = []
for r in df_rows:
zbinvs.append( np.apply_along_axis(lambda x: (x - mean) / stddev, 0, r.binv) )
idxs = []
for zbinv in zbinvs:
low_idx = None
high_idx = None
for i,v in enumerate(zbinv):
if low_idx == None and (v > -sigma_cutoff):
low_idx = i
if high_idx == None and (v > sigma_cutoff):
high_idx = i
if low_idx == None:
low_idx = 0
if high_idx == None:
high_idx = len(zbinv)
idxs.append((low_idx, high_idx))
print("hist_bin_count", "low_idx", "high_idx", "benchmark_name", "probe_ar_mode", "low_outlier_fraction", "high_outlier_fraction")
for row,idx in zip(df_rows, idxs):
print(len(row.binc), idx[0], idx[1], row.benchmark_name, row.p_ar, \
np.sum(row.binc[:idx[0]]) / np.sum(row.binc), \
np.sum(row.binc[idx[1]:]) / np.sum(row.binc))
out = []
if return_z_normalised_hists:
for row,zbinv,idx in zip(df_rows, zbinvs, idxs):
low_outliers = np.sum(row.binc[:idx[0]]) / np.sum(row.binc)
high_outliers = np.sum(row.binc[idx[1]:]) / np.sum(row.binc)
out.append( pd.Series([ row.binc[idx[0]:idx[1]], zbinv[idx[0]:idx[1]], low_outliers+high_outliers ], index=["zbinc", "zbinv", "zoutlier_frac"]) )
else:
for row,idx in zip(df_rows, idxs):
low_outliers = np.sum(row.binc[:idx[0]]) / np.sum(row.binc)
high_outliers = np.sum(row.binc[idx[1]:]) / np.sum(row.binc)
out.append( pd.Series([ row.binc[idx[0]:idx[1]], row.binv[idx[0]:idx[1]], low_outliers+high_outliers ], index=["zbinc", "zbinv", "zoutlier_frac"]) )
return out
def add_zscores(df, rows, sigma_cutoff, return_z_normalised_hists):
scores = get_zscore_hists(rows, sigma_cutoff, return_z_normalised_hists)
for row,score in zip(rows, scores):
df.at[row.name, "zbinc"] = score[0]
df.at[row.name, "zbinv"] = score[1]
df.at[row.name, "zoutlier_frac"] = score[2]
def add_zscores_to_ad0_ad3_pairs(df, sigma_cutoff, return_z_normalised_hists):
df["zbinc"] = pd.Series(dtype="object")
df["zbinv"] = pd.Series(dtype="object")
df["zoutlier_frac"] = pd.Series(dtype="object")
for i, row_ar0 in df.loc[df.p_ar == "ADAPTIVE_0"].iterrows():
row_ar3 = df.loc[(df.p_ar == "ADAPTIVE_3") & (df.benchmark_name == row_ar0.benchmark_name) & (df.cong == row_ar0.cong) & (df.nnodes == row_ar0.nnodes)]
if len(row_ar3) == 1:
row_ar3 = row_ar3.iloc[0]
else:
print("len row_ar3 not as expected: ", len(row_ar3))
exit(1)
rows = [row_ar0, row_ar3]
add_zscores(df, rows, sigma_cutoff, return_z_normalised_hists)
def calc_qcd(binc, binv):
idxq1 = int(0.25 * np.sum(binc))
idxq3 = int(0.75 * np.sum(binc))
q3 = -1
q1 = -1
for i,val in enumerate(np.cumsum(binc)):
if idxq1 < val and q1 == -1:
q1 = binv[i]
if idxq3 < val and q3 == -1:
q3 = binv[i]
return (q3 - q1) / (q3 + q1)
def calc_stddev(binc, binv):
mean = calc_mean(binc, binv)
sumv = 0
for i,v in enumerate(binv):
sumv += binc[i] * ((v - mean)**2)
return np.sqrt(sumv / (np.sum(binc) - 1) )
def calc_mean(binc, binv):
return np.sum(np.multiply(binc, binv)) / np.sum(binc)
def calc_mean_in_range(binc, binv, lowv, highv):
low_idx = np.where(binv == lowv)[0][0]
high_idx = np.where(binv == highv)[0][0]
return np.sum(np.multiply(binc[low_idx:high_idx], binv[low_idx:high_idx])) / np.sum(binc[low_idx:high_idx])
def calc_plotting_quantiles(df_row, use_zbins):
plotting_quantiles = ["h01","h25", "h50", "h75","h99"]
bv = "binv"
bc = "binc"
if use_zbins:
bv = "zbinv"
bc = "zbinc"
if not (np.isnan(df_row[bc]).any() and np.isnan(df_row[bv]).any()):
qs = get_quantiles(df_row[bv], df_row[bc])
return [qs[x] for x in plotting_quantiles]
def get_quantiles(binv, binc):
hmin = binv[0]
hmax = binv[-1]
hmean = np.sum(binc * binv) / np.sum(binc)
idx001 = int(0.001 * np.sum(binc))
idx01 = int(0.01 * np.sum(binc))
idx05 = int(0.05 * np.sum(binc))
idx10 = int(0.10 * np.sum(binc))
idx25 = int(0.25 * np.sum(binc))
idx50 = int(0.50 * np.sum(binc))
idx75 = int(0.75 * np.sum(binc))
idx90 = int(0.90 * np.sum(binc))
idx95 = int(0.95 * np.sum(binc))
idx99 = int(0.99 * np.sum(binc))
idx999 = int(0.999 * np.sum(binc))
h001 = 0
h01 = 0
h05 = 0
h10 = 0
h25 = 0
h50 = 0
h75 = 0
h90 = 0
h95 = 0
h99 = 0
h999 = 0
for i,val in enumerate(np.cumsum(binc)):
if idx001 < val and h001 == 0:
h001 = binv[i]
if idx01 < val and h01 == 0:
h01 = binv[i]
if idx05 < val and h05 == 0:
h05 = binv[i]
if idx10 < val and h10 == 0:
h10 = binv[i]
if idx25 < val and h25 == 0:
h25 = binv[i]
if idx50 < val and h50 == 0:
h50 = binv[i]
if idx75 < val and h75 == 0:
h75 = binv[i]
if idx90 < val and h90 == 0:
h90 = binv[i]
if idx95 < val and h95 == 0:
h95 = binv[i]
if idx99 < val and h99 == 0:
h99 = binv[i]
if idx999 < val and h999 == 0:
h999 = binv[i]
return {"hmin": hmin, "hmax": hmax, "hmean": hmean, "h001": h001, "h01": h01, "h05": h05, "h10": h10, "h25": h25, "h50": h50, "h75": h75, "h90": h90, "h95": h95, "h99": h99, "h999": h999}
def plot_zdata_data_probes_and_congs_individual(df, num_nodes):
plot_linewidth = 4
label_mapping = {"ADAPTIVE_0": "AD0", "ADAPTIVE_3": "AD3"}
color_mapping = {"ADAPTIVE_0": "red", "ADAPTIVE_3": "green"}
probe = "RR Two-sided BW"
df1 = df.loc[df.nnodes == num_nodes]
rows = []
rows.append(df1.loc[(df1.p_ar == "ADAPTIVE_0") & (df1.cong == "yes")].iloc[0])
rows.append(df1.loc[(df1.p_ar == "ADAPTIVE_3") & (df1.cong == "yes")].iloc[0])
# initialise plot
fig, ax = plt.subplots()
cl.init_plotting()
ax.set_ylabel('PDF')
ax.set_xlabel("Bandwidth (MiB/s/rank)")
name = probe
# Plot data histograms with dots
cl.hide_spines()
for d in rows:
resamples = get_resampled_kde(d.zbinv, d.zbinc)
line_style = "-"
line_name = label_mapping[d.p_ar]
sns.kdeplot(resamples, gridsize=500, ax=ax, label=line_name, linestyle=line_style, color=color_mapping[d.p_ar], linewidth=plot_linewidth)
if d.cong == "yes":
ax.axvline(d["mean"], linewidth=plot_linewidth-1, alpha=0.5, color=color_mapping[d.p_ar], linestyle="-")
ax.axvline(d["95p"], linewidth=plot_linewidth-1, alpha=0.5, color=color_mapping[d.p_ar], linestyle="--")
ax.get_legend().remove()
handles, labels = ax.get_legend_handles_labels()
fig.legend(handles, labels, frameon=False, loc='upper center', ncol=len(rows), columnspacing=1, handlelength=1.0, handletextpad=0.5, borderaxespad=0.0)
ax.text(230, 0.0075, "mean")
ax.text(410, 0.0075, "95 \%ile")
fig.subplots_adjust(bottom=0.2)
fig.savefig(name.replace(" ","_") + '.pdf', transparent=False, dpi=300, bbox_inches='tight')
def get_resampled_kde(binv, binc):
num_samples = 100000
print("Drawing", str(num_samples), "samples for plot. For final version this should be 100'000.")
np.random.seed(20201337)
resamples = np.random.choice(binv, size=num_samples, p=binc/binc.sum())
return resamples
def set_more_printout_pandas():
pd.set_option('display.max_rows', 500)
pd.set_option('display.max_columns', 500)
pd.set_option('display.width', 1000)
def pickout_data(df):
df2 = df.loc[(df.benchmark_name == "RR Two-sided BW") & (df.cong == "yes")]
return df2.loc[(df2.p_ar == "ADAPTIVE_0") | (df2.p_ar == "ADAPTIVE_3")]
# Main entry point
if __name__ == "__main__":
df = load_dataframe(sys.argv[1])
df = pickout_data(df)
if df.isnull().values.any():
print("There are NULL values in the dataframe. Aborting.")
exit(1)
sigma_cutoff = 3
return_z_normalised_hists = False
calc_quantiles_using_zhist = True
add_zscores_to_ad0_ad3_pairs(df, sigma_cutoff, return_z_normalised_hists)
df["mean"] = df.apply(func=lambda r: calc_mean(r.zbinc, r.zbinv), axis=1)
df["95p"] = df.apply(func=lambda r: get_quantiles(r.zbinv, r.zbinc)["h95"], axis=1)
df["qcd"] = df.apply(func=lambda r: calc_qcd(r.zbinc, r.zbinv), axis=1)
df["quantiles"] = df.apply(func=calc_plotting_quantiles, args=[calc_quantiles_using_zhist], axis=1)
df["quantile_spread"] = df.apply(func=lambda x: max(x.quantiles) - min(x.quantiles), axis=1)
df["median"] = df.apply(func=lambda x: x.quantiles[2], axis=1)
plot_zdata_data_probes_and_congs_individual(df, 512)
set_more_printout_pandas()
print(df[["benchmark_name", "nnodes", "net_nodes", "p_ar", "cong", "mean", "qcd", "median", "quantile_spread", "zoutlier_frac"]])
#!/bin/bash
set -x
msg_count=64
cong_proportion=0.5
target=gpcnet/network_load_test.c
sed -i -E 's/define CONGESTOR_NODE_FRAC [0-9]\.[0-9]+/define CONGESTOR_NODE_FRAC '${cong_proportion}'/g' ${target}
sed -i -E 's/define A2A_MSG_COUNT [0-9]+/define A2A_MSG_COUNT '${msg_count}'/g' ${target}
sed -i -E 's/define INCAST_MSG_COUNT [0-9]+/define INCAST_MSG_COUNT '${msg_count}'/g' ${target}
sed -i -E 's/define BCAST_MSG_COUNT [0-9]+/define BCAST_MSG_COUNT '${msg_count}'/g' ${target}
import numpy as np
import pandas as pd
import sys
import os
import glob
import pickle
import re
import datetime
name_mapping = {
"RR Two-sided Lat" : "p2p_latency",
"RR Get Lat" : "get_latency",
"RR Two-sided BW" : "p2p_bandwidth",
"RR Put BW" : "put_bandwidth",
"RR Two-sided BW+Sync" : "p2p_neighbors",
"Nat Two-sided BW" : "p2p_bandwidth_nat",
"Multiple Allreduce" : "allreduce_latency",
"Multiple Alltoall" : "a2a_bandwidth"
}
inv_name_mapping = {v: k for k, v in name_mapping.items()}
cong_name_mapping = {
"Alltoall" : "a2a_congestor",
"Two-sided Incast" : "p2p_incast_congestor",
"Put Incast" : "put_incast_congestor",
"Get Bcast" : "get_bcast_congestor",
}
month_mapping = {
"Jan" : 1,
"Feb" : 2,
"Mar" : 3,
"Apr" : 4,
"May" : 5,
"Jun" : 6,
"Jul" : 7,
"Aug" : 8,
"Sep" : 9,
"Oct" : 10,
"Nov" : 11,
"Dec" : 12
}
def save_dataframe(df, filename):
with open(filename, "wb") as f:
print("Saving data to", filename)
pickle.dump(df, f)
def loa