Commit 28ed4c37 authored by Neil McGlohon's avatar Neil McGlohon

Add model-net-synthetic-all to build

parent 716fddd4
......@@ -198,6 +198,7 @@ bin_PROGRAMS += src/network-workloads/model-net-dumpi-traces-dump
bin_PROGRAMS += src/network-workloads/model-net-synthetic
bin_PROGRAMS += src/network-workloads/model-net-synthetic-custom-dfly
bin_PROGRAMS += src/network-workloads/model-net-synthetic-slimfly
bin_PROGRAMS += src/network-workloads/model-net-synthetic-all
bin_PROGRAMS += src/network-workloads/model-net-synthetic-fattree
bin_PROGRAMS += src/network-workloads/model-net-synthetic-dfly-plus
bin_PROGRAMS += src/network-workloads/model-net-synthetic-dally-dfly
......
/*
* This test program generates some synthetic traffic patterns for the slim
* fly, dragonfly, and fat-tree network models. Currently it supports
* uniform random, worst-case (slim fly only), 1D nearest neighbor, 2D nearest
* neighbor, 3D nearest neighbor, gather, scatter, and bisection traffic
* patterns.
*/
#include "codes/model-net.h"
#include "codes/lp-io.h"
#include "codes/codes.h"
#include "codes/codes_mapping.h"
#include "codes/configuration.h"
#include "codes/lp-type-lookup.h"
#define PRINT_WORST_CASE_MATCH 0
FILE * slimfly_results_log_2=NULL;
FILE * slimfly_ross_csv_log=NULL;
static int net_id = 0;
static int offset = 2;
static int traffic = 1;
static double warm_up_time = 0.0;
static double arrival_time = 0.0;
static double load = 0.0; //percent utilization of each terminal's uplink when sending messages
static double MEAN_INTERVAL = 0.0;
static int payload_size = 0;
static double link_bandwidth = 0.0;
static int *worst_dest; //Array mapping worst case destination for each router
static int total_routers;
static char lp_io_dir[356] = {'\0'};
static lp_io_handle io_handle;
static unsigned int lp_io_use_suffix = 0;
static int do_lp_io = 0;
static int num_msgs = 20;
static int num_servers_per_rep = 0;
static int num_routers_per_grp = 0;
static int num_nodes = 0; // Number of terminals/compute nodes
static int num_servers = 0; // Number of servers/MPI processes
typedef struct svr_msg svr_msg;
typedef struct svr_state svr_state;
/* global variables for codes mapping */
static char group_name[MAX_NAME_LENGTH];
static char lp_type_name[MAX_NAME_LENGTH];
static int group_index, lp_type_index, rep_id, offset;
/* 2D rank communication heat map */
static int **comm_map;
/* Function for computing local and global connections for a given router id */
static void get_router_connections(int src_router_id, int* local_channels, int* global_channels);
/* Global variables for worst-case workload pairing */
static int num_local_channels;
static int num_global_channels;
int *X;
int *X_prime;
int X_size;
/* type of events */
enum svr_event
{
KICKOFF, /* kickoff event */
REMOTE, /* remote event */
LOCAL /* local event */
};
/* type of synthetic traffic */
enum TRAFFIC
{
UNIFORM = 1, /* sends message to a randomly selected node */
WORST_CASE = 2, /* sends messages in pairs configured to utilize same worst-case hop router */
NEAREST_NEIGHBOR_1D = 3, /* sends messages to next and previous nodes */
NEAREST_NEIGHBOR_2D = 4, /* sends messages to nearest neighbor nodes following 2D node mapping */
NEAREST_NEIGHBOR_3D = 5, /* sends messages to nearest neighbor nodes following 3D node mapping */
GATHER = 6, /* sends messages from all ranks to rank 0 */
SCATTER = 7, /* sends messages from rank 0 to all other ranks */
BISECTION = 8, /* sends messages between paired nodes in opposing bisection partitions */
ALL2ALL = 9, /* sends messages from each rank to all other ranks */
PING = 10 /* sends messages from rank 0 to rank 1 */
};
struct svr_state
{
int msg_sent_count; /* requests sent */
int msg_recvd_count; /* requests recvd */
int* msg_send_times; /* send time of all messages */
int* msg_recvd_times; /* recv time of all messages */
int local_recvd_count; /* number of local messages received */
tw_stime start_ts; /* time that we started sending requests */
tw_stime end_ts; /* time that we ended sending requests */
char output_buf[512]; /* buffer space for lp-io output data */
char output_buf2[65536]; /* buffer space for lp-io output data */
char output_buf3[65536]; /* buffer space for msg timing output data */
};
struct svr_msg
{
enum svr_event svr_event_type;
tw_lpid src; /* source of this request or ack */
int incremented_flag; /* helper for reverse computation */
model_net_event_return event_rc;
};
static void svr_init(
svr_state * ns,
tw_lp * lp);
static void svr_event(
svr_state * ns,
tw_bf * b,
svr_msg * m,
tw_lp * lp);
static void svr_rev_event(
svr_state * ns,
tw_bf * b,
svr_msg * m,
tw_lp * lp);
static void svr_finalize(
svr_state * ns,
tw_lp * lp);
tw_lptype svr_lp = {
(init_f) svr_init,
(pre_run_f) NULL,
(event_f) svr_event,
(revent_f) svr_rev_event,
(commit_f) NULL,
(final_f) svr_finalize,
(map_f) codes_mapping,
sizeof(svr_state),
};
/* setup for the ROSS event tracing
* can have a different function for rbev_trace_f and ev_trace_f
* but right now it is set to the same function for both
*/
void svr_event_collect(svr_msg *m, tw_lp *lp, char *buffer, int *collect_flag)
{
(void)lp;
(void)collect_flag;
int type = (int) m->svr_event_type;
memcpy(buffer, &type, sizeof(type));
}
/* can add in any model level data to be collected along with simulation engine data
* in the ROSS instrumentation. Will need to update the last field in
* svr_model_types[0] for the size of the data to save in each function call
*/
void svr_model_stat_collect(svr_state *s, tw_lp *lp, char *buffer)
{
(void)s;
(void)lp;
(void)buffer;
return;
}
st_model_types svr_model_types[] = {
{(ev_trace_f) svr_event_collect,
sizeof(int),
(model_stat_f) svr_model_stat_collect,
0,
NULL,
NULL,
0},
{NULL, 0, NULL, 0, NULL, NULL, 0}
};
static const st_model_types *svr_get_model_stat_types(void)
{
return(&svr_model_types[0]);
}
void svr_register_model_types()
{
st_model_type_register("nw-lp", svr_get_model_stat_types());
}
const tw_optdef app_opt [] =
{
TWOPT_GROUP("Model net synthetic traffic " ),
TWOPT_UINT("traffic", traffic, "1: Uniform Random, 2: Worst-Case, 3: 1D Nearest Neighbor, 4: 2D Nearest Neighbor, 5: 3D Nearest Neighbor, 6: Gather, 7: Scatter, 8: Bisection, 9: All-To-All, 1: Ping"),
TWOPT_UINT("num_messages", num_msgs, "Number of messages to be generated per terminal "),
TWOPT_UINT("payload_size", payload_size, "Size of data to be sent per terminal "),
TWOPT_STIME("arrival_time", arrival_time, "INTER-ARRIVAL TIME"),
TWOPT_STIME("warm_up_time", warm_up_time, "Time delay before starting stats collection. For generating accurate observed bandwidth calculations"),
TWOPT_STIME("load", load, "percentage of packet inter-arrival rate to simulate"),
TWOPT_CHAR("lp-io-dir", lp_io_dir, "Where to place io output (unspecified -> no output"),
TWOPT_UINT("lp-io-use-suffix", lp_io_use_suffix, "Whether to append uniq suffix to lp-io directory (default 0)"),
TWOPT_END(),
};
const tw_lptype* svr_get_lp_type()
{
return(&svr_lp);
}
static void svr_add_lp_type()
{
lp_type_register("nw-lp", svr_get_lp_type());
}
/* convert GiB/s and bytes to ns */
static tw_stime bytes_to_ns(uint64_t bytes, double GB_p_s)
{
tw_stime time;
/* bytes to GB */
time = ((double)bytes)/(1024.0*1024.0*1024.0);
/* MB to s */
time = time / GB_p_s;
/* s to ns */
time = time * 1000.0 * 1000.0 * 1000.0;
return(time);
}
/** Function checks to see if the provided src and dst routers are two hops away from one another and if they are, sets
* the intm input variable to the id of the intermediate router.
* @param[in] dest Local/relative ID of the destination router
* @param[in] src Local/relative ID of the source terminal
* @param[in/out] intm Local/relative ID of the intm router if it exists
*/
bool is_two_hops_away(int src, int dst, int *intm, int *count)
{
if(src == dst)
return false;
int *local_channel_src = (int*) malloc(num_local_channels*sizeof(int));
int *global_channel_src = (int*) malloc(num_global_channels*sizeof(int));
int *local_channel_dst = (int*) malloc(num_local_channels*sizeof(int));
int *global_channel_dst = (int*) malloc(num_global_channels*sizeof(int));
get_router_connections(src, local_channel_src, global_channel_src);
get_router_connections(dst, local_channel_dst, global_channel_dst);
int i,j;
int intm_count = 0;
bool return_val = false;
int temp_vals[6];
for(i=0;i<num_local_channels;i++)
{
for(j=0; j<num_global_channels;j++){
if(local_channel_src[i] == global_channel_dst[j])
{
temp_vals[intm_count] = local_channel_src[i];
intm_count++;
return_val = true;
}
}
}
for(i=0;i<num_global_channels;i++)
{
for(j=0; j<num_local_channels;j++){
if(global_channel_src[i] == local_channel_dst[j])
{
temp_vals[intm_count] = global_channel_src[i];
intm_count++;
return_val = true;
}
}
}
for(i=0;i<num_global_channels;i++)
{
for(j=0; j<num_global_channels;j++){
if(global_channel_src[i] == global_channel_dst[j])
{
temp_vals[intm_count] = global_channel_src[i];
intm_count++;
return_val = true;
}
}
}
for(i=0;i<num_local_channels;i++)
{
for(j=0; j<num_local_channels;j++){
if(local_channel_src[i] == local_channel_dst[j])
{
temp_vals[intm_count] = local_channel_src[i];
intm_count++;
return_val = true;
}
}
}
for(i=0;i<intm_count;i++){
intm[i] = temp_vals[i];
//printf("src:%d dst:%d intm router%d: %d\n", src, dst, i, intm[i]);
}
*count = intm_count;
return return_val;
}
/** Latest implementation of function to return an array mapping each router with it's corresponding worst-case router pair
* @param[in/out] *worst_dest Router ID to send all messages to
*/
void init_worst_case_mapping()
{
int pairings = 0;
int srcID = -1;
int* middled;
//for(int start=0; start<total_routers; start++){
// printf("start:%d\n",start);
int start = 0;
// variable to track if a router is an intermediate router for another pairing
middled = (int*)calloc(total_routers,sizeof(int));
// Loop over all routers
for(int src_idx=start; src_idx<total_routers+start; src_idx++){
int src = src_idx % total_routers;
if(src == srcID)
printf("src:%d total_routers:%d\n",src, total_routers);
// If no pairing yet
if(worst_dest[src] < 0){
// Check remaining routers
for(int dst_idx=src+total_routers/2; dst_idx<total_routers+src+total_routers/2; dst_idx++){
int dst = dst_idx % total_routers;
if(src == srcID)
printf("potential dst:%d\n",dst);
// Check if destination already has a pairing
if(worst_dest[dst] < 0 && dst != src){
int intm[6];
int len_intm = 0;
// Check if the destination is two hops from the source
if(is_two_hops_away(src, dst, intm, &len_intm)){
for(int intm_indx=0;intm_indx<len_intm;intm_indx++){
assert(intm[intm_indx]>=0);
// Check if the intm router already has a pairing
if(worst_dest[intm[intm_indx]] < 0){
// Check remaining routers for worst case pairing for intm router
for(int intm_dst_idx=intm[intm_indx]+total_routers/2; intm_dst_idx<total_routers+intm[intm_indx]+total_routers/2; intm_dst_idx++){
int intm_dst = intm_dst_idx % total_routers;
// Make sure intm_dst is not equal to dst or src
if(intm_dst != dst && intm_dst != src){
// Check if destination already has a pairing
if(worst_dest[intm_dst] < 0){
int intm2[6];
int len_intm2 = 0;
// Check if the destination is two hops from the source
if(is_two_hops_away(intm[intm_indx], intm_dst, intm2, &len_intm2)){
for(int intm2_indx=0; intm2_indx<len_intm2; intm2_indx++){
assert(intm2[intm2_indx]>=0);
// Check if the intm2 router already has been set as an intermediate router
if(middled[intm2[intm2_indx]] < 1){
// Check if the destination is either the src or dst router
if(intm2[intm2_indx] == src){
worst_dest[src] = dst;
worst_dest[dst] = src;
worst_dest[intm[intm_indx]] = intm_dst;
worst_dest[intm_dst] = intm[intm_indx];
middled[src]++;
middled[intm[intm_indx]]++;
pairings+=4;
//printf("T0 found pairings: %d<->%d<->%d & %d<->%d<->%d. Total pairings:%d\n",intm_dst, src, intm[intm_indx], src, intm[intm_indx], dst, pairings);
break;
}
if(intm2[intm2_indx] == dst){
worst_dest[src] = dst;
worst_dest[dst] = src;
worst_dest[intm[intm_indx]] = intm_dst;
worst_dest[intm_dst] = intm[intm_indx];
middled[dst]++;
middled[intm[intm_indx]]++;
pairings+=4;
//printf("T1 found pairings: %d<->%d<->%d & %d<->%d<->%d. Total pairings:%d\n",src, intm[intm_indx], dst, intm[intm_indx], dst, intm_dst, pairings);
break;
}
}else if(src == srcID)
printf("intm2:%d is already paired with %d\n",intm2[intm2_indx], worst_dest[intm2[intm2_indx]]);
}if(worst_dest[intm[intm_indx]] < 0){
if(src == srcID)
printf("No worst-case pairing found for the intermediate router:%d with src:%d and dst:%d\n",intm[intm_indx], src, dst);
}else{
break;
}
}else if(src == srcID)
printf("intm_dst:%d is not two hops away from intm:%d\n",intm_dst, intm[intm_indx]);
}else if(src == srcID)
printf("intm_dst:%d is already paired with %d\n",intm_dst, worst_dest[intm_dst]);
}else if(src == srcID)
printf("intm_dst:%d is equal to dst:%d\n", intm_dst, dst);
}if(worst_dest[intm[intm_indx]] < 0){
if(src == srcID){
printf("No worst-case pairing found for the intermediate router:%d with src:%d and dst:%d\n",intm[intm_indx], src, dst);
}
}else{
break;
}
}else{
if(src == srcID)
printf("dst:%d intm:%d has a pairing\n",dst, intm[intm_indx]);
}
}if(worst_dest[src] < 0){
if(src == srcID){
printf("No worst-case pairing found for src:%d and dst:%d\n", src, dst);
}
}else{
if(src == srcID){
printf("Found worst-case pairing found for src:%d and dst:%d\n", src, dst);
}
break;
}
}else{
if(src == srcID)
printf("dst:%d isn't two hops away\n",dst);
}
}else{
if(src == srcID)
printf("dst:%d has connection to %d\n",dst, worst_dest[dst]);
}
}
// Verify a destination was found
if(worst_dest[src] < 0){
printf("No worst-case pairing found for source router:%d\n",src);
//free(middled);
//break;
//exit(0);
}
}
}
//}
int no_pairing_count = 0;
int no_reverse_pairing_count = 0;
int *number_of_pairings = (int*)calloc(total_routers,sizeof(int));
int *number_of_reverse_pairings = (int*)calloc(total_routers,sizeof(int));
int *reverse_pairings = (int*)calloc(total_routers,sizeof(int));
for(int i=0; i<total_routers; i++){
reverse_pairings[i] = -1;
}
for(int i=0; i<total_routers; i++){
if (worst_dest[i] < 0){
no_pairing_count++;
//printf("no mapping for %d\n",i);
}
else{
reverse_pairings[worst_dest[i]]= i;
number_of_pairings[i]++;
if(number_of_pairings[i] > 1)
printf("router:%d has more than one pairing (%d pairings)\n",i,number_of_pairings[i]);
}
}
for(int i=0; i<total_routers; i++){
if(reverse_pairings[i] < 0){
no_reverse_pairing_count++;
//printf("no reverse mapping for %d\n",i);
}
else{
number_of_reverse_pairings[i]++;
if(number_of_reverse_pairings[i] > 1)
printf("router:%d has more than one reverse pairing (%d pairings)\n",i,number_of_reverse_pairings[i]);
}
}
printf("number of unpaired routers:%d\n",no_pairing_count);
printf("number of unpaired reverse routers:%d\n",no_reverse_pairing_count);
}
/** Older implementation of function to return an array mapping each router with it's corresponding worst-case router pair
* @param[in/out] *worst_dest Router ID to send all messages to
*/
void old_init_worst_case_mapping()
{
int i,j,k;
int r1,r2; //Routers to be paired
for(k=0;k<2;k++)
{
for(j=0;j<num_routers_per_grp-1;j+=2)
{
for(i=0;i<num_routers_per_grp;i++)
{
r1 = i + j*num_routers_per_grp + k*num_routers_per_grp*num_routers_per_grp;
r2 = i + (j+1)*num_routers_per_grp + k*num_routers_per_grp*num_routers_per_grp;
worst_dest[r1] = r2;
worst_dest[r2] = r1;
// printf("%d->%d, %d->%d\n",r1,worst_dest[r1],r2,worst_dest[r2]);
}
}
}
j = num_routers_per_grp-1;
for(i=0;i<num_routers_per_grp;i++)
{
r1 = i + j*num_routers_per_grp + 0*num_routers_per_grp*num_routers_per_grp;
r2 = i + j*num_routers_per_grp + 1*num_routers_per_grp*num_routers_per_grp;
worst_dest[r1] = r2;
worst_dest[r2] = r1;
// printf("%d->%d, %d->%d\n",r1,worst_dest[r1],r2,worst_dest[r2]);
}
}
static void issue_event(
svr_state * ns,
tw_lp * lp)
{
(void)ns;
tw_event *e;
svr_msg *m;
tw_stime kickoff_time;
/* each server sends a dummy event to itself that will kick off the real
* simulation
*/
configuration_get_value_double(&config, "PARAMS", "cn_bandwidth", NULL, &link_bandwidth);
if(!link_bandwidth) {
link_bandwidth = 4.7;
fprintf(stderr, "Bandwidth of channels not specified, setting to %lf\n", link_bandwidth);
}
if(arrival_time!=0)
{
MEAN_INTERVAL = arrival_time;
}
if(load != 0)
{
MEAN_INTERVAL = bytes_to_ns(payload_size, load*link_bandwidth);
if(traffic == NEAREST_NEIGHBOR_1D){
MEAN_INTERVAL = MEAN_INTERVAL * 2; //Because we are injecting two messages each interval
}else if(traffic == NEAREST_NEIGHBOR_2D){
MEAN_INTERVAL = MEAN_INTERVAL * 4;
}else if(traffic == NEAREST_NEIGHBOR_3D){
MEAN_INTERVAL = MEAN_INTERVAL * 6;
}else if(traffic == SCATTER){
MEAN_INTERVAL = MEAN_INTERVAL * (num_nodes - 1);
}else if(traffic == GATHER){
MEAN_INTERVAL = MEAN_INTERVAL * (num_nodes - 1);
}else if(traffic == ALL2ALL){
MEAN_INTERVAL = MEAN_INTERVAL * (num_nodes - 1);
}
}
unsigned int rng_calls = 0;
//kickoff_time = g_tw_lookahead + tw_rand_normal_sd(lp->rng, MEAN_INTERVAL, MEAN_INTERVAL*0.05, &rng_calls);
//kickoff_time = g_tw_lookahead + tw_rand_normal_sd(lp->rng, MEAN_INTERVAL, MEAN_INTERVAL*0.00001, &rng_calls);
kickoff_time = g_tw_lookahead + MEAN_INTERVAL;
/*if(lp->gid == 0){
printf("MEAN_INTERVAL:%f\n",MEAN_INTERVAL);
printf("kickoff_time:%f\n",kickoff_time);
}*/
e = tw_event_new(lp->gid, kickoff_time, lp);
m = tw_event_data(e);
m->svr_event_type = KICKOFF;
tw_event_send(e);
}
static void svr_init(
svr_state * ns,
tw_lp * lp)
{
// Initiailize comm_map 2D array
if(!lp->gid){
comm_map = (int**)malloc(num_nodes*sizeof(int*));
for(int i=0; i<num_nodes; i++){
comm_map[i] = (int*)calloc(num_nodes,sizeof(int));
}
}
ns->msg_send_times = (int*)calloc(num_msgs*2,sizeof(int));
ns->msg_recvd_times = (int*)calloc(num_msgs*2,sizeof(int));
issue_event(ns, lp);
return;
}
static void handle_kickoff_rev_event(
svr_state * ns,
tw_bf * b,
svr_msg * m,
tw_lp * lp)
{
(void)b;
(void)m;
(void)lp;
if(m->incremented_flag)
if(b->c1)
tw_rand_reverse_unif(lp->rng);
ns->msg_sent_count--;
model_net_event_rc2(lp, &m->event_rc);
tw_rand_reverse_unif(lp->rng);
}
static void handle_kickoff_event(
svr_state * ns,
tw_bf * b,
svr_msg * m,
tw_lp * lp)
{
(void)m;
if(traffic == SCATTER){
if(ns->msg_sent_count/(num_nodes-1) >= num_msgs){
m->incremented_flag = 1;
return;
}
}else{
if(ns->msg_sent_count >= num_msgs)
{
m->incremented_flag = 1;
return;
}
}
m->incremented_flag = 0;
char anno[MAX_NAME_LENGTH];
svr_msg * m_local = malloc(sizeof(svr_msg));
svr_msg * m_remote = malloc(sizeof(svr_msg));
m_local->svr_event_type = LOCAL;
m_local->src = lp->gid;
memcpy(m_remote, m_local, sizeof(svr_msg));
m_remote->svr_event_type = REMOTE;
ns->start_ts = tw_now(lp);
codes_mapping_get_lp_info(lp->gid, group_name, &group_index, lp_type_name, &lp_type_index, anno, &rep_id, &offset);
int num_server_lps = codes_mapping_get_lp_count(group_name, 1, "server", NULL, 0);
int src_terminal_id, src_router_id, dst_router_id;
int num_transfers = 0;
int *local_dest;
tw_lpid global_dest = -1;
// Compute current server's local/relative ID
int server_id = rep_id * num_server_lps + offset;
/* in case of uniform random traffic, send to a random destination. */
if(traffic == UNIFORM)
{
b->c1 = 1;
num_transfers = 1;
local_dest = (int*)malloc(num_transfers*sizeof(int));
local_dest[0] = tw_rand_integer(lp->rng, 0, num_nodes - 1);
}
else if(traffic == WORST_CASE)
{
num_transfers = 1;
local_dest = (int*)malloc(num_transfers*sizeof(int));
// Compute local id of source terminal (same local id as current server, assuming one server per terminal)
src_terminal_id = (rep_id * num_server_lps) + offset;
// Compute loacl id of source router
src_router_id = src_terminal_id / (num_server_lps);
// Only send messages if we have a worst-case pairing
if(worst_dest[src_router_id] < 0)
return;
// Get local id of destination router from precomputed worst-case mapping
dst_router_id = worst_dest[src_router_id];
// Get local id of destination terminal (same offset connection from dest router as src terminal from src router)
local_dest[0] = (dst_router_id * num_server_lps) + offset;
}
else if(traffic == NEAREST_NEIGHBOR_1D)
{
num_transfers = 2;
local_dest = (int*)malloc(num_transfers*sizeof(int));
// Neighbors in the first dimension
local_dest[0] = (server_id - 1);
if(local_dest[0] < 0)
local_dest[0] = num_nodes - 1;
local_dest[1] = (server_id + 1) % num_nodes;
}
else if(traffic == NEAREST_NEIGHBOR_2D)
{
num_transfers = 4;
local_dest = (int*)malloc(num_transfers*sizeof(int));
// Neighbors in the first dimension
local_dest[0] = (server_id - 1);
if(local_dest[0] < 0)
local_dest[0] = num_nodes - 1;
local_dest[1] = (server_id + 1) % num_nodes;
// Neighbors in the second dimension
local_dest[2] = (server_id + num_nodes/6) % num_nodes;
local_dest[3] = (server_id - num_nodes/6) % num_nodes;
if(local_dest[3] < 0)
local_dest[3] = num_nodes + local_dest[3];
}
else if(traffic == NEAREST_NEIGHBOR_3D)
{