diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index 4a019f6a1e8318fe162154109856c15363ec4f8d..52a7d716df74e500ea723e99c2d4a8f5aa08535e 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -3,8 +3,10 @@ Nikhil Jain, Abhinav Bhatele (LLNL) - Addition of direct scheme for setting up dragonfly network topology. - Network configuration setup for custom dragonfly model. - Topology generations scripts for custom dragonfly model. + - Bug fix for virtual channel deadlocks in custom dragonfly model. - Bug reporter for CODES network models. - Fat tree network setup and adaptive routing. + - Pending: Merging Express mesh model to master. Jens Domke (U. of Dresden) - Static routing in fat tree network model. diff --git a/codes/codes-workload.h b/codes/codes-workload.h index acba269c1aad654eaf0ce86b0aca10535a5d1102..3620e15cb485b022193f9d554dbf83eff70ac648 100644 --- a/codes/codes-workload.h +++ b/codes/codes-workload.h @@ -72,6 +72,7 @@ struct dumpi_trace_params { #ifdef ENABLE_CORTEX_PYTHON char cortex_script[MAX_NAME_LENGTH_WKLD]; char cortex_class[MAX_NAME_LENGTH_WKLD]; + char cortex_gen[MAX_NAME_LENGTH_WKLD]; #endif }; diff --git a/codes/model-net-lp.h b/codes/model-net-lp.h index b4ac1fb95f64ccfabde12eb86e4fb28b279bdf84..42686ba7a84b2acc8d31eb81dfa823e434f39952 100644 --- a/codes/model-net-lp.h +++ b/codes/model-net-lp.h @@ -22,6 +22,7 @@ extern "C" { #include "model-net.h" #include "model-net-sched.h" #include "net/dragonfly.h" +#include "net/dragonfly-custom.h" #include "net/slimfly.h" #include "net/fattree.h" #include "net/loggp.h" @@ -124,6 +125,7 @@ typedef struct model_net_wrap_msg { union { model_net_base_msg m_base; // base lp terminal_message m_dfly; // dragonfly + terminal_custom_message m_custom_dfly; // dragonfly-custom slim_terminal_message m_slim; // slimfly fattree_message m_fat; // fattree loggp_message m_loggp; // loggp diff --git a/configure.ac b/configure.ac index 6b9a0b9bb0286143b6c415e53127d2cbf1e72567..f54a3587dbd46c324e07046b1c1ee0319abbe12c 100755 --- a/configure.ac +++ b/configure.ac @@ -140,7 +140,7 @@ AC_ARG_WITH([boost],[AS_HELP_STRING([--with-boost@<:@=DIR@:>@], [location of Boost Python installation])]) if [ test "x${with_python}" != "x" -a "x${with_boost}" != "x"] ; then - AC_CHECK_FILES([${with_python}/lib/libpython2.7.a ${with_boost}/lib/libboost_python.a], + AC_CHECK_FILES([${with_python}/lib/libpython2.7.so ${with_boost}/lib/libboost_python.a], AM_CONDITIONAL(USE_PYTHON, true), AC_MSG_ERROR(Could not find Python and/or Boost-Python libraries)) PYTHON_CFLAGS="-I${with_python}/include -I${with_boost}/include" diff --git a/scripts/allocation_gen/config_alloc.conf b/scripts/allocation_gen/config_alloc.conf index 8e13c4d0b35df25e9bf7021cf18a7ee25e492e59..77bcb78ae0bda4429e0a244c2923596baa72f2e5 100644 --- a/scripts/allocation_gen/config_alloc.conf +++ b/scripts/allocation_gen/config_alloc.conf @@ -1,4 +1,4 @@ rand -792 -200 +8316 +2048 2048 diff --git a/scripts/gen-cray-topo/connections_general.c b/scripts/gen-cray-topo/connections_general.c index d006c70392b80cb75fa7602abf34ed791d055de3..a1700f06ad62660c57a4ecf33b75d4a508ff20f2 100755 --- a/scripts/gen-cray-topo/connections_general.c +++ b/scripts/gen-cray-topo/connections_general.c @@ -29,12 +29,15 @@ int main(int argc, char **argv) { int r = atoi(argv[2]); int c = atoi(argv[3]); + int total_routers = g * r * c; + FILE *intra = fopen(argv[4], "wb"); FILE *inter = fopen(argv[5], "wb"); int router = 0; int green = 0, black = 1; int groups = 0; + printf("\n Rows %d Cols %d Groups %d ", r, c, g); for(int rows = 0; rows < r; rows++) { for(int cols = 0; cols < c; cols++) { for(int cols1 = 0; cols1 < c; cols1++) { @@ -71,7 +74,7 @@ int main(int argc, char **argv) { } else { ndstg--; } - int gsize = 2, gs = 16; + int gsize = 2, gs = c; for(int row = 0; row < r; row++) { int srcrB = srcg * r * c + row * c, srcr; int dstrB = dstg * r * c + row * c, dstr; @@ -79,15 +82,19 @@ int main(int argc, char **argv) { int dstB = (nsrcg % (gs/2)) * 2; srcr = srcrB + srcB; dstr = dstrB + dstB; - for(int r = 0; r < 2; r++) { - for(int block = 0; block < gsize; block++) { - fwrite(&srcr, sizeof(int), 1, inter); - fwrite(&dstr, sizeof(int), 1, inter); - printf("INTER %d %d\n", srcr, dstr); + + if(srcr >= total_routers || dstr >= total_routers) + printf("\n connection between invalid routers src %d and dest %d ", srcr, dstr); + + for(int r = 0; r < 2; r++) { + for(int block = 0; block < gsize; block++) { + fwrite(&srcr, sizeof(int), 1, inter); + fwrite(&dstr, sizeof(int), 1, inter); + printf("INTER %d %d srcg %d destg %d srcrb %d dstrB %d \n", srcr, dstr, srcg, dstg, srcrB, dstrB); } srcr++; dstr++; - } + } } } } diff --git a/scripts/gen-cray-topo/connections_general_patched.c b/scripts/gen-cray-topo/connections_general_patched.c new file mode 100644 index 0000000000000000000000000000000000000000..e0e7da0c33e0a28037f28eaa240a3ac5979c21fe --- /dev/null +++ b/scripts/gen-cray-topo/connections_general_patched.c @@ -0,0 +1,103 @@ +////////////////////////////////////////////////////////////////////////////// +// Copyright (c) 2014, Lawrence Livermore National Security, LLC. +// Produced at the Lawrence Livermore National Laboratory. +// +// Written by: +// Nikhil Jain +// Abhinav Bhatele +// Peer-Timo Bremer +// +// LLNL-CODE-678961. All rights reserved. +// +// This file is part of Damselfly. For details, see: +// https://github.com/scalability-llnl/damselfly +// Please also read the LICENSE file for our notice and the LGPL. +////////////////////////////////////////////////////////////////////////////// + +#include "stdio.h" +#include "stdlib.h" + +//Usage ./binary num_groups num_rows num_columns intra_file inter_file + +int main(int argc, char **argv) { + if(argc < 3) { + printf("Correct usage: %s ", argv[0]); + exit(0); + } + + int g = atoi(argv[1]); + int r = atoi(argv[2]); + int c = atoi(argv[3]); + int g_p = atoi(argv[4]); + int r_p = atoi(argv[5]); + int c_p = atoi(argv[6]); + + int total_routers = g * r * c; + int routers_per_g = r * c; + + FILE *intra = fopen(argv[7], "wb"); + FILE *inter = fopen(argv[8], "wb"); + + int router = 0; + int green = 0, black = 1; + int groups = 0; + for(int rows = 0; rows < r; rows++) { + for(int cols = 0; cols < c; cols++) { + for(int cols1 = 0; cols1 < c; cols1++) { + if(cols1 != cols) { + int dest = (rows * c) + cols1; + for(int link = 0; link < c_p; link++) { + fwrite(&router, sizeof(int), 1, intra); + fwrite(&dest, sizeof(int), 1, intra); + fwrite(&green, sizeof(int), 1, intra); + printf("INTRA %d %d %d\n", router, dest, green); + } + } + } + for(int rows1 = 0; rows1 < r; rows1++) { + if(rows1 != rows) { + int dest = (rows1 * c) + cols; + for(int link = 0; link < r_p; link++) { + fwrite(&router, sizeof(int), 1, intra); + fwrite(&dest, sizeof(int), 1, intra); + fwrite(&black, sizeof(int), 1, intra); + printf("INTRA %d %d %d\n", router, dest, black); + } + } + } + router++; + } + } + + for(int srcg = 0; srcg < g; srcg++) { + for(int dstg = 0; dstg < g; dstg++) { + if(srcg != dstg) { + int nsrcg = srcg; + int ndstg = dstg; + if(srcg > dstg) { + nsrcg--; + } else { + ndstg--; + } + int startSrc = ndstg * g_p; + int startDst = nsrcg * g_p; + for(int link = 0; link < g_p; link++) { + int srcrB = srcg * routers_per_g, srcr; + int dstrB = dstg * routers_per_g, dstr; + srcr = srcrB + (startSrc + link) % routers_per_g; + dstr = dstrB + (startDst + link) % routers_per_g; + + if(srcr >= total_routers || dstr >= total_routers) + printf("\n connection between invalid routers src %d and dest %d ", srcr, dstr); + + fwrite(&srcr, sizeof(int), 1, inter); + fwrite(&dstr, sizeof(int), 1, inter); + printf("INTER %d %d srcg %d destg %d\n", srcr, dstr, srcg, dstg); + } + } + } + } + + fclose(intra); + fclose(inter); +} diff --git a/src/network-workloads/conf/dragonfly-custom/inter-9K-custom b/src/network-workloads/conf/dragonfly-custom/inter-9K-custom new file mode 100644 index 0000000000000000000000000000000000000000..a7bd637794e00bf421bc87f9210fd92e229030d4 Binary files /dev/null and b/src/network-workloads/conf/dragonfly-custom/inter-9K-custom differ diff --git a/src/network-workloads/conf/dragonfly-custom/intra-9K-custom b/src/network-workloads/conf/dragonfly-custom/intra-9K-custom new file mode 100644 index 0000000000000000000000000000000000000000..6987ec0907819def37338702295b0e3853a66ec3 Binary files /dev/null and b/src/network-workloads/conf/dragonfly-custom/intra-9K-custom differ diff --git a/src/network-workloads/conf/dragonfly-custom/modelnet-test-dragonfly-custom-768-nodes.conf.in b/src/network-workloads/conf/dragonfly-custom/modelnet-test-dragonfly-custom-768-nodes.conf.in index b8fecd3e625f72462db4ae77693fa73d7f2a20ec..87f6328f525a1269d540fa575e9a5f0e46fed0bc 100644 --- a/src/network-workloads/conf/dragonfly-custom/modelnet-test-dragonfly-custom-768-nodes.conf.in +++ b/src/network-workloads/conf/dragonfly-custom/modelnet-test-dragonfly-custom-768-nodes.conf.in @@ -49,9 +49,9 @@ PARAMS # number of global channels per router num_global_channels="10"; # network config file for intra-group connections - intra-group-connections="@abs_srcdir@/intra-custom-small"; + intra-group-connections="@abs_srcdir@/intra-theta"; # network config file for inter-group connections - inter-group-connections="@abs_srcdir@/inter-custom-small"; + inter-group-connections="@abs_srcdir@/inter-theta"; # routing protocol to be used routing="prog-adaptive"; } diff --git a/src/network-workloads/conf/dragonfly-custom/modelnet-test-dragonfly-custom.conf b/src/network-workloads/conf/dragonfly-custom/modelnet-test-dragonfly-custom.conf index be792312160f3ef9f4eb05c18eef1b5388c06a57..b2152ee8143137d5d511ca4bc9a802c693cc41e4 100644 --- a/src/network-workloads/conf/dragonfly-custom/modelnet-test-dragonfly-custom.conf +++ b/src/network-workloads/conf/dragonfly-custom/modelnet-test-dragonfly-custom.conf @@ -2,11 +2,11 @@ LPGROUPS { MODELNET_GRP { - repetitions="1600"; + repetitions="1520"; # name of this lp changes according to the model - nw-lp="4"; + nw-lp="8"; # these lp names will be the same for dragonfly-custom model - modelnet_dragonfly_custom="4"; + modelnet_dragonfly_custom="8"; modelnet_dragonfly_custom_router="1"; } } @@ -24,11 +24,11 @@ PARAMS # number of routers within each group # this is dictated by the dragonfly configuration files # intra-group rows for routers - num_router_rows="4"; + num_router_rows="1"; # intra-group columns for routers - num_router_cols="20"; + num_router_cols="40"; # number of groups in the network - num_groups="20"; + num_groups="38"; # buffer size in bytes for local virtual channels local_vc_size="8192"; #buffer size in bytes for global virtual channels @@ -38,20 +38,24 @@ PARAMS #bandwidth in GiB/s for local channels local_bandwidth="5.25"; # bandwidth in GiB/s for global channels - global_bandwidth="18.75"; + global_bandwidth="4.69"; # bandwidth in GiB/s for compute node-router channels cn_bandwidth="8.0"; +# Number of row channels + num_row_chans="2"; +# Number of column channels + num_col_chans="1"; # ROSS message size message_size="592"; # number of compute nodes connected to router, dictated by dragonfly config # file - num_cns_per_router="4"; + num_cns_per_router="8"; # number of global channels per router - num_global_channels="10"; + num_global_channels="4"; # network config file for intra-group connections - intra-group-connections="../src/network-workloads/conf/dragonfly-custom/intra-custom"; + intra-group-connections="/Users/mmubarak/Documents/software_development/codes/scripts/gen-cray-topo/intratest"; # network config file for inter-group connections - inter-group-connections="../src/network-workloads/conf/dragonfly-custom/inter-custom"; + inter-group-connections="/Users/mmubarak/Documents/software_development/codes/scripts/gen-cray-topo/intertest"; # routing protocol to be used routing="prog-adaptive"; } diff --git a/src/network-workloads/conf/dragonfly-custom/modelnet-test-dragonfly-theta.conf b/src/network-workloads/conf/dragonfly-custom/modelnet-test-dragonfly-theta.conf index a93f6a04006bdf46749ab15a943595cf08e30f9a..ef941ed99c67592f069bb5106d82e45681b0ed6d 100644 --- a/src/network-workloads/conf/dragonfly-custom/modelnet-test-dragonfly-theta.conf +++ b/src/network-workloads/conf/dragonfly-custom/modelnet-test-dragonfly-theta.conf @@ -37,7 +37,7 @@ PARAMS #bandwidth in GiB/s for local channels local_bandwidth="5.25"; # bandwidth in GiB/s for global channels - global_bandwidth="18.75"; + global_bandwidth="4.69"; # bandwidth in GiB/s for compute node-router channels cn_bandwidth="16.0"; # ROSS message size @@ -46,7 +46,7 @@ PARAMS # file num_cns_per_router="4"; # number of global channels per router - num_global_channels="10"; + num_global_channels="4"; # network config file for intra-group connections intra-group-connections="../src/network-workloads/conf/dragonfly-custom/intra-theta"; # network config file for inter-group connections diff --git a/src/network-workloads/model-net-mpi-replay.c b/src/network-workloads/model-net-mpi-replay.c index 21c9fe86d7b2456e8561e9d5c0baa2d9e67990a2..c52d8c3d99af0829ff3756032ebc7088414b51a9 100644 --- a/src/network-workloads/model-net-mpi-replay.c +++ b/src/network-workloads/model-net-mpi-replay.c @@ -13,26 +13,42 @@ #include "codes/model-net.h" #include "codes/rc-stack.h" #include "codes/quicklist.h" +#include "codes/quickhash.h" #include "codes/codes-jobmap.h" /* turning on track lp will generate a lot of output messages */ #define MN_LP_NM "modelnet_dragonfly_custom" +#define CONTROL_MSG_SZ 64 #define TRACK_LP -1 #define TRACE -1 #define MAX_WAIT_REQS 512 #define CS_LP_DBG 0 +#define EAGER_THRESHOLD 81920000 +#define RANK_HASH_TABLE_SZ 2000 +#define NOISE 3.0 +#define NW_LP_NM "nw-lp" #define lprintf(_fmt, ...) \ do {if (CS_LP_DBG) printf(_fmt, __VA_ARGS__);} while (0) #define MAX_STATS 65536 +#define PAYLOAD_SZ 1024 +static int msg_size_hash_compare( + void *key, struct qhash_head *link); + +int enable_msg_tracking = 0; + +int unmatched = 0; char workload_type[128]; char workload_file[8192]; char offset_file[8192]; static int wrkld_id; static int num_net_traces = 0; +static int num_dumpi_traces = 0; + static int alloc_spec = 0; -static double self_overhead = 10.0; +static tw_stime self_overhead = 10.0; +static tw_stime mean_interval = 100000; /* Doing LP IO*/ static char lp_io_dir[256] = {'\0'}; @@ -41,21 +57,23 @@ static unsigned int lp_io_use_suffix = 0; static int do_lp_io = 0; /* variables for loading multiple applications */ -/* Xu's additions start */ char workloads_conf_file[8192]; char alloc_file[8192]; int num_traces_of_job[5]; +tw_stime soft_delay_mpi = 2500; +tw_stime nic_delay = 1000; +tw_stime copy_per_byte_eager = 0.55; char file_name_of_job[5][8192]; struct codes_jobmap_ctx *jobmap_ctx; struct codes_jobmap_params_list jobmap_p; -/* Xu's additions end */ /* Variables for Cortex Support */ /* Matthieu's additions start */ #ifdef ENABLE_CORTEX_PYTHON -static char cortex_file[512]; -static char cortex_class[512]; +static char cortex_file[512] = "\0"; +static char cortex_class[512] = "\0"; +static char cortex_gen[512] = "\0"; #endif /* Matthieu's additions end */ @@ -65,9 +83,12 @@ typedef int32_t dumpi_req_id; static int net_id = 0; static float noise = 5.0; -static int num_net_lps = 0, num_mpi_lps = 0; +static int num_nw_lps = 0, num_mpi_lps = 0; + +static int num_syn_clients; FILE * workload_log = NULL; +FILE * msg_size_log = NULL; FILE * workload_agg_log = NULL; FILE * workload_meta_log = NULL; @@ -76,6 +97,9 @@ static uint64_t sample_bytes_written = 0; long long num_bytes_sent=0; long long num_bytes_recvd=0; +long long num_syn_bytes_sent = 0; +long long num_syn_bytes_recvd = 0; + double max_time = 0, max_comm_time = 0, max_wait_time = 0, max_send_time = 0, max_recv_time = 0; double avg_time = 0, avg_comm_time = 0, avg_wait_time = 0, avg_send_time = 0, avg_recv_time = 0; @@ -102,6 +126,12 @@ enum MPI_NW_EVENTS MPI_SEND_ARRIVED, MPI_SEND_ARRIVED_CB, // for tracking message times on sender MPI_SEND_POSTED, + MPI_REND_ARRIVED, + MPI_REND_ACK_ARRIVED, + CLI_BCKGND_FIN, + CLI_BCKGND_ARRIVE, + CLI_BCKGND_GEN, + CLI_NBR_FINISH, }; struct mpi_workload_sample @@ -145,6 +175,15 @@ struct pending_waits struct qlist_head ql; }; +struct msg_size_info +{ + int64_t msg_size; + int num_msgs; + tw_stime agg_latency; + tw_stime avg_latency; + struct qhash_head * hash_link; + struct qlist_head ql; +}; typedef struct mpi_msgs_queue mpi_msgs_queue; typedef struct completed_requests completed_requests; typedef struct pending_waits pending_waits; @@ -158,6 +197,9 @@ struct nw_state int app_id; int local_rank; + int is_finished; + int neighbor_completed; + struct rc_stack * processed_ops; struct rc_stack * matched_reqs; @@ -194,9 +236,18 @@ struct nw_state /* Pending wait operation */ struct pending_waits * wait_op; + /* Message size latency information */ + struct qhash_table * msg_sz_table; + struct qlist_head msg_sz_list; + + /* quick hash for maintaining message latencies */ + unsigned long num_bytes_sent; unsigned long num_bytes_recvd; + unsigned long syn_data; + unsigned long gen_data; + /* For sampling data */ int sampling_indx; int max_arr_size; @@ -242,9 +293,12 @@ struct nw_message } rc; }; +static void send_ack_back(nw_state* s, tw_bf * bf, nw_message * m, tw_lp * lp, mpi_msgs_queue * mpi_op); + +static void send_ack_back_rc(nw_state* s, tw_bf * bf, nw_message * m, tw_lp * lp); /* executes MPI isend and send operations */ static void codes_exec_mpi_send( - nw_state* s, tw_bf * bf, nw_message * m, tw_lp* lp, struct codes_workload_op * mpi_op); + nw_state* s, tw_bf * bf, nw_message * m, tw_lp* lp, struct codes_workload_op * mpi_op, int is_rend); /* execute MPI irecv operation */ static void codes_exec_mpi_recv( nw_state* s, tw_bf * bf, nw_message * m, tw_lp * lp, struct codes_workload_op * mpi_op); @@ -292,6 +346,286 @@ static void update_message_time_rc( /* conversion from seconds to eanaoseconds */ static tw_stime s_to_ns(tw_stime ns); +static void update_message_size_rc( + struct nw_state * ns, + tw_lp * lp, + tw_bf * bf, + struct nw_message * m) +{ + +} +/* update the message size */ +static void update_message_size( + struct nw_state * ns, + tw_lp * lp, + tw_bf * bf, + struct nw_message * m, + mpi_msgs_queue * qitem, + int is_eager, + int is_send) +{ + struct qhash_head * hash_link = NULL; + tw_stime msg_init_time = qitem->req_init_time; + + if(!ns->msg_sz_table) + ns->msg_sz_table = qhash_init(msg_size_hash_compare, quickhash_64bit_hash, RANK_HASH_TABLE_SZ); + + hash_link = qhash_search(ns->msg_sz_table, &(qitem->num_bytes)); + + if(is_send) + msg_init_time = m->fwd.sim_start_time; + + /* update hash table */ + if(!hash_link) + { + struct msg_size_info * msg_info = malloc(sizeof(struct msg_size_info)); + msg_info->msg_size = qitem->num_bytes; + msg_info->num_msgs = 1; + msg_info->agg_latency = tw_now(lp) - msg_init_time; + msg_info->avg_latency = msg_info->agg_latency; + qhash_add(ns->msg_sz_table, &(msg_info->msg_size), &(msg_info->hash_link)); + qlist_add(&msg_info->ql, &ns->msg_sz_list); + //printf("\n Msg size %d aggregate latency %f num messages %d ", m->fwd.num_bytes, msg_info->agg_latency, msg_info->num_msgs); + } + else + { + struct msg_size_info * tmp = qhash_entry(hash_link, struct msg_size_info, hash_link); + tmp->num_msgs++; + tmp->agg_latency += tw_now(lp) - msg_init_time; + tmp->avg_latency = (tmp->agg_latency / tmp->num_msgs); +// printf("\n Msg size %d aggregate latency %f num messages %d ", qitem->num_bytes, tmp->agg_latency, tmp->num_msgs); + } +} +static void notify_background_traffic_rc( + struct nw_state * ns, + tw_lp * lp, + tw_bf * bf, + struct nw_message * m) +{ + tw_rand_reverse_unif(lp->rng); +} + +static void notify_background_traffic( + struct nw_state * ns, + tw_lp * lp, + tw_bf * bf, + struct nw_message * m) +{ + struct codes_jobmap_id jid; + jid = codes_jobmap_to_local_id(ns->nw_id, jobmap_ctx); + + int num_jobs = codes_jobmap_get_num_jobs(jobmap_ctx); + + for(int other_id = 0; other_id < num_jobs; other_id++) + { + if(other_id == jid.job) + continue; + + struct codes_jobmap_id other_jid; + other_jid.job = other_id; + + int num_other_ranks = codes_jobmap_get_num_ranks(other_id, jobmap_ctx); + + lprintf("\n Other ranks %ld ", num_other_ranks); + tw_stime ts = (1.1 * g_tw_lookahead) + tw_rand_exponential(lp->rng, mean_interval/10000); + tw_lpid global_dest_id; + + for(int k = 0; k < num_other_ranks; k++) + { + other_jid.rank = k; + int intm_dest_id = codes_jobmap_to_global_id(other_jid, jobmap_ctx); + global_dest_id = codes_mapping_get_lpid_from_relative(intm_dest_id, NULL, NW_LP_NM, NULL, 0); + + tw_event * e; + struct nw_message * m_new; + e = tw_event_new(global_dest_id, ts, lp); + m_new = tw_event_data(e); + m_new->msg_type = CLI_BCKGND_FIN; + tw_event_send(e); + } + } + return; +} +static void notify_neighbor_rc( + struct nw_state * ns, + tw_lp * lp, + tw_bf * bf, + struct nw_message * m) +{ + if(bf->c0) + { + notify_background_traffic_rc(ns, lp, bf, m); + return; + } + + if(bf->c1) + { + tw_rand_reverse_unif(lp->rng); + } +} +static void notify_neighbor( + struct nw_state * ns, + tw_lp * lp, + tw_bf * bf, + struct nw_message * m) +{ + if(ns->local_rank == num_dumpi_traces - 1 + && ns->is_finished == 1 + && ns->neighbor_completed == 1) + { + printf("\n All workloads completed, notifying background traffic "); + bf->c0 = 1; + notify_background_traffic(ns, lp, bf, m); + return; + } + + struct codes_jobmap_id nbr_jid; + nbr_jid.job = ns->app_id; + tw_lpid global_dest_id; + + if(ns->is_finished == 1 && (ns->neighbor_completed == 1 || ns->local_rank == 0)) + { + bf->c1 = 1; + + printf("\n Local rank %d notifying neighbor %d ", ns->local_rank, ns->local_rank+1); + tw_stime ts = (1.1 * g_tw_lookahead) + tw_rand_exponential(lp->rng, mean_interval/10000); + nbr_jid.rank = ns->local_rank + 1; + + /* Send a notification to the neighbor about completion */ + int intm_dest_id = codes_jobmap_to_global_id(nbr_jid, jobmap_ctx); + global_dest_id = codes_mapping_get_lpid_from_relative(intm_dest_id, NULL, NW_LP_NM, NULL, 0); + + tw_event * e; + struct nw_message * m_new; + e = tw_event_new(global_dest_id, ts, lp); + m_new = tw_event_data(e); + m_new->msg_type = CLI_NBR_FINISH; + tw_event_send(e); + } +} +void finish_bckgnd_traffic_rc( + struct nw_state * ns, + tw_bf * b, + struct nw_message * msg, + tw_lp * lp) +{ + ns->is_finished = 0; + return; +} +void finish_bckgnd_traffic( + struct nw_state * ns, + tw_bf * b, + struct nw_message * msg, + tw_lp * lp) +{ + ns->is_finished = 1; + lprintf("\n LP %llu completed sending data %lld completed at time %lf ", lp->gid, ns->gen_data, tw_now(lp)); + return; +} + +void finish_nbr_wkld_rc( + struct nw_state * ns, + tw_bf * b, + struct nw_message * msg, + tw_lp * lp) +{ + ns->neighbor_completed = 0; + + notify_neighbor_rc(ns, lp, b, msg); +} + +void finish_nbr_wkld( + struct nw_state * ns, + tw_bf * b, + struct nw_message * msg, + tw_lp * lp) +{ + printf("\n Workload completed, notifying neighbor "); + ns->neighbor_completed = 1; + + notify_neighbor(ns, lp, b, msg); +} +static void gen_synthetic_tr_rc(nw_state * s, tw_bf * bf, nw_message * m, tw_lp * lp) +{ + if(bf->c0) + return; + + model_net_event_rc2(lp, &m->event_rc); + s->gen_data -= PAYLOAD_SZ; + + num_syn_bytes_sent -= PAYLOAD_SZ; + tw_rand_reverse_unif(lp->rng); + tw_rand_reverse_unif(lp->rng); + +} + +/* generate synthetic traffic */ +static void gen_synthetic_tr(nw_state * s, tw_bf * bf, nw_message * m, tw_lp * lp) +{ + if(s->is_finished == 1) + { + bf->c0 = 1; + return; + } + + /* Get job information */ + tw_lpid global_dest_id; + + struct codes_jobmap_id jid; + jid = codes_jobmap_to_local_id(s->nw_id, jobmap_ctx); + + int num_clients = codes_jobmap_get_num_ranks(jid.job, jobmap_ctx); + int dest_svr = tw_rand_integer(lp->rng, 0, num_clients - 1); + + if(dest_svr == s->local_rank) + { + dest_svr = (s->local_rank + 1) % num_clients; + } + + jid.rank = dest_svr; + + int intm_dest_id = codes_jobmap_to_global_id(jid, jobmap_ctx); + global_dest_id = codes_mapping_get_lpid_from_relative(intm_dest_id, NULL, NW_LP_NM, NULL, 0); + + nw_message remote_m; + remote_m.fwd.sim_start_time = tw_now(lp); + remote_m.fwd.dest_rank = dest_svr; + remote_m.msg_type = CLI_BCKGND_ARRIVE; + remote_m.fwd.num_bytes = PAYLOAD_SZ; + remote_m.fwd.app_id = s->app_id; + remote_m.fwd.src_rank = s->local_rank; + + m->event_rc = model_net_event(net_id, "synthetic-tr", global_dest_id, PAYLOAD_SZ, 0.0, + sizeof(nw_message), (const void*)&remote_m, + 0, NULL, lp); + + s->gen_data += PAYLOAD_SZ; + num_syn_bytes_sent += PAYLOAD_SZ; + + /* New event after MEAN_INTERVAL */ + tw_stime ts = mean_interval + tw_rand_exponential(lp->rng, NOISE); + tw_event * e; + nw_message * m_new; + e = tw_event_new(lp->gid, ts, lp); + m_new = tw_event_data(e); + m_new->msg_type = CLI_BCKGND_GEN; + tw_event_send(e); +} + +void arrive_syn_tr_rc(nw_state * s, tw_bf * bf, nw_message * m, tw_lp * lp) +{ +// printf("\n Data arrived %d total data %ld ", m->fwd.num_bytes, s->syn_data); + int data = m->fwd.num_bytes; + s->syn_data -= data; + num_syn_bytes_recvd -= data; +} +void arrive_syn_tr(nw_state * s, tw_bf * bf, nw_message * m, tw_lp * lp) +{ +// printf("\n Data arrived %d total data %ld ", m->fwd.num_bytes, s->syn_data); + int data = m->fwd.num_bytes; + s->syn_data += data; + num_syn_bytes_recvd += data; +} /* Debugging functions, may generate unused function warning */ static void print_waiting_reqs(int32_t * reqs, int count) { @@ -331,20 +665,32 @@ static int clear_completed_reqs(nw_state * s, int32_t * reqs, int count) { int i, matched = 0; + for( i = 0; i < count; i++) { struct qlist_head * ent = NULL; + struct completed_requests * current = NULL; + struct completed_requests * prev = NULL; + qlist_for_each(ent, &s->completed_reqs) { - struct completed_requests* current = - qlist_entry(ent, completed_requests, ql); + current = qlist_entry(ent, completed_requests, ql); + + if(prev) + rc_stack_push(lp, prev, free, s->matched_reqs); + if(current->req_id == reqs[i]) { ++matched; qlist_del(¤t->ql); - rc_stack_push(lp, current, free, s->matched_reqs); + prev = current; } + else + prev = NULL; } + + if(prev) + rc_stack_push(lp, prev, free, s->matched_reqs); } return matched; } @@ -497,6 +843,7 @@ static void codes_exec_mpi_wait_all_rc( } return; } + static void codes_exec_mpi_wait_all( nw_state* s, tw_bf * bf, @@ -605,10 +952,11 @@ static int rm_matching_rcv(nw_state * ns, qi = qlist_entry(ent, mpi_msgs_queue, ql); if(//(qi->num_bytes == qitem->num_bytes) //&& - ((qi->tag == qitem->tag) || qi->tag == -1) + ((qi->tag == qitem->tag) || qi->tag == -1) && ((qi->source_rank == qitem->source_rank) || qi->source_rank == -1)) { matched = 1; + //qitem->num_bytes = qi->num_bytes; break; } ++index; @@ -616,8 +964,19 @@ static int rm_matching_rcv(nw_state * ns, if(matched) { + if(enable_msg_tracking && qitem->num_bytes < EAGER_THRESHOLD) + { + update_message_size(ns, lp, bf, m, qitem, 1, 1); + } + if(qitem->num_bytes >= EAGER_THRESHOLD) + { + /* Matching receive found, need to notify the sender to transmit + * the data * (only works in sequential mode)*/ + bf->c10 = 1; + send_ack_back(ns, bf, m, lp, qitem); + } m->rc.saved_recv_time = ns->recv_time; - ns->recv_time += (tw_now(lp) - qi->req_init_time); + ns->recv_time += (tw_now(lp) - m->fwd.sim_start_time); if(qi->op_type == CODES_WK_IRECV) update_completed_queue(ns, bf, m, lp, qi->req_id); @@ -649,7 +1008,7 @@ static int rm_matching_send(nw_state * ns, (qi->tag == qitem->tag || qitem->tag == -1) && ((qi->source_rank == qitem->source_rank) || qitem->source_rank == -1)) { - qitem->num_bytes = qi->num_bytes; + //qi->num_bytes = qitem->num_bytes; matched = 1; break; } @@ -658,6 +1017,16 @@ static int rm_matching_send(nw_state * ns, if(matched) { + if(enable_msg_tracking && (qi->num_bytes < EAGER_THRESHOLD)) + update_message_size(ns, lp, bf, m, qi, 1, 0); + + if(qitem->num_bytes >= EAGER_THRESHOLD) + { + /* Matching receive found, need to notify the sender to transmit + * the data */ + bf->c10 = 1; + send_ack_back(ns, bf, m, lp, qi); + } m->rc.saved_recv_time = ns->recv_time; ns->recv_time += (tw_now(lp) - qitem->req_init_time); @@ -720,6 +1089,10 @@ static void codes_exec_mpi_recv_rc( tw_lp* lp) { ns->recv_time = m->rc.saved_recv_time; + + if(bf->c10) + send_ack_back_rc(ns, bf, m, lp); + if(m->fwd.found_match >= 0) { ns->recv_time = m->rc.saved_recv_time; @@ -825,12 +1198,35 @@ int get_global_id_of_job_rank(tw_lpid job_rank, int app_id) int global_rank = codes_jobmap_to_global_id(lid, jobmap_ctx); return global_rank; } +static void codes_exec_mpi_send_rc(nw_state * s, tw_bf * bf, nw_message * m, tw_lp * lp) +{ + if(enable_sampling) + { + int indx = s->sampling_indx; + + s->mpi_wkld_samples[indx].num_sends_sample--; + s->mpi_wkld_samples[indx].num_bytes_sample -= m->rc.saved_num_bytes; + + if(bf->c1) + { + s->sampling_indx--; + s->cur_interval_end -= sampling_interval; + } + } + model_net_event_rc2(lp, &m->event_rc); + if(m->op_type == CODES_WK_ISEND) + codes_issue_next_event_rc(lp); + s->num_sends--; + s->num_bytes_sent -= m->rc.saved_num_bytes; + num_bytes_sent -= m->rc.saved_num_bytes; +} /* executes MPI send and isend operations */ static void codes_exec_mpi_send(nw_state* s, tw_bf * bf, nw_message * m, tw_lp* lp, - struct codes_workload_op * mpi_op) + struct codes_workload_op * mpi_op, + int is_rend) { /* model-net event */ int global_dest_rank = mpi_op->u.send.dest_rank; @@ -840,14 +1236,16 @@ static void codes_exec_mpi_send(nw_state* s, global_dest_rank = get_global_id_of_job_rank(mpi_op->u.send.dest_rank, s->app_id); } - //printf("\n Sender rank %d global dest rank %d ", s->nw_id, global_dest_rank); + //printf("\n Sender rank %d global dest rank %d dest-rank %d rend %d", s->nw_id, global_dest_rank, mpi_op->u.send.dest_rank, is_rend); m->rc.saved_num_bytes = mpi_op->u.send.num_bytes; /* model-net event */ tw_lpid dest_rank = codes_mapping_get_lpid_from_relative(global_dest_rank, NULL, "nw-lp", NULL, 0); - - num_bytes_sent += mpi_op->u.send.num_bytes; - s->num_bytes_sent += mpi_op->u.send.num_bytes; + if(!is_rend) + { + num_bytes_sent += mpi_op->u.send.num_bytes; + s->num_bytes_sent += mpi_op->u.send.num_bytes; + } if(enable_sampling) { if(tw_now(lp) >= s->cur_interval_end) @@ -875,7 +1273,6 @@ static void codes_exec_mpi_send(nw_state* s, nw_message local_m; nw_message remote_m; - local_m.fwd.sim_start_time = tw_now(lp); local_m.fwd.dest_rank = mpi_op->u.send.dest_rank; local_m.fwd.src_rank = mpi_op->u.send.source_rank; local_m.op_type = mpi_op->op_type; @@ -884,14 +1281,51 @@ static void codes_exec_mpi_send(nw_state* s, local_m.fwd.num_bytes = mpi_op->u.send.num_bytes; local_m.fwd.req_id = mpi_op->u.send.req_id; local_m.fwd.app_id = s->app_id; - - remote_m = local_m; - remote_m.msg_type = MPI_SEND_ARRIVED; - - m->event_rc = model_net_event_mctx(net_id, &group_ratio, &group_ratio, - "test", dest_rank, mpi_op->u.send.num_bytes, self_overhead, + + + if(mpi_op->u.send.num_bytes < EAGER_THRESHOLD) + { + /* directly issue a model-net send */ + + tw_stime copy_overhead = copy_per_byte_eager * mpi_op->u.send.num_bytes; + local_m.fwd.sim_start_time = tw_now(lp); + + remote_m = local_m; + remote_m.msg_type = MPI_SEND_ARRIVED; + m->event_rc = model_net_event_mctx(net_id, &group_ratio, &group_ratio, + "test", dest_rank, mpi_op->u.send.num_bytes, (self_overhead + copy_overhead + soft_delay_mpi + nic_delay), sizeof(nw_message), (const void*)&remote_m, sizeof(nw_message), (const void*)&local_m, lp); - + } + else if (is_rend == 0) + { + /* Initiate the handshake. Issue a control message to the destination first. No local message, + * only remote message sent. */ + remote_m.fwd.sim_start_time = tw_now(lp); + remote_m.fwd.dest_rank = mpi_op->u.send.dest_rank; + remote_m.fwd.src_rank = mpi_op->u.send.source_rank; + remote_m.msg_type = MPI_SEND_ARRIVED; + remote_m.op_type = mpi_op->op_type; + remote_m.fwd.tag = mpi_op->u.send.tag; + remote_m.fwd.num_bytes = mpi_op->u.send.num_bytes; + remote_m.fwd.req_id = mpi_op->u.send.req_id; + remote_m.fwd.app_id = s->app_id; + + m->event_rc = model_net_event_mctx(net_id, &group_ratio, &group_ratio, + "test", dest_rank, CONTROL_MSG_SZ, (self_overhead + soft_delay_mpi + nic_delay), + sizeof(nw_message), (const void*)&remote_m, 0, NULL, lp); + } + else if(is_rend == 1) + { + /* initiate the actual data transfer, local completion message is sent + * for any blocking sends. */ + local_m.fwd.sim_start_time = mpi_op->sim_start_time; + remote_m = local_m; + remote_m.msg_type = MPI_REND_ARRIVED; + + m->event_rc = model_net_event_mctx(net_id, &group_ratio, &group_ratio, + "test", dest_rank, mpi_op->u.send.num_bytes, (self_overhead + soft_delay_mpi + nic_delay), + sizeof(nw_message), (const void*)&remote_m, sizeof(nw_message), (const void*)&local_m, lp); + } if(enable_debug) { if(mpi_op->op_type == CODES_WK_ISEND) @@ -904,7 +1338,7 @@ static void codes_exec_mpi_send(nw_state* s, tw_now(lp), s->app_id, s->nw_id, global_dest_rank, mpi_op->u.send.tag, mpi_op->u.send.num_bytes); } /* isend executed, now get next MPI operation from the queue */ - if(mpi_op->op_type == CODES_WK_ISEND) + if(mpi_op->op_type == CODES_WK_ISEND && !is_rend) codes_issue_next_event(lp); } @@ -961,7 +1395,7 @@ static void update_completed_queue(nw_state* s, req->req_id = req_id; qlist_add_tail(&req->ql, &s->completed_reqs); - /*if(lp->gid == TRACK) +/* if(lp->gid == TRACK) { printf("\n Forward mode adding %ld ", req_id); print_completed_queue(&s->completed_reqs); @@ -981,6 +1415,38 @@ static void update_completed_queue(nw_state* s, } } +static void send_ack_back_rc(nw_state* s, tw_bf * bf, nw_message * m, tw_lp * lp) +{ + /* Send an ack back to the sender */ + model_net_event_rc2(lp, &m->event_rc); +} +static void send_ack_back(nw_state* s, tw_bf * bf, nw_message * m, tw_lp * lp, mpi_msgs_queue * mpi_op) +{ + int global_dest_rank = mpi_op->source_rank; + + if(alloc_spec) + { + global_dest_rank = get_global_id_of_job_rank(mpi_op->source_rank, s->app_id); + } + + tw_lpid dest_rank = codes_mapping_get_lpid_from_relative(global_dest_rank, NULL, "nw-lp", NULL, 0); + /* Send a message back to sender indicating availability.*/ + nw_message remote_m; + remote_m.fwd.sim_start_time = mpi_op->req_init_time; + remote_m.fwd.dest_rank = mpi_op->dest_rank; + remote_m.fwd.src_rank = mpi_op->source_rank; + remote_m.op_type = mpi_op->op_type; + remote_m.msg_type = MPI_REND_ACK_ARRIVED; + remote_m.fwd.tag = mpi_op->tag; + remote_m.fwd.num_bytes = mpi_op->num_bytes; + remote_m.fwd.req_id = mpi_op->req_id; + +// printf("\n Op type %d dest rank %d ", mpi_op->op_type, mpi_op->dest_rank); + m->event_rc = model_net_event_mctx(net_id, &group_ratio, &group_ratio, + "test", dest_rank, CONTROL_MSG_SZ, (self_overhead + soft_delay_mpi + nic_delay), + sizeof(nw_message), (const void*)&remote_m, 0, NULL, lp); + +} /* reverse handler for updating arrival queue function */ static void update_arrival_queue_rc(nw_state* s, tw_bf * bf, @@ -990,7 +1456,11 @@ static void update_arrival_queue_rc(nw_state* s, s->num_bytes_recvd -= m->fwd.num_bytes; num_bytes_recvd -= m->fwd.num_bytes; - codes_local_latency_reverse(lp); + if(bf->c1) + codes_local_latency_reverse(lp); + + if(bf->c10) + send_ack_back_rc(s, bf, m, lp); if(m->fwd.found_match >= 0) { @@ -1053,22 +1523,27 @@ static void update_arrival_queue(nw_state* s, tw_bf * bf, nw_message * m, tw_lp { global_src_id = get_global_id_of_job_rank(m->fwd.src_rank, s->app_id); } - tw_event *e_callback = + + if(m->fwd.num_bytes < EAGER_THRESHOLD) + { + bf->c1 = 1; + tw_event *e_callback = tw_event_new(rank_to_lpid(global_src_id), codes_local_latency(lp), lp); - nw_message *m_callback = tw_event_data(e_callback); - m_callback->msg_type = MPI_SEND_ARRIVED_CB; - m_callback->fwd.msg_send_time = tw_now(lp) - m->fwd.sim_start_time; - tw_event_send(e_callback); - + nw_message *m_callback = tw_event_data(e_callback); + m_callback->msg_type = MPI_SEND_ARRIVED_CB; + m_callback->fwd.msg_send_time = tw_now(lp) - m->fwd.sim_start_time; + tw_event_send(e_callback); + } /* Now reconstruct the queue item */ mpi_msgs_queue * arrived_op = (mpi_msgs_queue *) malloc(sizeof(mpi_msgs_queue)); arrived_op->req_init_time = m->fwd.sim_start_time; arrived_op->op_type = m->op_type; arrived_op->source_rank = m->fwd.src_rank; - arrived_op->dest_rank = m->fwd.dest_rank; - arrived_op->num_bytes = m->fwd.num_bytes; arrived_op->tag = m->fwd.tag; + arrived_op->req_id = m->fwd.req_id; + arrived_op->num_bytes = m->fwd.num_bytes; + arrived_op->dest_rank = m->fwd.dest_rank; if(s->nw_id == (tw_lpid)TRACK_LP) printf("\n Send op arrived source rank %d num bytes %llu ", arrived_op->source_rank, @@ -1117,6 +1592,7 @@ void nw_test_init(nw_state* s, tw_lp* lp) s->nw_id = codes_mapping_get_lp_relative_id(lp->gid, 0, 0); s->mpi_wkld_samples = calloc(MAX_STATS, sizeof(struct mpi_workload_sample)); s->sampling_indx = 0; + s->is_finished = 0; if(!num_net_traces) num_net_traces = num_mpi_lps; @@ -1158,6 +1634,7 @@ void nw_test_init(nw_state* s, tw_lp* lp) #ifdef ENABLE_CORTEX_PYTHON strcpy(params_d.cortex_script, cortex_file); strcpy(params_d.cortex_class, cortex_class); + strcpy(params_d.cortex_gen, cortex_gen); #endif } @@ -1166,13 +1643,15 @@ void nw_test_init(nw_state* s, tw_lp* lp) double overhead; int rc = configuration_get_value_double(&config, "PARAMS", "self_msg_overhead", NULL, &overhead); - if(overhead) + if(rc == 0) self_overhead = overhead; INIT_QLIST_HEAD(&s->arrival_queue); INIT_QLIST_HEAD(&s->pending_recvs_queue); INIT_QLIST_HEAD(&s->completed_reqs); + INIT_QLIST_HEAD(&s->msg_sz_list); + s->msg_sz_table = NULL; /* Initialize the RC stack */ rc_stack_create(&s->processed_ops); rc_stack_create(&s->matched_reqs); @@ -1180,15 +1659,43 @@ void nw_test_init(nw_state* s, tw_lp* lp) assert(s->processed_ops != NULL); assert(s->matched_reqs != NULL); - /* clock starts ticking when the first event is processed */ s->start_time = tw_now(lp); - codes_issue_next_event(lp); s->num_bytes_sent = 0; s->num_bytes_recvd = 0; s->compute_time = 0; s->elapsed_time = 0; + + s->app_id = lid.job; + s->local_rank = lid.rank; + + if(strcmp(file_name_of_job[lid.job], "synthetic") == 0) + { + tw_event * e; + nw_message * m_new; + tw_stime ts = tw_rand_exponential(lp->rng, mean_interval/1000); + e = tw_event_new(lp->gid, ts, lp); + m_new = tw_event_data(e); + m_new->msg_type = CLI_BCKGND_GEN; + tw_event_send(e); + } + else + { + printf("\n Trace %s job id %d %d ", file_name_of_job[lid.job], s->app_id, s->local_rank); + strcpy(params_d.file_name, file_name_of_job[lid.job]); + params_d.num_net_traces = num_traces_of_job[lid.job]; + params = (char*)¶ms_d; +// printf("network LP nw id %d app id %d local rank %d generating events, lp gid is %ld \n", s->nw_id, +// s->app_id, s->local_rank, lp->gid); +#ifdef ENABLE_CORTEX_PYTHON + strcpy(params_d.cortex_script, cortex_file); + strcpy(params_d.cortex_class, cortex_class); +#endif + + wrkld_id = codes_workload_load("dumpi-trace-workload", params, s->app_id, s->local_rank); + codes_issue_next_event(lp); + } if(enable_sampling && sampling_interval > 0) { s->max_arr_size = MAX_STATS; @@ -1219,6 +1726,52 @@ void nw_test_event_handler(nw_state* s, tw_bf * bf, nw_message * m, tw_lp * lp) update_arrival_queue(s, bf, m, lp); break; + case MPI_REND_ARRIVED: + { + /* update time of messages */ + mpi_msgs_queue mpi_op; + mpi_op.op_type = m->op_type; + mpi_op.tag = m->fwd.tag; + mpi_op.num_bytes = m->fwd.num_bytes; + mpi_op.source_rank = m->fwd.src_rank; + mpi_op.dest_rank = m->fwd.dest_rank; + mpi_op.req_init_time = m->fwd.sim_start_time; + + update_message_size(s, lp, bf, m, &mpi_op, 0, 1); + + int global_src_id = m->fwd.src_rank; + if(alloc_spec) + { + global_src_id = get_global_id_of_job_rank(m->fwd.src_rank, s->app_id); + } + tw_event *e_callback = + tw_event_new(rank_to_lpid(global_src_id), + codes_local_latency(lp), lp); + nw_message *m_callback = tw_event_data(e_callback); + m_callback->msg_type = MPI_SEND_ARRIVED_CB; + m_callback->fwd.msg_send_time = tw_now(lp) - m->fwd.sim_start_time; + tw_event_send(e_callback); + + } + break; + case MPI_REND_ACK_ARRIVED: + { + /* reconstruct the op and pass it on for actual data transfer */ + int is_rend = 1; + + struct codes_workload_op mpi_op; + mpi_op.op_type = m->op_type; + mpi_op.u.send.tag = m->fwd.tag; + mpi_op.u.send.num_bytes = m->fwd.num_bytes; + mpi_op.u.send.source_rank = m->fwd.src_rank; + mpi_op.u.send.dest_rank = m->fwd.dest_rank; + mpi_op.sim_start_time = m->fwd.sim_start_time; + mpi_op.u.send.req_id = m->fwd.req_id; + + codes_exec_mpi_send(s, bf, m, lp, &mpi_op, is_rend); + } + break; + case MPI_SEND_ARRIVED_CB: update_message_time(s, bf, m, lp); break; @@ -1232,11 +1785,30 @@ void nw_test_event_handler(nw_state* s, tw_bf * bf, nw_message * m, tw_lp * lp) { update_completed_queue(s, bf, m, lp, m->fwd.req_id); } + else + tw_error(TW_LOC, "\n Invalid op type "); } break; case MPI_OP_GET_NEXT: get_next_mpi_operation(s, bf, m, lp); break; + + case CLI_BCKGND_GEN: + gen_synthetic_tr(s, bf, m, lp); + break; + + case CLI_BCKGND_ARRIVE: + arrive_syn_tr(s, bf, m, lp); + break; + + case CLI_NBR_FINISH: + finish_nbr_wkld(s, bf, m, lp); + break; + + case CLI_BCKGND_FIN: + finish_bckgnd_traffic(s, bf, m, lp); + break; + } } @@ -1246,6 +1818,12 @@ static void get_next_mpi_operation_rc(nw_state* s, tw_bf * bf, nw_message * m, t if(m->op_type == CODES_WK_END) { + s->is_finished = 0; + + if(bf->c9) + return; + + notify_neighbor_rc(s, lp, bf, m); return; } switch(m->op_type) @@ -1253,25 +1831,7 @@ static void get_next_mpi_operation_rc(nw_state* s, tw_bf * bf, nw_message * m, t case CODES_WK_SEND: case CODES_WK_ISEND: { - if(enable_sampling) - { - int indx = s->sampling_indx; - - s->mpi_wkld_samples[indx].num_sends_sample--; - s->mpi_wkld_samples[indx].num_bytes_sample -= m->rc.saved_num_bytes; - - if(bf->c1) - { - s->sampling_indx--; - s->cur_interval_end -= sampling_interval; - } - } - model_net_event_rc2(lp, &m->event_rc); - if(m->op_type == CODES_WK_ISEND) - codes_issue_next_event_rc(lp); - s->num_sends--; - s->num_bytes_sent -= m->rc.saved_num_bytes; - num_bytes_sent -= m->rc.saved_num_bytes; + codes_exec_mpi_send_rc(s, bf, m, lp); } break; @@ -1338,6 +1898,7 @@ static void get_next_mpi_operation_rc(nw_state* s, tw_bf * bf, nw_message * m, t static void get_next_mpi_operation(nw_state* s, tw_bf * bf, nw_message * m, tw_lp * lp) { //struct codes_workload_op * mpi_op = malloc(sizeof(struct codes_workload_op)); +// printf("\n App id %d local rank %d ", s->app_id, s->local_rank); struct codes_workload_op mpi_op; codes_workload_get_next(wrkld_id, s->app_id, s->local_rank, &mpi_op); @@ -1346,7 +1907,27 @@ static void get_next_mpi_operation(nw_state* s, tw_bf * bf, nw_message * m, tw_l if(mpi_op.op_type == CODES_WK_END) { s->elapsed_time = tw_now(lp) - s->start_time; - return; + s->is_finished = 1; + + if(!alloc_spec) + { + bf->c9 = 1; + return; + } + + /* Notify ranks from other job that checkpoint traffic has + * completed */ + int num_jobs = codes_jobmap_get_num_jobs(jobmap_ctx); + if(num_jobs <= 1) + { + bf->c9 = 1; + return; + } + + notify_neighbor(s, lp, bf, m); + printf("Client rank %d completed workload, local rank %d .\n", s->nw_id, s->local_rank); + + return; } switch(mpi_op.op_type) { @@ -1354,7 +1935,8 @@ static void get_next_mpi_operation(nw_state* s, tw_bf * bf, nw_message * m, tw_l case CODES_WK_ISEND: { s->num_sends++; - codes_exec_mpi_send(s, bf, m, lp, &mpi_op); + //printf("\n MPI SEND "); + codes_exec_mpi_send(s, bf, m, lp, &mpi_op, 0); } break; @@ -1362,12 +1944,14 @@ static void get_next_mpi_operation(nw_state* s, tw_bf * bf, nw_message * m, tw_l case CODES_WK_IRECV: { s->num_recvs++; + //printf("\n MPI RECV "); codes_exec_mpi_recv(s, bf, m, lp, &mpi_op); } break; case CODES_WK_DELAY: { + //printf("\n MPI DELAY "); s->num_delays++; if(disable_delay) codes_issue_next_event(lp); @@ -1379,6 +1963,7 @@ static void get_next_mpi_operation(nw_state* s, tw_bf * bf, nw_message * m, tw_l case CODES_WK_WAITSOME: case CODES_WK_WAITANY: { + //printf("\n MPI WAITANY WAITSOME "); s->num_waitsome++; codes_issue_next_event(lp); } @@ -1386,12 +1971,14 @@ static void get_next_mpi_operation(nw_state* s, tw_bf * bf, nw_message * m, tw_l case CODES_WK_WAITALL: { + //printf("\n MPI WAITALL "); s->num_waitall++; codes_exec_mpi_wait_all(s, bf, m, lp, &mpi_op); } break; case CODES_WK_WAIT: { + //printf("\n MPI WAIT "); s->num_wait++; codes_exec_mpi_wait(s, lp, &mpi_op); } @@ -1405,6 +1992,7 @@ static void get_next_mpi_operation(nw_state* s, tw_bf * bf, nw_message * m, tw_l case CODES_WK_ALLREDUCE: case CODES_WK_COL: { + //printf("\n MPI COL "); s->num_cols++; codes_issue_next_event(lp); } @@ -1439,13 +2027,36 @@ void nw_test_finalize(nw_state* s, tw_lp* lp) if(s->nw_id >= (tw_lpid)num_net_traces) return; } - int count_irecv = qlist_count(&s->pending_recvs_queue); - int count_isend = qlist_count(&s->arrival_queue); - if(count_irecv || count_isend) + struct msg_size_info * tmp_msg = NULL; + struct qlist_head * ent = NULL; + + if(s->local_rank == 0 && enable_msg_tracking) + fprintf(msg_size_log, "\n rank_id message_size num_messages avg_latency"); + + if(enable_msg_tracking) + { + qlist_for_each(ent, &s->msg_sz_list) + { + tmp_msg = qlist_entry(ent, struct msg_size_info, ql); + printf("\n Rank %d Msg size %d num_msgs %d agg_latency %f avg_latency %f", + s->local_rank, tmp_msg->msg_size, tmp_msg->num_msgs, tmp_msg->agg_latency, tmp_msg->avg_latency); + //fprintf(msg_size_log, "\n Rank %d Msg size %d num_msgs %d agg_latency %f avg_latency %f", + // s->local_rank, tmp_msg->msg_size, tmp_msg->num_msgs, tmp_msg->agg_latency, tmp_msg->avg_latency); + if(s->local_rank == 0) + { + fprintf(msg_size_log, "\n %d %d %d %f", + s->nw_id, tmp_msg->msg_size, tmp_msg->num_msgs, tmp_msg->avg_latency); + } + } + } + int count_irecv = 0, count_isend = 0; + count_irecv = qlist_count(&s->pending_recvs_queue); + count_isend = qlist_count(&s->arrival_queue); + if(count_irecv > 0 || count_isend > 0) { + unmatched = 1; printf("\n LP %llu unmatched irecvs %d unmatched sends %d Total sends %ld receives %ld collectives %ld delays %ld wait alls %ld waits %ld send time %lf wait %lf", lp->gid, count_irecv, count_isend, s->num_sends, s->num_recvs, s->num_cols, s->num_delays, s->num_waitall, s->num_wait, s->send_time, s->wait_time); - tw_error(TW_LOC, "\n Unmatched send and receive, terminating simulation"); } written += sprintf(s->output_buf + written, "\n %llu %llu %ld %ld %ld %ld %lf %lf %lf", lp->gid, s->nw_id, s->num_sends, s->num_recvs, s->num_bytes_sent, s->num_bytes_recvd, s->send_time, s->elapsed_time - s->compute_time, s->compute_time); @@ -1481,7 +2092,7 @@ void nw_test_finalize(nw_state* s, tw_lp* lp) avg_comm_time += (s->elapsed_time - s->compute_time); avg_wait_time += s->wait_time; avg_send_time += s->send_time; - avg_recv_time += s->recv_time; + avg_recv_time += s->recv_time; //printf("\n LP %ld Time spent in communication %llu ", lp->gid, total_time - s->compute_time); rc_stack_destroy(s->matched_reqs); @@ -1509,16 +2120,42 @@ void nw_test_event_handler_rc(nw_state* s, tw_bf * bf, nw_message * m, tw_lp * l } break; + case MPI_REND_ACK_ARRIVED: + { + codes_exec_mpi_send_rc(s, bf, m, lp); + } + break; + + case MPI_REND_ARRIVED: + codes_local_latency_reverse(lp); + break; + case MPI_OP_GET_NEXT: get_next_mpi_operation_rc(s, bf, m, lp); break; + + case CLI_BCKGND_GEN: + gen_synthetic_tr_rc(s, bf, m, lp); + break; + + case CLI_BCKGND_ARRIVE: + arrive_syn_tr_rc(s, bf, m, lp); + break; + + case CLI_NBR_FINISH: + finish_nbr_wkld_rc(s, bf, m, lp); + break; + + case CLI_BCKGND_FIN: + finish_bckgnd_traffic_rc(s, bf, m, lp); + break; } } const tw_optdef app_opt [] = { TWOPT_GROUP("Network workload test"), - TWOPT_CHAR("workload_type", workload_type, "workload type (either \"scalatrace\" or \"dumpi\")"), + TWOPT_CHAR("workload_type", workload_type, "dumpi"), TWOPT_CHAR("workload_file", workload_file, "workload file name"), TWOPT_CHAR("alloc_file", alloc_file, "allocation file name"), TWOPT_CHAR("workload_conf_file", workloads_conf_file, "workload config file name"), @@ -1526,7 +2163,7 @@ const tw_optdef app_opt [] = TWOPT_UINT("disable_compute", disable_delay, "disable compute simulation"), TWOPT_UINT("enable_mpi_debug", enable_debug, "enable debugging of MPI sim layer (works with sync=1 only)"), TWOPT_UINT("sampling_interval", sampling_interval, "sampling interval for MPI operations"), - TWOPT_UINT("enable_sampling", enable_sampling, "enable sampling"), + TWOPT_UINT("enable_sampling", enable_sampling, "enable sampling (only works in sequential mode)"), TWOPT_STIME("sampling_end_time", sampling_end_time, "sampling_end_time"), TWOPT_CHAR("lp-io-dir", lp_io_dir, "Where to place io output (unspecified -> no output"), TWOPT_UINT("lp-io-use-suffix", lp_io_use_suffix, "Whether to append uniq suffix to lp-io directory (default 0)"), @@ -1534,6 +2171,7 @@ const tw_optdef app_opt [] = #ifdef ENABLE_CORTEX_PYTHON TWOPT_CHAR("cortex-file", cortex_file, "Python file (without .py) containing the CoRtEx translation class"), TWOPT_CHAR("cortex-class", cortex_class, "Python class implementing the CoRtEx translator"), + TWOPT_CHAR("cortex-gen", cortex_gen, "Python function to pre-generate MPI events"), #endif TWOPT_END() }; @@ -1595,6 +2233,18 @@ void nw_lp_register_model() } /* end of ROSS event tracing setup */ +static int msg_size_hash_compare( + void *key, struct qhash_head *link) +{ + int64_t *in_size = (int64_t *)key; + struct msg_size_info *tmp; + + tmp = qhash_entry(link, struct msg_size_info, hash_link); + if (tmp->msg_size == *in_size) + return 1; + + return 0; +} int main( int argc, char** argv ) { int rank, nprocs; @@ -1636,14 +2286,21 @@ int main( int argc, char** argv ) while(!feof(name_file)) { ref = fscanf(name_file, "%d %s", &num_traces_of_job[i], file_name_of_job[i]); - if(ref!=EOF) + + if(ref != EOF && strcmp(file_name_of_job[i], "synthetic") == 0) + { + num_syn_clients = num_traces_of_job[i]; + num_net_traces += num_traces_of_job[i]; + } + else if(ref!=EOF) { if(enable_debug) printf("\n%d traces of app %s \n", num_traces_of_job[i], file_name_of_job[i]); num_net_traces += num_traces_of_job[i]; - i++; + num_dumpi_traces += num_traces_of_job[i]; } + i++; } fclose(name_file); assert(strlen(alloc_file) != 0); @@ -1685,6 +2342,17 @@ int main( int argc, char** argv ) return -1; } } + if(enable_msg_tracking) + { + msg_size_log = fopen("mpi-msg-sz-logs", "w+"); + + if(!msg_size_log) + { + printf("\n Error logging MPI operations... quitting "); + MPI_Finalize(); + return -1; + } + } char agg_log_name[512]; sprintf(agg_log_name, "mpi-aggregate-logs-%d.bin", rank); workload_agg_log = fopen(agg_log_name, "w+"); @@ -1704,8 +2372,11 @@ int main( int argc, char** argv ) codes_mapping_setup(); num_mpi_lps = codes_mapping_get_lp_count("MODELNET_GRP", 0, "nw-lp", NULL, 0); - num_net_lps = codes_mapping_get_lp_count("MODELNET_GRP", 1, MN_LP_NM, NULL, 0); - if (lp_io_dir[0]){ + + num_nw_lps = codes_mapping_get_lp_count("MODELNET_GRP", 1, + "nw-lp", NULL, 1); + + if (lp_io_dir[0]){ do_lp_io = 1; /* initialize lp io */ int flags = lp_io_use_suffix ? LP_IO_UNIQ_SUFFIX : 0; @@ -1757,6 +2428,10 @@ int main( int argc, char** argv ) assert(ret == 0 || !"lp_io_flush failure"); } model_net_report_stats(net_id); + + if(unmatched) + tw_error(TW_LOC, "\n Unmatched send and receive, terminating simulation"); + if(alloc_spec) codes_jobmap_destroy(jobmap_ctx); @@ -1764,12 +2439,3 @@ int main( int argc, char** argv ) return 0; } - -/* - * Local variables: - * c-indent-level: 4 - * c-basic-offset: 4 - * End: - * - * vim: ft=c ts=8 sts=4 sw=4 expandtab - */ diff --git a/src/network-workloads/model-net-synthetic-custom-dfly.c b/src/network-workloads/model-net-synthetic-custom-dfly.c index 8aa0945ab583bc127daebb4de98085786e9e4b95..2c27966d467c33b0771e34097db444524d44b62a 100644 --- a/src/network-workloads/model-net-synthetic-custom-dfly.c +++ b/src/network-workloads/model-net-synthetic-custom-dfly.c @@ -35,8 +35,8 @@ static lp_io_handle io_handle; static unsigned int lp_io_use_suffix = 0; static int do_lp_io = 0; static int num_msgs = 20; -static unsigned int sampling_interval = 800000; -static unsigned int sampling_end_time = 1600000; +static tw_stime sampling_interval = 800000; +static tw_stime sampling_end_time = 1600000; typedef struct svr_msg svr_msg; typedef struct svr_state svr_state; @@ -111,8 +111,8 @@ const tw_optdef app_opt [] = TWOPT_GROUP("Model net synthetic traffic " ), TWOPT_UINT("traffic", traffic, "UNIFORM RANDOM=1, NEAREST NEIGHBOR=2 "), TWOPT_UINT("num_messages", num_msgs, "Number of messages to be generated per terminal "), - TWOPT_UINT("sampling-interval", sampling_interval, "the sampling interval "), - TWOPT_UINT("sampling-end-time", sampling_end_time, "sampling end time "), + TWOPT_STIME("sampling-interval", sampling_interval, "the sampling interval "), + TWOPT_STIME("sampling-end-time", sampling_end_time, "sampling end time "), TWOPT_STIME("arrival_time", arrival_time, "INTER-ARRIVAL TIME"), TWOPT_CHAR("lp-io-dir", lp_io_dir, "Where to place io output (unspecified -> no output"), TWOPT_UINT("lp-io-use-suffix", lp_io_use_suffix, "Whether to append uniq suffix to lp-io directory (default 0)"), diff --git a/src/network-workloads/model-net-synthetic.c b/src/network-workloads/model-net-synthetic.c index 4bb6420856af62bb87074d54afc1cb39f5c48763..e0192781519db701332bd83aa6b3c0fb9d5a921f 100644 --- a/src/network-workloads/model-net-synthetic.c +++ b/src/network-workloads/model-net-synthetic.c @@ -35,8 +35,8 @@ static lp_io_handle io_handle; static unsigned int lp_io_use_suffix = 0; static int do_lp_io = 0; static int num_msgs = 20; -static unsigned int sampling_interval = 800000; -static unsigned int sampling_end_time = 1600000; +static tw_stime sampling_interval = 800000; +static tw_stime sampling_end_time = 1600000; typedef struct svr_msg svr_msg; typedef struct svr_state svr_state; @@ -146,8 +146,8 @@ const tw_optdef app_opt [] = TWOPT_GROUP("Model net synthetic traffic " ), TWOPT_UINT("traffic", traffic, "UNIFORM RANDOM=1, NEAREST NEIGHBOR=2 "), TWOPT_UINT("num_messages", num_msgs, "Number of messages to be generated per terminal "), - TWOPT_UINT("sampling-interval", sampling_interval, "the sampling interval "), - TWOPT_UINT("sampling-end-time", sampling_end_time, "sampling end time "), + TWOPT_STIME("sampling-interval", sampling_interval, "the sampling interval "), + TWOPT_STIME("sampling-end-time", sampling_end_time, "sampling end time "), TWOPT_STIME("arrival_time", arrival_time, "INTER-ARRIVAL TIME"), TWOPT_CHAR("lp-io-dir", lp_io_dir, "Where to place io output (unspecified -> no output"), TWOPT_UINT("lp-io-use-suffix", lp_io_use_suffix, "Whether to append uniq suffix to lp-io directory (default 0)"), diff --git a/src/networks/model-net/doc/README.fattree.txt b/src/networks/model-net/doc/README.fattree.txt index 925227df327c620c5a7c5a4d06d3f1f8646c2a13..a12ec67cbc2092e8af43528c6206188aa0dc463a 100644 --- a/src/networks/model-net/doc/README.fattree.txt +++ b/src/networks/model-net/doc/README.fattree.txt @@ -84,31 +84,27 @@ tar xzf fts.tgz cd fault_tolerance_simulation/ rm 0001-*.patch 0002-*.patch 0003-*.patch 0004-*.patch 0005-*.patch tar xzf $P_PATH/sar.patches.tgz - + wget http://downloads.openfabrics.org/management/opensm-3.3.20.tar.gz mv opensm-3.3.20.tar.gz opensm.tar.gz -wget http://downloads.openfabrics.org/ibutils/ibutils-1.5.7-0.2.gbd7e502.tar.gz -mv ibutils-1.5.7-0.2.gbd7e502.tar.gz ibutils.tar.gz -wget http://downloads.openfabrics.org/management/infiniband-diags-1.6.7.tar.gz -mv infiniband-diags-1.6.7.tar.gz infiniband-diags.tar.gz wget https://www.openfabrics.org/downloads/management/libibmad-1.3.12.tar.gz mv libibmad-1.3.12.tar.gz libibmad.tar.gz wget https://www.openfabrics.org/downloads/management/libibumad-1.3.10.2.tar.gz mv libibumad-1.3.10.2.tar.gz libibumad.tar.gz - + patch -p1 < $P_PATH/fts.patch - -./simuate.py -s + +./simulate.py -s 2. Add LFT creating scripts to the fall-in-place toolchain. cd $HOME/simulation/scripts patch -p1 < $P_PATH/lft.patch chmod +x post_process_* -chmod +x create_static_lft.sh +chmod +x create_static_lft.sh 3. Choose a routing algorithm which should be used by OpenSM (possible options: updn, dnup, ftree, lash, dor, torus-2QoS, dfsssp, sssp) -export OSM_ROUTING="ftree" +export OSM_ROUTING="ftree" ~/simulation/scripts/create_static_lft.sh routing_folder dot_file (here routing_folder and dot_file should be same as the one used during the run used to dump the topology) diff --git a/src/networks/model-net/dragonfly-custom.C b/src/networks/model-net/dragonfly-custom.C index 95c75cbed52a7ae6c6d22b206d4bc7b8dcea2d4c..c82a0a87bb607b77a6da955fb166e1c49ee0dc3e 100644 --- a/src/networks/model-net/dragonfly-custom.C +++ b/src/networks/model-net/dragonfly-custom.C @@ -63,7 +63,6 @@ struct InterGroupLink { int src, dest; }; - static int debug_slot_count = 0; static long term_ecount, router_ecount, term_rev_ecount, router_rev_ecount; static long packet_gen = 0, packet_fin = 0; @@ -91,6 +90,10 @@ static int router_magic_num = 0; /* terminal magic number */ static int terminal_magic_num = 0; +/* Hops within a group */ +static int num_intra_nonmin_hops = 4; +static int num_intra_min_hops = 2; + static FILE * dragonfly_log = NULL; static int sample_bytes_written = 0; @@ -99,6 +102,9 @@ static int sample_rtr_bytes_written = 0; static char cn_sample_file[MAX_NAME_LENGTH]; static char router_sample_file[MAX_NAME_LENGTH]; +//don't do overhead here - job of MPI layer +static tw_stime mpi_soft_overhead = 0; + typedef struct terminal_custom_message_list terminal_custom_message_list; struct terminal_custom_message_list { terminal_custom_message msg; @@ -137,6 +143,7 @@ struct dragonfly_param int num_cn; int intra_grp_radix; int num_col_chans; + int num_row_chans; int num_router_rows; int num_router_cols; int num_groups; @@ -233,6 +240,10 @@ struct terminal_state tw_stime last_buf_full; tw_stime busy_time; + + tw_stime max_latency; + tw_stime min_latency; + char output_buf[4096]; /* For LP suspend functionality */ int error_ct; @@ -269,7 +280,7 @@ typedef enum event_t /* whether the last hop of a packet was global, local or a terminal */ enum last_hop { - GLOBAL, + GLOBAL=1, LOCAL, TERMINAL, ROOT @@ -280,7 +291,7 @@ enum last_hop * for detecting load on global channels are not very accurate */ enum ROUTING_ALGO { - MINIMAL = 0, + MINIMAL = 1, NON_MINIMAL, ADAPTIVE, PROG_ADAPTIVE @@ -515,9 +526,11 @@ static void dragonfly_read_config(const char * anno, dragonfly_param *params){ fprintf(stderr, "Bandwidth of compute node channels not specified, setting to %lf\n", p->cn_bandwidth); } - p->router_delay = 50; - configuration_get_value_double(&config, "PARAMS", "router_delay", anno, + rc = configuration_get_value_double(&config, "PARAMS", "router_delay", anno, &p->router_delay); + if(rc) { + p->router_delay = 100; + } configuration_get_value(&config, "PARAMS", "cn_sample_file", anno, cn_sample_file, MAX_NAME_LENGTH); @@ -558,6 +571,11 @@ static void dragonfly_read_config(const char * anno, dragonfly_param *params){ // printf("\n Number of links connecting chassis not specified, setting to default value 3 "); p->num_col_chans = 3; } + rc = configuration_get_value_int(&config, "PARAMS", "num_row_chans", anno, &p->num_row_chans); + if(rc) { +// printf("\n Number of links connecting chassis not specified, setting to default value 3 "); + p->num_row_chans = 1; + } rc = configuration_get_value_int(&config, "PARAMS", "num_router_rows", anno, &p->num_router_rows); if(rc) { printf("\n Number of router rows not specified, setting to 6 "); @@ -568,7 +586,7 @@ static void dragonfly_read_config(const char * anno, dragonfly_param *params){ printf("\n Number of router columns not specified, setting to 16 "); p->num_router_cols = 16; } - p->intra_grp_radix = p->num_router_cols + (p->num_router_rows * p->num_col_chans); + p->intra_grp_radix = (p->num_router_cols * p->num_row_chans) + (p->num_router_rows * p->num_col_chans); p->num_routers = p->num_router_rows * p->num_router_cols; rc = configuration_get_value_int(&config, "PARAMS", "num_cns_per_router", anno, &p->num_cn); @@ -582,7 +600,7 @@ static void dragonfly_read_config(const char * anno, dragonfly_param *params){ printf("\n Number of global channels per router not specified, setting to 10 "); p->num_global_channels = 10; } - p->radix = p->num_router_cols + (p->num_col_chans * p->num_router_rows) + p->num_global_channels + p->num_cn; + p->radix = (p->num_router_cols * p->num_row_chans) + (p->num_col_chans * p->num_router_rows) + p->num_global_channels + p->num_cn; p->total_routers = p->num_groups * p->num_routers; p->total_terminals = p->total_routers * p->num_cn; @@ -627,7 +645,10 @@ static void dragonfly_read_config(const char * anno, dragonfly_param *params){ } FILE *systemFile = fopen(interFile, "rb"); if(!myRank) + { printf("Reading inter-group connectivity file: %s\n", interFile); + printf("\n Total routers %d total groups %d ", p->total_routers, p->num_groups); + } { vector< int > offsets; @@ -637,6 +658,7 @@ static void dragonfly_read_config(const char * anno, dragonfly_param *params){ for(int g = 0; g < connectionList.size(); g++) { connectionList[g].resize(p->num_groups); } + InterGroupLink newLink; while(fread(&newLink, sizeof(InterGroupLink), 1, systemFile) != 0) { @@ -668,7 +690,7 @@ static void dragonfly_read_config(const char * anno, dragonfly_param *params){ printf(" ( %d - ", it->first); for(int l = 0; l < it->second.size(); l++) { // offset is number of local connections - // type is blue or green according to Cray architecture + // type is black or green according to Cray architecture printf("%d,%d ", it->second[l].offset, it->second[l].type); } printf(")"); @@ -812,7 +834,9 @@ terminal_custom_init( terminal_state * s, s->router_id=(int)s->terminal_id / (s->params->num_cn); s->terminal_available_time = 0.0; s->packet_counter = 0; - + s->min_latency = INT_MAX; + s->max_latency = 0; + s->finished_msgs = 0; s->finished_chunks = 0; s->finished_packets = 0; @@ -881,8 +905,10 @@ void router_custom_setup(router_state * r, tw_lp * lp) "does not match with repetitions * dragonfly_router %d ", p->num_routers, p->total_routers, num_grp_reps * num_routers_per_mgrp); - r->router_id=mapping_rep_id + mapping_offset; + r->router_id = codes_mapping_get_lp_relative_id(lp->gid, 0, 0); r->group_id=r->router_id/p->num_routers; + + //printf("\n Local router id %d global id %d ", r->router_id, lp->gid); r->fwd_events = 0; r->rev_events = 0; @@ -975,7 +1001,7 @@ static tw_stime dragonfly_custom_packet_event( //e_new = tw_event_new(sender->gid, xfer_to_nic_time+offset, sender); //msg = tw_event_data(e_new); e_new = model_net_method_event_new(sender->gid, xfer_to_nic_time+offset, - sender, DRAGONFLY, (void**)&msg, (void**)&tmp_ptr); + sender, DRAGONFLY_CUSTOM, (void**)&msg, (void**)&tmp_ptr); strcpy(msg->category, req->category); msg->final_dest_gid = req->final_dest_lp; msg->total_size = req->msg_size; @@ -995,18 +1021,18 @@ static tw_stime dragonfly_custom_packet_event( if(is_last_pckt) /* Its the last packet so pass in remote and local event information*/ { - if(req->remote_event_size > 0) - { - msg->remote_event_size_bytes = req->remote_event_size; - memcpy(tmp_ptr, remote_event, req->remote_event_size); - tmp_ptr += req->remote_event_size; - } - if(req->self_event_size > 0) - { - msg->local_event_size_bytes = req->self_event_size; - memcpy(tmp_ptr, self_event, req->self_event_size); - tmp_ptr += req->self_event_size; - } + if(req->remote_event_size > 0) + { + msg->remote_event_size_bytes = req->remote_event_size; + memcpy(tmp_ptr, remote_event, req->remote_event_size); + tmp_ptr += req->remote_event_size; + } + if(req->self_event_size > 0) + { + msg->local_event_size_bytes = req->self_event_size; + memcpy(tmp_ptr, self_event, req->self_event_size); + tmp_ptr += req->self_event_size; + } } //printf("\n dragonfly remote event %d local event %d last packet %d %lf ", msg->remote_event_size_bytes, msg->local_event_size_bytes, is_last_pckt, xfer_to_nic_time); tw_event_send(e_new); @@ -1048,11 +1074,11 @@ static void router_credit_send(router_state * s, terminal_custom_message * msg, ts = g_tw_lookahead + p->credit_delay + tw_rand_unif(lp->rng); if (is_terminal) { - buf_e = model_net_method_event_new(dest, ts, lp, DRAGONFLY, + buf_e = model_net_method_event_new(dest, ts, lp, DRAGONFLY_CUSTOM, (void**)&buf_msg, NULL); buf_msg->magic = terminal_magic_num; } else { - buf_e = model_net_method_event_new(dest, ts, lp, DRAGONFLY_ROUTER, + buf_e = model_net_method_event_new(dest, ts, lp, DRAGONFLY_CUSTOM_ROUTER, (void**)&buf_msg, NULL); buf_msg->magic = router_magic_num; } @@ -1119,13 +1145,18 @@ static void packet_generate(terminal_state * s, tw_bf * bf, terminal_custom_mess int total_event_size; uint64_t num_chunks = msg->packet_size / p->chunk_size; + double cn_delay = s->params->cn_delay; + if (msg->packet_size % s->params->chunk_size) num_chunks++; if(!num_chunks) num_chunks = 1; - nic_ts = g_tw_lookahead + (num_chunks * s->params->cn_delay) + tw_rand_unif(lp->rng); + if(msg->packet_size < s->params->chunk_size) + cn_delay = bytes_to_ns(msg->packet_size % s->params->chunk_size, s->params->cn_bandwidth); + + nic_ts = g_tw_lookahead + (num_chunks * cn_delay) + tw_rand_unif(lp->rng); msg->packet_ID = lp->gid + g_tw_nlp * s->packet_counter; msg->my_N_hop = 0; @@ -1138,20 +1169,19 @@ static void packet_generate(terminal_state * s, tw_bf * bf, terminal_custom_mess msg->packet_ID, s->terminal_id, LLU(msg->dest_terminal_id), LLU(msg->packet_size), LLU(num_chunks)); - for(uint64_t i = 0; i < num_chunks; i++) + for(int i = 0; i < num_chunks; i++) { terminal_custom_message_list *cur_chunk = (terminal_custom_message_list*)malloc( sizeof(terminal_custom_message_list)); msg->origin_router_id = s->router_id; init_terminal_custom_message_list(cur_chunk, msg); - if(msg->remote_event_size_bytes + msg->local_event_size_bytes > 0) { cur_chunk->event_data = (char*)malloc( msg->remote_event_size_bytes + msg->local_event_size_bytes); } - void * m_data_src = model_net_method_get_edata(DRAGONFLY, msg); + void * m_data_src = model_net_method_get_edata(DRAGONFLY_CUSTOM, msg); if (msg->remote_event_size_bytes){ memcpy(cur_chunk->event_data, m_data_src, msg->remote_event_size_bytes); } @@ -1181,7 +1211,7 @@ static void packet_generate(terminal_state * s, tw_bf * bf, terminal_custom_mess bf->c5 = 1; ts = codes_local_latency(lp); terminal_custom_message *m; - tw_event* e = model_net_method_event_new(lp->gid, ts, lp, DRAGONFLY, + tw_event* e = model_net_method_event_new(lp->gid, ts, lp, DRAGONFLY_CUSTOM, (void**)&m, NULL); m->type = T_SEND; m->magic = terminal_magic_num; @@ -1189,7 +1219,7 @@ static void packet_generate(terminal_state * s, tw_bf * bf, terminal_custom_mess tw_event_send(e); } - total_event_size = model_net_get_msg_sz(DRAGONFLY) + + total_event_size = model_net_get_msg_sz(DRAGONFLY_CUSTOM) + msg->remote_event_size_bytes + msg->local_event_size_bytes; mn_stats* stat; stat = model_net_find_stats(msg->category, s->dragonfly_stats_array); @@ -1286,11 +1316,12 @@ static void packet_send(terminal_state * s, tw_bf * bf, terminal_custom_message codes_mapping_get_lp_info(lp->gid, lp_group_name, &mapping_grp_id, NULL, &mapping_type_id, NULL, &mapping_rep_id, &mapping_offset); codes_mapping_get_lp_id(lp_group_name, LP_CONFIG_NM_ROUT, NULL, 1, - s->router_id, 0, &router_id); + s->router_id / num_routers_per_mgrp, s->router_id % num_routers_per_mgrp, &router_id); +// printf("\n Local router id %d global router id %d ", s->router_id, router_id); // we are sending an event to the router, so no method_event here void * remote_event; e = model_net_method_event_new(router_id, ts, lp, - DRAGONFLY_ROUTER, (void**)&m, &remote_event); + DRAGONFLY_CUSTOM_ROUTER, (void**)&m, &remote_event); memcpy(m, &cur_entry->msg, sizeof(terminal_custom_message)); if (m->remote_event_size_bytes){ memcpy(remote_event, cur_entry->event_data, m->remote_event_size_bytes); @@ -1303,6 +1334,7 @@ static void packet_send(terminal_state * s, tw_bf * bf, terminal_custom_message m->magic = router_magic_num; m->path_type = -1; m->local_event_size_bytes = 0; + m->intm_rtr_id = -1; tw_event_send(e); @@ -1331,7 +1363,7 @@ static void packet_send(terminal_state * s, tw_bf * bf, terminal_custom_message bf->c3 = 1; terminal_custom_message *m_new; ts += tw_rand_unif(lp->rng); - e = model_net_method_event_new(lp->gid, ts, lp, DRAGONFLY, + e = model_net_method_event_new(lp->gid, ts, lp, DRAGONFLY_CUSTOM, (void**)&m_new, NULL); m_new->type = T_SEND; m_new->magic = terminal_magic_num; @@ -1408,12 +1440,19 @@ static void packet_arrive_rc(terminal_state * s, tw_bf * bf, terminal_custom_mes s->finished_packets--; } if(bf->c3) + { dragonfly_max_latency = msg->saved_available_time; - + } + + if(bf->c22) + { + s->max_latency = msg->saved_available_time; + } if(bf->c7) { //assert(!hash_link); - tw_rand_reverse_unif(lp->rng); + if(bf->c8) + tw_rand_reverse_unif(lp->rng); N_finished_msgs--; s->finished_msgs--; total_msg_sz -= msg->total_size; @@ -1447,9 +1486,9 @@ static void packet_arrive_rc(terminal_state * s, tw_bf * bf, terminal_custom_mes } static void send_remote_event(terminal_state * s, terminal_custom_message * msg, tw_lp * lp, tw_bf * bf, char * event_data, int remote_event_size) { - void * tmp_ptr = model_net_method_get_edata(DRAGONFLY, msg); + void * tmp_ptr = model_net_method_get_edata(DRAGONFLY_CUSTOM, msg); //tw_stime ts = g_tw_lookahead + bytes_to_ns(msg->remote_event_size_bytes, (1/s->params->cn_bandwidth)); - tw_stime ts = g_tw_lookahead + tw_rand_unif(lp->rng); + tw_stime ts = g_tw_lookahead + mpi_soft_overhead + tw_rand_unif(lp->rng); if (msg->is_pull){ bf->c4 = 1; struct codes_mctx mc_dst = @@ -1522,7 +1561,7 @@ static void packet_arrive(terminal_state * s, tw_bf * bf, terminal_custom_messag tw_event * buf_e; terminal_custom_message * buf_msg; buf_e = model_net_method_event_new(msg->intm_lp_id, ts, lp, - DRAGONFLY_ROUTER, (void**)&buf_msg, NULL); + DRAGONFLY_CUSTOM_ROUTER, (void**)&buf_msg, NULL); buf_msg->magic = router_magic_num; buf_msg->vc_index = msg->vc_index; buf_msg->output_chan = msg->output_chan; @@ -1601,7 +1640,7 @@ static void packet_arrive(terminal_state * s, tw_bf * bf, terminal_custom_messag /* Now retreieve the number of chunks completed from the hash and update * them */ - void *m_data_src = model_net_method_get_edata(DRAGONFLY, msg); + void *m_data_src = model_net_method_get_edata(DRAGONFLY_CUSTOM, msg); /* If an entry does not exist then create one */ if(!tmp) @@ -1643,11 +1682,20 @@ static void packet_arrive(terminal_state * s, tw_bf * bf, terminal_custom_messag tmp->remote_event_size = msg->remote_event_size_bytes; memcpy(tmp->remote_event_data, m_data_src, msg->remote_event_size_bytes); } + if(s->min_latency > tw_now(lp) - msg->travel_start_time) { + s->min_latency = tw_now(lp) - msg->travel_start_time; + } if (dragonfly_max_latency < tw_now( lp ) - msg->travel_start_time) { bf->c3 = 1; msg->saved_available_time = dragonfly_max_latency; dragonfly_max_latency = tw_now( lp ) - msg->travel_start_time; + s->max_latency = tw_now(lp) - msg->travel_start_time; } + if(s->max_latency < tw_now( lp ) - msg->travel_start_time) { + bf->c22 = 1; + msg->saved_available_time = s->max_latency; + s->max_latency = tw_now(lp) - msg->travel_start_time; + } /* If all chunks of a message have arrived then send a remote event to the * callee*/ //assert(tmp->num_chunks <= total_chunks); @@ -1663,7 +1711,10 @@ static void packet_arrive(terminal_state * s, tw_bf * bf, terminal_custom_messag s->finished_msgs++; //assert(tmp->remote_event_data && tmp->remote_event_size > 0); - send_remote_event(s, msg, lp, bf, tmp->remote_event_data, tmp->remote_event_size); + if(tmp->remote_event_data && tmp->remote_event_size > 0) { + bf->c8 = 1; + send_remote_event(s, msg, lp, bf, tmp->remote_event_data, tmp->remote_event_size); + } /* Remove the hash entry */ qhash_del(hash_link); rc_stack_push(lp, tmp, free_tmp, s->st); @@ -1785,32 +1836,32 @@ void dragonfly_custom_rsample_fin(router_state * s, "link traffic for each of the %d links (int64_t) \nsample end time (double) forward events per sample \nreverse events per sample ", p->radix, p->radix); fprintf(fp, "\n\nOrdering of links \n%d local (router-router same group) channels \n%d global (router-router remote group)" - " channels \n%d terminal channels", p->radix/2, p->radix/4, p->radix/4); + " channels \n%d terminal channels", p->intra_grp_radix, p->num_global_channels); fclose(fp); } - char rt_fn[MAX_NAME_LENGTH]; - if(strcmp(router_sample_file, "") == 0) - sprintf(rt_fn, "dragonfly-router-sampling-%ld.bin", g_tw_mynode); - else - sprintf(rt_fn, "%s-%ld.bin", router_sample_file, g_tw_mynode); - - int i = 0; + char rt_fn[MAX_NAME_LENGTH]; + if(strcmp(router_sample_file, "") == 0) + sprintf(rt_fn, "dragonfly-router-sampling-%ld.bin", g_tw_mynode); + else + sprintf(rt_fn, "%s-%ld.bin", router_sample_file, g_tw_mynode); + + int i = 0; - int size_sample = sizeof(tw_lpid) + p->radix * (sizeof(int64_t) + sizeof(tw_stime)) + sizeof(tw_stime) + 2 * sizeof(long); - FILE * fp = fopen(rt_fn, "a"); - fseek(fp, sample_rtr_bytes_written, SEEK_SET); + int size_sample = sizeof(tw_lpid) + p->radix * (sizeof(int64_t) + sizeof(tw_stime)) + sizeof(tw_stime) + 2 * sizeof(long); + FILE * fp = fopen(rt_fn, "a"); + fseek(fp, sample_rtr_bytes_written, SEEK_SET); - for(; i < s->op_arr_size; i++) - { - fwrite((void*)&(s->rsamples[i].router_id), sizeof(tw_lpid), 1, fp); - fwrite(s->rsamples[i].busy_time, sizeof(tw_stime), p->radix, fp); - fwrite(s->rsamples[i].link_traffic_sample, sizeof(int64_t), p->radix, fp); - fwrite((void*)&(s->rsamples[i].end_time), sizeof(tw_stime), 1, fp); - fwrite((void*)&(s->rsamples[i].fwd_events), sizeof(long), 1, fp); - fwrite((void*)&(s->rsamples[i].rev_events), sizeof(long), 1, fp); - } - sample_rtr_bytes_written += (s->op_arr_size * size_sample); - fclose(fp); + for(; i < s->op_arr_size; i++) + { + fwrite((void*)&(s->rsamples[i].router_id), sizeof(tw_lpid), 1, fp); + fwrite(s->rsamples[i].busy_time, sizeof(tw_stime), p->radix, fp); + fwrite(s->rsamples[i].link_traffic_sample, sizeof(int64_t), p->radix, fp); + fwrite((void*)&(s->rsamples[i].end_time), sizeof(tw_stime), 1, fp); + fwrite((void*)&(s->rsamples[i].fwd_events), sizeof(long), 1, fp); + fwrite((void*)&(s->rsamples[i].rev_events), sizeof(long), 1, fp); + } + sample_rtr_bytes_written += (s->op_arr_size * size_sample); + fclose(fp); } void dragonfly_custom_sample_init(terminal_state * s, tw_lp * lp) @@ -1920,18 +1971,19 @@ void dragonfly_custom_sample_fin(terminal_state * s, "\nbusy time (double)\nsample end time(double) \nforward events (long) \nreverse events (long)"); fclose(fp); } - char rt_fn[MAX_NAME_LENGTH]; - if(strncmp(cn_sample_file, "", 10) == 0) - sprintf(rt_fn, "dragonfly-cn-sampling-%ld.bin", g_tw_mynode); - else - sprintf(rt_fn, "%s-%ld.bin", cn_sample_file, g_tw_mynode); - FILE * fp = fopen(rt_fn, "a"); - fseek(fp, sample_bytes_written, SEEK_SET); - fwrite(s->sample_stat, sizeof(struct dfly_cn_sample), s->op_arr_size, fp); - fclose(fp); + char rt_fn[MAX_NAME_LENGTH]; + if(strncmp(cn_sample_file, "", 10) == 0) + sprintf(rt_fn, "dragonfly-cn-sampling-%ld.bin", g_tw_mynode); + else + sprintf(rt_fn, "%s-%ld.bin", cn_sample_file, g_tw_mynode); + + FILE * fp = fopen(rt_fn, "a"); + fseek(fp, sample_bytes_written, SEEK_SET); + fwrite(s->sample_stat, sizeof(struct dfly_cn_sample), s->op_arr_size, fp); + fclose(fp); - sample_bytes_written += (s->op_arr_size * sizeof(struct dfly_cn_sample)); + sample_bytes_written += (s->op_arr_size * sizeof(struct dfly_cn_sample)); } static void terminal_buf_update_rc(terminal_state * s, @@ -1964,7 +2016,7 @@ terminal_buf_update(terminal_state * s, if(s->in_send_loop == 0 && s->terminal_msgs[0] != NULL) { terminal_custom_message *m; bf->c1 = 1; - tw_event* e = model_net_method_event_new(lp->gid, ts, lp, DRAGONFLY, + tw_event* e = model_net_method_event_new(lp->gid, ts, lp, DRAGONFLY_CUSTOM, (void**)&m, NULL); m->type = T_SEND; m->magic = terminal_magic_num; @@ -2017,12 +2069,12 @@ dragonfly_custom_terminal_final( terminal_state * s, int written = 0; if(!s->terminal_id) - written = sprintf(s->output_buf, "# Format <# Flits/Packets finished> \n"); + written = sprintf(s->output_buf, "# Format <# Flits/Packets finished> \n"); - written += sprintf(s->output_buf + written, "%llu %u %llu %lf %ld %lf %lf\n", - LLU(lp->gid), s->terminal_id, LLU(s->total_msg_size), s->total_time, + written += sprintf(s->output_buf + written, "%llu %u %llu %lf %ld %lf %lf %lf %lf\n", + LLU(lp->gid), s->terminal_id, LLU(s->total_msg_size), s->total_time/s->finished_chunks, s->finished_packets, (double)s->total_hops/s->finished_chunks, - s->busy_time); + s->busy_time, s->max_latency, s->min_latency); lp_io_write(lp->gid, (char*)"dragonfly-msg-stats", written, s->output_buf); @@ -2073,7 +2125,7 @@ void dragonfly_custom_router_final(router_state * s, LLU(lp->gid), s->router_id / p->num_routers, s->router_id % p->num_routers); - for(int d = 0; d < p->num_routers + p->num_global_channels; d++) + for(int d = 0; d < p->radix; d++) written += sprintf(s->output_buf + written, " %lf", s->busy_time[d]); sprintf(s->output_buf + written, "\n"); @@ -2084,20 +2136,20 @@ void dragonfly_custom_router_final(router_state * s, { written = sprintf(s->output_buf2, "# Format "); written += sprintf(s->output_buf2 + written, "# Router ports in the order: %d local channels, %d global channels \n", - p->num_routers, p->num_global_channels); + p->intra_grp_radix, p->num_global_channels); } written += sprintf(s->output_buf2 + written, "\n %llu %d %d", LLU(lp->gid), s->router_id / p->num_routers, s->router_id % p->num_routers); - for(int d = 0; d < p->num_routers + p->num_global_channels; d++) + for(int d = 0; d < p->radix; d++) written += sprintf(s->output_buf2 + written, " %lld", LLD(s->link_traffic[d])); lp_io_write(lp->gid, (char*)"dragonfly-router-traffic", written, s->output_buf2); } -static int get_intra_router(int src_router_id, int dest_router_id, int num_rtrs_per_grp) +static vector get_intra_router(router_state * s, int src_router_id, int dest_router_id, int num_rtrs_per_grp) { /* Check for intra-group connections */ int src_rel_id = src_router_id % num_rtrs_per_grp; @@ -2107,26 +2159,47 @@ static int get_intra_router(int src_router_id, int dest_router_id, int num_rtrs_ map< int, vector > &curMap = intraGroupLinks[src_rel_id]; map< int, vector >::iterator it_src = curMap.begin(); + int offset = group_id * num_rtrs_per_grp; + vector intersection; /* If no direct connection exists then find an intermediate connection */ if(curMap.find(dest_rel_id) == curMap.end()) { + /*int src_col = src_rel_id % s->params->num_router_cols; + int src_row = src_rel_id / s->params->num_router_cols; + + int dest_col = dest_rel_id % s->params->num_router_cols; + int dest_row = dest_rel_id / s->params->num_router_cols; + + //row first, column second + int choice1 = src_row * s->params->num_router_cols + dest_col; + int choice2 = dest_row * s->params->num_router_cols + src_col; + intersection.push_back(offset + choice1); + intersection.push_back(offset + choice2);*/ map > &destMap = intraGroupLinks[dest_rel_id]; map< int, vector >::iterator it_dest = destMap.begin(); - - for(; it_src != curMap.end(); it_src++) { - if(destMap.find(it_src->first) != destMap.end()) - { - return (group_id * num_rtrs_per_grp) + it_src->first; - } + + while(it_src != curMap.end() && it_dest != destMap.end()) + { + if(it_src->first < it_dest->first) + it_src++; + else + if(it_dest->first < it_src->first) + it_dest++; + else { + intersection.push_back(offset + it_src->first); + it_src++; + it_dest++; + } } + } else { /* There is a direct connection */ - return dest_router_id; + intersection.push_back(dest_router_id); } - return -1; + return intersection; } /* get the next stop for the current packet * determines if it is a router within a group, a router in another group @@ -2149,6 +2222,7 @@ get_next_stop(router_state * s, dest_group_id = dest_router_id / s->params->num_routers; int origin_grp_id = msg->origin_router_id / s->params->num_routers; + int select_chan = -1; /* If the packet has arrived at the destination router */ if(dest_router_id == local_router_id) { @@ -2158,15 +2232,17 @@ get_next_stop(router_state * s, /* If the packet has arrived at the destination group */ if(s->group_id == dest_group_id) { - int next_stop = get_intra_router(local_router_id, dest_router_id, s->params->num_routers); - assert(next_stop != -1); + bf->c19 = 1; + vector next_stop = get_intra_router(s, local_router_id, dest_router_id, s->params->num_routers); + assert(!next_stop.empty()); + select_chan = tw_rand_integer(lp->rng, 0, next_stop.size() - 1); /* If there is a direct connection between */ - if(msg->last_hop == GLOBAL && next_stop == dest_router_id) - msg->my_l_hop++; + //if(msg->last_hop == GLOBAL && next_stop[select_chan] == dest_router_id) + // msg->my_l_hop++; - codes_mapping_get_lp_id(lp_group_name, LP_CONFIG_NM_ROUT, s->anno, 0, next_stop, - 0, &router_dest_id); + codes_mapping_get_lp_id(lp_group_name, LP_CONFIG_NM_ROUT, s->anno, 0, next_stop[select_chan] / num_routers_per_mgrp, + next_stop[select_chan] % num_routers_per_mgrp, &router_dest_id); return router_dest_id; } @@ -2178,8 +2254,6 @@ get_next_stop(router_state * s, || s->router_id == msg->intm_rtr_id || (routing == PROG_ADAPTIVE && do_chan_selection)) { - int select_chan = -1; - if(adap_chan >= 0) select_chan = adap_chan; else @@ -2206,27 +2280,31 @@ get_next_stop(router_state * s, //printf("\n Dest router id %d %d !!! ", dest_router_id, msg->intm_rtr_id); if(s->router_id == msg->saved_src_dest) { - dest_lp = connectionList[dest_group_id][my_grp_id][msg->saved_src_chan]; + //dest_lp = connectionList[dest_group_id][my_grp_id][msg->saved_src_chan]; + dest_lp = interGroupLinks[s->router_id][dest_group_id][0].dest; } else { /* Connection within the group */ - dest_lp = get_intra_router(local_router_id, msg->saved_src_dest, s->params->num_routers); - assert(dest_lp != -1); + bf->c21 = 1; + vector dests = get_intra_router(s, local_router_id, msg->saved_src_dest, s->params->num_routers); + assert(!dests.empty()); + select_chan = tw_rand_integer(lp->rng, 0, dests.size() - 1); /* If there is a direct connection */ /* Handling cases where one hop can be skipped. */ if((msg->last_hop == GLOBAL || s->router_id == msg->intm_rtr_id) - && dest_lp == msg->saved_src_dest) + && dests[select_chan] == msg->saved_src_dest) { - msg->my_l_hop++; + //msg->my_l_hop++; if(msg->packet_ID == LLU(TRACK_PKT)) printf("\n Packet %llu local hops being incremented %d ", msg->packet_ID, msg->my_l_hop); } + dest_lp = dests[select_chan]; } - codes_mapping_get_lp_id(lp_group_name, LP_CONFIG_NM_ROUT, s->anno, 0, dest_lp, - 0, &router_dest_id); + codes_mapping_get_lp_id(lp_group_name, LP_CONFIG_NM_ROUT, s->anno, 0, dest_lp / num_routers_per_mgrp, + dest_lp % num_routers_per_mgrp, &router_dest_id); return router_dest_id; } @@ -2239,44 +2317,36 @@ get_output_port( router_state * s, int next_stop) { int output_port = -1; + int rand_offset = -1; int terminal_id = codes_mapping_get_lp_relative_id(msg->dest_terminal_id, 0, 0); const dragonfly_param *p = s->params; + + int local_router_id = codes_mapping_get_lp_relative_id(next_stop, 0, 0); + int src_router = s->router_id; + int dest_router = local_router_id; if((tw_lpid)next_stop == msg->dest_terminal_id) { + /* Make a random number selection (only for reverse computation) */ + int rand_sel = tw_rand_integer(lp->rng, 0, terminal_id); output_port = p->intra_grp_radix + p->num_global_channels + ( terminal_id % p->num_cn); } else { - int local_router_id = codes_mapping_get_lp_relative_id(next_stop, 0, 0); int intm_grp_id = local_router_id / p->num_routers; + int rand_offset = -1; if(intm_grp_id != s->group_id) { - bf->c15 = 1; /* traversing a global channel */ - int rand_offset = -1; - int src_router = s->router_id; - int dest_router = local_router_id; - vector &curVec = interGroupLinks[src_router][intm_grp_id]; + if(interGroupLinks[src_router][intm_grp_id].size() == 0) + printf("\n Source router %d intm_grp_id %d ", src_router, intm_grp_id); + assert(interGroupLinks[src_router][intm_grp_id].size() > 0); - if(interGroupLinks[src_router][intm_grp_id].size() > 1) - { - if(interGroupLinks[src_router][intm_grp_id].size() > 2) - tw_error(TW_LOC, "\n Model currently functional for two global links from a router to the same group "); - - /* Note: we are not using ROSS random number generator here for - * channel selection - * because it will be hard to handle this with reverse handlers - * for adaptive/prog-adaptive routings. */ - if((src_router + intm_grp_id) % 2) - rand_offset = 1; - else - rand_offset = 0; - } + rand_offset = tw_rand_integer(lp->rng, 0, interGroupLinks[src_router][intm_grp_id].size()-1); assert(rand_offset >= 0); @@ -2298,21 +2368,21 @@ get_output_port( router_state * s, if(src_col == dest_col) { - int last_port = s->last_sent_chan[dest_row]; - output_port = p->num_router_cols + (dest_row * p->num_col_chans) + last_port; - s->last_sent_chan[dest_row] =( s->last_sent_chan[dest_row] + 1) % s->params->num_col_chans; + int offset = tw_rand_integer(lp->rng, 0, p->num_col_chans -1); + output_port = p->num_router_cols * p->num_row_chans + dest_row * p->num_col_chans + offset; assert(output_port < p->intra_grp_radix); } else if(src_row == dest_row) { - output_port = dest_col; - assert(output_port < s->params->num_router_cols); + int offset = tw_rand_integer(lp->rng, 0, p->num_row_chans -1); + output_port = dest_col * p->num_row_chans + offset; + assert(output_port < (s->params->num_router_cols * p->num_row_chans)); } else { - tw_error(TW_LOC, "\n Invalid dragonfly connectivity src row %d dest row %d src col %d dest col %d", - src_row, dest_row, src_col, dest_col); + tw_error(TW_LOC, "\n Invalid dragonfly connectivity src row %d dest row %d src col %d dest col %d src %d dest %d", + src_row, dest_row, src_col, dest_col, intragrp_rtr_id, intra_rtr_id); } } @@ -2338,13 +2408,17 @@ static void do_local_adaptive_routing(router_state * s, tw_error(TW_LOC, "\n Invalid local routing my grp id %d dest_gid %d intm_gid %d intm rid %d", my_grp_id, dest_grp_id, intm_grp_id, intm_router_id); - int next_min_stop = get_intra_router(s->router_id, dest_router_id, s->params->num_routers); - int next_nonmin_stop = get_intra_router(s->router_id, intm_router_id, s->params->num_routers); + int min_chan=-1, nonmin_chan=-1; + vector next_min_stops = get_intra_router(s, s->router_id, dest_router_id, s->params->num_routers); + vector next_nonmin_stops = get_intra_router(s, s->router_id, intm_router_id, s->params->num_routers); - codes_mapping_get_lp_id(lp_group_name, LP_CONFIG_NM_ROUT, s->anno, 0, next_min_stop, - 0, &min_rtr_id); - codes_mapping_get_lp_id(lp_group_name, LP_CONFIG_NM_ROUT, s->anno, 0, next_nonmin_stop, - 0, &nonmin_rtr_id); + min_chan = tw_rand_integer(lp->rng, 0, next_min_stops.size() - 1); + nonmin_chan = tw_rand_integer(lp->rng, 0, next_nonmin_stops.size() - 1); + + codes_mapping_get_lp_id(lp_group_name, LP_CONFIG_NM_ROUT, s->anno, 0, next_min_stops[min_chan] / num_routers_per_mgrp, + next_min_stops[min_chan] % num_routers_per_mgrp, &min_rtr_id); + codes_mapping_get_lp_id(lp_group_name, LP_CONFIG_NM_ROUT, s->anno, 0, next_nonmin_stops[nonmin_chan] / num_routers_per_mgrp, + next_nonmin_stops[nonmin_chan] % num_routers_per_mgrp, &nonmin_rtr_id); min_port = get_output_port(s, msg, lp, bf, min_rtr_id); nonmin_port = get_output_port(s, msg, lp, bf, nonmin_rtr_id); @@ -2362,12 +2436,14 @@ static void do_local_adaptive_routing(router_state * s, int local_stop = -1; tw_lpid global_stop; - if(nonmin_port_count >= min_port_count) +// if(nonmin_port_count * num_intra_nonmin_hops > min_port_count * num_intra_min_hops) + if(nonmin_port_count > min_port_count) { msg->path_type = MINIMAL; } else { +// printf("\n Nonmin port count %ld min port count %ld ", nonmin_port_count, min_port_count); msg->path_type = NON_MINIMAL; } } @@ -2387,12 +2463,12 @@ static int do_global_adaptive_routing( router_state * s, int num_min_chans = connectionList[my_grp_id][dest_grp_id].size(); int num_nonmin_chans = connectionList[my_grp_id][intm_grp_id].size(); - int min_chan_a, min_chan_b, nonmin_chan_a, nonmin_chan_b; int min_rtr_a, min_rtr_b, nonmin_rtr_a, nonmin_rtr_b; - int dest_rtr_a, dest_rtr_b; + vector dest_rtr_as, dest_rtr_bs; int min_port_a, min_port_b, nonmin_port_a, nonmin_port_b; tw_lpid min_rtr_a_id, min_rtr_b_id, nonmin_rtr_a_id, nonmin_rtr_b_id; + bool noIntraA, noIntraB; /* two possible routes to minimal destination */ min_chan_a = tw_rand_integer(lp->rng, 0, num_min_chans - 1); @@ -2401,23 +2477,51 @@ static int do_global_adaptive_routing( router_state * s, if(min_chan_a == min_chan_b && num_min_chans > 1) min_chan_b = (min_chan_a + 1) % num_min_chans; + int chana1 = 0; + //chana1 = tw_rand_integer(lp->rng, 0, interGroupLinks[s->router_id][dest_grp_id].size()-1); + //chana1=0; + min_rtr_a = connectionList[my_grp_id][dest_grp_id][min_chan_a]; - - if(num_min_chans > 1) + noIntraA = false; + if(min_rtr_a == s->router_id) { + noIntraA = true; + min_rtr_a = interGroupLinks[s->router_id][dest_grp_id][chana1].dest; + } + if(num_min_chans > 1) { + noIntraB = false; min_rtr_b = connectionList[my_grp_id][dest_grp_id][min_chan_b]; - - dest_rtr_a = get_intra_router(s->router_id, min_rtr_a, s->params->num_routers); + + if(min_rtr_b == s->router_id) { + noIntraB = true; + min_rtr_b = interGroupLinks[s->router_id][dest_grp_id][chana1].dest; + } + } + + if(noIntraA) { + dest_rtr_as.push_back(min_rtr_a); + } else { + dest_rtr_as = get_intra_router(s, s->router_id, min_rtr_a, s->params->num_routers); + } - codes_mapping_get_lp_id(lp_group_name, LP_CONFIG_NM_ROUT, s->anno, 0, dest_rtr_a, - 0, &min_rtr_a_id); + int dest_rtr_b_sel; + int dest_rtr_a_sel = tw_rand_integer(lp->rng, 0, dest_rtr_as.size() - 1); + + codes_mapping_get_lp_id(lp_group_name, LP_CONFIG_NM_ROUT, s->anno, 0, dest_rtr_as[dest_rtr_a_sel] / num_routers_per_mgrp, + dest_rtr_as[dest_rtr_a_sel] % num_routers_per_mgrp, &min_rtr_a_id); min_port_a = get_output_port(s, msg, lp, bf, min_rtr_a_id); if(num_min_chans > 1) { - dest_rtr_b = get_intra_router(s->router_id, min_rtr_b, s->params->num_routers); - codes_mapping_get_lp_id(lp_group_name, LP_CONFIG_NM_ROUT, s->anno, 0, dest_rtr_b, - 0, &min_rtr_b_id); + bf->c10 = 1; + if(noIntraB) { + dest_rtr_bs.push_back(min_rtr_b); + } else { + dest_rtr_bs = get_intra_router(s, s->router_id, min_rtr_b, s->params->num_routers); + } + dest_rtr_b_sel = tw_rand_integer(lp->rng, 0, dest_rtr_bs.size() - 1); + codes_mapping_get_lp_id(lp_group_name, LP_CONFIG_NM_ROUT, s->anno, 0, dest_rtr_bs[dest_rtr_b_sel] / num_routers_per_mgrp, + dest_rtr_bs[dest_rtr_b_sel] % num_routers_per_mgrp, &min_rtr_b_id); min_port_b = get_output_port(s, msg, lp, bf, min_rtr_b_id); } @@ -2429,20 +2533,47 @@ static int do_global_adaptive_routing( router_state * s, nonmin_chan_b = (nonmin_chan_a + 1) % num_nonmin_chans; nonmin_rtr_a = connectionList[my_grp_id][intm_grp_id][nonmin_chan_a]; - - if(num_nonmin_chans > 1) + noIntraA = false; + if(nonmin_rtr_a == s->router_id) { + bf->c25=1; + noIntraA = true; + nonmin_rtr_a = interGroupLinks[s->router_id][intm_grp_id][0].dest; + } + + if(num_nonmin_chans > 1) { nonmin_rtr_b = connectionList[my_grp_id][intm_grp_id][nonmin_chan_b]; + noIntraB = false; + if(nonmin_rtr_b == s->router_id) { + bf->c26=1; + noIntraB = true; + nonmin_rtr_b = interGroupLinks[s->router_id][intm_grp_id][0].dest; + } + } - dest_rtr_a = get_intra_router(s->router_id, nonmin_rtr_a, s->params->num_routers); - codes_mapping_get_lp_id(lp_group_name, LP_CONFIG_NM_ROUT, s->anno, 0, dest_rtr_a, - 0, &nonmin_rtr_a_id); + if(noIntraA) { + dest_rtr_as.clear(); + dest_rtr_as.push_back(nonmin_rtr_a); + } else { + dest_rtr_as = get_intra_router(s, s->router_id, nonmin_rtr_a, s->params->num_routers); + } + dest_rtr_a_sel = tw_rand_integer(lp->rng, 0, dest_rtr_as.size() - 1); + + codes_mapping_get_lp_id(lp_group_name, LP_CONFIG_NM_ROUT, s->anno, 0, dest_rtr_as[dest_rtr_a_sel] / num_routers_per_mgrp, + dest_rtr_as[dest_rtr_a_sel] % num_routers_per_mgrp, &nonmin_rtr_a_id); nonmin_port_a = get_output_port(s, msg, lp, bf, nonmin_rtr_a_id); if(num_nonmin_chans > 1) { - dest_rtr_b = get_intra_router(s->router_id, nonmin_rtr_b, s->params->num_routers); - codes_mapping_get_lp_id(lp_group_name, LP_CONFIG_NM_ROUT, s->anno, 0, dest_rtr_b, - 0, &nonmin_rtr_b_id); + bf->c11 = 1; + if(noIntraB) { + dest_rtr_bs.clear(); + dest_rtr_bs.push_back(nonmin_rtr_b); + } else { + dest_rtr_bs = get_intra_router(s, s->router_id, nonmin_rtr_b, s->params->num_routers); + } + dest_rtr_b_sel = tw_rand_integer(lp->rng, 0, dest_rtr_bs.size() - 1); + codes_mapping_get_lp_id(lp_group_name, LP_CONFIG_NM_ROUT, s->anno, 0, dest_rtr_bs[dest_rtr_b_sel] / num_routers_per_mgrp, + dest_rtr_bs[dest_rtr_b_sel] % num_routers_per_mgrp, &nonmin_rtr_b_id); nonmin_port_b = get_output_port(s, msg, lp, bf, nonmin_rtr_b_id); } /*randomly select two minimal routes and two non-minimal routes */ @@ -2487,7 +2618,7 @@ static int do_global_adaptive_routing( router_state * s, int next_min_count = -1, next_nonmin_count = -1; /* First compare which of the nonminimal ports has less congestions */ - if(nonmin_port_b_count && nonmin_port_a_count > nonmin_port_b_count) + if(num_nonmin_chans > 1 && nonmin_port_a_count > nonmin_port_b_count) { next_nonmin_count = nonmin_port_b_count; next_nonmin_stop = nonmin_chan_b; @@ -2498,7 +2629,7 @@ static int do_global_adaptive_routing( router_state * s, next_nonmin_stop = nonmin_chan_a; } /* do the same for minimal ports */ - if(min_port_b_count && min_port_a_count > min_port_b_count) + if(num_min_chans > 1 && min_port_a_count > min_port_b_count) { next_min_count = min_port_b_count; next_min_stop = min_chan_b; @@ -2553,14 +2684,33 @@ static void router_packet_receive_rc(router_state * s, if(bf->c20) { - tw_rand_reverse_unif(lp->rng); - tw_rand_reverse_unif(lp->rng); - tw_rand_reverse_unif(lp->rng); - tw_rand_reverse_unif(lp->rng); + for(int i = 0; i < 8; i++) + tw_rand_reverse_unif(lp->rng); + + //tw_rand_reverse_unif(lp->rng); + + if(bf->c10) + { + tw_rand_reverse_unif(lp->rng); + tw_rand_reverse_unif(lp->rng); + } + if(bf->c11) + { + tw_rand_reverse_unif(lp->rng); + tw_rand_reverse_unif(lp->rng); + } + } + if(bf->c6) + { + for(int i = 0; i < 4; i++) + tw_rand_reverse_unif(lp->rng); } if(bf->c19) tw_rand_reverse_unif(lp->rng); + if(bf->c21) + tw_rand_reverse_unif(lp->rng); + tw_rand_reverse_unif(lp->rng); if(bf->c2) { tw_rand_reverse_unif(lp->rng); terminal_custom_message_list * tail = return_tail(s->pending_msgs[output_port], s->pending_msgs_tail[output_port], output_chan); @@ -2588,15 +2738,6 @@ router_packet_receive( router_state * s, { router_ecount++; - bf->c1 = 0; - bf->c2 = 0; - bf->c3 = 0; - bf->c4 = 0; - bf->c5 = 0; - bf->c15 = 0; - bf->c19 = 0; - bf->c20 = 0; - tw_stime ts; int next_stop = -1, output_port = -1, output_chan = -1, adap_chan = -1; @@ -2627,21 +2768,20 @@ router_packet_receive( router_state * s, * intermediate router ID which is in the same group. */ if(src_grp_id != dest_grp_id) { - bf->c15 = 1; intm_router_id = tw_rand_integer(lp->rng, 0, s->params->total_routers - 1); } else - intm_router_id = (src_grp_id * s->params->num_routers) + tw_rand_integer(lp->rng, 0, s->params->num_routers - 1); + intm_router_id = (src_grp_id * s->params->num_routers) + + (((s->router_id % s->params->num_routers) + + tw_rand_integer(lp->rng, 1, s->params->num_routers - 1)) % s->params->num_routers); /* For global adaptive routing, we make sure that a different group * is selected. For local adaptive routing, if the same router as self is * selected then we choose the neighboring router. */ if(src_grp_id != dest_grp_id - && (intm_router_id / s->params->num_routers) == local_grp_id) - intm_router_id = (s->router_id + s->params->num_routers) % s->params->total_routers; - else if(intm_router_id == s->router_id) - intm_router_id = (src_grp_id * s->params->num_routers) + ((s->router_id + 1) % s->params->num_routers); - + && (intm_router_id / s->params->num_routers) == local_grp_id) + intm_router_id = (s->router_id + s->params->num_routers) % s->params->total_routers; + /* progressive adaptive routing is only triggered when packet has to traverse a * global channel. It doesn't make sense to use it within a group */ if(dest_grp_id != src_grp_id && @@ -2660,6 +2800,7 @@ router_packet_receive( router_state * s, (routing == ADAPTIVE || routing == PROG_ADAPTIVE) && cur_chunk->msg.last_hop == TERMINAL) { + bf->c6 = 1; do_local_adaptive_routing(s, lp, &(cur_chunk->msg), bf, dest_router_id, intm_router_id); } @@ -2679,8 +2820,8 @@ router_packet_receive( router_state * s, printf("\n Packet %llu local hops being reset from %d to 2", msg->packet_ID, msg->my_l_hop); /* Reset any existing hop additions */ - if(routing == PROG_ADAPTIVE) - cur_chunk->msg.my_l_hop = 2; + //if(routing == PROG_ADAPTIVE) + // cur_chunk->msg.my_l_hop = 2; cur_chunk->msg.intm_rtr_id = intm_router_id; cur_chunk->msg.nonmin_done = 0; @@ -2722,6 +2863,19 @@ router_packet_receive( router_state * s, output_chan = 0; if(output_port < s->params->intra_grp_radix) { + if(cur_chunk->msg.my_g_hop == 1) { + if(routing == PROG_ADAPTIVE && cur_chunk->msg.my_l_hop < 4) { + cur_chunk->msg.my_l_hop = 4; + } else if(cur_chunk->msg.my_l_hop < 2) { + cur_chunk->msg.my_l_hop = 2; + } + } else if (cur_chunk->msg.my_g_hop == 2) { + if(routing == PROG_ADAPTIVE && cur_chunk->msg.my_l_hop < 6) { + cur_chunk->msg.my_l_hop = 6; + } else if(cur_chunk->msg.my_l_hop < 4) { + cur_chunk->msg.my_l_hop = 4; + } + } output_chan = cur_chunk->msg.my_l_hop; max_vc_size = s->params->local_vc_size; cur_chunk->msg.my_l_hop++; @@ -2736,7 +2890,7 @@ router_packet_receive( router_state * s, cur_chunk->msg.my_N_hop++; if(output_port >= s->params->radix) - tw_error(TW_LOC, "\n Output port greater than router radix"); + tw_error(TW_LOC, "\n Output port greater than router radix %d ", output_port); if(output_chan >= s->params->num_vcs || output_chan < 0) printf("\n Packet %llu Output chan %d output port %d my rid %d dest rid %d path %d my gid %d dest gid %d", @@ -2745,7 +2899,7 @@ router_packet_receive( router_state * s, assert(output_chan < s->params->num_vcs && output_chan >= 0); if(msg->remote_event_size_bytes > 0) { - void *m_data_src = model_net_method_get_edata(DRAGONFLY_ROUTER, msg); + void *m_data_src = model_net_method_get_edata(DRAGONFLY_CUSTOM_ROUTER, msg); cur_chunk->event_data = (char*)malloc(msg->remote_event_size_bytes); memcpy(cur_chunk->event_data, m_data_src, msg->remote_event_size_bytes); } @@ -2762,7 +2916,7 @@ router_packet_receive( router_state * s, terminal_custom_message *m; ts = codes_local_latency(lp); tw_event *e = model_net_method_event_new(lp->gid, ts, lp, - DRAGONFLY_ROUTER, (void**)&m, NULL); + DRAGONFLY_CUSTOM_ROUTER, (void**)&m, NULL); m->type = R_SEND; m->magic = router_magic_num; m->vc_index = output_port; @@ -2817,7 +2971,6 @@ static void router_packet_send_rc(router_state * s, } s->next_output_available_time[output_port] = msg->saved_available_time; - prepend_to_terminal_custom_message_list(s->pending_msgs[output_port], s->pending_msgs_tail[output_port], output_chan, cur_entry); @@ -2887,7 +3040,10 @@ router_packet_send( router_state * s, num_chunks = 1; double bytetime = delay; - + + if(cur_entry->msg.packet_size == 0) + bytetime = bytes_to_ns(CREDIT_SIZE, bandwidth); + if((cur_entry->msg.packet_size % s->params->chunk_size) && (cur_entry->msg.chunk_id == num_chunks - 1)) bytetime = bytes_to_ns(cur_entry->msg.packet_size % s->params->chunk_size, bandwidth); @@ -2905,11 +3061,11 @@ router_packet_send( router_state * s, assert(cur_entry->msg.next_stop == cur_entry->msg.dest_terminal_id); e = model_net_method_event_new(cur_entry->msg.next_stop, s->next_output_available_time[output_port] - tw_now(lp), lp, - DRAGONFLY, (void**)&m, &m_data); + DRAGONFLY_CUSTOM, (void**)&m, &m_data); } else { e = model_net_method_event_new(cur_entry->msg.next_stop, s->next_output_available_time[output_port] - tw_now(lp), lp, - DRAGONFLY_ROUTER, (void**)&m, &m_data); + DRAGONFLY_CUSTOM_ROUTER, (void**)&m, &m_data); } memcpy(m, &cur_entry->msg, sizeof(terminal_custom_message)); if (m->remote_event_size_bytes){ @@ -2968,7 +3124,7 @@ router_packet_send( router_state * s, bf->c3 = 1; terminal_custom_message *m_new; ts += g_tw_lookahead + tw_rand_unif(lp->rng); - e = model_net_method_event_new(lp->gid, ts, lp, DRAGONFLY_ROUTER, + e = model_net_method_event_new(lp->gid, ts, lp, DRAGONFLY_CUSTOM_ROUTER, (void**)&m_new, NULL); m_new->type = R_SEND; m_new->magic = router_magic_num; @@ -3040,7 +3196,7 @@ static void router_buf_update(router_state * s, tw_bf * bf, terminal_custom_mess bf->c2 = 1; terminal_custom_message *m; tw_stime ts = codes_local_latency(lp); - tw_event *e = model_net_method_event_new(lp->gid, ts, lp, DRAGONFLY_ROUTER, + tw_event *e = model_net_method_event_new(lp->gid, ts, lp, DRAGONFLY_CUSTOM_ROUTER, (void**)&m, NULL); m->type = R_SEND; m->vc_index = indx; @@ -3053,7 +3209,6 @@ static void router_buf_update(router_state * s, tw_bf * bf, terminal_custom_mess void router_custom_event(router_state * s, tw_bf * bf, terminal_custom_message * msg, tw_lp * lp) { - s->fwd_events++; rc_stack_gc(lp, s->st); @@ -3191,10 +3346,10 @@ struct model_net_method dragonfly_custom_method = dragonfly_custom_report_stats, NULL, NULL, - (event_f)dragonfly_custom_sample_fn, - (revent_f)dragonfly_custom_sample_rc_fn, + NULL,//(event_f)dragonfly_custom_sample_fn, + NULL,//(revent_f)dragonfly_custom_sample_rc_fn, (init_f)dragonfly_custom_sample_init, - (final_f)dragonfly_custom_sample_fin + NULL,//(final_f)dragonfly_custom_sample_fin }; struct model_net_method dragonfly_custom_router_method = @@ -3211,9 +3366,9 @@ struct model_net_method dragonfly_custom_router_method = NULL, // not yet supported NULL, NULL, - (event_f)dragonfly_custom_rsample_fn, - (revent_f)dragonfly_custom_rsample_rc_fn, + NULL,//(event_f)dragonfly_custom_rsample_fn, + NULL,//(revent_f)dragonfly_custom_rsample_rc_fn, (init_f)dragonfly_custom_rsample_init, - (final_f)dragonfly_custom_rsample_fin + NULL,//(final_f)dragonfly_custom_rsample_fin }; } diff --git a/src/networks/model-net/model-net-lp.c b/src/networks/model-net/model-net-lp.c index 6b98aedd5e677bbd4bc15e6a888eda1c47bd3a32..54570d7d632375f79d173b123abb55f1ee8a9390 100644 --- a/src/networks/model-net/model-net-lp.c +++ b/src/networks/model-net/model-net-lp.c @@ -18,6 +18,7 @@ /**** BEGIN SIMULATION DATA STRUCTURES ****/ int model_net_base_magic; +int mn_sample_enabled = 0; // message-type specific offsets - don't want to get bitten later by alignment // issues... @@ -36,7 +37,6 @@ static const char * annos[CONFIGURATION_MAX_ANNOS]; static model_net_base_params all_params[CONFIGURATION_MAX_ANNOS]; static tw_stime mn_sample_interval = 0.0; -static int mn_sample_enabled = 0; static tw_stime mn_sample_end = 0.0; typedef struct model_net_base_state { @@ -323,9 +323,9 @@ void model_net_base_configure(){ msg_offsets[DRAGONFLY_ROUTER] = offsetof(model_net_wrap_msg, msg.m_dfly); msg_offsets[DRAGONFLY_CUSTOM] = - offsetof(model_net_wrap_msg, msg.m_dfly); + offsetof(model_net_wrap_msg, msg.m_custom_dfly); msg_offsets[DRAGONFLY_CUSTOM_ROUTER] = - offsetof(model_net_wrap_msg, msg.m_dfly); + offsetof(model_net_wrap_msg, msg.m_custom_dfly); msg_offsets[SLIMFLY] = offsetof(model_net_wrap_msg, msg.m_slim); msg_offsets[FATTREE] = diff --git a/src/networks/model-net/torus.c b/src/networks/model-net/torus.c index cd0cc1af3108914c1cb44012159a805b59e8f1ca..27b628a2e6f4eaa5439d0bf07b4d52711181b2ea 100644 --- a/src/networks/model-net/torus.c +++ b/src/networks/model-net/torus.c @@ -1694,7 +1694,7 @@ static void torus_report_stats() if(!g_tw_mynode) { - printf(" Average number of hops traversed %f average message latency %lf us maximum message latency %lf us finished packets %lld finished hops %lld \n", + printf(" Average number of hops traversed %f average packet latency %lf us maximum packet latency %lf us finished packets %lld finished hops %lld \n", (float)avg_hops/total_finished_packets, avg_time/(total_finished_packets*1000), max_time/1000, total_finished_packets, avg_hops); } } diff --git a/src/util/patches/fts.patch b/src/util/patches/fts.patch index c50a705e34a8c146fe2644705680a07f325ba181..6ef6b098d2f85a783d603e992a0073a6a7a5e92e 100644 --- a/src/util/patches/fts.patch +++ b/src/util/patches/fts.patch @@ -626,13 +626,109 @@ diff -Nur scripts.orig/createIBNet.py scripts/createIBNet.py + #sys.exit('\nFinish!') + sys.exit(0) -diff -Nur scripts.orig/simulate.py scripts/simulate.py ---- scripts.orig/simulate.py 2016-08-17 14:02:19.306717000 -0700 -+++ scripts/simulate.py 2016-08-17 13:10:45.369042000 -0700 -@@ -724,21 +724,23 @@ - os.system("sed -i -e 's#\"/usr/bin\":##g' %s/bin/ibdiagnet" % ofeddir) +diff -u scripts/simulate.py scripts/simulate.py +--- scripts/simulate.py 2016-08-17 13:10:45.369042000 -0700 ++++ scripts/simulate.py 2017-04-25 19:24:20.959463076 +0900 +@@ -610,24 +610,24 @@ + os.mkdir(os.path.join(instdir, 'libibmad')) + if not os.path.exists(os.path.join(instdir, 'opensm')): + os.mkdir(os.path.join(instdir, 'opensm')) +- if not os.path.exists(os.path.join(instdir, 'infiniband-diags')): +- os.mkdir(os.path.join(instdir, 'infiniband-diags')) ++ #if not os.path.exists(os.path.join(instdir, 'infiniband-diags')): ++ # os.mkdir(os.path.join(instdir, 'infiniband-diags')) + if not os.path.exists(os.path.join(instdir, 'ibsim')): + os.mkdir(os.path.join(instdir, 'ibsim')) +- if not os.path.exists(os.path.join(instdir, 'ibutils')): +- os.mkdir(os.path.join(instdir, 'ibutils')) +- if not os.path.exists(os.path.join(instdir, 'omnet')): +- os.mkdir(os.path.join(instdir, 'omnet')) +- if not os.path.exists(os.path.join(instdir, 'ibmodel')): +- os.mkdir(os.path.join(instdir, 'ibmodel')) ++ #if not os.path.exists(os.path.join(instdir, 'ibutils')): ++ # os.mkdir(os.path.join(instdir, 'ibutils')) ++ #if not os.path.exists(os.path.join(instdir, 'omnet')): ++ # os.mkdir(os.path.join(instdir, 'omnet')) ++ #if not os.path.exists(os.path.join(instdir, 'ibmodel')): ++ # os.mkdir(os.path.join(instdir, 'ibmodel')) + os.system('tar xzf %s.tar.gz -C %s --strip-components 1' % (os.path.join(currdir, 'libibumad'), 'libibumad')) + os.system('tar xzf %s.tar.gz -C %s --strip-components 1' % (os.path.join(currdir, 'libibmad'), 'libibmad')) + os.system('tar xzf %s.tar.gz -C %s --strip-components 1' % (os.path.join(currdir, 'opensm'), 'opensm')) +- os.system('tar xzf %s.tar.gz -C %s --strip-components 1' % (os.path.join(currdir, 'infiniband-diags'), 'infiniband-diags')) ++ #os.system('tar xzf %s.tar.gz -C %s --strip-components 1' % (os.path.join(currdir, 'infiniband-diags'), 'infiniband-diags')) + os.system('tar xzf %s.tar.gz -C %s --strip-components 1' % (os.path.join(currdir, 'ibsim'), 'ibsim')) +- os.system('tar xzf %s.tar.gz -C %s --strip-components 1' % (os.path.join(currdir, 'ibutils'), 'ibutils')) +- os.system('tar xzf %s.tar.gz -C %s --strip-components 1' % (os.path.join(currdir, 'omnet'), 'omnet')) +- os.system('tar xzf %s.tar.gz -C %s --strip-components 1' % (os.path.join(currdir, 'ibmodel'), 'ibmodel')) ++ #os.system('tar xzf %s.tar.gz -C %s --strip-components 1' % (os.path.join(currdir, 'ibutils'), 'ibutils')) ++ #os.system('tar xzf %s.tar.gz -C %s --strip-components 1' % (os.path.join(currdir, 'omnet'), 'omnet')) ++ #os.system('tar xzf %s.tar.gz -C %s --strip-components 1' % (os.path.join(currdir, 'ibmodel'), 'ibmodel')) + + # patching opensm + for root, dirs, files in os.walk(currdir): +@@ -681,64 +681,70 @@ + os.system('make install >install.log 2>&1') os.chdir(instdir) +- print "Step 4..." +- os.chdir(os.path.join(instdir, 'infiniband-diags')) +- os.system('(aclocal; libtoolize --force; autoheader; autoconf) >/dev/null 2>&1') +- os.system('./autogen.sh >autogen.log 2>&1') +- os.system('./configure %s --with-perl-installdir=%s >configure.log 2>&1' % (glob_config_opts, os.path.join(ofeddir, 'perl'))) +- os.system('./autogen.sh >autogen.log 2>&1') +- os.system('./configure %s --with-perl-installdir=%s >configure.log 2>&1' % (glob_config_opts, os.path.join(ofeddir, 'perl'))) +- os.system('make >make.log 2>&1') +- os.system('make install >install.log 2>&1') +- os.chdir(instdir) ++ print '...... skipping step 4 .....' ++ if 0: ++ print "Step 4..." ++ os.chdir(os.path.join(instdir, 'infiniband-diags')) ++ os.system('(aclocal; libtoolize --force; autoheader; autoconf) >/dev/null 2>&1') ++ os.system('./autogen.sh >autogen.log 2>&1') ++ os.system('./configure %s --with-perl-installdir=%s >configure.log 2>&1' % (glob_config_opts, os.path.join(ofeddir, 'perl'))) ++ os.system('./autogen.sh >autogen.log 2>&1') ++ os.system('./configure %s --with-perl-installdir=%s >configure.log 2>&1' % (glob_config_opts, os.path.join(ofeddir, 'perl'))) ++ os.system('make >make.log 2>&1') ++ os.system('make install >install.log 2>&1') ++ os.chdir(instdir) + + print "Step 5..." + os.chdir(os.path.join(instdir, 'ibsim')) + os.system('make IB_DEV_DIR=%s prefix=%s all install >make.log 2>&1' % (instdir, ofeddir)) + os.chdir(instdir) + +- print "Step 6..." +- #os.chdir(os.path.join(instdir, 'ibutils')) ++ print '...... skipping step 6 .....' ++ if 0: ++ print "Step 6..." ++ #os.chdir(os.path.join(instdir, 'ibutils')) + # if socket.gethostname().find("t2a") >= 0: + # os.system("for x in `grep -R 2.2.6 . | cut -d ':' -f1 | sort -u`; do sed -i -e 's/2.2.6b/2.2.6/g' $x; done") + # os.system("for x in `grep -R Debian-2.2.6-2ubuntu1 . | cut -d ':' -f1 | sort -u`; do sed -i -e 's/ Debian-2.2.6-2ubuntu1//g' $x; done") + # os.system("for x in `grep -R 1.3017 . | cut -d ':' -f1 | sort -u`; do sed -i -e 's/1.3017/1.3012/g' $x; done") + # elif socket.gethostname().find("rc0") >= 0: +- os.chdir(os.path.join(instdir, 'ibutils', 'ibdiag')) +- os.system('(aclocal; libtoolize --force; autoconf) >/dev/null 2>&1') +- os.chdir(os.path.join(instdir, 'ibutils', 'ibdm')) +- os.system('(aclocal; libtoolize --force; autoheader; autoconf) >/dev/null 2>&1') +- os.chdir(os.path.join(instdir, 'ibutils', 'ibis')) +- os.system('(aclocal; libtoolize --force; autoheader; autoconf) >/dev/null 2>&1') +- os.chdir(os.path.join(instdir, 'ibutils', 'ibmgtsim')) +- os.system('(aclocal; libtoolize --force; autoheader; autoconf) >/dev/null 2>&1') +- os.chdir(os.path.join(instdir, 'ibutils')) +- os.system('(aclocal; libtoolize --force; autoconf) >/dev/null 2>&1') +- os.system('./autogen.sh >autogen.log 2>&1') +- os.system('./configure %s --with-osm=%s >configure.log 2>&1' % (glob_config_opts, ofeddir)) +- os.system('./autogen.sh >autogen.log 2>&1') +- os.system('./configure %s --with-osm=%s >configure.log 2>&1' % (glob_config_opts, ofeddir)) +- os.system('make >make.log 2>&1') +- os.system('make check >check.log 2>&1') +- os.system('make install >install.log 2>&1') +- os.system("sed -i -e 's#\"/usr/bin\":##g' %s/bin/ibdiagnet" % ofeddir) +- os.chdir(instdir) +- - print "Step 7..." - os.chdir(os.path.join(instdir, 'omnet')) - os.environ['PATH'] = '%s:%s' % (os.path.join(instdir, 'omnet', 'bin'), os.getenv('PATH')) @@ -648,6 +744,26 @@ diff -Nur scripts.orig/simulate.py scripts/simulate.py - os.environ['LD_LIBRARY_PATH'] = '%s:%s' % (os.path.join(instdir, 'omnet', 'lib'), os.getenv('LD_LIBRARY_PATH')) - os.system('opp_makemake -f --deep -O out') - os.system('make MODE=release') ++ os.chdir(os.path.join(instdir, 'ibutils', 'ibdiag')) ++ os.system('(aclocal; libtoolize --force; autoconf) >/dev/null 2>&1') ++ os.chdir(os.path.join(instdir, 'ibutils', 'ibdm')) ++ os.system('(aclocal; libtoolize --force; autoheader; autoconf) >/dev/null 2>&1') ++ os.chdir(os.path.join(instdir, 'ibutils', 'ibis')) ++ os.system('(aclocal; libtoolize --force; autoheader; autoconf) >/dev/null 2>&1') ++ os.chdir(os.path.join(instdir, 'ibutils', 'ibmgtsim')) ++ os.system('(aclocal; libtoolize --force; autoheader; autoconf) >/dev/null 2>&1') ++ os.chdir(os.path.join(instdir, 'ibutils')) ++ os.system('(aclocal; libtoolize --force; autoconf) >/dev/null 2>&1') ++ os.system('./autogen.sh >autogen.log 2>&1') ++ os.system('./configure %s --with-osm=%s >configure.log 2>&1' % (glob_config_opts, ofeddir)) ++ os.system('./autogen.sh >autogen.log 2>&1') ++ os.system('./configure %s --with-osm=%s >configure.log 2>&1' % (glob_config_opts, ofeddir)) ++ os.system('make >make.log 2>&1') ++ os.system('make check >check.log 2>&1') ++ os.system('make install >install.log 2>&1') ++ os.system("sed -i -e 's#\"/usr/bin\":##g' %s/bin/ibdiagnet" % ofeddir) ++ os.chdir(instdir) ++ + print '...... skipping omnet install .....' + if 0: + print "Step 7..." @@ -668,10 +784,124 @@ diff -Nur scripts.orig/simulate.py scripts/simulate.py print "Step 9..." os.system('cp %s/*.py %s/' % (currdir, scriptdir)) -@@ -925,39 +927,40 @@ - log.close() +@@ -832,24 +838,26 @@ + routing_failed = True + osm_log.close() + +- print "Scan fabric..." +- scanlog = os.path.join(ofedoutdir, 'scan.log') +- cmd = '%s %s %s >%s 2>&1 &' % (os.path.join(scriptdir, 'scanFabric.sh' ), instdir, ofedoutdir, scanlog) +- log.write('%s %s\n' % (time.asctime(), cmd)) +- os.system(cmd) +- while True: +- if os.path.exists(scanlog): +- break +- time.sleep(1) +- scan_log = open(scanlog, 'r') +- gstr = '' +- while True: +- str = scan_log.read() +- gstr = gstr + str +- if gstr.find('SUBNET SCANNED') != -1: +- break +- time.sleep(1) +- scan_log.close() ++ # doesnt work in this mode since we removed ibdiagnet/ibnetdiscover ++ if 0: ++ print "Scan fabric..." ++ scanlog = os.path.join(ofedoutdir, 'scan.log') ++ cmd = '%s %s %s >%s 2>&1 &' % (os.path.join(scriptdir, 'scanFabric.sh' ), instdir, ofedoutdir, scanlog) ++ log.write('%s %s\n' % (time.asctime(), cmd)) ++ os.system(cmd) ++ while True: ++ if os.path.exists(scanlog): ++ break ++ time.sleep(1) ++ scan_log = open(scanlog, 'r') ++ gstr = '' ++ while True: ++ str = scan_log.read() ++ gstr = gstr + str ++ if gstr.find('SUBNET SCANNED') != -1: ++ break ++ time.sleep(1) ++ scan_log.close() + + print "Stop ibsim/opensm..." + toKill = [['startOpenSM.pid', signal.SIGTERM], ['startSIM.pid', signal.SIGTERM], ['opensm.pid', signal.SIGKILL], ['ibsim.pid', signal.SIGKILL]] +@@ -867,6 +875,9 @@ return + if routing.find('tofu') > -1: ++ log.write( '\nERROR: tofu not supported in this modus!\n' ) ++ log.close() ++ return + print "Replace with tofu routing..." + LFTtoFDBSlog = os.path.join(testdir, 'LFTtoFDBSlog.log') + cmd = '%s --lft %s --ibnd %s --fdbs %s >%s 2>&1 &' % (os.path.join(scriptdir, 'IBNetLFTtoFDBS.py'), os.path.join(ofedoutdir, 'topo.lft'), os.path.join(ofedoutdir, 'ibnetdiscover.log'), os.path.join(ofedoutdir, 'ibdiagnet.fdbs'), LFTtoFDBSlog) +@@ -896,68 +907,71 @@ + log.close() + return + +- print "Check the connectivity of the system..." +- checklog = os.path.join(testdir, 'checkConnectivity.log') +- cmd = '%s %s >%s 2>&1 &' % (os.path.join(scriptdir, 'checkConnectivity.py'), ofedoutdir, checklog) +- log.write('%s %s\n' % (time.asctime(), cmd)) +- os.system(cmd) +- while True: +- if os.path.exists(checklog): +- break +- time.sleep(1) +- check_log = open(checklog, 'r') +- gstr = '' +- while True: +- str = check_log.read() +- gstr = gstr + str +- if gstr.find('Finish') != -1 or gstr.find('Usage') != -1 or gstr.find('Error') != -1: +- break +- time.sleep(1) +- routing_failed = False +- gstr = gstr.splitlines() +- for line in gstr: +- if line.find('Error') != -1 or line.find('Usage') != -1: +- log.write( '%s\n' % (line) ) +- routing_failed = True +- check_log.close() +- if routing_failed: +- log.write( '\nERROR: routing failed, connectivity broken!\n' ) +- log.close() +- return ++ # doesnt work in this mode since we removed ibdiagnet/ibnetdiscover ++ if 0: ++ print "Check the connectivity of the system..." ++ checklog = os.path.join(testdir, 'checkConnectivity.log') ++ cmd = '%s %s >%s 2>&1 &' % (os.path.join(scriptdir, 'checkConnectivity.py'), ofedoutdir, checklog) ++ log.write('%s %s\n' % (time.asctime(), cmd)) ++ os.system(cmd) ++ while True: ++ if os.path.exists(checklog): ++ break ++ time.sleep(1) ++ check_log = open(checklog, 'r') ++ gstr = '' ++ while True: ++ str = check_log.read() ++ gstr = gstr + str ++ if gstr.find('Finish') != -1 or gstr.find('Usage') != -1 or gstr.find('Error') != -1: ++ break ++ time.sleep(1) ++ routing_failed = False ++ gstr = gstr.splitlines() ++ for line in gstr: ++ if line.find('Error') != -1 or line.find('Usage') != -1: ++ log.write( '%s\n' % (line) ) ++ routing_failed = True ++ check_log.close() ++ if routing_failed: ++ log.write( '\nERROR: routing failed, connectivity broken!\n' ) ++ log.close() ++ return + - print "Transform fabric into omnet++ format..." - omnetfiles = os.path.join(testdir, 'omnet') - translog = os.path.join(testdir, 'ibsim2omnet.log') @@ -742,23 +972,21 @@ diff -Nur scripts.orig/simulate.py scripts/simulate.py log.write('\nFinished!\n') log.close() -@@ -1138,5 +1141,6 @@ - else: - parser.print_help() +@@ -1140,3 +1154,4 @@ - sys.exit('\nFinish!') + #sys.exit('\nFinish!') + sys.exit(0) -diff -Nur scripts.orig/startOpenSM.sh scripts/startOpenSM.sh ---- scripts.orig/startOpenSM.sh 2016-08-17 14:02:30.943729000 -0700 -+++ scripts/startOpenSM.sh 2016-08-17 13:59:43.760997000 -0700 +diff -u scripts/startOpenSM.sh scripts/startOpenSM.sh +--- scripts/startOpenSM.sh 2016-08-17 13:59:43.760997000 -0700 ++++ scripts/startOpenSM.sh 2017-04-25 18:59:13.484892065 +0900 @@ -39,7 +39,10 @@ DEBUG="" if [ -n $2 ]; then - DEBUG="-D ${2}" -+ DEBUG="-D ${2} -d2" ++ DEBUG="-D 0x43 -d2 --once" +fi +if [ -f ${4}/../batch_jobs.conf ]; then + DEBUG="${DEBUG} --batch_job_file ${4}/../batch_jobs.conf" diff --git a/src/util/patches/lft.patch b/src/util/patches/lft.patch index b4f50ed0da52b1f3ee47f27c0f18a7eb22c9cda4..4338d4feee5276259bc9a302e961439a04ccc430 100644 --- a/src/util/patches/lft.patch +++ b/src/util/patches/lft.patch @@ -1,6 +1,6 @@ -diff -Nur scripts.orig/create_static_lft.sh scripts/create_static_lft.sh ---- scripts.orig/create_static_lft.sh 1969-12-31 16:00:00.000000000 -0800 -+++ scripts/create_static_lft.sh 2016-08-16 11:08:06.058810000 -0700 +diff -u scripts/create_static_lft.sh 2016-08-16 11:08:06.058810000 -0700 scripts/create_static_lft.sh +--- scripts/create_static_lft.sh 2016-08-16 11:08:06.058810000 -0700 ++++ scripts/create_static_lft.sh 2017-04-25 19:59:16.977561809 +0900 @@ -0,0 +1,47 @@ +#!/bin/bash + @@ -37,21 +37,21 @@ diff -Nur scripts.orig/create_static_lft.sh scripts/create_static_lft.sh +$HOME/simulation/scripts/simulate.py -n ${SIM_DIR} -r ${OSM_ROUTING} -p exchange +if [ "x$?" != "x0" ]; then exit -1; fi + -+mv ${SIM_DIR}/ofedout/ibdiagnet.fdbs ${SIM_DIR}/ ++mv ${SIM_DIR}/ofedout/opensm.fdbs ${SIM_DIR}/ +mv ${SIM_DIR}/ofedout/opensm-subnet.lst ${SIM_DIR}/ +echo "running post_process_lfts.py" -+$HOME/simulation/scripts/post_process_lfts.py ${SIM_DIR}/ibdiagnet.fdbs ${SIM_DIR}/opensm-subnet.lst ${SIM_DIR}/ ++$HOME/simulation/scripts/post_process_lfts.py ${SIM_DIR}/opensm.fdbs ${SIM_DIR}/opensm-subnet.lst ${SIM_DIR}/ +if [ "x$?" != "x0" ]; then exit -1; fi +echo "Done with script" + +#if [ -z ${KEEP_INTERMEDIATE} ]; then -+# rm -rf ./checkConnectivity.log ./log.txt ./ofedout/ ./${WRITE_TOPOLOGY_DOT_FILE}.dot ./topo.* ./ibdiagnet.fdbs ./opensm-subnet.lst ++# rm -rf ./log.txt ./ofedout/ ./${WRITE_TOPOLOGY_DOT_FILE}.dot ./topo.* ./opensm.fdbs ./opensm-subnet.lst +#fi + +exit 0 -diff -Nur scripts.orig/get_static_lft_for_codes.sh scripts/get_static_lft_for_codes.sh ---- scripts.orig/get_static_lft_for_codes.sh 1969-12-31 16:00:00.000000000 -0800 -+++ scripts/get_static_lft_for_codes.sh 2016-08-16 11:08:06.058810000 -0700 +diff -u scripts/get_static_lft_for_codes.sh scripts/get_static_lft_for_codes.sh +--- scripts/get_static_lft_for_codes.sh 2016-08-16 11:08:06.058810000 -0700 ++++ scripts/get_static_lft_for_codes.sh 2017-04-25 19:59:39.901542081 +0900 @@ -0,0 +1,36 @@ +#!/bin/bash + @@ -79,13 +79,13 @@ diff -Nur scripts.orig/get_static_lft_for_codes.sh scripts/get_static_lft_for_co +$HOME/simulation/scripts/simulate.py -n ${SIM_DIR} -r ${OSM_ROUTING} -p exchange +if [ "x$?" != "x0" ]; then exit -1; fi + -+mv ${SIM_DIR}/ofedout/ibdiagnet.fdbs ${SIM_DIR}/ ++mv ${SIM_DIR}/ofedout/opensm.fdbs ${SIM_DIR}/ +mv ${SIM_DIR}/ofedout/opensm-subnet.lst ${SIM_DIR}/ -+$HOME/simulation/scripts/post_process_lfts.py ${SIM_DIR}/ibdiagnet.fdbs ${SIM_DIR}/opensm-subnet.lst ${SIM_DIR}/ ++$HOME/simulation/scripts/post_process_lfts.py ${SIM_DIR}/opensm.fdbs ${SIM_DIR}/opensm-subnet.lst ${SIM_DIR}/ +if [ "x$?" != "x0" ]; then exit -1; fi + +#if [ -z ${KEEP_INTERMEDIATE} ]; then -+# rm -rf ./checkConnectivity.log ./log.txt ./ofedout/ ./${WRITE_TOPOLOGY_DOT_FILE}.dot ./topo.* ./ibdiagnet.fdbs ./opensm-subnet.lst ++# rm -rf ./log.txt ./ofedout/ ./${WRITE_TOPOLOGY_DOT_FILE}.dot ./topo.* ./opensm.fdbs ./opensm-subnet.lst +#fi + +exit 0 @@ -117,9 +117,9 @@ diff -Nur scripts.orig/post_process_dot.sh scripts/post_process_dot.sh +rm -f ${PATH_TO_DOT}.dot.* + +exit 0 -diff -Nur scripts.orig/post_process_lfts.py scripts/post_process_lfts.py ---- scripts.orig/post_process_lfts.py 1969-12-31 16:00:00.000000000 -0800 -+++ scripts/post_process_lfts.py 2016-08-15 15:41:29.189179000 -0700 +diff -u scripts/post_process_lfts.py scripts/post_process_lfts.py +--- scripts/post_process_lfts.py 2016-08-15 15:41:29.189179000 -0700 ++++ scripts/post_process_lfts.py 2017-04-25 19:12:03.552153747 +0900 @@ -0,0 +1,62 @@ +#!/usr/bin/env python + @@ -136,7 +136,7 @@ diff -Nur scripts.orig/post_process_lfts.py scripts/post_process_lfts.py + + outdir = os.path.normpath(sys.argv[3]) +except: -+ sys.exit('Usage: post_process_lfts.py ./ibdiagnet.fdbs ./opensm-subnet.lst') ++ sys.exit('Usage: post_process_lfts.py ./opensm.fdbs ./opensm-subnet.lst') + +if not os.path.exists(fdbsFile) or not os.path.exists(lstFile): + sys.exit('ERR: file %s or %s does not exist' % (fdbsFile, lstFile)) @@ -183,3 +183,1943 @@ diff -Nur scripts.orig/post_process_lfts.py scripts/post_process_lfts.py +out.close() + +sys.exit(0) +only in patch2: +unchanged: +--- scripts.orig/checkConnectivity.py 2017-04-25 16:23:23.472601516 +0900 ++++ scripts/checkConnectivity.py 1970-01-01 09:00:00.000000000 +0900 +@@ -1,165 +0,0 @@ +-#!/usr/bin/env python +- +-import sys, re, os, random, operator, colorsys, math +- +-class showUnicastForwarding(object): +- subnet = {} +- hcaTable = [] +- disconn = 0 +- +- def parse_lstFile(self, fileName=''): +- lstFile = open( fileName, 'r' ) +- +- network = {} +- for line in lstFile: +- p = re.compile('{\s+([a-zA-Z0-9_-]+)\s+Ports:(\w+)\s+SystemGUID:(\w+)\s+NodeGUID:(\w+)\s+PortGUID:(\w+)\s+VenID:(\w+)\s+DevID:(\w+)\s+Rev:(\w+)\s+{(.+)}\s+LID:(\w+)\s+PN:(\w+)\s+}\s+{\s+([a-zA-Z0-9_-]+)\s+Ports:(\w+)\s+SystemGUID:(\w+)\s+NodeGUID:(\w+)\s+PortGUID:(\w+)\s+VenID:(\w+)\s+DevID:(\w+)\s+Rev:(\w+)\s+{(.+)}\s+LID:(\w+)\s+PN:(\w+)\s+}\s+.+') +- if p.match(line): +- m = p.match(line) +- node1, ports1, sguid1, nguid1, pguid1, vid1, did1, rev1, name1, lid1, pn1 = m.group(1), int(m.group(2),16), m.group(3), m.group(4), m.group(5), \ +- m.group(6), m.group(7), m.group(8), m.group(9), int(m.group(10),16), int(m.group(11),16) +- node2, ports2, sguid2, nguid2, pguid2, vid2, did2, rev2, name2, lid2, pn2 = m.group(12), int(m.group(13),16), m.group(14), m.group(15), m.group(16), \ +- m.group(17), m.group(18), m.group(19), m.group(20), int(m.group(21),16), int(m.group(22),16) +- +- nguid1, nguid2 = nguid1.lower(), nguid2.lower() +- +- if node1.find('CA') > -1: +- node1, ports1, sguid1, nguid1, pguid1, vid1, did1, rev1, name1, lid1, pn1, node2, ports2, sguid2, nguid2, pguid2, vid2, did2, rev2, name2, lid2, pn2 = \ +- node2, ports2, sguid2, nguid2, pguid2, vid2, did2, rev2, name2, lid2, pn2, node1, ports1, sguid1, nguid1, pguid1, vid1, did1, rev1, name1, lid1, pn1 +- if node2.find('CA') > -1: +- #nguid2 = name2 +- self.hcaTable.append([nguid2, pn2]) +- +- if network.has_key(nguid1): +- network[nguid1][pn1] = {'rnguid':nguid2, 'rpn':pn2} +- else: +- network[nguid1] = {'lid':lid1} +- network[nguid1][pn1] = {'rnguid':nguid2, 'rpn':pn2} +- +- if network.has_key(nguid2): +- network[nguid2][pn2] = {'rnguid':nguid1, 'rpn':pn1} +- else: +- network[nguid2] = {'lid':lid2} +- network[nguid2][pn2] = {'rnguid':nguid1, 'rpn':pn1} +- +- self.subnet = network +- lstFile.close() +- +- +- def parse_fdbsFile(self, fileName=''): +- network = self.subnet +- nguid = None +- lft = {} +- +- fdbsFile = open( fileName, 'r' ) +- for line in fdbsFile: +- p = re.compile('osm_ucast_mgr_dump_ucast_routes: Switch 0x(\w+)') +- if p.match(line): +- if nguid: +- network[nguid]['lft'] = lft +- lft = {} +- nguid = p.match(line).group(1).lower() +- +- p = re.compile('0x(\w+)\s+:\s+(\d+)\s+.*') +- if p.match(line): +- lid, port = int(p.match(line).group(1), 16), int(p.match(line).group(2)) +- lft[lid] = port +- +- if nguid: +- network[nguid]['lft'] = lft +- +- fdbsFile.close() +- +- +- def calc_connectivity(self): +- network = self.subnet +- +- hopMatrix = {} +- for hcaSrc, portSrc in self.hcaTable: +- +- srcLid = network[hcaSrc]['lid'] +- hopMatrix[srcLid] = {} +- +- for hcaDest, portDest in self.hcaTable: +- if hcaSrc == hcaDest: +- continue +- +- curr = network[hcaSrc][portSrc]['rnguid'] +- destLid = network[hcaDest]['lid'] +- hop = 1 +- +- while (curr != hcaDest): +- if not network[curr]['lft'].has_key(destLid): +- print "No connection between: ", hcaSrc, hcaDest +- self.disconn += 1 +- break +- exitPort = network[curr]['lft'][destLid] +- curr = network[curr][exitPort]['rnguid'] +- hop += 1 +- if hop > 64: +- print 'Error: hop count bigger than 64 => there is something wrong with the LFTs!' +- return +- +- hopMatrix[srcLid][destLid] = hop +- +- if self.disconn > 0: +- print 'Error: %s Hca<->Hca routes are disconnected due to faults in the routing table (out of %s connections in total)' % (self.disconn, len(self.hcaTable)*(len(self.hcaTable)-1)) +- else: +- minHop = 2^32-1 +- maxHop, sumHop, count, sumSqare = 0, 0, 0, 0 +- for srcLid in hopMatrix.keys(): +- for dstLid in hopMatrix[srcLid].keys(): +- hop = hopMatrix[srcLid][dstLid] +- sumHop += hop +- sumSqare += hop*hop +- count += 1 +- if minHop > hop: +- minHop = hop +- if maxHop < hop: +- maxHop = hop +- +- print 'Hop Count for the Network [min, max, avg, stddev]: [', minHop, maxHop, (1.0*sumHop)/count, math.sqrt(((1.0*sumSqare)/count)-math.pow((1.0*sumHop)/count, 2)), ']' +- +- +- def __init__(self): +- if len(sys.argv) == 2: +- path = os.path.normpath(sys.argv[1]) +- if not os.path.exists(path): +- print "Error: input directory does not exist or is not accessable" +- print "" +- self.printHelp() +- self.inputDir = os.path.join(path) +- +- lstFile = os.path.join(self.inputDir, 'ibdiagnet.lst') +- if not os.path.exists(lstFile): +- print "Error: %s file does not exist or is not accessable" % lstFile +- print "" +- self.printHelp() +- +- fdbsFile = os.path.join(self.inputDir, 'ibdiagnet.fdbs') +- if not os.path.exists(fdbsFile): +- print "Error: %s file does not exist or is not accessable" % fdbsFile +- print "" +- self.printHelp() +- +- else: +- self.printHelp() +- +- self.parse_lstFile(lstFile) +- self.parse_fdbsFile(fdbsFile) +- self.calc_connectivity() +- +- +- def printHelp(self): +- print "Usage: ", sys.argv[0], " " +- print "" +- print " Example:" +- print " ", sys.argv[0], " $OSM_CACHE_DIR/ibdiagnetout" +- print "" +- print " Hint to generate *.lst and *.fdbs outpout:" +- print " ibdiagnet [-r] -o " +- sys.exit('') +- +-if __name__ == "__main__": +- app = showUnicastForwarding() +- sys.exit('\nFinish!') +- +only in patch2: +unchanged: +--- scripts.orig/IBNetLFTtoFDBS.py 2017-04-25 16:23:23.640601410 +0900 ++++ scripts/IBNetLFTtoFDBS.py 1970-01-01 09:00:00.000000000 +0900 +@@ -1,161 +0,0 @@ +-#!/usr/bin/env python +- +-import sys, re, os +-from optparse import OptionParser +- +-class IBNetLFTtoFDBS(object): +- maxLID = 0 +- +- def read_ibnetdiscoverFile(self, fileName): +- if not os.path.exists(fileName): +- sys.exit("Error: file specified by '-ibnd' not found; check your config") +- +- # parse ibnetdiscover.log to get the topology of the network +- network = {} +- guid = 0 +- ibnetFile = open(fileName, 'r') +- for line in ibnetFile: +- p = re.compile('^sysimgguid=.*') +- if p.match(line): +- guid = 0 +- p = re.compile('^Switch\s+(\d+)\s"S-(\w+)"\s+.*#\s+"(.*)".*\s+lid\s+(\d+)\s*.*$') +- if p.match(line): +- m = p.match(line) +- guid, numPorts, name, lid = int(m.group(2), 16), int(m.group(1)), m.group(3), int(m.group(4)) +- if network.has_key(guid): +- print "Error: same switch guid found multiple times in ibnetdiscover.log" +- else: +- network[guid] = {'lid':lid, 'name':name, 'numPorts':numPorts, 'ports':{}} +- if lid > self.maxLID: self.maxLID = lid +- p = re.compile('^\[(\d+)\]\s+"(\w)-(\w+)"\[(\d+)\].*#\s+"(.*)"\s+lid\s+(\d+).*') +- if guid != 0 and p.match(line): +- # ibnetdiscover OR ibnetdiscover -s +- m = p.match(line) +- lport, rtype, rguid, rport, rname, rlid = int(m.group(1)), m.group(2), int(m.group(3), 16), int(m.group(4)), m.group(5), int(m.group(6)) +- if network[guid]['ports'].has_key(lport): +- print "Error: same port of one switch found multiple times in ibnetdiscover.log" +- else: +- network[guid]['ports'][lport] = {'type':rtype, 'guid':rguid, 'port':rport, 'name':rname, 'lid':rlid} +- if rlid > self.maxLID: self.maxLID = rlid +- ibnetFile.close() +- return network +- +- +- def read_lftFile(self, fileName): +- if not os.path.exists(fileName): +- sys.exit("Error: file specified by '-lft' not found; check your config") +- +- # names are unique if they come out of createIBNet.py +- lft = {} +- swname = '' +- lftFile = open(fileName, 'r') +- for line in lftFile: +- p = re.compile('^S(\S*)\s+\d+\s+\[(.*)\]') +- if p.match(line): +- m = p.match(line) +- swname, adj_list = 'S%s'%m.group(1), m.group(2).split(',') +- lft[swname] = {'name':swname, 'lid':-1, 'adj_list':adj_list, 'lft':{}} +- +- p = re.compile('^(\S*)\s+(\d+)') +- if p.match(line): +- m = p.match(line) +- remote, route_thru_port = m.group(1), int(m.group(2)) +- lft[swname]['lft'][remote] = {'lid':-1, 'arti_port':route_thru_port, 'real_port':-1} +- +- p = re.compile('^(\S*)\s+UNREACHABLE') +- if p.match(line): +- m = p.match(line) +- remote = m.group(1) +- lft[swname]['lft'][remote] = {'lid':-1, 'arti_port':-1, 'real_port':-1} +- lftFile.close() +- return lft +- +- +- def mergeLFTandIBNETD(self, network, lft): +- for guid in network.keys(): +- name, lid = network[guid]['name'], network[guid]['lid'] +- lft[name]['lid'] = lid +- lft[name]['guid'] = guid +- lft[name]['lft'][name]['real_port'] = 0 +- +- for port in network[guid]['ports'].keys(): +- rtype, rname, rlid = network[guid]['ports'][port]['type'], network[guid]['ports'][port]['name'], network[guid]['ports'][port]['lid'] +- lft[name]['lft'][rname]['lid'] = rlid +- if rtype == 'H': +- if lft[name]['lft'][rname]['arti_port'] != 0: +- sys.exit("Error: arti_port for local Hca should be 0") +- if lft[name]['lft'][rname]['real_port'] != -1: +- sys.exit("Error: real_port for local Hca specified twice") +- lft[name]['lft'][rname]['real_port'] = port +- else: +- arti_port = lft[name]['lft'][rname]['arti_port'] +- for remote in lft[name]['lft'].keys(): +- if lft[name]['lft'][remote]['arti_port'] == arti_port: +- lft[name]['lft'][remote]['real_port'] = port +- +- # add known LIDs to other LFTs +- for name2 in lft.keys(): +- if name2 == name: continue +- for rname2 in lft[name2]['lft'].keys(): +- if rname2 == rname: +- lft[name2]['lft'][rname2]['lid'] = rlid +- +- #print '\n', name, lft[name]['guid'] +- #for port in network[guid]['ports'].keys(): print port, network[guid]['ports'][port] +- #for rname in lft[name]['lft'].keys(): +- # print rname, lft[name]['lft'][rname]['lid'], lft[name]['lft'][rname]['real_port'], lft[name]['lft'][rname]['arti_port'] +- return lft +- +- +- def write_fdbsFile(self, fileName, lft): +- if not os.path.exists(fileName): +- print "Warning: file specified by '-fdbs' does not exist; take action to move it to the right place for further processing" +- +- fdbsFile = open(fileName, 'w') +- for switch in lft.keys(): +- guid = lft[switch]['guid'] +- fdbsFile.write('osm_ucast_mgr_dump_ucast_routes: Switch 0x%016x\n' % guid) +- fdbsFile.write('LID : Port : Hops : Optimal\n') +- for lid in xrange(1, self.maxLID+1): +- remote = '' +- for rname in lft[switch]['lft'].keys(): +- if lft[switch]['lft'][rname]['lid'] == lid: +- remote = rname +- break +- +- if lft[switch]['lft'][remote]['real_port'] != -1: +- fdbsFile.write('0x%04X : %03i : 00 : yes\n' % (lid, lft[switch]['lft'][remote]['real_port'])) +- else: +- fdbsFile.write('0x%04X : UNREACHABLE\n' % lid) +- fdbsFile.write('\n') +- fdbsFile.close() +- +- +- def __init__(self): +- parser = OptionParser(description='Convert rounting created by createIBNet.py into real LFTs which match IBsim output (ibdiagnet.fdbs).') +- parser.add_option('--lft', nargs=1, help='*.lft file created by createIBNet.py', \ +- metavar='', dest='lftFile') +- parser.add_option('--ibnd', nargs=1, help='ibnetdiscover.log file (or equivalent) created by `ibnetdiscover -s > ibnetdiscover.log`', \ +- metavar='', dest='ibnetdiscoverFile') +- parser.add_option('--fdbs', nargs=1, help='output file: this script writes/replaces *.fdbs (similar to the one you get with `ibdiagnet -o `)', \ +- metavar='', dest='fdbsFile') +- (options, args) = parser.parse_args() +- +- if options.ibnetdiscoverFile == None or options.lftFile == None or options.fdbsFile == None: +- parser.print_usage() +- sys.exit() +- +- ibnetdiscoverFile = options.ibnetdiscoverFile +- lftFile = options.lftFile +- fdbsFile = options.fdbsFile +- +- network = self.read_ibnetdiscoverFile(ibnetdiscoverFile) +- lfts = self.read_lftFile(lftFile) +- realLFTs = self.mergeLFTandIBNETD(network, lfts) +- self.write_fdbsFile(fdbsFile, realLFTs) +- +- +-if __name__ == "__main__": +- app = IBNetLFTtoFDBS() +- sys.exit('\nFinish!') +- +only in patch2: +unchanged: +--- scripts.orig/ibsim2omnet.py 2017-04-25 16:23:23.712601364 +0900 ++++ scripts/ibsim2omnet.py 1970-01-01 09:00:00.000000000 +0900 +@@ -1,774 +0,0 @@ +-#!/usr/bin/env python +- +-import sys, re, os, random, operator +- +-class ibsim2omnet(object): +- inputDir = '' +- outputPrefix = '' +- LFTs = {} +- switchList = [] +- switches = {} +- hcaList = [] +- hopMatrix = {} +- vlMap = {} +- lidList = [] +- has_vltable = False +- max_lid = 0 +- +- vl_avail = 8 +- sizeOneVLinCredits = 128 +- width = 4 # 4x +- speed = 2.5 # 2.5 Gbit/s +- creditSize = 64 # bytes in one buffer element +- mtu = 2048 # maximum size [in Bytes] of the payload in one packet +- totalBufferSize = int(vl_avail * sizeOneVLinCredits) # the total buffer size in credits +- #credMinTime = 4 # time between checing VL update and injecting an update [in usec]; also huge influence on sim time +- vlHighLimit = 4 # IB Vl Arb High Limit +- msgLength = mtu +- +- +- trafficDist = 'trfUniform' +- dstSeqMode = 'dstRandom' #'dstSeqLoop' +- +- LRH = 8 # local route header in byte +- GRH = 0 # we dont need it (40 byte in theory) +- BTH = 12 +- ETH = 0 # not needed +- ICRC = 4 +- VCRC = 2 +- +- pktLenByte = LRH + GRH + BTH + ETH + mtu + ICRC + VCRC # simple pkt (IBA 5.4) +- +- vlout = 'vltable_dump' # possible: vltable_dump (extra file) or vltable_print (within opensm.log) +- pattern = 'rndAll' # possible: nextNeighbor, rndAll, oneShift, allShift, oneExchange, allExchange +- +- cputime = 24*60*60 +- simtime = 1 +- +- +- def FloydWarshall(self, graph): +- # initialized to infinity (0 and w have been initialized already) +- for i in graph.keys(): +- for j in graph.keys(): +- if not graph[i].has_key(j): +- graph[i][j] = 4294967295 +- +- # run Floyd-Warshall algorithm +- for k in graph.keys(): +- for i in graph.keys(): +- for j in graph.keys(): +- if graph[i][k] + graph[k][j] < graph[i][j]: +- graph[i][j] = graph[i][k] + graph[k][j] +- +- +- def read_ibdiagnet(self): +- if not os.path.exists(os.path.join(self.inputDir, 'ibdiagnet.fdbs')): +- print "Error: ibdiagnet.fdbs not found" +- print "" +- self.printHelp() +- +- # parse ibdiagnet.fdbs to get the LFTs for each switch +- guid = 0 +- max_lid = 0 +- fdbsFile = open(os.path.join(self.inputDir, 'ibdiagnet.fdbs'), 'r') +- for line in fdbsFile: +- p = re.compile('^$') +- if p.match(line): +- guid = 0 +- if self.max_lid < max_lid: +- self.max_lid = max_lid +- max_lid = 0 +- p = re.compile('.* Switch 0x(\w+)$') +- if p.match(line): +- m = p.match(line) +- guid = int(m.group(1), 16) # hex to int +- if self.LFTs.has_key(guid): +- print "Error: same switch guid found multiple times in ibdiagnet.fdbs" +- else: +- self.switchList.append(guid) +- self.LFTs[guid] = [] +- p = re.compile('^0x(\w*)\s.*') +- if guid != 0 and p.match(line): +- max_lid = max_lid + 1 +- p = re.compile('^0x(\w+)\s+:\s+(\d+)\s+:.*') +- if p.match(line): +- m = p.match(line) +- lid, port = int(m.group(1), 16), int(m.group(2)) +- p = re.compile('^0x(\w+)\s+:\s+UNREACHABLE.*') +- if p.match(line): +- m = p.match(line) +- lid, port = int(m.group(1), 16), 255 +- self.LFTs[guid].append([lid, port]) +- fdbsFile.close() +- #for i in xrange(0,50): +- # print '0x%04X' % (i) +- +- +- def read_ibnetdiscover(self): +- if not os.path.exists(os.path.join(self.inputDir, 'ibnetdiscover.log')): +- print "Error: ibnetdiscover.log not found" +- print "" +- self.printHelp() +- +- # parse ibnetdiscover.log to get the topology of the network +- guid = 0 +- ibnetFile = open(os.path.join(self.inputDir, 'ibnetdiscover.log'), 'r') +- for line in ibnetFile: +- p = re.compile('^sysimgguid=.*') +- if p.match(line): +- guid = 0 +- p = re.compile('^Switch\s+(\d+)\s"S-(\w+)"\s+.*#\s+"(.*)".*\s+lid\s+(\d+)\s*.*$') +- if p.match(line): +- m = p.match(line) +- guid, numPorts, name, lid = int(m.group(2), 16), int(m.group(1)), m.group(3), int(m.group(4)) +- if self.switches.has_key(guid): +- print "Error: same switch guid found multiple times in ibnetdiscover.log" +- else: +- self.switches[guid] = {'lid':lid, 'name':name, 'numPorts':numPorts, 'ports':{}, 'numHCA':0} +- self.hopMatrix[guid] = {guid: 0} +- p = re.compile('^\[(\d+)\]\s+"(\w)-(\w+)"\[(\d+)\].*#\s+"(.*)"\s+lid\s+(\d+)\s+(\d+)x(\w+)') +- if guid != 0 and p.match(line): +- # ibnetdiscover OR ibnetdiscover -s +- m = p.match(line) +- lport, rtype, rguid, rport, rname, rlid, rlinkwidth, rlinkfreq = int(m.group(1)), m.group(2), int(m.group(3), 16), int(m.group(4)), m.group(5), int(m.group(6)), int(m.group(7)), m.group(8) +- if rlinkfreq == "SDR": rlinkspeed = 2.5 # 8b/10b encoding +- elif rlinkfreq == "DDR": rlinkspeed = 5.0 # 8b/10b encoding +- elif rlinkfreq == "QDR": rlinkspeed = 10.0 # 8b/10b encoding +- elif rlinkfreq == "FDR10": rlinkspeed = 10.3125 # 64b/66b encoding +- elif rlinkfreq == "FDR": rlinkspeed = 14.0625 # 64b/66b encoding +- elif rlinkfreq == "EDR": rlinkspeed = 25.78125 # 64b/66b encoding +- if self.switches[guid]['ports'].has_key(lport): +- print "Error: same port of one switch found multiple times in ibnetdiscover.log" +- else: +- self.switches[guid]['ports'][lport] = {'type':rtype, 'guid':rguid, 'port':rport, 'name':rname, 'lid':rlid, 'width':rlinkwidth, 'speed':rlinkspeed} +- if rtype == "H": +- self.hcaList.append([rguid, rlid, rport, rlinkwidth, rlinkspeed]) +- self.switches[guid]['numHCA'] += 1 +- elif rtype == "S" and not self.hopMatrix[guid].has_key(rguid): +- self.hopMatrix[guid][rguid] = 1 +- +- ibnetFile.close() +- +- +- def read_opensmlog(self): +- # parse opensm.log to get the slid/dlid to SL mapping +- vltable_found = False +- if self.vlout.find('vltable_print') > -1: +- try: +- osmFile = open(os.path.join(self.inputDir, 'opensm.log'), 'r') +- except IOError: +- print "Error: opensm.log not found" +- print "" +- self.printHelp() +- for line in osmFile: +- p = re.compile('.*Virtual Lanes available:\s(\d*)') +- if p.match(line): +- self.vl_avail = int(p.match(line).group(1)) +- self.totalBufferSize = int(self.vl_avail * self.sizeOneVLinCredits) +- p = re.compile('.*vltable_print:\s*route from') +- if not vltable_found and p.match(line): +- vltable_found = True +- p = re.compile('.*vltable_print:\s+route from src_lid=(\d+) to dest_lid=(\d+) on vl=(\d+)') +- if p.match(line): +- m = p.match(line) +- slid, dlid, vl = int(m.group(1)), int(m.group(2)), int(m.group(3)) +- if not self.vlMap.has_key(slid): +- self.lidList.append(slid) +- self.vlMap[slid] = [] +- self.vlMap[slid].append([dlid, vl]) +- self.lidList.sort() # sort in-place +- self.has_vltable = vltable_found +- osmFile.close() +- else: +- try: +- osmFile = open(os.path.join(self.inputDir, 'opensm.log'), 'r') +- for line in osmFile: +- p = re.compile('.*Virtual Lanes available:\s+(\d+)') +- if p.match(line): +- self.vl_avail = int(p.match(line).group(1)) +- self.totalBufferSize = int(self.vl_avail * self.sizeOneVLinCredits) +- osmFile.close() +- except IOError: +- self.has_vltable = False +- try: +- vlFile = open(os.path.join(self.inputDir, 'vltable.log'), 'r') +- for line in vlFile: +- p = re.compile('(\d+)\s+(\d+)\s+(\d+)') +- if p.match(line): +- m = p.match(line) +- slid, dlid, vl = int(m.group(1)), int(m.group(2)), int(m.group(3)) +- if not self.vlMap.has_key(slid): +- self.lidList.append(slid) +- self.vlMap[slid] = [] +- if vl > self.vl_avail - 1: +- self.vl_avail = vl + 1 +- self.totalBufferSize = int(self.vl_avail * self.sizeOneVLinCredits) +- self.vlMap[slid].append([dlid, vl]) +- self.lidList.sort() # sort in-place +- self.has_vltable = True +- vlFile.close() +- except IOError: +- self.has_vltable = False +- +- +- def write_omnet_vlt(self): +- # write *.vlt (VL table) file +- path, prefix = os.path.split(os.path.normpath(self.outputPrefix)) +- vltFile = open(os.path.join(path, prefix+'.vlt'), 'w') +- if self.has_vltable: +- for lid in self.lidList: +- # sort vl table (just in case, and because python is fun) +- self.vlMap[lid] = sorted(self.vlMap[lid], key=lambda dlid_vl_comb: dlid_vl_comb[0]) # sort NOT in-place +- # fill holes in the list +- for nlid in xrange(1, self.max_lid+1): # 0..(largest lid) +- if [x[0] for x in self.vlMap[lid]].count(nlid) < 1: +- self.vlMap[lid].insert(nlid-1, [nlid, 255]) +- # write fake vl=255 for lid=0 in the begining, stupid lists +- vltFile.write("%s: 255 %s\n" % (lid, " ".join([str(x[1]) for x in self.vlMap[lid]]))) +- else: +- print "Warning: opensm.log does not contain vltable info or vltable.log does not exist; assume vl=0 for all slid/dlid" +- for lid in xrange(1, self.max_lid+1): +- vltFile.write("%s: 255 %s\n" % (lid, " ".join(['0' for x in range(1, self.max_lid+1)]))) +- vltFile.close() +- +- +- def get_hcaList_for_shift(self): +- # get the hop count from all sw to all sw +- self.FloydWarshall(self.hopMatrix) +- # get sw list and sort by name +- swList0 = self.hopMatrix.keys() +- swList0.sort() +- # drop all sw which have no HCA +- swList = [] +- for swGUID in swList0: +- if self.switches[swGUID]['numHCA'] != 0: +- swList.append(swGUID) +- swList0 = [] +- # add new HCAs to the list of 'sorted' HCAs; HCAs will be grouped by switch and 'close' to each other in the topology +- hcaList = [] +- while len(swList) > 0: +- swGUID = swList.pop(0) +- # insert all HCA of this switch +- for port in self.switches[swGUID]['ports'].keys(): +- if self.switches[swGUID]['ports'][port]['type'] == "H": +- hcaList.append([self.switches[swGUID]['ports'][port]['guid'], self.switches[swGUID]['ports'][port]['lid']]) +- +- # search next sw which is closest to the current and move him into position 0 +- nextSW = None +- maxHOP = 4294967295 +- for sw in swList: +- if self.hopMatrix[swGUID][sw] < maxHOP: +- nextSW = sw +- maxHOP = self.hopMatrix[swGUID][sw] +- if nextSW != None: +- swList.remove(nextSW) +- swList.insert(0, nextSW) +- return hcaList +- +- +- def write_omnet_dst(self): +- # write *.dst (destination sequence table) file +- path, prefix = os.path.split(os.path.normpath(self.outputPrefix)) +- dstFile = open(os.path.join(path, prefix+'.dst'), 'w') +- if self.pattern.find('nextNeighbor') > -1: +- for hca in self.hcaList: +- guid, lid, port, w, s = hca +- i = self.hcaList.index(hca) +- next_guid, next_lid, next_port, w, s = self.hcaList[(i+1) % len(self.hcaList)] +- dstFile.write('%s: %s\n' % (lid, next_lid)) +- elif self.pattern.find('rndAll') > -1: +- for hca in self.hcaList: +- guid, lid, port, w, s = hca +- list = [str(hca[1]) for hca in self.hcaList if (hca[1] != lid)] +- if self.dstSeqMode.find('dstSeqLoop') > -1: +- random.shuffle(list) +- elif self.dstSeqMode.find('dstRandom') > -1: +- list.sort() +- else: +- print "WARNING: unknown dstSeqMode found..." +- dstFile.write('%s: %s\n' % (lid, " ".join(list))) +- elif self.pattern.find('oneShift') > -1 or self.pattern.find('oneExchange') > -1: +- # get the actual shift +- shift = 0 +- p = re.compile('shift(\d+)') +- if p.match(self.dstSeqMode): +- m = p.match(self.dstSeqMode) +- shift = int(m.group(1)) +- p = re.compile('exchange(\d+)') +- if p.match(self.dstSeqMode): +- m = p.match(self.dstSeqMode) +- shift = int(m.group(1)) +- +- # add new HCAs to the list of 'sorted' HCAs; HCAs will be grouped by switch and 'close' to each other in the topology +- hcaList = self.get_hcaList_for_shift() +- +- # write the shift/exchange pattern for every HCA +- pos = 0 +- llen = len(hcaList) +- if self.pattern.find('oneShift') > -1: +- shift = shift % llen +- for hcaGUID, hcaLID in hcaList: +- dstFile.write('%s: %s\n' % (hcaLID, hcaList[(llen + pos + shift) % llen][1])) +- pos += 1 +- elif self.pattern.find('oneExchange') > -1: +- shift = max(shift % int(llen/2.0), 1) +- for hcaGUID, hcaLID in hcaList: +- dstFile.write('%s: %s %s\n' % (hcaLID, hcaList[(llen + pos - shift) % llen][1], hcaList[(llen + pos + shift) % llen][1])) +- pos += 1 +- elif self.pattern.find('allShift') > -1 or self.pattern.find('allExchange') > -1: +- # add new HCAs to the list of 'sorted' HCAs; HCAs will be grouped by switch and 'close' to each other in the topology +- hcaList = self.get_hcaList_for_shift() +- llen = len(hcaList) +- dbl_hcaList = [] +- dbl_hcaList.extend(hcaList) +- dbl_hcaList.extend(hcaList) +- +- pos = 0 +- # write the shift/exchange pattern for every HCA +- if self.pattern.find('allShift') > -1: +- for hcaGUID, hcaLID in hcaList: +- dstFile.write('%s: %s\n' % (hcaLID, " ".join([str(x[1]) for x in dbl_hcaList[pos+1:pos+llen]]))) +- pos += 1 +- elif self.pattern.find('allExchange') > -1: +- for hcaGUID, hcaLID in hcaList: +- dstFile.write('%s:' % hcaLID) +- for shift in range(1, int(llen/2.0)): +- dstFile.write(' %s %s' % (dbl_hcaList[llen + pos - shift][1], dbl_hcaList[pos + shift][1])) +- if llen % 2 == 0: +- dstFile.write(' %s' % dbl_hcaList[pos + int(llen/2.0)][1]) +- dstFile.write('\n') +- pos += 1 +- else: +- print "WARNING: unknown pattern found..." +- +- dstFile.close() +- +- +- def write_omnet_fdbs(self): +- # write *.fdbs file +- path, prefix = os.path.split(os.path.normpath(self.outputPrefix)) +- fdbsFile = open(os.path.join(path, prefix+'.fdbs'), 'w') +- for guid in self.switchList: +- # write fake LID=0 in the begining, stupid lists +- fdbsFile.write("%s: 0 %s\n" % (self.switchList.index(guid), " ".join([str(x[1]) for x in self.LFTs[guid]]))) +- fdbsFile.close() +- +- +- def write_omnet_ned(self): +- # head of *.ned file +- path, prefix = os.path.split(os.path.normpath(self.outputPrefix)) +- nedFile = open(os.path.join(path, prefix+'.ned'), 'w') +- #nedFile.write("package ibmodel.networks.%s;\n\n" % (prefix)) +- #nedFile.write("import ibmodel.src.Switch;\n") +- #nedFile.write("import ibmodel.src.HCA;\n\n") +- nedFile.write("import src.Switch;\n") +- nedFile.write("import src.HCA;\n") +- nedFile.write("import src.BandwidthControler;\n") +- nedFile.write("import src.SendRecvControler;\n") +- nedFile.write("import src.GlobalDLControler;\n\n") +- nedFile.write('module %s\n{\n\t@display("bgb=946,315");\n' % (prefix)) +- +- # write submodules (i.e. hca or switch) +- nedFile.write('\tsubmodules:\n') +- nedFile.write('\t\tbandwidthControler: BandwidthControler;\n') +- nedFile.write('\t\tsendrecvControler: SendRecvControler;\n') +- nedFile.write('\t\tglobalDLControler: GlobalDLControler;\n') +- for guid in self.switchList: +- nedFile.write('\t\tS_%016x: Switch\t//LID=%s Name="%s"\n\t\t{\n' % (guid, self.switches[guid]['lid'], self.switches[guid]['name'])) +- nedFile.write('\t\t\tparameters:\n') +- nedFile.write('\t\t\t\tnumSwitchPorts = %s;\n' % (self.switches[guid]['numPorts']+1)) # we also need port 0, or would have to convert the fdbs tables +- nedFile.write('\t\t\t\tfdbIndex = %s;\n' % (self.switchList.index(guid))) +- nedFile.write('\t\t\t\t//DRGroups = \" \";\n') +- nedFile.write('\t\t\tgates:\n') +- nedFile.write('\t\t\t\tout[%s];\n' % (self.switches[guid]['numPorts']+1)) +- nedFile.write('\t\t\t\tin[%s];\n' % (self.switches[guid]['numPorts']+1)) +- nedFile.write('\t\t}\n') +- for hcaPort in [port for port in self.switches[guid]['ports'].keys() if self.switches[guid]['ports'][port]['type'] == "H"]: +- nedFile.write('\t\tH_%016x_%s: HCA\t//LID=%s Name="%s"\n\t\t{\n' % (self.switches[guid]['ports'][hcaPort]['guid'], self.switches[guid]['ports'][hcaPort]['port'], self.switches[guid]['ports'][hcaPort]['lid'], self.switches[guid]['ports'][hcaPort]['name'])) +- nedFile.write('\t\t\tparameters:\n') +- nedFile.write('\t\t\t\tgen.srcLid = %s;\n' % (self.switches[guid]['ports'][hcaPort]['lid'])) +- nedFile.write('\t\t\t\tgen.dstLid = %s; //just a fake, the real dstLids are in a file\n' % (self.switches[guid]['ports'][hcaPort]['lid'])) +- nedFile.write('\t\t\t\tgen.vlSeqIndex = %s;\n' % (self.switches[guid]['ports'][hcaPort]['lid'])) +- nedFile.write('\t\t\t\tgen.dstSeqIndex = %s;\n' % (self.switches[guid]['ports'][hcaPort]['lid'])) +- nedFile.write('\t\t}\n') +- nedFile.write('\n') +- +- # www.openfabrics.org/archives/2007infiniband/10_W.L.%2520Gore%2520%26%2520Associates.pdf +- # we assume fastest 3m copper cables +- linkdelay = 43 # in ns +- +- # write connections (i.e. hca<->sw or sw<->sw) +- nedFile.write('\n\tconnections:\n') +- for guid in self.switchList: +- nedFile.write('\t\t//S_%016x --> SWs\n' % (guid)) +- #nedFile.write('\t\tS_%016x.out[0] --> { delay = %sns; } --> S_%016x.in[0];\n' % (guid, linkdelay, guid)) # managment port, we assign it as self port (not needed with @loose) +- for swPort in [port for port in self.switches[guid]['ports'].keys() if self.switches[guid]['ports'][port]['type'] == "S"]: +- nedFile.write('\t\tS_%016x.out[%s] --> { delay = %sns; } --> S_%016x.in[%s];\n' % (guid, swPort, linkdelay, self.switches[guid]['ports'][swPort]['guid'], self.switches[guid]['ports'][swPort]['port'])) +- nedFile.write('\t\t//S_%016x --> HCAs\n' % (guid)) +- for swPort in [port for port in self.switches[guid]['ports'].keys() if self.switches[guid]['ports'][port]['type'] == "H"]: +- nedFile.write('\t\tS_%016x.out[%s] --> { delay = %sns; } --> H_%016x_%s.in;\n' % (guid, swPort, linkdelay, self.switches[guid]['ports'][swPort]['guid'], self.switches[guid]['ports'][swPort]['port'])) +- nedFile.write('\t\t//HCAs --> S_%016x\n' % (guid)) +- for swPort in [port for port in self.switches[guid]['ports'].keys() if self.switches[guid]['ports'][port]['type'] == "H"]: +- nedFile.write('\t\tH_%016x_%s.out --> { delay = %sns; } --> S_%016x.in[%s];\n' % (self.switches[guid]['ports'][swPort]['guid'], self.switches[guid]['ports'][swPort]['port'], linkdelay, guid, swPort)) +- nedFile.write('\n') +- +- # end of *.ned file +- nedFile.write('}\n\n') +- nedFile.write('network FABRIC extends %s\n{\n\tparameters:\n}\n' % (prefix)) +- nedFile.close() +- +- +- def write_omnet_ini(self): +- path, prefix = os.path.split(os.path.normpath(self.outputPrefix)) +- iniFile = open(os.path.join(path, prefix+'.ini'), 'w') +- +- iniFile.write('[General]\n') +- iniFile.write('network = FABRIC\t# this line is for Cmdenv\n') +- iniFile.write('cmdenv-express-mode = true\n') +- iniFile.write('record-eventlog = false\n') +- iniFile.write('cmdenv-interactive = true\n') +- iniFile.write('print-undisposed = false\n') +- iniFile.write('debug-on-errors = true\n') +- iniFile.write('\n') +- +- iniFile.write('#######################\n# SWITCH #\n#######################\n') +- iniFile.write('**.S_*.fdbsVecFile = "%s.fdbs"\n' % (prefix)) +- iniFile.write('**.S_**.ibuf.maxBeingSent = 2\n') +- iniFile.write('**.S_**.ibuf.totalBufferSize = %s\t# in credits\n' % self.totalBufferSize) +- iniFile.write('**.S_**.ibuf.maxStatic* = %s\t#Max num of credits reserved for VL*\n' % int(self.totalBufferSize / self.vl_avail)) +- #iniFile.write('**.S_**.ibuf.maxStatic0 = %s\t#Max num of credits reserved for VL*\n' % int(self.totalBufferSize / self.vl_avail)) +- #iniFile.write('**.S_**.ibuf.maxStatic* = %s\t#Max num of credits reserved for VL*\n' % int(32)) +- iniFile.write('include ./%s.sw\n' % (prefix)) +- iniFile.write('\n') +- +- iniFile.write('#######################\n# HCA #\n#######################\n') +- # start a bit late to wait for the flow control update of the VLs +- iniFile.write('**.H_**.gen.genStartTime = 0.0000001s\n') +- iniFile.write('**.H_**.gen**.virtualLaneFile = true\n') +- iniFile.write('**.H_**.gen**.vlSeqVecFile = "%s.vlt"\n' % (prefix)) +- iniFile.write('**.H_**.ibuf.maxBeingSent = 2\n') +- iniFile.write('**.H_**.ibuf.totalBufferSize = %s\t# in credits\n' % self.totalBufferSize) +- iniFile.write('**.H_**.ibuf.maxStatic* = %s\t#Max num of credits reserved for VL*\n' % int(self.totalBufferSize / self.vl_avail)) +- #iniFile.write('**.H_**.ibuf.maxStatic0 = %s\t#Max num of credits reserved for VL*\n' % int(self.totalBufferSize / self.vl_avail)) +- #iniFile.write('**.H_**.ibuf.maxStatic* = %s\t#Max num of credits reserved for VL*\n' % int(32)) +- #iniFile.write('**.H_**.sink.pciExpWidth = 8\t# Sink must not be too efficient\n') +- #iniFile.write('**.H_**.sink.pciExpTransferRate = 2.5\t# 2.5 equals PCI 1.1, 5.0 equals PCI 2.0, and 8.0 PCI 3.0\n') +- iniFile.write('**.H_**.sink.hiccupDuration_us = ${hdur=0.01}\n') +- iniFile.write('**.H_**.sink.hiccupDelay_us = 0.1\n') +- iniFile.write('include ./%s.hca\n' % (prefix)) +- iniFile.write('\n') +- +- iniFile.write('#######################\n# Traffic #\n#######################\n') +- iniFile.write('**.H_**.gen.trafficDist = "%s"\n' % self.trafficDist) +- if self.trafficDist.find('trfFlood') > -1: +- self.msgLength = 65536 +- iniFile.write('**.H_**.gen**.floodVLs = "0 1 2 3 4 5 6 7"\n') +- #iniFile.write('**.H_**.gen**.msgLength = 65536\n') +- elif self.trafficDist.find('trfLengthMix') > -1: +- iniFile.write('**.H_**.gen**.PktLenDist = "%s"\n' % int(self.pktLenByte)) +- iniFile.write('**.H_**.gen**.PktLenProb = "10"\t# some list with percentages\n') +- elif self.trafficDist.find('trfUniform') > -1: +- iniFile.write('**.H_**.gen.intraBurstDelay = 0\t# sleep time between bursts in nsec\n') +- iniFile.write('**.H_**.gen.interBurstDelay = 0\t# between packets of same burst in nsec\n') +- iniFile.write('**.H_**.gen.burstLength = 1\t# the length of the burst in packets\n') +- iniFile.write('**.H_**.gen.burstNumber = 1\t# Number of burst overall\n') +- iniFile.write('**.H_**.gen.slowStart = 0\t# start with a lower intraburstdelay\n') +- if self.pattern.find('oneShift') > -1 or self.pattern.find('oneExchange') > -1: +- iniFile.write('**.H_**.gen.dstSeqMode = "%s"\n' % 'dstSeqLoop') +- else: +- iniFile.write('**.H_**.gen.dstSeqMode = "%s"\n' % self.dstSeqMode) +- if self.dstSeqMode.find('dstLid') > -1: +- iniFile.write('**.H_**.gen.dstLid = 0\n') +- elif self.dstSeqMode.find('dstSeqOnce') > -1 or self.dstSeqMode.find('dstSeqLoop') > -1 \ +- or self.dstSeqMode.find('dstRandom') > -1 or self.dstSeqMode.find('shift') > -1 or self.dstSeqMode.find('exchange') > -1: +- iniFile.write('**.H_**.gen.dstSeqVecFile = "%s.dst"\n' % (prefix)) +- iniFile.write('**.H_**.gen.sizeSeqVecFile = ""\t# VecFile for Msg sizes; nothing to do w/ dstSeqVecFile\n') +- elif self.dstSeqMode.find('dstHotSpot') > -1: +- iniFile.write('**.H_**.gen.dynamicHSFile = false\t# no file provided\n') +- iniFile.write('**.H_**.gen.dynHSSeqVecFile = ""\n') +- iniFile.write('**.H_**.gen.dstHotSpotPerc = 0\n') +- iniFile.write('**.H_**.gen.dstBurstHSDel = 0.5\n') +- iniFile.write('**.H_**.gen.dstBurstHSInterDel = 0.5\n') +- if self.dstSeqMode.find('dstSeqOnce') > -1 and (self.pattern.find('allShift') > -1 or self.pattern.find('allExchange') > -1): +- if len(self.hcaList) >= 500: # small msg for big cluster, otherwise long runtime +- self.msgLength = self.msgLength * 1 +- else: +- self.msgLength = self.msgLength * 10 +- iniFile.write('**.H_**.gen**.msgLength = %s\t# msgLen == mtu here\n' % self.msgLength) +- iniFile.write('**.H_**.gen.dstHotSpotPerc = 0\n') +- iniFile.write('\n') +- +- iniFile.write('#######################\n# Generator (gen) #\n#######################\n') +- iniFile.write('**.gen.GenModel = 0\n') +- iniFile.write('**.gen.width = %s\t# if not set before use default 4x\n' % self.width) +- iniFile.write('**.gen.speed = %s\t# if not set before use default 2.5\n' % self.speed) +- iniFile.write('**.gen.creditSize = %s\t# bytes in one buffer element\n' % self.creditSize) +- iniFile.write('**.gen.mtu = %s\t# number of credits in one mtu\n' % int(self.mtu / self.creditSize)) +- iniFile.write('**.gen.packetlengthbytes = %s\t# packetlength in bytes\n' % int(self.pktLenByte)) +- iniFile.write('**.gen.packetlength = %s\t# packetlength in number of credits\n' % int((self.pktLenByte + self.creditSize - 1) / self.creditSize)) +- iniFile.write('**.gen**.msgLengthInMTUs = %s\n' % int(self.msgLength / self.mtu)) +- iniFile.write('\n') +- +- iniFile.write('#######################\n# Input Buffer (ibuf) #\n#######################\n') +- iniFile.write('**.ibuf.width = %s\t# if not set before use default 4x\n' % self.width) +- iniFile.write('**.ibuf.recordVectors = 0\n') +- iniFile.write('\n') +- +- iniFile.write('#######################\n# Output Buffer (obuf)#\n#######################\n') +- iniFile.write('**.obuf.width = %s\t# if not set before use default 4x\n' % self.width) +- iniFile.write('**.obuf.speed = %s\t# if not set before use default 2.5\n' % self.speed) +- iniFile.write('**.obuf.recordVectors = 0\n') +- iniFile.write('**.S_**.obuf.size = 78\t# the number of credits the Q can store in credits for switches\n') +- iniFile.write('**.H_**.obuf.size = 56\t# in credits for HCAs\n') +- #iniFile.write('**.obuf.credMinTime = %s\t# time between VL Credit packets in usec\n' % self.credMinTime) +- iniFile.write('\n') +- +- iniFile.write('#######################\n# IB traffic sink #\n#######################\n') +- iniFile.write('**.sink.recordVectors = 0\n') +- iniFile.write('**.sink.creditSize = %s\t# credit size in bytes\n' % self.creditSize) +- iniFile.write('\n') +- +- iniFile.write('#######################\n# IB VL Arbiter #\n#######################\n') +- iniFile.write('**.maxVL = %s\t# start count at 0\n' % (self.vl_avail - 1)) +- iniFile.write('**.vlarb.recordVectors = 0\n') +- iniFile.write('**.vlarb.width = %s\t# if not set before use default 4x\n' % self.width) +- iniFile.write('**.vlarb.vlHighLimit = %s\t# IB VL Arb High Limit\n' % self.vlHighLimit) +- if self.has_vltable: +- iniFile.write('**.vlarb.highVLArbEntries = "%s"\n' % (" ".join([("%s:64" % x) for x in range(0,self.vl_avail)]))) +- iniFile.write('**.vlarb.lowVLArbEntries = "%s"\n' % (" ".join([("%s:4" % x) for x in range(0,self.vl_avail)]))) +- else: +- iniFile.write('**.vlarb.highVLArbEntries = "%s"\n' % (" ".join([("%s:255" % x) for x in range(0,self.vl_avail)]))) +- iniFile.write('**.vlarb.lowVLArbEntries = "%s"\n' % (" ".join([("%s:32" % x) for x in range(0,self.vl_avail)]))) +- iniFile.write('\n') +- +- iniFile.write('#######################\n# Congestion control #\n#######################\n') +- iniFile.write('**.cc_enabled = false\n') +- iniFile.write('**.S_**.ccmgr.Victim_Mask = 0\n') +- iniFile.write('\n') +- +- iniFile.write('#######################\n# Runs #\n#######################\n') +- if len(self.hcaList) >= 900: +- self.cputime = 4*24*60*60 # 4 days instead +- iniFile.write('cpu-time-limit = ${cputime=%s}s\n' % (self.cputime)) +- iniFile.write('sim-time-limit = ${simtime=%s}s\n' % (self.simtime)) +- iniFile.write('**.logInterval = ${logint=%s}s\n' % (self.simtime/1000.0)) +- iniFile.write('**.vector-recording = false\n') # very very very important +- iniFile.write('**.recordVectors = 0\n') +- iniFile.write('\n') +- +- iniFile.write('#######################\n# predefined stuff #\n#######################\n') +- #iniFile.write('include ../../src/modules.ini\n') +- if os.getenv('IBMODELDIR'): +- iniFile.write('include %s\n' % os.path.join(os.getenv('IBMODELDIR'), 'src', 'modules.ini')) +- else: +- iniFile.write('include %s\n' % os.path.join('..', '..', 'src', 'modules.ini')) +- iniFile.write('\n') +- +- iniFile.close() +- +- +- def write_omnet_sw(self): +- # write special (maybe changing) configs for switches +- path, prefix = os.path.split(os.path.normpath(self.outputPrefix)) +- swFile = open(os.path.join(path, prefix+'.sw'), 'w') +- for guid in self.switchList: +- widthBin = {1: 0, 4: 0, 8: 0, 12: 0} +- speedBin = {2.5: 0, 5.0: 0, 10.0: 0, 12.5: 0, 17.05: 0, 31.25: 0} +- for port in self.switches[guid]['ports'].keys(): +- widthBin[ self.switches[guid]['ports'][port]['width'] ] = widthBin[ self.switches[guid]['ports'][port]['width'] ] + 1 +- speedBin[ self.switches[guid]['ports'][port]['speed'] ] = speedBin[ self.switches[guid]['ports'][port]['speed'] ] + 1 +- +- mostFilling = 0 +- mostFilledWidthBin = 0 +- highestWidthFilling = 0 +- for width in widthBin.keys(): +- if widthBin[width] > mostFilling: +- mostFilling = widthBin[width] +- mostFilledWidthBin = width +- if widthBin[width] != 0 and width > highestWidthFilling: +- highestWidthFilling = width +- mostFilling = 0 +- mostFilledSpeedBin = 0 +- highestSpeedFilling = 0 +- for speed in speedBin.keys(): +- if speedBin[speed] > mostFilling: +- mostFilling = speedBin[speed] +- mostFilledSpeedBin = speed +- if speedBin[speed] != 0 and speed > highestSpeedFilling: +- highestSpeedFilling = speed +- +- for width in widthBin.keys(): +- if widthBin[width] == 0 or width == mostFilledWidthBin: +- continue +- for port in self.switches[guid]['ports'].keys(): +- if self.switches[guid]['ports'][port]['width'] == width: +- swFile.write('**.S_%016x**.port[%s]**.width = %s\n' % (guid, port, width)) +- swFile.write('**.S_%016x**.width = %s\n' % (guid, mostFilledWidthBin)) # link width 1x, 4x, 8x, or 12x (somehow 1x is not supported by the model) +- +- for speed in speedBin.keys(): +- if speedBin[speed] == 0 or speed == mostFilledSpeedBin: +- continue +- for port in self.switches[guid]['ports'].keys(): +- if self.switches[guid]['ports'][port]['speed'] == speed: +- swFile.write('**.S_%016x**.port[%s]**.speed = %s\n' % (guid, port, speed)) +- swFile.write('**.S_%016x**.speed = %s\n' % (guid, mostFilledSpeedBin)) # link speed 2.5, 5.0, or 10.0, ... (higher for FDR) +- +- if highestWidthFilling == 4 and highestSpeedFilling == 2.5: +- ISWDelay = 64 - 4 +- elif highestWidthFilling == 4 and highestSpeedFilling == 5.0: +- ISWDelay = 32 - 2 +- elif highestWidthFilling == 4 and highestSpeedFilling == 10.0: +- ISWDelay = 16 - 1 +- elif highestWidthFilling == 8 and highestSpeedFilling == 2.5: +- ISWDelay = 32 - 2 +- elif highestWidthFilling == 8 and highestSpeedFilling == 5.0: +- ISWDelay = 16 - 1 +- elif highestWidthFilling == 8 and highestSpeedFilling == 10.0: +- ISWDelay = 8 - 1 # correct??? +- else: +- sys.exit("Error: supit width/speed combination -> ISWDelay unknown") +- swFile.write('**.S_%016x**.ISWDelay = %s\t# in ns\n' % (guid, ISWDelay)) +- swFile.write('**.S_%016x**.VSWDelay = 60\t# in ns\n' % guid) +- +- if highestSpeedFilling == 2.5: coreFreq = 200 +- elif highestSpeedFilling == 5.0: coreFreq = 400 +- elif highestSpeedFilling == 10.0: coreFreq = 800 +- else: sys.exit("Error: unknown coreFreq") +- swFile.write('**.S_%016x**.vlarb.coreFreq_MH = %s\t# switch core frequency should be 200, 400, and 800 for SDR, DDR, and QDR respectively\n' % (guid, coreFreq)) +- +- # see IBA 7.9.4 C7-54 for calc of credMinTime +- # (assume: we want to send a FCP after every 5 data credits) +- # 8 bit / speed_in_bit/s -> 1 symbol cycle in sec; convert to microsec +- credMinTime = ((8 / (highestSpeedFilling * 10**9)) * (10**6) * 64) * 5 +- swFile.write('**.S_%016x**.obuf.credMinTime = %s\n' % (guid, credMinTime)) +- swFile.close() +- +- +- def write_omnet_hca(self): +- # write special (maybe changing) configs for switches +- path, prefix = os.path.split(os.path.normpath(self.outputPrefix)) +- hcaFile = open(os.path.join(path, prefix+'.hca'), 'w') +- +- widthBin = {1: 0, 4: 0, 8: 0, 12: 0} +- speedBin = {2.5: 0, 5.0: 0, 10.0: 0, 12.5: 0, 17.05: 0, 31.25: 0} +- pciBin = {"1X/2.5": [8,2.5], "1X/5.0": [8,2.5], "1X/10.0": [8,2.5], \ +- "4X/2.5": [8,2.5], "4X/5.0": [8,5.0], "4X/10.0": [8,5.0], "4X/12.5": [16,5.0], "4X/17.05": [16,5.0], \ +- "8X/2.5": [8,2.5], "8X/5.0": [8,5.0], "8X/10.0": [16,5.0], "8X/12.5": [16,8], "8X/17.05": [16,8]} +- for hca in self.hcaList: +- guid, lid, port, width, speed = hca +- widthBin[ width ] = widthBin[ width ] + 1 +- speedBin[ speed ] = speedBin[ speed ] + 1 +- +- mostFilling = 0 +- mostFilledWidthBin = 0 +- for width in widthBin.keys(): +- if widthBin[width] > mostFilling: +- mostFilling = widthBin[width] +- mostFilledWidthBin = width +- mostFilling = 0 +- mostFilledSpeedBin = 0 +- for speed in speedBin.keys(): +- if speedBin[speed] > mostFilling: +- mostFilling = speedBin[speed] +- mostFilledSpeedBin = speed +- +- for width in widthBin.keys(): +- if widthBin[width] == 0 or width == mostFilledWidthBin: +- continue +- for hca in self.hcaList: +- guid, lid, port, hcawidth, hcaspeed = hca +- if hcawidth == width: +- hcaFile.write('**.H_%016x_%s**.width = %s\n' % (guid, port, width)) +- hcaFile.write('**.H_**.width = %s\n' % (mostFilledWidthBin)) +- +- for speed in speedBin.keys(): +- if speedBin[speed] == 0 or speed == mostFilledSpeedBin: +- continue +- for hca in self.hcaList: +- guid, lid, port, hcawidth, hcaspeed = hca +- if hcaspeed == speed: +- hcaFile.write('**.H_%016x_%s**.speed = %s\n' % (guid, port, speed)) +- if speed == 2.5: coreFreq = 125 +- elif speed == 5.0: coreFreq = 250 +- elif speed == 10.0: coreFreq = 500 +- else: sys.exit("Error: unknown coreFreq") +- if pciBin.has_key("%sX/%s" % (hcawidth, hcaspeed)): +- pciwidth, pcispeed = pciBin["%sX/%s" % (hcawidth, hcaspeed)] +- else: sys.exit("Error: unknown pci width/speed") +- hcaFile.write('**.H_%016x_%s**.vlarb.coreFreq_MH = %s\n' % (guid, port, coreFreq)) +- credMinTime = ((8 / (speed * 10**9)) * (10**6) * 64) * 5 +- hcaFile.write('**.H_%016x_%s**.obuf.credMinTime = %s\n' % (guid, port, credMinTime)) +- hcaFile.write('**.H_%016x_%s**.sink.pciExpWidth = %s\n' % (guid, port, pciwidth)) +- hcaFile.write('**.H_%016x_%s**.sink.pciExpTransferRate = %s\n' % (guid, port, pcispeed)) +- +- hcaFile.write('**.H_**.speed = %s\n' % (mostFilledSpeedBin)) +- +- if mostFilledSpeedBin == 2.5: coreFreq = 125 +- elif mostFilledSpeedBin == 5.0: coreFreq = 250 +- elif mostFilledSpeedBin == 10.0: coreFreq = 500 +- else: sys.exit("Error: unknown coreFreq") +- hcaFile.write('**.H_**.vlarb.coreFreq_MH = %s\t# HCAs frequency is the wire frequency, should be 125, 250, and 500 for SDR, DDR, and QDR respectively\n' % coreFreq) +- +- # see calc for sw (its the same) +- credMinTime = ((8 / (mostFilledSpeedBin * 10**9)) * (10**6) * 64) * 5 +- hcaFile.write('**.H_**.obuf.credMinTime = %s\n' % credMinTime) +- +- if pciBin.has_key("%sX/%s" % (mostFilledWidthBin, mostFilledSpeedBin)): +- pciwidth, pcispeed = pciBin["%sX/%s" % (mostFilledWidthBin, mostFilledSpeedBin)] +- else: sys.exit("Error: unknown pci width/speed") +- hcaFile.write('**.H_**.sink.pciExpWidth = %s\n' % pciwidth) +- hcaFile.write('**.H_**.sink.pciExpTransferRate = %s\n' % pcispeed) +- +- +- def printHelp(self): +- print "Usage: ", sys.argv[0], " " +- print "" +- print " Example:" +- print " ", sys.argv[0], " $OSM_CACHE_DIR/ibdiagnetout ./new-omnet-network oneShift shift12" +- print " Possible pattern:" +- print " nextNeighbor, rndAll, oneShift, allShift, oneExchange, allExchange" +- print " Possible fine tuning of the traffic:" +- print " none, dstRandom, dstSeqLoop, dstSeqOnce, shift[0-9999], exchange[0-9999]" +- print "" +- print " Hint to generate IB outpout:" +- print " opensm.log with vltable_print() output" +- print " ibdiagnet -o " +- print " ibnetdiscover -s > /ibnetdiscover.log" +- sys.exit('') +- +- +- def __init__(self): +- if len(sys.argv) > 4: +- path = os.path.normpath(sys.argv[1]) +- if not os.path.exists(path): +- print "Error: input directory does not exist or is not accessable" +- print "" +- self.printHelp() +- self.inputDir = os.path.join(path) +- +- path, prefix = os.path.split( os.path.normpath(sys.argv[2]) ) +- if path == '': +- path = os.getcwd() +- self.outputPrefix = os.path.join(path, prefix) +- +- self.pattern = sys.argv[3] +- self.dstSeqMode = sys.argv[4] +- else: +- self.printHelp() +- +- self.read_ibdiagnet() +- self.read_ibnetdiscover() +- self.read_opensmlog() +- +- self.write_omnet_vlt() +- self.write_omnet_dst() +- self.write_omnet_fdbs() +- self.write_omnet_ned() +- self.write_omnet_ini() +- self.write_omnet_sw() +- self.write_omnet_hca() +- +- +-if __name__ == "__main__": +- app = ibsim2omnet() +- sys.exit('\nFinish!') +only in patch2: +unchanged: +--- scripts.orig/lostConnectionsBeforeRerouting.py 2017-04-25 16:23:23.804601307 +0900 ++++ scripts/lostConnectionsBeforeRerouting.py 1970-01-01 09:00:00.000000000 +0900 +@@ -1,231 +0,0 @@ +-#!/usr/bin/env python +- +-import sys, re, os, random, operator, colorsys, copy +- +-class lostConnections(object): +- subnet = {} +- hcaTable = [] +- disconn = 0 +- killedSwitches = [] +- killedLinks = [] +- bisected = False +- +- +- def parse_topoLog(self, fileName=''): +- read_next_line_for_killed_switches = False +- read_next_line_for_killed_links = False +- +- self.bisected = False +- logFile = open( fileName, 'r' ) +- for line in logFile: +- if read_next_line_for_killed_switches: +- tmpSwitchList = line.split(';') +- self.killedSwitches = [sw.strip() for sw in tmpSwitchList] +- read_next_line_for_killed_switches = False +- continue +- +- if read_next_line_for_killed_links: +- tmpLinkList = line.split(';') +- for link in tmpLinkList: +- link = link.strip() +- linkAtoB = link.split('<->') +- linkAtoB.sort() +- self.killedLinks.append("%s<->%s" % (linkAtoB[0], linkAtoB[1])) +- read_next_line_for_killed_links = False +- continue +- +- p = re.compile('List of dead (\w+):') +- if p.match(line): +- m = p.match(line) +- if m.group(1).find('switches') > -1: +- read_next_line_for_killed_switches = True +- elif m.group(1).find('links') > -1: +- read_next_line_for_killed_links = True +- else: +- sys.exit("Error: strange output, don't understand the following line:\n%s" % line) +- continue +- +- if line.find('led to a disconnected network') > -1: +- self.bisected = True +- logFile.close() +- +- +- def parse_lstFile(self, fileName=''): +- lstFile = open( fileName, 'r' ) +- +- killedSwitches = copy.deepcopy(self.killedSwitches) +- killedLinks = copy.deepcopy(self.killedLinks) +- killedLinksForLater = {} +- +- network = {} +- for line in lstFile: +- p = re.compile('{\s+([a-zA-Z0-9_-]+)\s+Ports:(\w+)\s+SystemGUID:(\w+)\s+NodeGUID:(\w+)\s+PortGUID:(\w+)\s+VenID:(\w+)\s+DevID:(\w+)\s+Rev:(\w+)\s+{(.+)}\s+LID:(\w+)\s+PN:(\w+)\s+}\s+{\s+([a-zA-Z0-9_-]+)\s+Ports:(\w+)\s+SystemGUID:(\w+)\s+NodeGUID:(\w+)\s+PortGUID:(\w+)\s+VenID:(\w+)\s+DevID:(\w+)\s+Rev:(\w+)\s+{(.+)}\s+LID:(\w+)\s+PN:(\w+)\s+}\s+.+') +- if p.match(line): +- m = p.match(line) +- node1, ports1, sguid1, nguid1, pguid1, vid1, did1, rev1, name1, lid1, pn1 = m.group(1), int(m.group(2),16), m.group(3), m.group(4), m.group(5), \ +- m.group(6), m.group(7), m.group(8), m.group(9), int(m.group(10),16), int(m.group(11),16) +- node2, ports2, sguid2, nguid2, pguid2, vid2, did2, rev2, name2, lid2, pn2 = m.group(12), int(m.group(13),16), m.group(14), m.group(15), m.group(16), \ +- m.group(17), m.group(18), m.group(19), m.group(20), int(m.group(21),16), int(m.group(22),16) +- +- nguid1, nguid2 = nguid1.lower(), nguid2.lower() +- +- if node1.find('CA') > -1: +- node1, ports1, sguid1, nguid1, pguid1, vid1, did1, rev1, name1, lid1, pn1, node2, ports2, sguid2, nguid2, pguid2, vid2, did2, rev2, name2, lid2, pn2 = \ +- node2, ports2, sguid2, nguid2, pguid2, vid2, did2, rev2, name2, lid2, pn2, node1, ports1, sguid1, nguid1, pguid1, vid1, did1, rev1, name1, lid1, pn1 +- +- connectionKilled = False +- +- if killedSwitches.count(name1.strip()) > 0 or killedSwitches.count(name2.strip()) > 0: +- connectionKilled = True +- +- linkAtoB = [name1.strip(), name2.strip()] +- linkAtoB.sort() +- link = "%s<->%s" % (linkAtoB[0], linkAtoB[1]) +- if killedLinks.count(link) > 0: +- if killedLinksForLater.has_key(link): +- killedLinksForLater[link].append([nguid1, pn1, nguid2, pn2]) +- else: +- killedLinksForLater[link] = [[nguid1, pn1, nguid2, pn2]] +- #link1 = "%s<->%s" % (name1.strip(), name2.strip()) +- #link2 = "%s<->%s" % (name2.strip(), name1.strip()) +- #if killedLinks.count(link1) > 0: +- # killedLinks.remove(link1) +- # connectionKilled = True +- #if killedLinks.count(link2) > 0: +- # killedLinks.remove(link2) +- # connectionKilled = True +- +- if node2.find('CA') > -1: +- self.hcaTable.append([nguid2, pn2]) +- +- if network.has_key(nguid1): +- network[nguid1][pn1] = {'rnguid':nguid2, 'rpn':pn2, 'killed':connectionKilled} +- else: +- network[nguid1] = {'lid':lid1} +- network[nguid1][pn1] = {'rnguid':nguid2, 'rpn':pn2, 'killed':connectionKilled} +- +- if network.has_key(nguid2): +- network[nguid2][pn2] = {'rnguid':nguid1, 'rpn':pn1, 'killed':connectionKilled} +- else: +- network[nguid2] = {'lid':lid2} +- network[nguid2][pn2] = {'rnguid':nguid1, 'rpn':pn1, 'killed':connectionKilled} +- +- for link in killedLinksForLater.keys(): +- if killedLinks.count(link) > len(killedLinksForLater[link]): +- sys.exit('Error: morere links killed by createIBNet than available; can not be true.') +- parallelLinkSet = killedLinksForLater[link][:] +- random.shuffle(parallelLinkSet) +- for i in range(killedLinks.count(link)): +- nguid1, pn1, nguid2, pn2 = parallelLinkSet[i] +- network[nguid1][pn1]['killed'] = True +- network[nguid2][pn2]['killed'] = True +- +- self.subnet = network +- lstFile.close() +- +- +- def parse_fdbsFile(self, fileName=''): +- network = self.subnet +- nguid = None +- lft = {} +- +- fdbsFile = open( fileName, 'r' ) +- for line in fdbsFile: +- p = re.compile('osm_ucast_mgr_dump_ucast_routes: Switch 0x(\w+)') +- if p.match(line): +- if nguid: +- network[nguid]['lft'] = lft +- lft = {} +- nguid = p.match(line).group(1).lower() +- +- p = re.compile('0x(\w+)\s+:\s+(\d+)\s+.*') +- if p.match(line): +- lid, port = int(p.match(line).group(1), 16), int(p.match(line).group(2)) +- lft[lid] = port +- if nguid: +- network[nguid]['lft'] = lft +- fdbsFile.close() +- +- +- def calc_lostConnections(self): +- network = self.subnet +- +- for hcaSrc, portSrc in self.hcaTable: +- for hcaDest, portDest in self.hcaTable: +- if hcaSrc == hcaDest: +- continue +- curr = network[hcaSrc][portSrc]['rnguid'] +- destLid = network[hcaDest]['lid'] +- +- if network[hcaSrc][portSrc]['killed']: +- self.disconn += 1 +- continue +- +- while (curr != hcaDest): +- if not network[curr]['lft'].has_key(destLid): +- print "Error: No connection between: ", hcaSrc, hcaDest, " (this should not happen, or initial routing is broken)" +- self.disconn += 1 +- break +- exitPort = network[curr]['lft'][destLid] +- if network[curr][exitPort]['killed']: +- self.disconn += 1 +- break +- curr = network[curr][exitPort]['rnguid'] +- +- if not self.bisected: +- print '%s of %s Hca<->Hca routes are disconnected due to dead network components (before reroute)' % (self.disconn, len(self.hcaTable)*(len(self.hcaTable)-1)) +- else: +- print '%s of %s Hca<->Hca routes are disconnected due to dead network components (before reroute) and faults caused network bisection' % (self.disconn, len(self.hcaTable)*(len(self.hcaTable)-1)) +- +- +- def __init__(self): +- if len(sys.argv) == 3: +- path = os.path.normpath(sys.argv[1]) +- if not os.path.exists(path): +- print "Error: input directory does not exist or is not accessable" +- print "" +- self.printHelp() +- self.inputDir = os.path.join(path) +- +- lstFile = os.path.join(self.inputDir, 'ibdiagnet.lst') +- if not os.path.exists(lstFile): +- print "Error: %s file does not exist or is not accessable" % lstFile +- print "" +- self.printHelp() +- +- fdbsFile = os.path.join(self.inputDir, 'ibdiagnet.fdbs') +- if not os.path.exists(fdbsFile): +- print "Error: %s file does not exist or is not accessable" % fdbsFile +- print "" +- self.printHelp() +- +- topoLog = os.path.normpath(sys.argv[2]) +- if not os.path.exists(topoLog): +- print "Error: %s file does not exist or is not accessable" % topoLog +- print "" +- self.printHelp() +- +- else: +- self.printHelp() +- +- self.parse_topoLog(topoLog) +- self.parse_lstFile(lstFile) +- self.parse_fdbsFile(fdbsFile) +- self.calc_lostConnections() +- +- +- def printHelp(self): +- print "Usage: ", sys.argv[0], " " +- print "" +- print " Example:" +- print " ", sys.argv[0], " $OSM_CACHE_DIR/ibdiagnetout ./topo_w_failure.log" +- print "" +- print " Hint to generate *.lst, *.fdbs and log outpout:" +- print " ibdiagnet [-r] -o " +- print " createIBNet.py -t ... > topo_w_failure.log" +- sys.exit('') +- +-if __name__ == "__main__": +- app = lostConnections() +- sys.exit('\nFinish!') +- +only in patch2: +unchanged: +--- scripts.orig/scanFabric.sh 2017-04-25 17:46:35.148894936 +0900 ++++ scripts/scanFabric.sh 1970-01-01 09:00:00.000000000 +0900 +@@ -1,38 +0,0 @@ +-#!/bin/sh +- +-trap cleanup 2 11 15 +- +-cleanup() +-{ +- kill -9 ${sub} +- exit 0 +-} +- +-#0. Init +-if [ -z $2 ]; then +- export OFEDDIR=/home/domke/DiplArbeit/ofed +- export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${OFEDDIR}/lib +- export IBSIMLIB=${OFEDDIR}/lib/umad2sim/libumad2sim.so +- export OSM_TMP_DIR=/home/domke/DiplArbeit/tmp +- export OSM_CACHE_DIR=/home/domke/DiplArbeit/tmp +-else +- export OFEDDIR=$1/ofed +- export PATH=${OFEDDIR}/bin:${OFEDDIR}/sbin:$PATH +- export LD_LIBRARY_PATH=${OFEDDIR}/lib:$LD_LIBRARY_PATH +- export IBSIMLIB=${OFEDDIR}/lib/umad2sim/libumad2sim.so +- export OSM_TMP_DIR=$2 +- export OSM_CACHE_DIR=$2 +-fi +- +-#1. Run Simulator: +-cd $OFEDDIR +-LD_PRELOAD=${IBSIMLIB} ./bin/ibdiagnet -o $OSM_TMP_DIR +- +-cd $OFEDDIR +-LD_PRELOAD=${IBSIMLIB} ./sbin/ibnetdiscover -s > $OSM_TMP_DIR/ibnetdiscover.log +- +-echo '' +-echo 'SUBNET SCANNED' +- +-exit 0 +- +only in patch2: +unchanged: +--- scripts.orig/simLostConn.py 2017-04-25 16:23:23.892601252 +0900 ++++ scripts/simLostConn.py 1970-01-01 09:00:00.000000000 +0900 +@@ -1,541 +0,0 @@ +-#!/usr/bin/env python +- +-import sys, os, random, time, subprocess, signal, socket, re, math +-from optparse import OptionParser +- +-class buildExperiment(object): +- def copyOldBaseData(self, oldDir, newDir): +- if not os.path.exists(oldDir): +- print 'Warning: %s missing' % oldDir +- return False +- if not os.path.exists(os.path.join(oldDir, 'omnet.log')): +- print 'Warning: simulation for base data did not finish; omnet.log missing (for %s).' % oldDir +- return False +- +- if not os.path.exists(newDir): os.makedirs(newDir) +- os.system("cp %s/topo.net %s/" % (oldDir, newDir)) +- os.system("cp %s/*.lst %s/" % (os.path.join(oldDir, 'ofedout'), newDir)) +- os.system("cp %s/*.fdbs %s/" % (os.path.join(oldDir, 'ofedout'), newDir)) +- return True +- +- +- def buildSH(self, scriptsDir, expDir, task): +- runsh = os.path.join(expDir, 'lostConnSim_%s.sh' % os.path.basename(expDir)) +- run = open(runsh, 'w') +- run.write('#!/bin/bash\n') +- run.write('cd %s\n' % os.path.join(expDir)) +- run.write('%s/simLostConn.py -n `pwd` -t "%s"\n' % (scriptsDir, task)) +- run.close() +- +- +- def __init__(self): +- if os.uname()[1].find('tgx-login1') > -1 or os.uname()[1].find('nid0') > -1: +- homedir = os.path.join('/', 'work', 'A2401644') +- else: homedir = os.getenv('HOME') +- instdir = os.path.join(homedir, 'simulation') +- scriptdir = os.path.join(instdir, 'scripts') +- expdir = os.path.join(instdir, 'experiments') +- lostdir = os.path.join(instdir, 'lostConnExp') +- pattern = 'exchange' +- routings = ['minhop', 'sssp', 'dfsssp', 'ftree', 'updn', 'dnup', 'dor', 'lash', 'torus-2QoS', 'tofu'] +- faultType = 'lnf' +- faultyLinks = '0' +- rseed = '1' +- +- if not os.path.exists(expdir): +- sys.exit('Error: directory with base data is missing.') +- if not os.path.exists(lostdir): +- os.mkdir(lostdir) +- +- # 2d Mesh (25 switches + 240 links), balanced +- #routings = ['dfsssp', 'dor', 'lash', 'torus-2QoS'] +- sw, hca, links = 25, 275, 240 +- task = '-wid 4 -spe QDR -t 2D-Mesh -d1 5 -d2 5 -ml -s %s -sp 36 -n %s -np 1' % (sw, hca) +- newDirName = '2d_mesh_s%s_n%s_l%s' % (sw, hca, links) +- oldDirName = '%s_rs%s' % (newDirName, rseed) +- for routing in routings: +- oldDir = os.path.join(expdir, oldDirName, faultType, faultyLinks, routing, pattern) +- newDir = os.path.join(lostdir, newDirName, routing) +- if not self.copyOldBaseData(oldDir, newDir): continue +- self.buildSH(scriptdir, newDir, task) +- +- # 2d Torus (25 switches + 300 links), balanced +- #routings = ['dfsssp', 'lash', 'torus-2QoS'] +- sw, hca, links = 25, 275, 300 +- task = '-wid 4 -spe QDR -t 2D-Torus -d1 5 -d2 5 -ml -s %s -sp 36 -n %s -np 1' % (sw, hca) +- newDirName = '2d_torus_s%s_n%s_l%s' % (sw, hca, links) +- oldDirName = '%s_rs%s' % (newDirName, rseed) +- for routing in routings: +- oldDir = os.path.join(expdir, oldDirName, faultType, faultyLinks, routing, pattern) +- newDir = os.path.join(lostdir, newDirName, routing) +- if not self.copyOldBaseData(oldDir, newDir): continue +- self.buildSH(scriptdir, newDir, task) +- +- # 3d Mesh (27 switches + 216 links), balanced +- #routings = ['dfsssp', 'dor', 'lash', 'torus-2QoS'] +- sw, hca, links = 27, 270, 216 +- task = '-wid 4 -spe QDR -t 3D-Mesh -d1 3 -d2 3 -d3 3 -ml -s %s -sp 36 -n %s -np 1' % (sw, hca) +- newDirName = '3d_mesh_s%s_n%s_l%s' % (sw, hca, links) +- oldDirName = '%s_rs%s' % (newDirName, rseed) +- for routing in routings: +- oldDir = os.path.join(expdir, oldDirName, faultType, faultyLinks, routing, pattern) +- newDir = os.path.join(lostdir, newDirName, routing) +- if not self.copyOldBaseData(oldDir, newDir): continue +- self.buildSH(scriptdir, newDir, task) +- +- # 3d torus (27 switches + 324 links), balanced +- #routings = ['dfsssp', 'dor', 'lash', 'torus-2QoS'] +- sw, hca, links = 27, 270, 324 +- task = '-wid 4 -spe QDR -t 3D-Torus -d1 3 -d2 3 -d3 3 -ml -s %s -sp 36 -n %s -np 1' % (sw, hca) +- newDirName = '3d_torus_s%s_n%s_l%s' % (sw, hca, links) +- oldDirName = '%s_rs%s' % (newDirName, rseed) +- for routing in routings: +- oldDir = os.path.join(expdir, oldDirName, faultType, faultyLinks, routing, pattern) +- newDir = os.path.join(lostdir, newDirName, routing) +- if not self.copyOldBaseData(oldDir, newDir): continue +- self.buildSH(scriptdir, newDir, task) +- +- # kautz (24 switches + 288 links), balanced +- #routings = ['dfsssp', 'lash'] +- sw, hca, links = 24, 264, 288 +- task = '-wid 4 -spe QDR -t Kautz -kb 2 -kn 4 -ml -sp 36 -n %s -np 1' % (hca) +- newDirName = 'kautz_s%s_n%s_l%s' % (sw, hca, links) +- oldDirName = '%s_rs%s' % (newDirName, rseed) +- for routing in routings: +- oldDir = os.path.join(expdir, oldDirName, faultType, faultyLinks, routing, pattern) +- newDir = os.path.join(lostdir, newDirName, routing) +- if not self.copyOldBaseData(oldDir, newDir): continue +- self.buildSH(scriptdir, newDir, task) +- +- # random (32 switches + 256 links), balanced +- #routings = ['dfsssp', 'dor', 'lash'] +- sw, hca, links = 32, 256, 256 +- task = '-wid 4 -spe QDR -t Random -s %s -sp 36 -n %s -np 1 -cl %s' % (sw, hca, links) +- newDirName = 'random_s%s_n%s_l%s' % (sw, hca, links) +- oldDirName = '%s_rs%s' % (newDirName, rseed) +- for routing in routings: +- oldDir = os.path.join(expdir, oldDirName, faultType, faultyLinks, routing, pattern) +- newDir = os.path.join(lostdir, newDirName, routing) +- if not self.copyOldBaseData(oldDir, newDir): continue +- self.buildSH(scriptdir, newDir, task) +- +- # k-ary-n (32 switches + 256 links), balanced +- #routings = ['minhop', 'sssp', 'dfsssp', 'ftree', 'updn', 'dnup', 'dor', 'lash'] +- sw, hca, links = 32, 256, 256 +- task = '-wid 4 -spe QDR -t k-ary-n-Tree -tk 16 -tn 2 -sp 36 -n %s' % (hca) +- newDirName = 'krntree_s%s_n%s_l%s' % (sw, hca, links) +- oldDirName = '%s_rs%s' % (newDirName, rseed) +- for routing in routings: +- oldDir = os.path.join(expdir, oldDirName, faultType, faultyLinks, routing, pattern) +- newDir = os.path.join(lostdir, newDirName, routing) +- if not self.copyOldBaseData(oldDir, newDir): continue +- self.buildSH(scriptdir, newDir, task) +- +- # xgft (33 switches + 242 links), balanced +- #routings = ['minhop', 'sssp', 'dfsssp', 'ftree', 'updn', 'dnup', 'dor', 'lash'] +- sw, hca, links = 33, 264, 242 +- task = '-wid 4 -spe QDR -t XGFT -xh 1 -xm 22 -xw 11 -sp 36 -n %s -np 1' % (hca) +- newDirName = 'xgft_s%s_n%s_l%s' % (sw, hca, links) +- oldDirName = '%s_rs%s' % (newDirName, rseed) +- for routing in routings: +- oldDir = os.path.join(expdir, oldDirName, faultType, faultyLinks, routing, pattern) +- newDir = os.path.join(lostdir, newDirName, routing) +- if not self.copyOldBaseData(oldDir, newDir): continue +- self.buildSH(scriptdir, newDir, task) +- +- # dragonfly (40 switches + 276 links), balanced +- #routings = ['dfsssp', 'lash'] +- sw, hca, links = 40, 280, 276 +- task = '-wid 4 -spe QDR -t Dragonfly -fa 10 -fp 5 -fh 5 -fg 4 -sp 36 -n %s -np 1' % (hca) +- newDirName = 'dragonfly_s%s_n%s_l%s' % (sw, hca, links) +- oldDirName = '%s_rs%s' % (newDirName, rseed) +- for routing in routings: +- oldDir = os.path.join(expdir, oldDirName, faultType, faultyLinks, routing, pattern) +- newDir = os.path.join(lostdir, newDirName, routing) +- if not self.copyOldBaseData(oldDir, newDir): continue +- self.buildSH(scriptdir, newDir, task) +- +- # cascade (1 group w/ 768 hca + 96 switches + 960 links) +- #routings = ['dfsssp', 'lash'] +- sw, hca, links = 96, 768, 960 +- task = '-wid 4 -spe QDR -t Cascade -cg 1 -np 1' +- newDirName = 'cascade_s%s_n%s_l%s' % (sw, hca, links) +- oldDirName = '%s_rs%s' % (newDirName, rseed) +- for routing in routings: +- oldDir = os.path.join(expdir, oldDirName, faultType, faultyLinks, routing, pattern) +- newDir = os.path.join(lostdir, newDirName, routing) +- if not self.copyOldBaseData(oldDir, newDir): continue +- self.buildSH(scriptdir, newDir, task) +- +- # tofu (288 sw/hca + 1440 links) +- #routings = ['dor', 'tofu'] +- sw, hca, links = 288, 288, 1440 +- task = '-wid 4 -spe QDR -t Tofu -d1 4 -d2 3 -d3 2 -n %s -np 1 -ari' % (hca) +- newDirName = 'tofu_s%s_n%s_l%s' % (sw, hca, links) +- oldDirName = '%s_rs%s' % (newDirName, rseed) +- for routing in routings: +- oldDir = os.path.join(expdir, oldDirName, faultType, faultyLinks, routing, pattern) +- newDir = os.path.join(lostdir, newDirName, routing) +- if not self.copyOldBaseData(oldDir, newDir): continue +- self.buildSH(scriptdir, newDir, task) +- +- # MMS (50 sw + 175*3 sw/sw-links) +- #routings = ['dfsssp', 'lash'] +- sw, hca, links = 50, 700, 525 +- task = '-wid 4 -spe QDR -t MMS -mms 7 -ml -s %s -sp 36 -n %s -np 1' % (sw, hca) +- newDirName = 'mms_s%s_n%s_l%s' % (sw, hca, links) +- oldDirName = '%s_rs%s' % (newDirName, rseed) +- for routing in routings: +- oldDir = os.path.join(expdir, oldDirName, faultType, faultyLinks, routing, pattern) +- newDir = os.path.join(lostdir, newDirName, routing) +- if not self.copyOldBaseData(oldDir, newDir): continue +- self.buildSH(scriptdir, newDir, task) +- +- # tsubame2.0 +- #routings = ['dfsssp', 'ftree', 'updn', 'dnup', 'lash'] +- sw, hca, links = 258, 1555, 3621 +- network = os.path.join(instdir, 'real_networks', 'tsubame2.0.txt') +- rootguid = os.path.join(instdir, 'real_networks', 'tsubame2.0_rootguids.conf') +- task = '-t load -i %s -rid %s -sp 36 -np 1' % (network, rootguid) +- newDirName = 'tsubame' +- oldDirName = '%s_rs%s' % (newDirName, rseed) +- for routing in routings: +- oldDir = os.path.join(expdir, oldDirName, faultType, faultyLinks, routing, pattern) +- newDir = os.path.join(lostdir, newDirName, routing) +- if not self.copyOldBaseData(oldDir, newDir): continue +- self.buildSH(scriptdir, newDir, task) +- +- # tsubame2.0 rail2 +- #routings = ['dfsssp', 'ftree', 'updn', 'dnup', 'lash'] +- sw, hca, links = 243, 1407, 3384 +- network = os.path.join(instdir, 'real_networks', 'tsubame2.0_rail2.txt') +- rootguid = os.path.join(instdir, 'real_networks', 'tsubame2.0_rail2_rootguids.conf') +- task = '-t load -i %s -rid %s -sp 36 -np 1' % (network, rootguid) +- newDirName = 'tsubame_rail2' +- oldDirName = '%s_rs%s' % (newDirName, rseed) +- for routing in routings: +- oldDir = os.path.join(expdir, oldDirName, faultType, faultyLinks, routing, pattern) +- newDir = os.path.join(lostdir, newDirName, routing) +- if not self.copyOldBaseData(oldDir, newDir): continue +- self.buildSH(scriptdir, newDir, task) +- +- # 3d torus (150 switches + 1050 links) +- #routings = ['dfsssp', 'lash', 'torus-2QoS'] +- sw, hca, links = 150, 1050, 1800 +- task = '-wid 4 -spe QDR -t 3D-Torus -d1 6 -d2 5 -d3 5 -ml -s %s -sp 36 -n %s -np 1' % (sw, hca) +- newDirName = '3d_torus_s%s_n%s_l%s' % (sw, hca, links) +- oldDirName = '%s_rs%s' % (newDirName, rseed) +- for routing in routings: +- oldDir = os.path.join(expdir, oldDirName, faultType, faultyLinks, routing, pattern) +- newDir = os.path.join(lostdir, newDirName, routing) +- if not self.copyOldBaseData(oldDir, newDir): continue +- self.buildSH(scriptdir, newDir, task) +- +- # 3d torus ( switches + links) +- #routings = ['dfsssp', 'lash', 'torus-2QoS'] +- sw, hca, links = 343, 2058, 5145 +- task = '-wid 4 -spe QDR -t 3D-Torus -d1 7 -d2 7 -d3 7 -ml -s %s -sp 36 -n %s -np 1' % (sw, hca) +- newDirName = '3d_torus_s%s_n%s_l%s' % (sw, hca, links) +- oldDirName = '%s_rs%s' % (newDirName, rseed) +- for routing in routings: +- oldDir = os.path.join(expdir, oldDirName, faultType, faultyLinks, routing, pattern) +- newDir = os.path.join(lostdir, newDirName, routing) +- if not self.copyOldBaseData(oldDir, newDir): continue +- self.buildSH(scriptdir, newDir, task) +- +- # kautz (150 switches + 1500 links) +- #routings = ['dfsssp', 'lash'] +- sw, hca, links = 150, 1050, 1500 +- task = '-wid 4 -spe QDR -t Kautz -kb 5 -kn 3 -ml -sp 36 -n %s -np 1' % (hca) +- newDirName = 'kautz_s%s_n%s_l%s' % (sw, hca, links) +- oldDirName = '%s_rs%s' % (newDirName, rseed) +- for routing in routings: +- oldDir = os.path.join(expdir, oldDirName, faultType, faultyLinks, routing, pattern) +- newDir = os.path.join(lostdir, newDirName, routing) +- if not self.copyOldBaseData(oldDir, newDir): continue +- self.buildSH(scriptdir, newDir, task) +- +- # kautz (392 switches + 5488 links) +- #routings = ['dfsssp', 'lash'] +- sw, hca, links = 392, 2352, 5488 +- task = '-wid 4 -spe QDR -t Kautz -kb 7 -kn 3 -ml -sp 36 -n %s -np 1' % (hca) +- newDirName = 'kautz_s%s_n%s_l%s' % (sw, hca, links) +- oldDirName = '%s_rs%s' % (newDirName, rseed) +- for routing in routings: +- oldDir = os.path.join(expdir, oldDirName, faultType, faultyLinks, routing, pattern) +- newDir = os.path.join(lostdir, newDirName, routing) +- if not self.copyOldBaseData(oldDir, newDir): continue +- self.buildSH(scriptdir, newDir, task) +- +- # k-ary-n (300 switches + 2000 links) +- #routings = ['dfsssp', 'ftree', 'updn', 'dnup'] +- sw, hca, links = 300, 1100, 2000 +- task = '-wid 4 -spe QDR -t k-ary-n-Tree -tk 10 -tn 3 -sp 36 -n %s' % (hca) +- newDirName = 'krntree_s%s_n%s_l%s' % (sw, hca, links) +- oldDirName = '%s_rs%s' % (newDirName, rseed) +- for routing in routings: +- oldDir = os.path.join(expdir, oldDirName, faultType, faultyLinks, routing, pattern) +- newDir = os.path.join(lostdir, newDirName, routing) +- if not self.copyOldBaseData(oldDir, newDir): continue +- self.buildSH(scriptdir, newDir, task) +- +- # k-ary-n (588 switches + 5488 links) +- #routings = ['dfsssp', 'ftree', 'updn', 'dnup'] +- sw, hca, links = 588, 2156, 5488 +- task = '-wid 4 -spe QDR -t k-ary-n-Tree -tk 14 -tn 3 -sp 36 -n %s' % (hca) +- newDirName = 'krntree_s%s_n%s_l%s' % (sw, hca, links) +- oldDirName = '%s_rs%s' % (newDirName, rseed) +- for routing in routings: +- oldDir = os.path.join(expdir, oldDirName, faultType, faultyLinks, routing, pattern) +- newDir = os.path.join(lostdir, newDirName, routing) +- if not self.copyOldBaseData(oldDir, newDir): continue +- self.buildSH(scriptdir, newDir, task) +- +- # dragonfly (150 switches + 1515 links) +- #routings = ['dfsssp', 'lash'] +- sw, hca, links = 180, 1080, 1515 +- task = '-wid 4 -spe QDR -t Dragonfly -fa 12 -fp 6 -fh 6 -fg 15 -sp 36 -n %s -np 1' % (hca) +- newDirName = 'dragonfly_s%s_n%s_l%s' % (sw, hca, links) +- oldDirName = '%s_rs%s' % (newDirName, rseed) +- for routing in routings: +- oldDir = os.path.join(expdir, oldDirName, faultType, faultyLinks, routing, pattern) +- newDir = os.path.join(lostdir, newDirName, routing) +- if not self.copyOldBaseData(oldDir, newDir): continue +- self.buildSH(scriptdir, newDir, task) +- +- # dragonfly (322 switches + 3105 links) +- #routings = ['dfsssp', 'lash'] +- sw, hca, links = 322, 2254, 3105 +- task = '-wid 4 -spe QDR -t Dragonfly -fa 14 -fp 7 -fh 7 -fg 23 -sp 36 -n %s -np 1' % (hca) +- newDirName = 'dragonfly_s%s_n%s_l%s' % (sw, hca, links) +- oldDirName = '%s_rs%s' % (newDirName, rseed) +- for routing in routings: +- oldDir = os.path.join(expdir, oldDirName, faultType, faultyLinks, routing, pattern) +- newDir = os.path.join(lostdir, newDirName, routing) +- if not self.copyOldBaseData(oldDir, newDir): continue +- self.buildSH(scriptdir, newDir, task) +- +- # cascade (2 group w/ 1536 hca + 192 switches + 1924 links) +- #routings = ['dfsssp', 'lash'] +- sw, hca, links = 192, 1536, 1924 +- task = '-wid 4 -spe QDR -t Cascade -cg 2 -np 1' +- newDirName = 'cascade_s%s_n%s_l%s' % (sw, hca, links) +- oldDirName = '%s_rs%s' % (newDirName, rseed) +- for routing in routings: +- oldDir = os.path.join(expdir, oldDirName, faultType, faultyLinks, routing, pattern) +- newDir = os.path.join(lostdir, newDirName, routing) +- if not self.copyOldBaseData(oldDir, newDir): continue +- self.buildSH(scriptdir, newDir, task) +- +- # cascade (3 group w/ 2304 hca + 288 switches + 2892 links) +- #routings = ['dfsssp', 'lash'] +- sw, hca, links = 288, 2304, 2892 +- task = '-wid 4 -spe QDR -t Cascade -cg 3 -np 1' +- newDirName = 'cascade_s%s_n%s_l%s' % (sw, hca, links) +- oldDirName = '%s_rs%s' % (newDirName, rseed) +- for routing in routings: +- oldDir = os.path.join(expdir, oldDirName, faultType, faultyLinks, routing, pattern) +- newDir = os.path.join(lostdir, newDirName, routing) +- if not self.copyOldBaseData(oldDir, newDir): continue +- self.buildSH(scriptdir, newDir, task) +- +- # tofu (1080 sw/hca + 5400 links) +- #routings = ['tofu'] +- sw, hca, links = 1080, 1080, 5400 +- task = '-wid 4 -spe QDR -t Tofu -d1 6 -d2 5 -d3 3 -n %s -np 1' % (hca) +- newDirName = 'tofu_s%s_n%s_l%s' % (sw, hca, links) +- oldDirName = '%s_rs%s' % (newDirName, rseed) +- for routing in routings: +- oldDir = os.path.join(expdir, oldDirName, faultType, faultyLinks, routing, pattern) +- newDir = os.path.join(lostdir, newDirName, routing) +- if not self.copyOldBaseData(oldDir, newDir): continue +- self.buildSH(scriptdir, newDir, task) +- +- # tofu (2100 sw/hca + 10500 links) +- #routings = ['tofu'] +- sw, hca, links = 2100, 2100, 10500 +- task = '-wid 4 -spe QDR -t Tofu -d1 7 -d2 5 -d3 5 -n %s -np 1' % (hca) +- newDirName = 'tofu_s%s_n%s_l%s' % (sw, hca, links) +- oldDirName = '%s_rs%s' % (newDirName, rseed) +- for routing in routings: +- oldDir = os.path.join(expdir, oldDirName, faultType, faultyLinks, routing, pattern) +- newDir = os.path.join(lostdir, newDirName, routing) +- if not self.copyOldBaseData(oldDir, newDir): continue +- self.buildSH(scriptdir, newDir, task) +- +- os.system("find %s -iname '*.sh' | xargs chmod +x" % lostdir) +- sys.exit() +- +- +-class runSim(object): +- def __init__(self, experimentdir, task): +- if os.uname()[1].find('tgx-login1') > -1 or os.uname()[1].find('nid0') > -1: +- homedir = os.path.join('/', 'work', 'A2401644') +- else: homedir = os.getenv('HOME') +- instdir = os.path.join(homedir, 'simulation') +- scriptdir = os.path.join(instdir, 'scripts') +- testdir = experimentdir +- +- log = open(os.path.join(testdir, 'lostPaths.log'), 'w', 0) +- +- faultType = '-lnf' +- for faultyLinks in [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18]: +- fault = '%s %s' % (faultType, faultyLinks) +- for rseed in xrange(100): +- seed = '-rs %s' % rseed +- os.system("%s/createIBNet.py %s %s %s -idc -o %s/tmp.net >%s/topo.log 2>&1" % (scriptdir, task, fault, seed, testdir, testdir)) +- log.write("faulttype: %s faults: %s rseed: %s\n" % (faultType, faultyLinks, rseed)) +- proc = subprocess.Popen([os.path.join(scriptdir,'lostConnectionsBeforeRerouting.py'), testdir, os.path.join(testdir,'topo.log')], stdout=subprocess.PIPE, stderr=subprocess.PIPE) +- out, err = proc.communicate() +- log.write("%s\n" % out) +- faultType = '-snf' +- for faultySwitches in [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]: +- fault = '%s %s' % (faultType, faultySwitches) +- for rseed in xrange(100): +- seed = '-rs %s' % rseed +- os.system("%s/createIBNet.py %s %s %s -idc -o %s/tmp.net >%s/topo.log 2>&1" % (scriptdir, task, fault, seed, testdir, testdir)) +- log.write("faulttype: %s faults: %s rseed: %s\n" % (faultType, faultySwitches, rseed)) +- proc = subprocess.Popen([os.path.join(scriptdir,'lostConnectionsBeforeRerouting.py'), testdir, os.path.join(testdir,'topo.log')], stdout=subprocess.PIPE, stderr=subprocess.PIPE) +- out, err = proc.communicate() +- log.write("%s\n" % out) +- +- log.write('\nFinished!\n') +- log.close() +- +- +-class analyzeExperiment(object): +- def __init__(self): +- if os.uname()[1].find('tgx-login1') > -1 or os.uname()[1].find('nid0') > -1: +- homedir = os.path.join('/', 'work', 'A2401644') +- else: homedir = os.getenv('HOME') +- instdir = os.path.join(homedir, 'simulation') +- scriptdir = os.path.join(instdir, 'scripts') +- lostdir = os.path.join(instdir, 'lostConnExp') +- result = os.path.join(instdir, 'resultLostConn.gp') +- +- res = open(result, 'w') +- res.write('# experiment | routing | faultType | faults | maxConn | numBisection | lostConn_min | lostConn_max | lostConn_avg | lostConn_stddev | 75% ConfidenceInterval LB | 75% ConfidenceInterval UB | 95% ConfidenceInterval LB | 95% ConfidenceInterval UB | 97.5% ConfidenceInterval LB | 97.5% ConfidenceInterval UB | 99% ConfidenceInterval LB | 99% ConfidenceInterval UB | 99.9% ConfidenceInterval LB | 99.9% ConfidenceInterval UB | percentLost | lowerExtrem | lowerQuartile | median | upperQuartile | upperExtrem\n') +- +- for root, dirs, files in os.walk(lostdir): +- if files.count('lostPaths.log') == 0: continue +- print 'Processing: ', root +- +- rest, routing = os.path.split(root) +- rest, exp = os.path.split(rest) +- LCRes = {} +- +- log = open(os.path.join(root, 'lostPaths.log'), 'r') +- logDone = False +- lineNr = 0 +- for line in log: +- lineNr += 1 +- p = re.compile('faulttype:\s+-(\w+)\s+faults:\s+(\d+)\s+rseed:\s+(\d+).*') +- if p.match(line): +- m = p.match(line) +- faultType, faults, rseed = m.group(1), m.group(2), m.group(3) +- if not LCRes.has_key(faultType): +- LCRes[faultType] = {faults: {'maxConn':0, 'LC_min':math.pow(2,64)-1, 'LC_max':0, 'LC_count':0, 'LC_sum':0.0, 'LC_sumSqare':0.0, 'LC_bisected':0, 'listLC':[]}} +- if not LCRes[faultType].has_key(faults): +- LCRes[faultType][faults] = {'maxConn':0, 'LC_min':math.pow(2,64)-1, 'LC_max':0, 'LC_count':0, 'LC_sum':0.0, 'LC_sumSqare':0.0, 'LC_bisected':0, 'listLC':[]} +- +- +- p = re.compile('(\d+)\s+of\s+(\d+)\s+Hca<->Hca.*') +- if p.match(line): +- m = p.match(line) +- lostConn, maxConn = int(m.group(1)), int(m.group(2)) +- if not LCRes.has_key(faultType) or not LCRes[faultType].has_key(faults): +- sys.exit('Error: file corrupted; faultType=%s or faults=%s not in database.' % (faultType, faults)) +- +- if LCRes[faultType][faults]['maxConn'] == 0: LCRes[faultType][faults]['maxConn'] = maxConn +- elif LCRes[faultType][faults]['maxConn'] != maxConn: sys.exit('Error: file corrupted; maxConn inconsitent; lineNr: %s.' % lineNr) +- +- if lostConn < LCRes[faultType][faults]['LC_min']: LCRes[faultType][faults]['LC_min'] = lostConn +- if LCRes[faultType][faults]['LC_max'] < lostConn: LCRes[faultType][faults]['LC_max'] = lostConn +- LCRes[faultType][faults]['LC_count'] += 1 +- LCRes[faultType][faults]['LC_sum'] += lostConn +- LCRes[faultType][faults]['LC_sumSqare'] += math.pow(lostConn, 2) +- LCRes[faultType][faults]['listLC'].append(lostConn) +- +- if line.find('and faults caused network bisection') > -1: +- LCRes[faultType][faults]['LC_bisected'] += 1 +- +- if line.find('Finished') > -1: +- logDone = True +- +- if not logDone: +- print 'Warning: either simulation failed or is running for ', root +- continue +- +- for faultType in LCRes.keys(): +- for faults in LCRes[faultType]: +- maxConn = LCRes[faultType][faults]['maxConn'] +- lostConn_min = LCRes[faultType][faults]['LC_min'] +- lostConn_max = LCRes[faultType][faults]['LC_max'] +- lostConn_sum = LCRes[faultType][faults]['LC_sum'] +- lostConn_sumSqare = LCRes[faultType][faults]['LC_sumSqare'] +- count = LCRes[faultType][faults]['LC_count'] +- lostConn_bisected = LCRes[faultType][faults]['LC_bisected'] +- lostConn_avg = lostConn_sum / count +- lostConn_stddev = math.sqrt((lostConn_sumSqare/count)-math.pow(lostConn_sum/count, 2)) +- # http://www.stat.purdue.edu/~mccabe/ips4tab/bmtables.pdf +- confi75lb = lostConn_avg - 1.15 * (lostConn_stddev / math.sqrt(count)) +- confi75ub = lostConn_avg + 1.15 * (lostConn_stddev / math.sqrt(count)) +- confi95lb = lostConn_avg - 1.96 * (lostConn_stddev / math.sqrt(count)) +- confi95ub = lostConn_avg + 1.96 * (lostConn_stddev / math.sqrt(count)) +- confi975lb = lostConn_avg - 2.24 * (lostConn_stddev / math.sqrt(count)) +- confi975ub = lostConn_avg + 2.24 * (lostConn_stddev / math.sqrt(count)) +- confi99lb = lostConn_avg - 2.58 * (lostConn_stddev / math.sqrt(count)) +- confi99ub = lostConn_avg + 2.58 * (lostConn_stddev / math.sqrt(count)) +- confi999lb = lostConn_avg - 3.29 * (lostConn_stddev / math.sqrt(count)) +- confi999ub = lostConn_avg + 3.29 * (lostConn_stddev / math.sqrt(count)) +- percentLost = lostConn_avg / maxConn * 100.0 +- +- listLostConn = LCRes[faultType][faults]['listLC'][:] +- listLostConn.sort() +- lowerExtrem = listLostConn[0] +- upperExtrem = listLostConn[-1] +- if len(listLostConn)%2 == 0: +- median = (listLostConn[len(listLostConn)/2-1] + listLostConn[len(listLostConn)/2]) / 2.0 +- lowerhalfList = listLostConn[:len(listLostConn)/2][:] +- upperhalfList = listLostConn[len(listLostConn)/2:][:] +- else: +- median = 1.0 * listLostConn[len(listLostConn)/2] +- lowerhalfList = listLostConn[:len(listLostConn)/2][:] +- upperhalfList = listLostConn[len(listLostConn)/2+1:][:] +- if len(lowerhalfList)%2 == 0: +- lowerQuartile = (lowerhalfList[len(lowerhalfList)/2-1] + lowerhalfList[len(lowerhalfList)/2]) / 2.0 +- else: +- lowerQuartile = 1.0 * lowerhalfList[len(lowerhalfList)/2] +- if len(upperhalfList)%2 == 0: +- upperQuartile = (upperhalfList[len(upperhalfList)/2-1] + upperhalfList[len(upperhalfList)/2]) / 2.0 +- else: +- upperQuartile = 1.0 * upperhalfList[len(upperhalfList)/2] +- +- res.write('%s \t%s \t%s \t%s \t%s \t%s \t%s \t%s \t%s \t%s \t%s \t%s \t%s \t%s \t%s \t%s \t%s \t%s \t%s \t%s \t%s \t%s \t%s \t%s \t%s \t%s\n' % (exp, routing, faultType, faults, maxConn, lostConn_bisected, lostConn_min, lostConn_max, lostConn_avg, lostConn_stddev, confi75lb, confi75ub, confi95lb, confi95ub, confi975lb, confi975ub, confi99lb, confi99ub, confi999lb, confi999ub, percentLost, lowerExtrem, lowerQuartile, median, upperQuartile, upperExtrem)) +- +- log.close() +- +- res.close() +- +- +-if __name__ == "__main__": +- parser = OptionParser() +- +- parser.add_option("-b", "--build", action="store_true", dest="build", help="build the experiemts", default=False) +- parser.add_option("-a", "--analyze", action="store_true", dest="analyze", help="analyze the experiemt results", default=False) +- parser.add_option("-n", "--network", dest="network", help="path to the topology file", default=None) +- parser.add_option("-t", "--task", dest="task", help="comm. to produce topology", default=None) +- +- (options, args) = parser.parse_args() +- +- if options.build: +- app = buildExperiment() +- elif options.analyze: +- app = analyzeExperiment() +- elif options.network and options.task: +- app = runSim(options.network, options.task) +- else: +- parser.print_help() +- +- sys.exit('\nFinish!') +- diff --git a/src/util/rc-stack.c b/src/util/rc-stack.c index ce1b650b404c6f9d4f1fbd3da78ca10831634895..612b271bcf576cc2b7decac3ad1dcb19acee9d26 100644 --- a/src/util/rc-stack.c +++ b/src/util/rc-stack.c @@ -38,6 +38,9 @@ void rc_stack_create(struct rc_stack **s){ case OPTIMISTIC: ss->mode = RC_OPT; break; + case OPTIMISTIC_REALTIME: + ss->mode = RC_OPT; + break; case OPTIMISTIC_DEBUG: ss->mode = RC_OPT_DBG; break; diff --git a/src/workload/methods/codes-dumpi-trace-nw-wrkld.c b/src/workload/methods/codes-dumpi-trace-nw-wrkld.c index c90898d4e9b81f4f2755044639901d1a2de79bf8..dfa2315fd525557fa374ee4384f1c00d298b3ff7 100644 --- a/src/workload/methods/codes-dumpi-trace-nw-wrkld.c +++ b/src/workload/methods/codes-dumpi-trace-nw-wrkld.c @@ -23,12 +23,12 @@ #include #endif #define PROFILE_TYPE cortex_dumpi_profile* -#define UNDUMPI_OPEN cortex_undumpi_open +//#define UNDUMPI_OPEN cortex_undumpi_open #define DUMPI_START_STREAM_READ cortex_dumpi_start_stream_read #define UNDUMPI_CLOSE cortex_undumpi_close #else #define PROFILE_TYPE dumpi_profile* -#define UNDUMPI_OPEN undumpi_open +//#define UNDUMPI_OPEN undumpi_open #define DUMPI_START_STREAM_READ dumpi_start_stream_read #define UNDUMPI_CLOSE undumpi_close #endif @@ -646,7 +646,11 @@ int dumpi_trace_nw_workload_load(const char* params, int app_id, int rank) else sprintf(file_name, "%s%d.bin", dumpi_params->file_name, rank); #ifdef ENABLE_CORTEX - profile = cortex_undumpi_open(file_name, app_id, dumpi_params->num_net_traces, rank); + if(strcmp(dumpi_params->file_name,"none") == 0) { + profile = cortex_undumpi_open(NULL, app_id, dumpi_params->num_net_traces, rank); + } else { + profile = cortex_undumpi_open(file_name, app_id, dumpi_params->num_net_traces, rank); + } #else profile = undumpi_open(file_name); #endif @@ -728,7 +732,11 @@ int dumpi_trace_nw_workload_load(const char* params, int app_id, int rank) #ifdef ENABLE_CORTEX #ifdef ENABLE_CORTEX_PYTHON - libundumpi_populate_callbacks(CORTEX_PYTHON_TRANSLATION, transarr); + if(dumpi_params->cortex_script[0] != 0) { + libundumpi_populate_callbacks(CORTEX_PYTHON_TRANSLATION, transarr); + } else { + libundumpi_populate_callbacks(CORTEX_MPICH_TRANSLATION, transarr); + } #else libundumpi_populate_callbacks(CORTEX_MPICH_TRANSLATION, transarr); #endif @@ -738,7 +746,16 @@ int dumpi_trace_nw_workload_load(const char* params, int app_id, int rank) //dumpi_free_header(trace_header); #ifdef ENABLE_CORTEX_PYTHON - cortex_python_set_module(dumpi_params->cortex_script,dumpi_params->cortex_class); + if(dumpi_params->cortex_script[0] != 0) { + if(dumpi_params->cortex_class[0] != 0) { + cortex_python_set_module(dumpi_params->cortex_script, dumpi_params->cortex_class); + } else { + cortex_python_set_module(dumpi_params->cortex_script, NULL); + } + if(dumpi_params->cortex_gen[0] != 0) { + cortex_python_call_generator(profile, dumpi_params->cortex_gen); + } + } #endif int finalize_reached = 0; diff --git a/tests/Makefile.subdir b/tests/Makefile.subdir index 2cca8b2916a97301c43de8662656157f30e8b1fb..fa608fbf2fd8a2e4fd6065911cd77e9646822774 100644 --- a/tests/Makefile.subdir +++ b/tests/Makefile.subdir @@ -28,13 +28,17 @@ TESTS += tests/lp-io-test.sh \ tests/map-ctx-test.sh \ tests/modelnet-test.sh \ tests/modelnet-test-torus.sh \ + tests/modelnet-test-torus-traces.sh \ tests/modelnet-test-loggp.sh \ tests/modelnet-test-dragonfly.sh \ tests/modelnet-test-slimfly.sh \ tests/modelnet-test-dragonfly-synthetic.sh \ + tests/modelnet-test-dragonfly-traces.sh \ tests/modelnet-test-dragonfly-custom-synthetic.sh \ + tests/modelnet-test-dragonfly-custom-traces.sh \ tests/modelnet-test-fattree-synthetic.sh \ tests/modelnet-test-slimfly-synthetic.sh \ + tests/modelnet-test-slimfly-traces.sh \ tests/modelnet-p2p-bw-loggp.sh \ tests/modelnet-prio-sched-test.sh @@ -57,9 +61,16 @@ EXTRA_DIST += tests/lp-io-test.sh \ tests/expected/mapping_test.out \ tests/modelnet-test.sh \ tests/modelnet-test-torus.sh \ + tests/modelnet-test-torus-traces.sh \ tests/modelnet-test-loggp.sh \ tests/modelnet-test-dragonfly.sh \ + tests/modelnet-test-dragonfly-synthetic.sh \ + tests/modelnet-test-dragonfly-traces.sh \ + tests/modelnet-test-dragonfly-custom-synthetic.sh \ + tests/modelnet-test-dragonfly-custom-traces.sh \ tests/modelnet-test-slimfly.sh \ + tests/modelnet-test-slimfly-synthetic.sh \ + tests/modelnet-test-slimfly-traces.sh \ tests/modelnet-p2p-bw-loggp.sh \ tests/modelnet-prio-sched-test.sh \ tests/conf/concurrent_msg_recv.conf \ @@ -107,4 +118,4 @@ tests_modelnet_simplep2p_test_SOURCES = tests/modelnet-simplep2p-test.c tests_modelnet_p2p_bw_SOURCES = tests/modelnet-p2p-bw.c tests_concurrent_msg_recv_SOURCES = tests/concurrent-msg-recv.c tests_modelnet_test_collective_SOURCES = tests/modelnet-test-collective.c -tests_modelnet_prio_sched_test_SOURCES = tests/modelnet-prio-sched-test.c +tests_modelnet_prio_sched_test_SOURCES = tests/modelnet-prio-sched-test.c \ No newline at end of file diff --git a/tests/modelnet-test-dragonfly-custom-synthetic.sh b/tests/modelnet-test-dragonfly-custom-synthetic.sh index 030ddd30486ab59bc980bf6e95367056df795620..e5c2e38cff2057ac9447dc6643610379a8ce0066 100755 --- a/tests/modelnet-test-dragonfly-custom-synthetic.sh +++ b/tests/modelnet-test-dragonfly-custom-synthetic.sh @@ -1,13 +1,7 @@ #!/bin/bash -if [ -z $srcdir ]; then - echo srcdir variable not set. - exit 1 - fi - src/network-workloads/model-net-synthetic-custom-dfly --sync=1 --num_messages=1 -- src/network-workloads/conf/dragonfly-custom/modelnet-test-dragonfly-1728-nodes.conf -mpirun -np 2 src/network-workloads/model-net-mpi-replay --sync=3 --num_net_traces=27 --workload_file=/tmp/df_AMG_n27_dumpi/dumpi-2014.03.03.14.55.00- --workload_type="dumpi" -- src/network-workloads/conf/dragonfly-custom/modelnet-test-dragonfly-1728-nodes.conf diff --git a/tests/modelnet-test-dragonfly-custom-traces.sh b/tests/modelnet-test-dragonfly-custom-traces.sh new file mode 100755 index 0000000000000000000000000000000000000000..fcb2fb1db1956b3ac9e0042e3e3bcff57a796d80 --- /dev/null +++ b/tests/modelnet-test-dragonfly-custom-traces.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +if [ -z $srcdir ]; then + echo srcdir variable not set. + exit 1 +fi + +mpirun -np 2 src/network-workloads/model-net-mpi-replay --sync=3 --num_net_traces=27 --disable_compute=1 --workload_file=/tmp/df_AMG_n27_dumpi/dumpi-2014.03.03.14.55.00- --workload_type="dumpi" -- src/network-workloads/conf/dragonfly-custom/modelnet-test-dragonfly-1728-nodes.conf diff --git a/tests/modelnet-test-dragonfly-synthetic.sh b/tests/modelnet-test-dragonfly-synthetic.sh index dbed882d6487241fab9ee013d0e7433b785544af..8859ecbfda88a62adcb5423ad096bdf8cd9d9238 100755 --- a/tests/modelnet-test-dragonfly-synthetic.sh +++ b/tests/modelnet-test-dragonfly-synthetic.sh @@ -1,14 +1,3 @@ #!/bin/bash -if [ -z $srcdir ]; then - echo srcdir variable not set. - exit 1 - fi - -source $srcdir/tests/download-traces.sh - src/network-workloads/model-net-synthetic --sync=1 --num_messages=1 -- $srcdir/src/network-workloads/conf/modelnet-synthetic-dragonfly.conf - -mpirun -np 2 src/network-workloads/model-net-mpi-replay --sync=3 --num_net_traces=27 --workload_file=/tmp/df_AMG_n27_dumpi/dumpi-2014.03.03.14.55.00- --workload_type="dumpi" -- $srcdir/src/network-workloads/conf/modelnet-mpi-test-dfly-amg-216.conf - - diff --git a/tests/modelnet-test-dragonfly-traces.sh b/tests/modelnet-test-dragonfly-traces.sh new file mode 100755 index 0000000000000000000000000000000000000000..cce52bd7d75c3651bddfe07f971fe0796d1ab0d3 --- /dev/null +++ b/tests/modelnet-test-dragonfly-traces.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +if [ -z $srcdir ]; then + echo srcdir variable not set. + exit 1 + fi + +source $srcdir/tests/download-traces.sh + +mpirun -np 2 src/network-workloads/model-net-mpi-replay --disable_compute=1 --sync=3 --num_net_traces=27 --workload_file=/tmp/df_AMG_n27_dumpi/dumpi-2014.03.03.14.55.00- --workload_type="dumpi" -- $srcdir/src/network-workloads/conf/modelnet-mpi-test-dfly-amg-216.conf + + diff --git a/tests/modelnet-test-slimfly-synthetic.sh b/tests/modelnet-test-slimfly-synthetic.sh index 391c2d1d660962735f2a9d0b436a440e93d3aa08..a610e63a569d6bc00903a260660a37fc48f18275 100755 --- a/tests/modelnet-test-slimfly-synthetic.sh +++ b/tests/modelnet-test-slimfly-synthetic.sh @@ -1,13 +1,4 @@ #!/bin/bash -if [ -z $srcdir ]; then - echo srcdir variable not set. - exit 1 - fi - -source $srcdir/tests/download-traces.sh - src/network-workloads/model-net-synthetic-slimfly --sync=1 -- $srcdir/src/network-workloads/conf/modelnet-synthetic-slimfly-min.conf -mpirun -np 2 src/network-workloads/model-net-mpi-replay --sync=3 --num_net_traces=27 --workload_file=/tmp/df_AMG_n27_dumpi/dumpi-2014.03.03.14.55.00- --workload_type="dumpi" -- $srcdir/src/network-workloads/conf/modelnet-mpi-test-slimfly-min.conf - diff --git a/tests/modelnet-test-slimfly-traces.sh b/tests/modelnet-test-slimfly-traces.sh new file mode 100755 index 0000000000000000000000000000000000000000..0809325aa844765178046bc8809b9a9e561ee281 --- /dev/null +++ b/tests/modelnet-test-slimfly-traces.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +if [ -z $srcdir ]; then + echo srcdir variable not set. + exit 1 + fi + +source $srcdir/tests/download-traces.sh + +mpirun -np 2 src/network-workloads/model-net-mpi-replay --disable_compute=1 --sync=3 --num_net_traces=27 --workload_file=/tmp/df_AMG_n27_dumpi/dumpi-2014.03.03.14.55.00- --workload_type="dumpi" -- $srcdir/src/network-workloads/conf/modelnet-mpi-test-slimfly-min.conf + diff --git a/tests/modelnet-test-torus-traces.sh b/tests/modelnet-test-torus-traces.sh new file mode 100755 index 0000000000000000000000000000000000000000..73926f7a37dfd58fc4b701906dd280946711d990 --- /dev/null +++ b/tests/modelnet-test-torus-traces.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +if [ -z $srcdir ]; then + echo srcdir variable not set. + exit 1 + fi + +source $srcdir/tests/download-traces.sh + +mpirun -np 2 src/network-workloads/model-net-mpi-replay --sync=3 --disable_compute=1 --num_net_traces=27 --workload_file=/tmp/df_AMG_n27_dumpi/dumpi-2014.03.03.14.55.00- --workload_type="dumpi" -- $srcdir/src/network-workloads/conf/modelnet-mpi-test-torus.conf + diff --git a/tests/modelnet-test-torus.sh b/tests/modelnet-test-torus.sh index e85d49155b481092fa8517f74f521afec92b205c..942a881efb24179cf365da51499747b172948631 100755 --- a/tests/modelnet-test-torus.sh +++ b/tests/modelnet-test-torus.sh @@ -1,13 +1,4 @@ #!/bin/bash -if [ -z $srcdir ]; then - echo srcdir variable not set. - exit 1 - fi - -source $srcdir/tests/download-traces.sh - tests/modelnet-test --sync=1 -- tests/conf/modelnet-test-torus.conf -mpirun -np 2 src/network-workloads/model-net-mpi-replay --sync=3 --num_net_traces=27 --workload_file=/tmp/df_AMG_n27_dumpi/dumpi-2014.03.03.14.55.00- --workload_type="dumpi" -- $srcdir/src/network-workloads/conf/modelnet-mpi-test-torus.conf -