Merge branch 'merged-branch-v1' into 'master'

Merged branch v1

See merge request !30

Makefile.am

@@ -60,7 +60,6 @@ endif
 if USE_DUMPI
 AM_CPPFLAGS += ${DUMPI_CFLAGS} -DUSE_DUMPI=1
 src_libcodes_la_SOURCES += src/workload/methods/codes-dumpi-trace-nw-wrkld.c
-LDADD += ${DUMPI_LIBS}
 if USE_CORTEX
 if USE_PYTHON
 if USE_CORTEX_PYTHON
@@ -73,5 +72,6 @@ endif
 AM_CPPFLAGS += ${CORTEX_CFLAGS} -DENABLE_CORTEX=1
 LDADD += ${CORTEX_LIBS}
 endif
+LDADD += ${DUMPI_LIBS}
 endif
 

codes/codes-mpi-replay.h

+/*
+ * Copyright (C) 2017 University of Chicago.
+ * See COPYRIGHT notice in top-level directory.
+ *
+ */
+
+
+#ifndef CODES_MPI_REPLAY_H
+#define CODES_MPI_REPLAY_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <mpi.h>
+
+int modelnet_mpi_replay(MPI_Comm comm, int* argc, char*** argv );
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* CODES_H */
+
+/*
+ * Local variables:
+ *  c-indent-level: 4
+ *  c-basic-offset: 4
+ * End:
+ *
+ * vim: ft=c ts=8 sts=4 sw=4 expandtab
+ */

codes/codes-workload.h

@@ -194,7 +194,7 @@ struct codes_workload_op
             /* TODO: not sure why source rank is here */
             int source_rank;/* source rank of MPI send message */
             int dest_rank; /* dest rank of MPI send message */
-            int64_t num_bytes; /* number of bytes to be transferred over the network */
+            uint64_t num_bytes; /* number of bytes to be transferred over the network */
             int16_t data_type; /* MPI data type to be matched with the recv */
             int count; /* number of elements to be received */
             int tag; /* tag of the message */
@@ -204,7 +204,7 @@ struct codes_workload_op
             /* TODO: not sure why source rank is here */
             int source_rank;/* source rank of MPI recv message */
             int dest_rank;/* dest rank of MPI recv message */
-            int num_bytes; /* number of bytes to be transferred over the network */
+            uint64_t num_bytes; /* number of bytes to be transferred over the network */
             int16_t data_type; /* MPI data type to be matched with the send */
             int count; /* number of elements to be sent */
             int tag; /* tag of the message */

codes/codes.h

@@ -14,6 +14,9 @@ extern "C" {
 
 #include <ross.h>
 #include <assert.h>
+#include <mpi.h>
+
+extern MPI_Comm MPI_COMM_CODES;
 
 // for printf conversions: shorthand for cast to long long unsigned format (llu)
 #define LLU(x) ((unsigned long long)(x))

codes/lp-type-lookup.h

@@ -21,8 +21,8 @@ const tw_lptype* lp_type_lookup(const char* name);
 /* register an LP with CODES/ROSS */
 void lp_type_register(const char* name, const tw_lptype* type);
 
-void trace_type_register(const char* name, const st_trace_type* type);
-const st_trace_type* trace_type_lookup(const char* name);
+void st_model_type_register(const char* name, const st_model_types* type);
+const st_model_types* st_model_type_lookup(const char* name);
 #ifdef __cplusplus
 }
 #endif

codes/model-net-method.h

@@ -63,8 +63,8 @@ struct model_net_method
     revent_f mn_sample_rc_fn;
     init_f mn_sample_init_fn;
     final_f mn_sample_fini_fn;
-    void (*mn_trace_register)(st_trace_type *base_type);
-    const st_trace_type* (*mn_get_trace_type)();
+    void (*mn_model_stat_register)(st_model_types *base_type);
+    const st_model_types* (*mn_get_model_stat_types)();
 };
 
 extern struct model_net_method * method_array[];

codes/model-net.h

@@ -18,6 +18,11 @@ extern "C" {
 #include <codes/codes-mapping-context.h>
 #include <stdint.h>
 
+#ifdef ENABLE_CORTEX
+#include <cortex/cortex.h>
+#include <cortex/topology.h>
+#endif
+
 #define PULL_MSG_SIZE 128
 
 #define MAX_NAME_LENGTH 256
@@ -388,6 +393,7 @@ void model_net_set_msg_param(
 
 /* returns pointer to LP information for simplenet module */
 const tw_lptype* model_net_get_lp_type(int net_id);
+const st_model_types* model_net_get_model_stat_type(int net_id);
 
 DEPRECATED
 uint64_t model_net_get_packet_size(int net_id);
@@ -406,6 +412,11 @@ void model_net_print_stats(tw_lpid lpid, mn_stats mn_stats_array[]);
 /* find model-net statistics */
 mn_stats* model_net_find_stats(char const * category, mn_stats mn_stats_array[]);
 
+#ifdef ENABLE_CORTEX
+/* structure that gives access to the topology functions */
+extern cortex_topology model_net_topology;
+#endif
+
 #ifdef __cplusplus
 }
 #endif

codes/net/dragonfly.h

@@ -91,6 +91,8 @@ struct terminal_message
    tw_stime saved_hist_start_time;
    tw_stime saved_sample_time;
    tw_stime msg_start_time;
+   tw_stime saved_busy_time_ross;
+   tw_stime saved_fin_chunks_ross;
 
    int saved_hist_num;
    int saved_occupancy;

codes/net/fattree.h

@@ -3,6 +3,9 @@
 
 #include <ross.h>
 
+/* Functions used for ROSS event tracing */
+extern void fattree_register_evtrace();
+
 /* Global variable for modelnet output directory name */
 extern char *modelnet_stats_dir;
 

src/Makefile.subdir

-lib_LTLIBRARIES += src/libcodes.la
+lib_LTLIBRARIES += src/libcodes.la src/libcodes_mpi_replay.la
 
 BUILT_SOURCES += src/modelconfig/configparser.h \
                  src/modelconfig/configlex.h
@@ -95,6 +95,7 @@ nobase_include_HEADERS = \
 	codes/net/simplenet-upd.h \
 	codes/net/simplep2p.h \
 	codes/net/torus.h \
+  codes/codes-mpi-replay.h \
 	codes/configfile.h
 
 
@@ -139,6 +140,7 @@ src_libcodes_la_SOURCES =  \
 	src/util/jobmap-impl/jobmap-list.c\
 	src/util/jobmap-impl/jobmap-identity.c\
 	src/util/codes-mapping-context.c \
+  src/util/codes-comm.c \
     src/workload/codes-workload.c \
     src/workload/methods/codes-iolang-wrkld.c \
     src/workload/methods/codes-checkpoint-wrkld.c \
@@ -160,6 +162,8 @@ src_libcodes_la_SOURCES =  \
 	src/networks/model-net/model-net-sched-impl.h \
 	src/networks/model-net/model-net-sched-impl.c 
 
+src_libcodes_mpi_replay_la_SOURCES = \
+  src/network-workloads/model-net-mpi-replay.c
 
 #codes/codes-nw-workload.h
 #src/network-workload/codes-nw-workload.c
@@ -175,6 +179,7 @@ src_util_templates_lp_template_dummy_main_SOURCES = \
 # get rid of annoying unused function in template
 
 bin_PROGRAMS += src/workload/codes-workload-dump
+bin_PROGRAMS += src/networks/model-net/topology-test
 bin_PROGRAMS += src/network-workloads/model-net-mpi-replay
 bin_PROGRAMS += src/network-workloads/model-net-dumpi-traces-dump
 bin_PROGRAMS += src/network-workloads/model-net-synthetic
@@ -187,10 +192,13 @@ src_workload_codes_workload_dump_SOURCES = \
 
 src_network_workloads_model_net_dumpi_traces_dump_SOURCES = src/network-workloads/model-net-dumpi-traces-dump.c 
 src_network_workloads_model_net_synthetic_slimfly_SOURCES = src/network-workloads/model-net-synthetic-slimfly.c
-src_network_workloads_model_net_mpi_replay_SOURCES = src/network-workloads/model-net-mpi-replay.c 
+src_network_workloads_model_net_mpi_replay_SOURCES = \
+	src/network-workloads/model-net-mpi-replay.c \
+  src/network-workloads/model-net-mpi-replay-main.c
+src_network_workloads_model_net_mpi_replay_CFLAGS = $(AM_CFLAGS)
 src_network_workloads_model_net_synthetic_SOURCES = src/network-workloads/model-net-synthetic.c 
 src_network_workloads_model_net_synthetic_custom_dfly_SOURCES = src/network-workloads/model-net-synthetic-custom-dfly.c 
-
+src_networks_model_net_topology_test_SOURCES = src/networks/model-net/topology-test.c
 
 #bin_PROGRAMS += src/network-workload/codes-nw-test
 

src/network-workloads/conf/dragonfly-custom/modelnet-test-dragonfly-1728-nodes.conf.in

@@ -41,7 +41,7 @@ PARAMS
 # bandwidth in GiB/s for compute node-router channels 
    cn_bandwidth="16.0";
 # ROSS message size 
-   message_size="592";
+   message_size="608";
 # number of compute nodes connected to router, dictated by dragonfly config
 # file
    num_cns_per_router="2";

src/network-workloads/conf/dragonfly-custom/modelnet-test-dragonfly-custom.conf

@@ -46,7 +46,7 @@ PARAMS
 # Number of column channels
    num_col_chans="1";
 # ROSS message size 
-   message_size="592";
+   message_size="608";
 # number of compute nodes connected to router, dictated by dragonfly config
 # file
    num_cns_per_router="8";

src/network-workloads/conf/dragonfly-custom/modelnet-test-dragonfly-edison.conf

@@ -41,7 +41,7 @@ PARAMS
 # bandwidth in GiB/s for compute node-router channels 
    cn_bandwidth="8.0";
 # ROSS message size 
-   message_size="592";
+   message_size="608";
 # number of compute nodes connected to router, dictated by dragonfly config
 # file
    num_cns_per_router="4";

src/network-workloads/conf/dragonfly-custom/modelnet-test-dragonfly-theta.conf

@@ -41,7 +41,7 @@ PARAMS
 # bandwidth in GiB/s for compute node-router channels 
    cn_bandwidth="16.0";
 # ROSS message size 
-   message_size="592";
+   message_size="608";
 # number of compute nodes connected to router, dictated by dragonfly config
 # file
    num_cns_per_router="4";

src/network-workloads/conf/modelnet-mpi-test-dfly-amg-216.conf

@@ -23,6 +23,6 @@ PARAMS
    local_bandwidth="5.25";
    global_bandwidth="4.7";
    cn_bandwidth="5.25";
-   message_size="592";
+   message_size="608";
    routing="adaptive";
 }

src/network-workloads/conf/modelnet-mpi-test-dragonfly.conf

@@ -23,7 +23,7 @@ PARAMS
    local_bandwidth="5.25";
    global_bandwidth="4.7";
    cn_bandwidth="5.25";
-   message_size="592";
+   message_size="608";
    routing="minimal";
    self_msg_overhead = "10";
 }

src/network-workloads/conf/modelnet-mpi-test-slimfly-min.conf

@@ -31,6 +31,6 @@ PARAMS
    cn_bandwidth="9.0";
    router_delay="0";
    link_delay="0";
-   message_size="592";
+   message_size="608";
    routing="minimal";
 }

src/network-workloads/conf/modelnet-mpi-test-torus.conf

@@ -10,7 +10,7 @@ LPGROUPS
 PARAMS
 {
    packet_size="512";
-   message_size="592";
+   message_size="608";
    modelnet_order=( "torus" );
    # scheduler options
    modelnet_scheduler="fcfs";

src/network-workloads/model-net-dumpi-traces-dump.c

@@ -22,9 +22,8 @@ static int wrkld_id;
 typedef struct nw_state nw_state;
 typedef struct nw_message nw_message;
 
-static int net_id = 0;
 static float noise = 5.0;
-static int num_net_lps, num_nw_lps;
+static int num_net_lps;
 long long num_bytes_sent=0;
 long long num_bytes_recvd=0;
 
@@ -43,10 +42,6 @@ long total_sends = 0;
 long total_recvs = 0;
 long total_delays = 0;
 
-/* global variables for codes mapping */
-static char lp_group_name[MAX_NAME_LENGTH], lp_type_name[MAX_NAME_LENGTH], annotation[MAX_NAME_LENGTH];
-static int mapping_grp_id, mapping_type_id, mapping_rep_id, mapping_offset;
-
 enum MPI_NW_EVENTS
 {
 	MPI_OP_GET_NEXT=1,
@@ -187,6 +182,8 @@ void nw_test_event_handler(nw_state* s, tw_bf * bf, nw_message * m, tw_lp * lp)
 
 void nw_test_event_handler_rc(nw_state* s, tw_bf * bf, nw_message * m, tw_lp * lp)
 {
+        (void)bf;
+
         codes_workload_get_next_rc(wrkld_id, 0, (int)s->nw_id, &m->op);
         if(m->op.op_type == CODES_WK_END)
                 return;
@@ -272,6 +269,8 @@ void nw_test_event_handler_rc(nw_state* s, tw_bf * bf, nw_message * m, tw_lp * l
 
 static void get_next_mpi_operation(nw_state* s, tw_bf * bf, nw_message * m, tw_lp * lp)
 {
+        (void)bf;
+
 		struct codes_workload_op mpi_op;
     		codes_workload_get_next(wrkld_id, 0, (int)s->nw_id, &mpi_op);
 		memcpy(&m->op, &mpi_op, sizeof(struct codes_workload_op));
@@ -366,7 +365,7 @@ void nw_test_finalize(nw_state* s, tw_lp* lp)
 		total_delays += s->num_delays;
 		total_collectives += s->num_cols;
 		
-		printf("\n LP %ld total sends %ld receives %ld wait_alls %ld waits %ld ", lp->gid, s->num_sends,s->num_recvs, s->num_waitall, s->num_wait);
+		printf("\n LP %llu total sends %ld receives %ld wait_alls %ld waits %ld ", lp->gid, s->num_sends,s->num_recvs, s->num_waitall, s->num_wait);
 	        avg_time += s->total_time;
 		avg_compute_time += s->compute_time;
                 avg_comm_time += (s->total_time - s->compute_time);
@@ -409,8 +408,6 @@ static void nw_add_lp_type()
 int main( int argc, char** argv )
 {
   int rank, nprocs;
-  int num_nets;
-  int* net_ids;
 
   g_tw_ts_end = s_to_ns(60*60*24*365); /* one year, in nsecs */
 
@@ -446,7 +443,6 @@ int main( int argc, char** argv )
     double total_avg_send_time;
     double total_avg_wait_time;
     double total_avg_recv_time;
-    double total_avg_col_time;
     double total_avg_comp_time;
     long overall_sends, overall_recvs, overall_waits, overall_cols;
 	
@@ -466,11 +462,10 @@ int main( int argc, char** argv )
    MPI_Reduce(&total_collectives, &overall_cols, 1, MPI_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
 
    if(!g_tw_mynode)
-	printf("\n Total bytes sent %lld recvd %lld \n avg runtime %lf \n avg comm time %lf avg compute time %lf \n avg collective time %lf avg send time %lf \n avg recv time %lf \n avg wait time %lf \n total sends %ld total recvs %ld total waits %ld total collectives %ld ", total_bytes_sent, total_bytes_recvd, 
+	printf("\n Total bytes sent %lld recvd %lld \n avg runtime %lf \n avg comm time %lf avg compute time %lf \n avg send time %lf \n avg recv time %lf \n avg wait time %lf \n total sends %ld total recvs %ld total waits %ld total collectives %ld ", total_bytes_sent, total_bytes_recvd, 
 			avg_run_time/num_net_lps,
 			avg_comm_run_time/num_net_lps,
 			total_avg_comp_time/num_net_lps,
-			total_avg_col_time/num_net_lps,
 			total_avg_send_time/num_net_lps,
 			total_avg_recv_time/num_net_lps,
 			total_avg_wait_time/num_net_lps,

src/network-workloads/model-net-mpi-replay-main.c

+/*
+ * Copyright (C) 2014 University of Chicago.
+ * See COPYRIGHT notice in top-level directory.
+ *
+ */
+#include <mpi.h>
+#include "codes/codes-mpi-replay.h"
+
+
+int main(int argc, char** argv) {
+  MPI_Init(&argc,&argv);
+  
+//  int rank, size;
+//  MPI_Comm_rank(MPI_COMM_WORLD,&rank);
+//  MPI_Comm_size(MPI_COMM_WORLD,&size);
+
+//  MPI_Comm comm;
+//  MPI_Comm_split(MPI_COMM_WORLD, rank < 2, rank, &comm);
+
+//  if(rank < 2)
+//  	modelnet_mpi_replay(comm,&argc,&argv);
+
+   modelnet_mpi_replay(MPI_COMM_WORLD,&argc,&argv);
+  int flag;
+  MPI_Finalized(&flag);
+	if(!flag) MPI_Finalize();
+	return 0;
+}
+
+/*
+ * Local variables:
+ *  c-indent-level: 4
+ *  c-basic-offset: 4
+ * End:
+ *
+ * vim: ft=c ts=8 sts=4 sw=4 expandtab
+ */

src/network-workloads/model-net-synthetic-custom-dfly.c

@@ -28,7 +28,7 @@ static int num_routers_per_grp = 0;
 static int num_nodes_per_grp = 0;
 static int num_nodes_per_cn = 0;
 static int num_groups = 0;
-static int num_nodes = 0;
+static unsigned long long num_nodes = 0;
 
 static char lp_io_dir[256] = {'\0'};
 static lp_io_handle io_handle;
@@ -76,6 +76,7 @@ struct svr_msg
     enum svr_event svr_event_type;
     tw_lpid src;          /* source of this request or ack */
     int incremented_flag; /* helper for reverse computation */
+    model_net_event_return event_rc;
 };
 
 static void svr_init(
@@ -133,6 +134,7 @@ static void issue_event(
     svr_state * ns,
     tw_lp * lp)
 {
+    (void)ns;
     tw_event *e;
     svr_msg *m;
     tw_stime kickoff_time;
@@ -172,7 +174,7 @@ static void handle_kickoff_rev_event(
     if(b->c1)
         tw_rand_reverse_unif(lp->rng);
 
-    model_net_event_rc(net_id, lp, PAYLOAD_SZ);
+    model_net_event_rc2(lp, &m->event_rc);
 	ns->msg_sent_count--;
     tw_rand_reverse_unif(lp->rng);
 }
@@ -239,6 +241,9 @@ static void handle_remote_rev_event(
             svr_msg * m,
             tw_lp * lp)
 {
+        (void)b;
+        (void)m;
+        (void)lp;
         ns->msg_recvd_count--;
 }
 
@@ -248,6 +253,9 @@ static void handle_remote_event(
 	    svr_msg * m,
 	    tw_lp * lp)
 {
+        (void)b;
+        (void)m;
+        (void)lp;
 	ns->msg_recvd_count++;
 }
 
@@ -257,6 +265,9 @@ static void handle_local_rev_event(
                 svr_msg * m,
                 tw_lp * lp)
 {
+        (void)b;
+        (void)m;
+        (void)lp;
 	ns->local_recvd_count--;
 }
 
@@ -266,6 +277,9 @@ static void handle_local_event(
                 svr_msg * m,
                 tw_lp * lp)
 {
+        (void)b;
+        (void)m;
+        (void)lp;
     ns->local_recvd_count++;
 }
 /* convert ns to seconds */
@@ -358,10 +372,10 @@ int main(
             return 0;
     }
 
-    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
-    MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
+    MPI_Comm_rank(MPI_COMM_CODES, &rank);
+    MPI_Comm_size(MPI_COMM_CODES, &nprocs);
 
-    configuration_load(argv[2], MPI_COMM_WORLD, &config);
+    configuration_load(argv[2], MPI_COMM_CODES, &config);
 
     model_net_register();
     svr_add_lp_type();
@@ -401,12 +415,12 @@ int main(
     {
         do_lp_io = 1;
         int flags = lp_io_use_suffix ? LP_IO_UNIQ_SUFFIX : 0;
-        int ret = lp_io_prepare(lp_io_dir, flags, &io_handle, MPI_COMM_WORLD);
+        int ret = lp_io_prepare(lp_io_dir, flags, &io_handle, MPI_COMM_CODES);
         assert(ret == 0 || !"lp_io_prepare failure");
     }
     tw_run();
     if (do_lp_io){
-        int ret = lp_io_flush(io_handle, MPI_COMM_WORLD);
+        int ret = lp_io_flush(io_handle, MPI_COMM_CODES);
         assert(ret == 0 || !"lp_io_flush failure");