Merge branch 'master' of xgitlab.cels.anl.gov:codes/codes into cortex

README-vis.md

+## README for using ROSS instrumentation in CODES
+
+For details about the ROSS instrumentation, see the [ROSS Instrumentation blog post](http://carothersc.github.io/ROSS/feature/instrumentation.html) on the ROSS webpage.
+The instrumentation will be merged into the master branch of the ROSS repo very soon.  
+
+There are currently 3 types of instrumentation: GVT-based, real time, and event tracing.  See the ROSS documentation for more info on
+the specific options or use `--help` with your model.  The GVT-based and real time sampling do not require any changes to your model code.
+The event tracing will run without any changes, but some additions to the model code is needed in order to get specific model event types.  
+This document describes how to do it.
+
+### Register LP event tracing function
+
+The examples here are based on the server LP for the synthetic workload generation for dragonfly (`src/network-workloads/model-net-synthetic.c`).
+
+As described in the ROSS Vis documentation, we need to first add our function that will save the event type (and any other desired data) to the
+buffer location provided by ROSS.
+```C
+void svr_event_collect(svr_msg *m, tw_lp *lp, char *buffer)
+{
+    int type = (int) m->svr_event_type;
+    memcpy(buffer, &type, sizeof(type));
+}
+```
+
+Then we need to create a `st_trace_type` struct with the pointer and size information.
+```C
+st_trace_type svr_trace_types[] = {
+    {(rbev_trace_f) svr_event_collect,
+    sizeof(int),
+    (ev_trace_f) svr_event_collect,
+    sizeof(int)},
+    {0}
+}
+```
+
+And a function to return this struct
+```C
+static const st_trace_type *svr_get_trace_types(void)
+{
+    return(&svr_trace_types[0]);
+}
+```
+
+As a reminder, there are two types of event tracing the full event trace (`ev_trace_f`) or only events that trigger rollbacks (`rbev_trace_f`).  
+It's set up so that you can have different functions for both types of event tracing, or you can use the same function for both.
+Immediately after each function pointer is a `size_t` type that takes the amount of data that the function will be placing in the buffer,
+so ROSS can appropriately handle things.
+
+If you have multiple LPs, you can do a `st_trace_type` for each LP, or you can reuse.  *Note*: You can only reuse `st_trace_type` and the event type collection
+function for LPs that use the same type of message struct.  For example, the dragonfly terminal and router LPs both use the `terminal_message` struct, so they can
+use the same functions for event tracing.  However the model net base LP uses the `model_net_wrap_msg` struct, so it gets its own event collection function and 
+`st_trace_type` struct, in order to read the event type correctly from the model.  
+
+`codes_mapping_init()` was changed to register the function pointers when it is setting up the LP types.  So for CODES models, you need to add a register function:
+```C
+void svr_register_trace()
+{
+    trace_type_register("server", svr_get_trace_types());
+}
+```
+
+`trace_type_register(const char* name, const st_trace_type* type)` is part of the API and lets CODES know the pointers for LP initialization.
+
+Now in the main function, you call the register function *before* calling `codes_mapping_setup()`.
+```C
+if (g_st_ev_trace)
+    svr_register_trace();
+```
+
+`g_st_ev_trace` is a ROSS flag for determining if event tracing is turned on.  
+
+That's all you need to add for each LP.  
+
+### Model Net LPs
+In addition to the dragonfly synthetic server LP, I've already added in the necessary changes for both the model net base LP type and dragonfly (both router and terminal LPs),
+so no other changes need to be made to those LPs. (Unless you want to collect some additional data.) 
+For any other network LPs that are based on the model net base LP type, there are a few additional details to know.
+There are two fields added to the `model_net_method` struct for pointers to the trace registration functions for each LP.
+
+```C
+void (*mn_trace_register)(st_trace_type *base_type);
+const st_trace_type* (*mn_get_trace_type)();
+```
+
+For example, right now, both the dragonfly router and terminal LPs use the same `st_trace_type dragonfly_trace_types` struct and the following function to return its pointer:
+```C
+static const st_trace_type *dragonfly_get_trace_types(void)
+{
+    return(&dragonfly_trace_types[0]);
+}
+```
+
+They have different register functions:
+```C
+static void dragonfly_register_trace(st_trace_type *base_type)
+{
+    trace_type_register(LP_CONFIG_NM_TERM, base_type);
+}
+
+static void router_register_trace(st_trace_type *base_type)
+{
+    trace_type_register(LP_CONFIG_NM_ROUT, base_type);
+}
+```
+
+And then the following additions to their `model_net_method` structs:
+```C
+struct model_net_method dragonfly_method =
+{
+    // the fields already in the struct
+    ...
+    // event tracing additions
+    .mn_trace_register = dragonfly_register_trace,
+    .mn_get_trace_type = dragonfly_get_trace_types,
+};
+
+struct model_net_method dragonfly_router_method =
+{
+    // the fields already in the struct
+    ...
+    // event tracing additions
+    .mn_trace_register = router_register_trace,
+    .mn_get_trace_type = dragonfly_get_trace_types,
+};
+```
+
+Any other LPs built off of the model net LP, can be changed in the same way.

codes/lp-type-lookup.h

@@ -21,6 +21,8 @@ const tw_lptype* lp_type_lookup(const char* name);
 /* register an LP with CODES/ROSS */
 void lp_type_register(const char* name, const tw_lptype* type);
 
+void trace_type_register(const char* name, const st_trace_type* type);
+const st_trace_type* trace_type_lookup(const char* name);
 #ifdef __cplusplus
 }
 #endif

codes/model-net-method.h

@@ -63,6 +63,8 @@ struct model_net_method
     revent_f mn_sample_rc_fn;
     init_f mn_sample_init_fn;
     final_f mn_sample_fini_fn;
+    void (*mn_trace_register)(st_trace_type *base_type);
+    const st_trace_type* (*mn_get_trace_type)();
 };
 
 extern struct model_net_method * method_array[];

codes/net/dragonfly.h

@@ -13,6 +13,9 @@ extern "C" {
 
 #include <ross.h>
 
+extern void dragonfly_register_evcol();
+extern void router_register_evcol();
+
 typedef struct terminal_message terminal_message;
 
 /* this message is used for both dragonfly compute nodes and routers */

configure.ac

@@ -197,3 +197,5 @@ fi
 AC_CONFIG_FILES([Makefile])
 
 AC_OUTPUT([maint/codes.pc])
+AC_OUTPUT([src/network-workloads/conf/dragonfly-custom/modelnet-test-dragonfly-1728-nodes.conf])
+

src/network-workloads/conf/dragonfly-custom/modelnet-test-dragonfly-1728-nodes.conf.in

+LPGROUPS
+{
+   MODELNET_GRP
+   {
+      repetitions="864";
+# name of this lp changes according to the model
+      nw-lp="2";
+# these lp names will be the same for dragonfly-custom model
+      modelnet_dragonfly_custom="2";
+      modelnet_dragonfly_custom_router="1";
+   }
+}
+PARAMS
+{
+# packet size in the network 
+   packet_size="1024";
+   modelnet_order=( "dragonfly_custom","dragonfly_custom_router" );
+   # scheduler options
+   modelnet_scheduler="fcfs";
+# chunk size in the network (when chunk size = packet size, packets will not be
+# divided into chunks)
+   chunk_size="1024";
+   # modelnet_scheduler="round-robin";
+   # number of routers within each group
+   # this is dictated by the dragonfly configuration files
+   num_router_rows="6";
+   # number of router columns 
+   num_router_cols="16";
+   # number of groups in the network
+   num_groups="9";
+# buffer size in bytes for local virtual channels 
+   local_vc_size="8192";
+#buffer size in bytes for global virtual channels 
+   global_vc_size="16384";
+#buffer size in bytes for compute node virtual channels 
+   cn_vc_size="8192";
+#bandwidth in GiB/s for local channels 
+   local_bandwidth="5.25";
+# bandwidth in GiB/s for global channels 
+   global_bandwidth="18.75";
+# bandwidth in GiB/s for compute node-router channels 
+   cn_bandwidth="16.0";
+# ROSS message size 
+   message_size="592";
+# number of compute nodes connected to router, dictated by dragonfly config
+# file
+   num_cns_per_router="2";
+# number of global channels per router 
+   num_global_channels="10";
+# network config file for intra-group connections 
+   intra-group-connections="@abs_srcdir@/intra-theta";
+# network config file for inter-group connections
+   inter-group-connections="@abs_srcdir@/inter-theta";
+# routing protocol to be used 
+   routing="adaptive";
+}

src/network-workloads/conf/dragonfly-custom/modelnet-test-dragonfly-custom-768-nodes.conf.in

+LPGROUPS
+{
+   MODELNET_GRP
+   {
+      repetitions="768";
+# name of this lp changes according to the model
+      nw-lp="2";
+# these lp names will be the same for dragonfly-custom model
+      modelnet_dragonfly_custom="2";
+      modelnet_dragonfly_custom_router="1";
+   }
+}
+PARAMS
+{
+# packet size in the network 
+   packet_size="1024";
+   modelnet_order=( "dragonfly_custom","dragonfly_custom_router" );
+   # scheduler options
+   modelnet_scheduler="fcfs";
+# chunk size in the network (when chunk size = packet size, packets will not be
+# divided into chunks)
+   chunk_size="1024";
+   # modelnet_scheduler="round-robin";
+   # number of routers within each group
+   # this is dictated by the dragonfly configuration files
+   # intra-group rows for routers
+   num_router_rows="16";
+   # intra-group columns for routers
+   num_router_cols="6";
+   # number of groups in the network
+   num_groups="8";
+# buffer size in bytes for local virtual channels 
+   local_vc_size="8192";
+#buffer size in bytes for global virtual channels 
+   global_vc_size="16384";
+#buffer size in bytes for compute node virtual channels 
+   cn_vc_size="8192";
+#bandwidth in GiB/s for local channels 
+   local_bandwidth="5.25";
+# bandwidth in GiB/s for global channels 
+   global_bandwidth="18.75";
+# bandwidth in GiB/s for compute node-router channels 
+   cn_bandwidth="8.0";
+# ROSS message size 
+   message_size="592";
+# number of compute nodes connected to router, dictated by dragonfly config
+# file
+   num_cns_per_router="2";
+# number of global channels per router 
+   num_global_channels="10";
+# network config file for intra-group connections 
+   intra-group-connections="@abs_srcdir@/intra-custom-small";
+# network config file for inter-group connections
+   inter-group-connections="@abs_srcdir@/inter-custom-small";
+# routing protocol to be used 
+   routing="prog-adaptive";
+}

src/network-workloads/conf/modelnet-mpi-test-fattree.conf

+LPGROUPS
+{
+   MODELNET_GRP
+   {
+      repetitions="32";     # repetitions = Ne = total # of edge switches. For type0 Ne = Np*Ns = ceil(N/Ns*(k/2))*(k/2) = ceil(N/(k/2)^2)*(k/2)
+      nw-lp="4";
+      modelnet_fattree="4";
+      fattree_switch="3";
+   }
+}
+PARAMS
+{
+   ft_type="0";
+   packet_size="512";
+   message_size="592";
+   chunk_size="512";
+   modelnet_scheduler="fcfs";
+   #modelnet_scheduler="round-robin";
+   modelnet_order=( "fattree" );
+   num_levels="3";
+   switch_count="32";       # = repititions
+   switch_radix="8";
+   router_delay="90";
+   terminal_radix="1";
+   soft_delay="1000";
+   vc_size="65536";
+   cn_vc_size="65536";
+   link_bandwidth="12.5";
+   cn_bandwidth="12.5";
+   routing="adaptive";
+   rail_routing="adaptive";
+}

src/network-workloads/conf/modelnet-mpi-test-slimfly-min.conf

+LPGROUPS
+{
+   MODELNET_GRP
+   {
+      repetitions="50";
+      nw-lp="3";
+      modelnet_slimfly="3";
+      slimfly_router="1";
+   }
+}
+PARAMS
+{
+   packet_size="256";
+   modelnet_order=( "slimfly" );
+   # scheduler options
+   modelnet_scheduler="fcfs";
+   chunk_size="256";
+   # modelnet_scheduler="round-robin";
+   num_vcs="4";
+   num_routers="5";
+   num_terminals="3";
+   global_channels="5";
+   local_channels="2";
+   generator_set_X=("1","4");				#   : Subgraph 0 generator set
+   generator_set_X_prime=("2","3");			#   : Subgraph 1 generator set
+   local_vc_size="25600";
+   global_vc_size="25600";
+   cn_vc_size="25600";
+   local_bandwidth="9.0";
+   global_bandwidth="9.0";
+   cn_bandwidth="9.0";
+   router_delay="0";
+   link_delay="0";
+   message_size="592";
+   routing="minimal";
+}

src/network-workloads/conf/modelnet-mpi-test-torus.conf

@@ -10,7 +10,7 @@ LPGROUPS
 PARAMS
 {
    packet_size="512";
-   message_size="560";
+   message_size="592";
    modelnet_order=( "torus" );
    # scheduler options
    modelnet_scheduler="fcfs";

src/network-workloads/model-net-mpi-replay.c

@@ -604,8 +604,9 @@ static int rm_matching_rcv(nw_state * ns,
 
     qlist_for_each(ent, &ns->pending_recvs_queue){
         qi = qlist_entry(ent, mpi_msgs_queue, ql);
-        if((qi->num_bytes == qitem->num_bytes)
-                && ((qi->tag == qitem->tag) || qi->tag == -1)
+        if(//(qi->num_bytes == qitem->num_bytes)
+                //&& 
+                ((qi->tag == qitem->tag) || qi->tag == -1)
                 && ((qi->source_rank == qitem->source_rank) || qi->source_rank == -1))
         {
             matched = 1;
@@ -1443,9 +1444,11 @@ void nw_test_finalize(nw_state* s, tw_lp* lp)
 		int count_irecv = qlist_count(&s->pending_recvs_queue);
         int count_isend = qlist_count(&s->arrival_queue);
 		if(count_irecv || count_isend)
-        printf("\n LP %llu unmatched irecvs %d unmatched sends %d Total sends %ld receives %ld collectives %ld delays %ld wait alls %ld waits %ld send time %lf wait %lf",
-			lp->gid, count_irecv, count_isend, s->num_sends, s->num_recvs, s->num_cols, s->num_delays, s->num_waitall, s->num_wait, s->send_time, s->wait_time);
-
+        {
+            printf("\n LP %llu unmatched irecvs %d unmatched sends %d Total sends %ld receives %ld collectives %ld delays %ld wait alls %ld waits %ld send time %lf wait %lf",
+			    lp->gid, count_irecv, count_isend, s->num_sends, s->num_recvs, s->num_cols, s->num_delays, s->num_waitall, s->num_wait, s->send_time, s->wait_time);
+            tw_error(TW_LOC, "\n Unmatched send and receive, terminating simulation");
+        }
         written += sprintf(s->output_buf + written, "\n %llu %llu %ld %ld %ld %ld %lf %lf %lf", lp->gid, s->nw_id, s->num_sends, s->num_recvs, s->num_bytes_sent,
                 s->num_bytes_recvd, s->send_time, s->elapsed_time - s->compute_time, s->compute_time);
         lp_io_write(lp->gid, "mpi-replay-stats", written, s->output_buf);

src/network-workloads/model-net-synthetic.c

@@ -15,6 +15,7 @@
 #include "codes/codes_mapping.h"
 #include "codes/configuration.h"
 #include "codes/lp-type-lookup.h"
+#include "codes/net/dragonfly.h"
 
 #define PAYLOAD_SZ 2048
 
@@ -105,6 +106,34 @@ tw_lptype svr_lp = {
     sizeof(svr_state),
 };
 
+/* setup for the ROSS event tracing
+ * can have a different function for  rbev_trace_f and ev_trace_f
+ * but right now it is set to the same function for both
+ */
+void svr_event_collect(svr_msg *m, tw_lp *lp, char *buffer)
+{
+    int type = (int) m->svr_event_type;
+    memcpy(buffer, &type, sizeof(type));
+}
+
+st_trace_type svr_trace_types[] = {
+    {(rbev_trace_f) svr_event_collect,
+     sizeof(int),
+     (ev_trace_f) svr_event_collect,
+     sizeof(int)},
+    {0}
+};
+
+static const st_trace_type  *svr_get_trace_types(void)
+{
+    return(&svr_trace_types[0]);
+}
+
+void svr_register_trace()
+{
+    trace_type_register("server", svr_get_trace_types());
+}
+
 const tw_optdef app_opt [] =
 {
         TWOPT_GROUP("Model net synthetic traffic " ),
@@ -364,6 +393,9 @@ int main(
     model_net_register();
     svr_add_lp_type();
 
+    if (g_st_ev_trace)
+        svr_register_trace();
+
     codes_mapping_setup();
 
     net_ids = model_net_configure(&num_nets);

src/networks/model-net/dragonfly-custom.C

@@ -22,7 +22,7 @@
 
 #define DUMP_CONNECTIONS 0
 #define CREDIT_SIZE 8
-#define DFLY_HASH_TABLE_SIZE 5000
+#define DFLY_HASH_TABLE_SIZE 4999
 
 // debugging parameters
 #define TRACK -1

src/networks/model-net/dragonfly.c

@@ -33,7 +33,7 @@
 #define COLLECTIVE_COMPUTATION_DELAY 5700
 #define DRAGONFLY_FAN_OUT_DELAY 20.0
 #define WINDOW_LENGTH 0
-#define DFLY_HASH_TABLE_SIZE 5000
+#define DFLY_HASH_TABLE_SIZE 4999
 
 // debugging parameters
 #define TRACK -1
@@ -3264,6 +3264,37 @@ tw_lptype dragonfly_lps[] =
    {NULL, NULL, NULL, NULL, NULL, NULL, NULL, 0},
 };
 
+/* For ROSS event tracing */
+void dragonfly_event_collect(terminal_message *m, tw_lp *lp, char *buffer)
+{
+    int type = (int) m->type;
+    memcpy(buffer, &type, sizeof(type));
+}
+
+st_trace_type dragonfly_trace_types[] = {
+    {(rbev_trace_f) dragonfly_event_collect,
+     sizeof(int),
+     (ev_trace_f) dragonfly_event_collect,
+     sizeof(int)},
+    {0}
+};
+
+static const st_trace_type  *dragonfly_get_trace_types(void)
+{
+    return(&dragonfly_trace_types[0]);
+}
+
+static void dragonfly_register_trace(st_trace_type *base_type)
+{
+    trace_type_register(LP_CONFIG_NM_TERM, base_type);
+}
+
+static void router_register_trace(st_trace_type *base_type)
+{
+    trace_type_register(LP_CONFIG_NM_ROUT, base_type);
+}
+/*** END of ROSS event tracing additions */
+
 /* returns the dragonfly lp type for lp registration */
 static const tw_lptype* dragonfly_get_cn_lp_type(void)
 {
@@ -3299,7 +3330,9 @@ struct model_net_method dragonfly_method =
     .mn_sample_fn = (void*)dragonfly_sample_fn,    
     .mn_sample_rc_fn = (void*)dragonfly_sample_rc_fn,
     .mn_sample_init_fn = (void*)dragonfly_sample_init,
-    .mn_sample_fini_fn = (void*)dragonfly_sample_fin
+    .mn_sample_fini_fn = (void*)dragonfly_sample_fin,
+    .mn_trace_register = dragonfly_register_trace,
+    .mn_get_trace_type = dragonfly_get_trace_types,
 };
 
 struct model_net_method dragonfly_router_method =
@@ -3318,5 +3351,7 @@ struct model_net_method dragonfly_router_method =
     .mn_sample_fn = (void*)dragonfly_rsample_fn,
     .mn_sample_rc_fn = (void*)dragonfly_rsample_rc_fn,
     .mn_sample_init_fn = (void*)dragonfly_rsample_init,
-    .mn_sample_fini_fn = (void*)dragonfly_rsample_fin
+    .mn_sample_fini_fn = (void*)dragonfly_rsample_fin,
+    .mn_trace_register = router_register_trace,
+    .mn_get_trace_type = dragonfly_get_trace_types,
 };