Adding dragonfly network model with codes, model-net-test case run time...

Adding dragonfly network model with codes, model-net-test case run time parameters are updated (see README_MN_TEST.txt for details)

codes/dragonfly.h

+/*
+ * * Copyright (C) 2011, University of Chicago
+ * *
+ * * See COPYRIGHT notice in top-level directory.
+ * */
+
+#ifndef INC_dragonfly_h
+#define INC_dragonfly_h
+
+#include <ross.h>
+
+#include "codes/codes_mapping.h"
+#include "codes/codes.h"
+#include "codes/model-net.h"
+#include "codes/model-net-method.h"
+
+#define CHUNK_SIZE 32.0
+#define CREDIT_SIZE 8
+#define MEAN_PROCESS 1.0
+#define MAX_NAME_LENGTH 256
+
+// debugging parameters
+#define TRACK 235221
+#define PRINT_ROUTER_TABLE 1
+#define DEBUG 1
+
+// arrival rate
+static double MEAN_INTERVAL=200.0;
+/* radix of a dragonfly router = number of global channels + number of
+ * compute node channels + number of local router channels */
+static int radix=0;
+
+/* configurable parameters, coming from the codes config file*/
+/* number of virtual channels, number of routers comes from the
+ * config file, number of compute nodes, global channels and group
+ * is calculated from these configurable parameters */
+int num_vcs, num_routers, num_cn, num_global_channels, num_groups;
+
+/* configurable parameters, global channel, local channel and 
+ * compute node bandwidth */
+double global_bandwidth, local_bandwidth, cn_bandwidth;
+
+/*configurable parameters, global virtual channel size, local
+ * virtual channel size and compute node channel size */
+int global_vc_size, local_vc_size, cn_vc_size;
+
+/* global variables for codes mapping */
+char lp_group_name[MAX_NAME_LENGTH], lp_type_name[MAX_NAME_LENGTH];
+int mapping_grp_id, mapping_type_id, mapping_rep_id, mapping_offset;
+
+/* sets up the dragonfly initial network parameters like number of dragonfly groups, 
+ * number of compute nodes, number of global channels, bandwidth etc. */
+static void dragonfly_setup(const void* net_params);
+
+/* reports dragonfly statistics like average packet delay, average number of hops
+ * traversed and maximum packet latency */
+static void dragonfly_report_stats();
+
+/* dragonfly packet event method called by modelnet, this method triggers the packet
+ * generate event of dragonfly and attached remote and local events to the last packet
+ * of the message */
+static void dragonfly_packet_event(char* category, tw_lpid final_dest_lp, int packet_size, int remote_event_size, const void* remote_event, int self_event_size, const void* self_event, tw_lp *sender, int is_last_pckt);
+
+/* returns dragonfly message size */
+static int dragonfly_get_msg_sz(void);
+
+/* reverse handler for dragonfly packet event */
+static void dragonfly_packet_event_rc(tw_lp *sender);
+
+/* returns the lp type of dragonfly compute node (terminal) */
+static const tw_lptype* dragonfly_get_cn_lp_type(void);
+
+/* returns the lp type of dragonfly router */
+static const tw_lptype* dragonfly_get_router_lp_type(void);
+
+/* data structure for dragonfly statistics */
+struct model_net_method dragonfly_method =
+{
+    .method_name = "dragonfly",
+    .mn_setup = dragonfly_setup,
+    .model_net_method_packet_event = dragonfly_packet_event,
+    .model_net_method_packet_event_rc = dragonfly_packet_event_rc,
+    .mn_get_lp_type = dragonfly_get_cn_lp_type,
+    .mn_get_msg_sz = dragonfly_get_msg_sz,
+    .mn_report_stats = dragonfly_report_stats,
+};
+
+/* handles terminal and router events like packet generate/send/receive/buffer */
+typedef enum event_t event_t;
+
+typedef struct terminal_state terminal_state;
+typedef struct terminal_message terminal_message;
+typedef struct router_state router_state;
+
+/* dragonfly compute node data structure */
+struct terminal_state
+{
+   unsigned long long packet_counter;
+
+   // Dragonfly specific parameters
+   unsigned int router_id;
+   unsigned int terminal_id;
+
+   // Each terminal will have an input and output channel with the router
+   int* vc_occupancy; // NUM_VC
+   int* output_vc_state;
+   tw_stime terminal_available_time;
+   tw_stime next_credit_available_time;
+// Terminal generate, sends and arrival T_SEND, T_ARRIVAL, T_GENERATE
+// Router-Router Intra-group sends and receives RR_LSEND, RR_LARRIVE
+// Router-Router Inter-group sends and receives RR_GSEND, RR_GARRIVE
+};
+/* terminal event type (1-4) */
+enum event_t
+{
+  T_GENERATE=1,
+  T_ARRIVE,
+  T_SEND,
+  T_BUFFER,
+  R_SEND,
+  R_ARRIVE,
+  R_BUFFER
+};
+/* status of a virtual channel can be idle, active, allocated or wait for credit */
+enum vc_status
+{
+   VC_IDLE,
+   VC_ACTIVE,
+   VC_ALLOC,
+   VC_CREDIT
+};
+
+/* whether the last hop of a packet was global, local or a terminal */
+enum last_hop
+{
+   GLOBAL,
+   LOCAL,
+   TERMINAL
+};
+
+/* three forms of routing algorithms available, adaptive routing is not
+ * accurate and fully functional in the current version as the formulas
+ * for detecting load on global channels are not very accurate */
+enum ROUTING_ALGO
+{
+   MINIMAL,
+   NON_MINIMAL,
+   ADAPTIVE
+};
+
+/* this message is used for both dragonfly compute nodes and routers */
+struct terminal_message
+{
+  /* flit travel start time*/
+  tw_stime travel_start_time;
+ /* packet ID of the flit  */
+  unsigned long long packet_ID;
+  /* event type of the flit */
+  short  type;
+  /* category: comes from codes */
+  char category[MAX_NAME_LENGTH];
+  /* final destination LP ID, this comes from codes can be a server or any other LP type*/
+  tw_lpid final_dest_gid;
+  /*sending LP ID from CODES, can be a server or any other LP type */
+  tw_lpid sender_lp;
+ /* destination terminal ID of the dragonfly */
+  unsigned int dest_terminal_id;
+  /* source terminal ID of the dragonfly */
+  unsigned int src_terminal_id;
+  /* number of hops traversed by the packet */
+  short my_N_hop;
+  /* Intermediate LP ID from which this message is coming */
+  unsigned int intm_lp_id;
+  short old_vc;
+  short saved_vc;
+  /* last hop of the message, can be a terminal, local router or global router */
+  short last_hop;
+  // For buffer message
+   short vc_index;
+   int input_chan;
+   int output_chan;
+   
+   tw_stime saved_available_time;
+   tw_stime saved_credit_time;
+
+   int intm_group_id;
+   short chunk_id;
+   int packet_size;
+   int remote_event_size_bytes;
+   int local_event_size_bytes;
+};
+
+struct router_state
+{
+   unsigned int router_id;
+   unsigned int group_id;
+  
+   int* global_channel; 
+   tw_stime* next_output_available_time;
+   tw_stime* next_credit_available_time;
+   int* vc_occupancy;
+   int* output_vc_state;
+};
+
+short routing = MINIMAL;
+
+int minimal_count, nonmin_count;
+int adaptive_threshold;
+int head_delay;
+int num_packets;
+int num_chunks;
+
+static tw_stime         dragonfly_total_time = 0;
+static tw_stime         dragonfly_max_latency = 0;
+
+int num_vc;
+int num_terminal=0, num_router=0;
+int total_routers, total_terminals, total_mpi_procs;
+
+static long long       total_hops = 0;
+static long long       N_finished_packets = 0;
+
+#endif

codes/model-net.h

@@ -34,10 +34,15 @@ struct simplenet_param
 
 struct dragonfly_param
 {
-  char* name;
   int num_routers; /*Number of routers in a group*/
-  int num_nodes; /*Number of compute nodes connected to a router*/
-  int num_chans; /*Number of global channels connected to each router*/
+  double local_bandwidth;/* bandwidth of the router-router channels within a group */
+  double global_bandwidth;/* bandwidth of the inter-group router connections */
+  double cn_bandwidth;/* bandwidth of the compute node channels connected to routers */
+  int num_vcs; /* number of virtual channels */
+  int local_vc_size; /* buffer size of the router-router channels */
+  int global_vc_size; /* buffer size of the global channels */
+  int cn_vc_size; /* buffer size of the compute node channels */
+  short routing; /* minimal or non-minimal routing */
 };
 
 struct torus_param
@@ -49,10 +54,7 @@ struct torus_param
   int num_vc; /* number of virtual channels for each torus link */
   float mean_process;/* mean process time for each flit  */
 };
-/* NOTE: the following auxilliary functions are probably wrong; just leaving
- * these examples from simplenet for reference purposes.
- *
- * In general we need to figure out how to pass configuration information to
+ /* In general we need to figure out how to pass configuration information to
  * the methods and we need to be able to calculate ross event message size.
  */
 /*Initialize the network by specifying the network parameters. The 

codes/torus.h

+/*
+ * * Copyright (C) 2011, University of Chicago
+ * *
+ * * See COPYRIGHT notice in top-level directory.
+ * */
+
 #ifndef INC_torus_h
 #define INC_torus_h
 
@@ -16,21 +22,35 @@
 #define MAX_NAME_LENGTH 256
 #define TRACE -1 
 
+/* Torus network model implementation of codes, implements the modelnet API */
+
 // Total number of nodes in torus, calculate in main
 int N_nodes = 1;
+/* Link bandwidth for each torus link, configurable from the config file */
 double link_bandwidth;
+/* buffer size of each torus link, configurable */
 int buffer_size;
+/* number of virtual channels for each torus link, configurable */
 int num_vc;
+/* number of torus dimensions, configurable */
 int n_dims;
+/* length of each torus dimension, configurable */
 int * dim_length;
+/* factor, used in torus coordinate calculation */
 int * factor;
+/* half length of each dimension, used in torus coordinates calculation */
 int * half_length;
 
+/* codes mapping group name, lp type name */
 char grp_name[MAX_NAME_LENGTH], type_name[MAX_NAME_LENGTH];
+/* codes mapping group id, lp type id, repetition id and offset */
 int grp_id, lp_type_id, rep_id, offset;
 
+/* nodes event enumeration, packet generation, send, receive and buffer event types */
 typedef enum nodes_event_t nodes_event_t;
+/* state of a torus compute node (come up with a better name instead of compute node?)*/
 typedef struct nodes_state nodes_state;
+/* torus message--- can be a packet or a flit */
 typedef struct nodes_message nodes_message;
 
 /* Issues a torus packet event call */
@@ -44,11 +64,16 @@ static void torus_packet_event(
 		       const void* self_event,
 		       tw_lp *sender,
 		       int is_last_pckt);
-
+/* torus reverse event handler */
 static void torus_packet_event_rc(tw_lp *sender);
+/* torus setup function, sets up configurable parameters like torus dimensions,
+ * length of each dimension, channel bandwidth, buffer size etc. */
 static void torus_setup(const void* net_params);
+/* returns size of the torus message */
 static int torus_get_msg_sz(void);
+/* returns torus lp type */
 static const tw_lptype* torus_get_lp_type(void);
+/* reports torus statistics */
 static void torus_report_stats(void);
 
 /* data structure for torus statistics */
@@ -63,6 +88,7 @@ struct model_net_method torus_method =
    .mn_report_stats = torus_report_stats,
 };
 
+/* event type of each torus message, can be packet generate, flit arrival, flit send or credit */
 enum nodes_event_t
 {
   GENERATE = 1,
@@ -71,62 +97,87 @@ enum nodes_event_t
   CREDIT,
 };
 
+/* state of a torus node */
 struct nodes_state
 {
+  /* counts the number of packets sent from this compute node */
   unsigned long long packet_counter;            
+  /* availability time of each torus link */
   tw_stime** next_link_available_time; 
+  /* availability of each torus credit link */
   tw_stime** next_credit_available_time;
+  /* next flit generate time */
   tw_stime** next_flit_generate_time;
+  /* buffer size for each torus virtual channel */
   int** buffer;
+  /* coordinates of the current torus node */
   int* dim_position;
+  /* neighbor LP ids for this torus node */
   int* neighbour_minus_lpID;
   int* neighbour_plus_lpID;
-  int source_dim;
-  int direction;
 };
 
 struct nodes_message
 {
+  /* category: comes from codes message */
   char category[CATEGORY_NAME_MAX];
+  /* time the packet was generated */
   tw_stime travel_start_time;
+  /* for reverse event computation*/
   tw_stime saved_available_time;
 
+  /* packet ID */
   unsigned long long packet_ID;
+  /* event type of the message */
   nodes_event_t	 type;
 
+  /* for reverse computation */
   int saved_src_dim;
   int saved_src_dir;
 
+  /* coordinates of the destination torus nodes */
   int* dest;
 
+  /* final destination LP ID, comes from codes, can be a server or any other I/O LP type */
   tw_lpid final_dest_gid;
+  /* destination torus node of the message */
   tw_lpid dest_lp;
+  /* LP ID of the sender, comes from codes, can be a server or any other I/O LP type */
   tw_lpid sender_lp;
 
+  /* number of hops traversed by the packet */
   int my_N_hop;
+  /* source dimension of the message */
   int source_dim;
+  /* source direction of the message */
   int source_direction;
+  /* next torus hop that the packet will traverse */
   int next_stop;
+  /* size of the torus packet */
   int packet_size;
+  /* chunk id of the flit (distinguishes flits) */
   short chunk_id;
 
-  // for codes local and remote events
+  /* for codes local and remote events, only carried by the last packet of the message */
   int local_event_size_bytes;
   int remote_event_size_bytes;
 };
 
+/* for calculating torus model statistics, average and maximum travel time of a packet */
 tw_stime         average_travel_time = 0;
 tw_stime         total_time = 0;
 tw_stime         max_latency = 0;
 
+/* indicates delays calculated through the bandwidth calculation of the torus link */
 float head_delay=0.0;
 float credit_delay = 0.0;
 
+/* number of finished packets on each PE */
 static long long       N_finished_packets = 0;
+/* total number of hops traversed by a message on each PE */
 static long long       total_hops = 0;
-// run time arguments
-int num_packets;
+
+/* number of chunks/flits in each torus packet, calculated through the size of each flit (32 bytes by default) */
 int num_chunks;
-int packet_offset = 0;
 
 #endif

src/models/Makefile.subdir

@@ -3,11 +3,13 @@ lib_LIBRARIES += src/libcodes-net.a
 nobase_include_HEADERS = \
  codes/model-net.h	\
  codes/torus.h		\
+ codes/dragonfly.h	\
  codes/model-net-method.h
 
 src_libcodes_net_a_SOURCES = \
  src/models/networks/model-net/model-net-method.h \
  src/models/networks/model-net/model-net.c \
  src/models/networks/model-net/simplenet-upd.c \
- src/models/networks/model-net/torus.c
+ src/models/networks/model-net/torus.c \
+ src/models/networks/model-net/dragonfly.c
 

src/models/networks/model-net/MN_TODO

@@ -6,3 +6,8 @@ end of the simulation.
 
 2- Mapping:
 - Update the mapping code to handle remainder (where number of ROSS LPs are not evenly divisible by the number of PEs).
+- Modify the get_lp_info function to return the LP count of the given LP? Can help with local lp count calculation.
+- Remove the group id and type id arguments from get_lp_info function.
+
+Make the model-net test case running for two different types of networks?
+- Use the I/O activity trace logs from BG/Q and BG/P for I/O modelling.

src/models/networks/model-net/README_MN_TEST.txt

@@ -4,9 +4,12 @@ will be executed in sequential mode.
 You can also run the test program manually in parallel (conservative or
 optimistic) mode as follows:
 
-mpiexec -n 4 tests/modelnet-test --sync=2 (optional --nkp=n) tests/modelnet-test.conf
+mpiexec -n 4 tests/modelnet-test --sync=2 --num_servers=n (optional --nkp=n) tests/modelnet-test.conf
     <or>
-mpiexec -n 4 tests/modelnet-test --sync=3 (optional --nkp=n) tests/modelnet-test.conf
+mpiexec -n 4 tests/modelnet-test --sync=3 --num_servers=n (optional --nkp=n) tests/modelnet-test.conf
+
+Note: The num_servers runtime argument should match the number of server repetitions specified in the modelnet-test config file.
 
 (1)- To run the modelnet test with the simplenet network plugin, use tests/modelnet-test.conf (default setting).
 (2)- To run the modelnet test with the torus network plugin, use tests/modelnet-test-torus.conf file.
+

src/models/networks/model-net/torus.c

@@ -19,6 +19,7 @@ static void torus_setup(const void* net_params)
        dim_length[i] = t_param->dim_length[i]; /* TODO, read comma separated values from files */
 }
 
+/* torus packet reverse event */
 static void torus_packet_event_rc(tw_lp *sender)
 {
   codes_local_latency_reverse(sender);
@@ -75,6 +76,7 @@ static void torus_packet_event(char* category, tw_lpid final_dest_lp, int packet
    //printf("\n torus remote event %d local event %d last packet %d %lf ", msg->remote_event_size_bytes, msg->local_event_size_bytes, is_last_pckt, xfer_to_nic_time);
     tw_event_send(e_new);
 }
+
 /*Initialize the torus model, this initialization part is borrowed from Ning's torus model */
 static void torus_init( nodes_state * s, 
 	   tw_lp * lp )
@@ -328,8 +330,8 @@ static void credit_send( nodes_state * s,
     tw_event_send( buf_e );
 
 }
-// send a packet from one torus node to another torus node
-// A packet can be up to 256 bytes on BG/L and BG/P and up to 512 bytes on BG/Q
+/* send a packet from one torus node to another torus node
+ A packet can be up to 256 bytes on BG/L and BG/P and up to 512 bytes on BG/Q */
 static void packet_send( nodes_state * s, 
 	         tw_bf * bf, 
 		 nodes_message * msg, 
@@ -460,6 +462,8 @@ static void packet_arrive( nodes_state * s,
    }
 }
 
+/* reports torus statistics like average packet latency, maximum packet latency and average
+ * number of torus hops traversed by the packet */
 static void torus_report_stats()
 {
     long long avg_hops, total_finished_packets;
@@ -475,6 +479,7 @@ static void torus_report_stats()
        printf(" Average number of hops traversed %f average message latency %lf us maximum message latency %lf us \n", (float)avg_hops/total_finished_packets, avg_time/(total_finished_packets*1000), max_time/1000);
      }
 }
+/* finalize the torus node and free all event buffers available */
 void
 final( nodes_state * s, tw_lp * lp )
 {
@@ -490,7 +495,7 @@ static void packet_buffer_process( nodes_state * s, tw_bf * bf, nodes_message *
    s->buffer[ msg->source_direction + ( msg->source_dim * 2 ) ][  0 ]--;
 }
 
-/* reverse handler */
+/* reverse handler for torus node */
 static void node_rc_handler(nodes_state * s, tw_bf * bf, nodes_message * msg, tw_lp * lp)
 {
   switch(msg->type)
@@ -547,7 +552,7 @@ static void node_rc_handler(nodes_state * s, tw_bf * bf, nodes_message * msg, tw
      }
 }
 
-/* forward event handler */
+/* forward event handler for torus node event */
 static void event_handler(nodes_state * s, tw_bf * bf, nodes_message * msg, tw_lp * lp)
 {
  *(int *) bf = (int) 0;
@@ -570,7 +575,7 @@ static void event_handler(nodes_state * s, tw_bf * bf, nodes_message * msg, tw_l
   break;
  }
 }
-
+/* event types */
 tw_lptype torus_lp =
 {
   (init_f) torus_init,

tests/modelnet-test-dragonfly.conf

+LPGROUPS
+{
+   MODELNET_GRP
+   {
+      repetitions="72";
+      server="1";
+      modelnet_dragonfly="1";
+   }
+  DRAGONFLY_ROUTER
+   {
+     repetitions="36";
+     dragonfly_router="1";
+   }
+}
+PARAMS
+{
+   packet_size="512";
+   modelnet="dragonfly";
+   num_vcs="1";
+   num_routers="4";
+   local_vc_size="16384";
+   global_vc_size="32768";
+   cn_vc_size="16384";
+   local_bandwidth="5.25";
+   global_bandwidth="4.7";
+   cn_bandwidth="5.25";
+   message_size="1024";
+   routing="nonminimal";
+}

tests/modelnet-test.c

@@ -6,7 +6,7 @@
 
 /* SUMMARY:
  *
- * This is a test harness for the simplenet module.  It sets up a number of
+ * This is a test harness for the modelnet module.  It sets up a number of
  * servers, each of which is paired up with a simplenet LP to serve as the
  * NIC.  Each server exchanges a sequence of requests and acks with one peer
  * and measures the throughput in terms of payload bytes (ack size) moved
@@ -24,11 +24,11 @@
 #include "codes/configuration.h"
 #include "codes/lp-type-lookup.h"
 
-#define NUM_SERVERS 16  /* number of servers */
 #define NUM_REQS 1000  /* number of requests sent by each server */
 #define PAYLOAD_SZ 2048 /* size of simulated data payload, bytes  */
 
 static int net_id = 0;
+static int num_servers = 16;
 
 typedef struct svr_msg svr_msg;
 typedef struct svr_state svr_state;
@@ -59,12 +59,6 @@ struct svr_msg
     int incremented_flag; /* helper for reverse computation */
 };
 
-const tw_optdef app_opt[] = {
-    TWOPT_GROUP("Simple Network Test Model"),
-    TWOPT_END()
-};
-
-
 static void svr_init(
     svr_state * ns,
     tw_lp * lp);
@@ -138,6 +132,14 @@ static void handle_req_rev_event(
     tw_bf * b,
     svr_msg * m,
     tw_lp * lp);
+
+const tw_optdef app_opt [] =
+{
+	TWOPT_GROUP("Model net test case" ),
+	TWOPT_UINT("num_servers", num_servers, "num_servers"),
+	TWOPT_END()
+};
+
 int main(
     int argc,
     char **argv)
@@ -153,9 +155,9 @@ int main(
     tw_opt_add(app_opt);
     tw_init(&argc, &argv);
 
-    if(argc != 2)
+    if(argc < 2)
     {
-	    printf("\n Usage: mpirun <args> --sync=2/3 mapping_file_name.conf");
+	    printf("\n Usage: mpirun <args> --sync=2/3 mapping_file_name.conf (optional --nkp) num_servers");
 	    MPI_Finalize();
 	    return 0;
     }
@@ -174,13 +176,13 @@ int main(
     g_tw_mapping=CUSTOM;
     g_tw_custom_initial_mapping=&codes_mapping_init;
     g_tw_custom_lp_global_to_local_map=&codes_mapping_to_lp;
+    g_tw_events_per_pe = 2048 * codes_mapping_get_lps_for_pe();
 
     if(!message_size)
     {
 	    message_size = 256;
 	    printf("\n Warning: ross message size not defined, resetting it to %d", message_size);
     }
-    printf("\n Initializing %d lps on %d ", codes_mapping_get_lps_for_pe(), g_tw_mynode);
     tw_define_lps(codes_mapping_get_lps_for_pe(), message_size, 0 );
     //g_tw_events_per_pe = 2 * NUM_REQS * (codes_mapping_get_lps_for_pe());
 
@@ -231,7 +233,7 @@ static void svr_init(
      * simulation
      */
 
-    printf("\n Initializing servers %d ", (int)lp->gid);
+//    printf("\n Initializing servers %d ", (int)lp->gid);
     /* skew each kickoff event slightly to help avoid event ties later on */
     kickoff_time = g_tw_lookahead + tw_rand_unif(lp->rng); 
 
@@ -347,7 +349,7 @@ static void handle_kickoff_event(
     ns->start_ts = tw_now(lp);