Merging fat tree network model

.gitignore

@@ -36,3 +36,8 @@
 
 # generated files from test runs
 ross.csv
+
+install-mastiff/include/codes/model-net-method.h
+configure.ac
+configure.ac
+configure.ac
\ No newline at end of file

codes/model-net-lp.h

@@ -23,6 +23,7 @@ extern "C" {
 #include "model-net-sched.h"
 #include "net/dragonfly.h"
 #include "net/slimfly.h"
+#include "net/fattree.h"
 #include "net/loggp.h"
 #include "net/simplenet-upd.h"
 #include "net/simplep2p.h"
@@ -123,7 +124,8 @@ typedef struct model_net_wrap_msg {
     union {
         model_net_base_msg      m_base;  // base lp
         terminal_message        m_dfly;  // dragonfly
-        slim_terminal_message   m_slim; // slimfly
+        slim_terminal_message   m_slim;  // slimfly
+	fattree_message		m_fat;   // fattree
         loggp_message           m_loggp; // loggp
         sn_message              m_snet;  // simplenet
         sp_message              m_sp2p;  // simplep2p

codes/model-net.h

@@ -61,6 +61,7 @@ typedef struct mn_stats mn_stats;
     X(SIMPLEP2P, "modelnet_simplep2p", "simplep2p", &simplep2p_method)\
     X(TORUS,     "modelnet_torus",     "torus",     &torus_method)\
     X(SLIMFLY,   "modelnet_slimfly",   "slimfly",   &slimfly_method)\
+    X(FATTREE,   "modelnet_fattree",   "fattree",   &fattree_method)\
     X(DRAGONFLY, "modelnet_dragonfly", "dragonfly", &dragonfly_method)\
     X(DRAGONFLY_ROUTER, "modelnet_dragonfly_router", "dragonfly_router", &dragonfly_router_method)\
     X(LOGGP,     "modelnet_loggp",     "loggp",     &loggp_method)\

codes/net/fattree.h

+#ifndef FATTREE_H
+#define FATTREE_H
+
+#include <ross.h>
+
+/* Global variable for modelnet output directory name */
+char *modelnet_stats_dir;
+
+typedef struct fattree_message fattree_message;
+
+/* this message is used for both fattree compute nodes and routers */
+struct fattree_message
+{
+  /* magic number */
+  int magic;
+  /* flit travel start time*/
+  tw_stime travel_start_time;
+ /* packet ID of the flit  */
+  unsigned long long packet_ID;
+  /* event type of the flit */
+  short  type;
+  /* category: comes from codes */
+  char category[CATEGORY_NAME_MAX];
+  /* final destination LP ID, this comes from codes can be a server or any other LP type*/
+  tw_lpid final_dest_gid;
+  /*sending LP ID from CODES, can be a server or any other LP type */
+  tw_lpid sender_lp;
+  tw_lpid sender_mn_lp; // source modelnet id
+ /* destination terminal ID of the message */
+//  int dest_num; replaced with dest_terminal_id
+  tw_lpid dest_terminal_id;
+  /* source terminal ID of the fattree */
+  unsigned int src_terminal_id;
+  /* Intermediate LP ID from which this message is coming */
+  unsigned int intm_lp_id;
+  short saved_vc;
+  short saved_off;
+  int last_hop;
+  int intm_id; //to find which port I connect to sender with
+
+  /* message originating router id */
+  unsigned int origin_switch_id;
+
+  /* number of hops traversed by the packet */
+  short my_N_hop;
+
+  // For buffer message
+  short vc_index;
+  short vc_off;
+  int is_pull;
+  model_net_event_return event_rc;
+  uint64_t pull_size;
+
+  /* for reverse computation */    
+  int path_type; 
+  tw_stime saved_available_time;
+  tw_stime saved_credit_time;
+  uint64_t packet_size;
+  tw_stime msg_start_time;
+  tw_stime saved_busy_time;
+  tw_stime saved_sample_time;
+  tw_stime saved_avg_time;
+  tw_stime saved_rcv_time;
+  tw_stime saved_total_time;
+
+  /* For routing */
+  uint64_t chunk_id;
+  uint64_t total_size;
+  uint64_t message_id;
+   
+  /* meta data to aggregate packets into a message at receiver */
+  uint64_t msg_size;
+  uint64_t src_nic;
+  uint64_t uniq_id;
+  uint64_t saved_size;
+
+  int remote_event_size_bytes;
+  int local_event_size_bytes;
+
+};
+
+#endif /* end of include guard: FATTREE_H */
+

src/Makefile.subdir

@@ -25,6 +25,7 @@ EXTRA_DIST += src/iokernellang/codesparser.y.in \
 			  src/network-workloads/conf/allocation-random.conf \
 			  src/network-workloads/conf/modelnet-synthetic-dragonfly.conf \
 			  src/network-workloads/conf/modelnet-synthetic-slimfly-min.conf \
+			  src/network-workloads/conf/modelnet-synthetic-fattree.conf \
 			  src/networks/model-net/doc/README \
 			  src/networks/model-net/doc/README.dragonfly.txt \
 			  src/networks/model-net/doc/README.loggp.txt \
@@ -88,6 +89,7 @@ nobase_include_HEADERS = \
 	codes/model-net-inspect.h \
 	codes/net/dragonfly.h \
 	codes/net/slimfly.h \
+	codes/net/fattree.h \
 	codes/net/loggp.h \
 	codes/net/simplenet-upd.h \
 	codes/net/simplep2p.h \
@@ -148,6 +150,7 @@ src_libcodes_a_SOURCES =  \
 	src/networks/model-net/torus.c \
 	src/networks/model-net/dragonfly.c \
 	src/networks/model-net/slimfly.c \
+	src/networks/model-net/fattree.c \
 	src/networks/model-net/loggp.c \
 	src/networks/model-net/simplep2p.c \
 	src/networks/model-net/model-net-lp.c \
@@ -174,14 +177,17 @@ bin_PROGRAMS += src/network-workloads/model-net-mpi-replay
 bin_PROGRAMS += src/network-workloads/model-net-dumpi-traces-dump
 bin_PROGRAMS += src/network-workloads/model-net-synthetic
 bin_PROGRAMS += src/network-workloads/model-net-synthetic-slimfly
+bin_PROGRAMS += src/network-workloads/model-net-synthetic-fattree
 
 src_workload_codes_workload_dump_SOURCES = \
  src/workload/codes-workload-dump.c
 
 src_network_workloads_model_net_mpi_replay_SOURCES = src/network-workloads/model-net-mpi-replay.c
+
 src_network_workloads_model_net_synthetic_SOURCES = src/network-workloads/model-net-synthetic.c
 src_network_workloads_model_net_synthetic_slimfly_SOURCES = src/network-workloads/model-net-synthetic-slimfly.c
 src_network_workloads_model_net_dumpi_traces_dump_SOURCES = src/network-workloads/model-net-dumpi-traces-dump.c
+
 #bin_PROGRAMS += src/network-workload/codes-nw-test
 
 #src_network_workload_codes_nw_test_SOURCES = \

src/network-workloads/conf/modelnet-synthetic-fattree-summit-k36-n3564.conf

+LPGROUPS
+{
+   MODELNET_GRP
+   {
+      repetitions="198";     # repetitions = Ne = total # of edge switches. For type0 Ne = Np*Ns = ceil(N/Ns*(k/2))*(k/2) = ceil(N/(k/2)^2)*(k/2)
+      server="18";
+      modelnet_fattree="18";
+      fattree_switch="3";
+   }
+}
+PARAMS
+{
+   ft_type="0";
+   packet_size="512";
+   message_size="512";
+   chunk_size="32";
+   modelnet_scheduler="fcfs";
+   #modelnet_scheduler="round-robin";
+   modelnet_order=( "fattree" );
+   num_levels="3";
+   switch_count="198";       # = repititions
+   switch_radix="36";
+   router_delay="60";
+   soft_delay="1000";
+   vc_size="65536";
+   cn_vc_size="65536";
+   link_bandwidth="12.5";
+   cn_bandwidth="12.5";
+}

src/network-workloads/conf/modelnet-synthetic-fattree.conf

+LPGROUPS
+{
+   MODELNET_GRP
+   {
+      repetitions="4";				#Number of leaf level switches
+      server="2";
+      modelnet_fattree="2";
+      fattree_switch="2";
+   }
+}
+PARAMS
+{
+   ft_type="1";							# Only appears to be one type (type=0) in fattree.c
+   num_levels="2";						# Must be 1 < num_levels < 4
+   switch_count="2,4";					# Some sort of csv string. I'm thinking it's the number of switches per level
+   switch_radix="4,4";						# Another csv string. I'm thinking it's the radix of the switches per level. All switches within same level have same radix
+   packet_size="512";
+   modelnet_order=( "fattree" );
+   # scheduler options
+   modelnet_scheduler="fcfs";
+   chunk_size="32";
+   # modelnet_scheduler="round-robin";
+   num_vcs="1";
+   vc_size="16384";
+   cn_vc_size="16384";
+   link_bandwidth="5";
+   cn_bandwidth="5";
+   message_size="512";
+   routing="minimal";
+   router_delay="50";
+   soft_delay="1000";
+}

src/network-workloads/conf/tracer-fattree-type2.conf

+LPGROUPS
+{
+   MODELNET_GRP
+   {
+      repetitions="16";
+      server="4";
+      modelnet_fattree="4";
+      fattree_switch="3";
+   }
+}
+PARAMS
+{
+   ft_type="1";
+   packet_size="512";
+   message_size="512";
+   chunk_size="32";
+   modelnet_scheduler="fcfs";
+   #modelnet_scheduler="round-robin";
+   modelnet_order=( "fattree" );
+   num_levels="3";
+   switch_count="16";
+   switch_radix="8";
+   router_delay="60";
+   soft_delay="1000";
+   vc_size="65536";
+   cn_vc_size="65536";
+   link_bandwidth="4.7";
+   cn_bandwidth="5.25";
+}

src/network-workloads/model-net-synthetic-fattree.c

+/*
+ * Copyright (C) 2015 University of Chicago.
+ * See COPYRIGHT notice in top-level directory.
+ *
+ */
+
+/*
+* The test program generates some synthetic traffic patterns for the model-net network models.
+* currently it only support the fat tree network model uniform random and nearest neighbor traffic patterns.
+*/
+
+#include "codes/model-net.h"
+#include "codes/lp-io.h"
+#include "codes/net/fattree.h"
+#include "codes/codes.h"
+#include "codes/codes_mapping.h"
+#include "codes/configuration.h"
+#include "codes/lp-type-lookup.h"
+
+#define PAYLOAD_SZ 512
+
+#define PARAMS_LOG 1
+
+static int net_id = 0;
+static int offset = 2;
+static int traffic = 1;
+static double arrival_time = 1000.0;
+static double load = 0.0;	//Percent utilization of terminal uplink
+static double MEAN_INTERVAL = 0.0;
+/* whether to pull instead of push */
+
+static int num_servers_per_rep = 0;
+static int num_routers_per_grp = 0;
+static int num_nodes_per_grp = 0;
+
+static int num_groups = 0;
+static int num_nodes = 0;
+
+typedef struct svr_msg svr_msg;
+typedef struct svr_state svr_state;
+
+/* global variables for codes mapping */
+static char group_name[MAX_NAME_LENGTH];
+static char lp_type_name[MAX_NAME_LENGTH];
+static int group_index, lp_type_index, rep_id, offset;
+
+/* convert GiB/s and bytes to ns */
+ static tw_stime bytes_to_ns(uint64_t bytes, double GB_p_s)
+ {
+     tw_stime time;
+ 
+     /* bytes to GB */
+     time = ((double)bytes)/(1024.0*1024.0*1024.0);
+     /* GiB to s */
+     time = time / GB_p_s;
+     /* s to ns */
+     time = time * 1000.0 * 1000.0 * 1000.0;
+ 
+     return(time);
+ }
+
+/* type of events */
+enum svr_event
+{
+    KICKOFF,	   /* kickoff event */
+    REMOTE,        /* remote event */
+    LOCAL      /* local event */
+};
+
+/* type of synthetic traffic */
+enum TRAFFIC
+{
+	UNIFORM = 1, /* sends message to a randomly selected node */
+	NEAREST_GROUP = 2, /* sends message to the node connected to the neighboring router */
+	NEAREST_NEIGHBOR = 3 /* sends message to the next node (potentially connected to the same router) */
+};
+
+struct svr_state
+{
+    int msg_sent_count;   /* requests sent */
+    int msg_recvd_count;  /* requests recvd */
+    int local_recvd_count; /* number of local messages received */
+    tw_stime start_ts;    /* time that we started sending requests */
+    tw_stime end_ts;      /* time that we ended sending requests */
+};
+
+struct svr_msg
+{
+    enum svr_event svr_event_type;
+    tw_lpid src;          /* source of this request or ack */
+    int incremented_flag; /* helper for reverse computation */
+};
+
+static void svr_init(
+    svr_state * ns,
+    tw_lp * lp);
+static void svr_event(
+    svr_state * ns,
+    tw_bf * b,
+    svr_msg * m,
+    tw_lp * lp);
+static void svr_rev_event(
+    svr_state * ns,
+    tw_bf * b,
+    svr_msg * m,
+    tw_lp * lp);
+static void svr_finalize(
+    svr_state * ns,
+    tw_lp * lp);
+
+tw_lptype svr_lp = {
+    (init_f) svr_init,
+    (pre_run_f) NULL,
+    (event_f) svr_event,
+    (revent_f) svr_rev_event,
+    (commit_f) NULL,
+    (final_f)  svr_finalize,
+    (map_f) codes_mapping,
+    sizeof(svr_state),
+};
+
+const tw_optdef app_opt [] =
+{
+        TWOPT_GROUP("Model net synthetic traffic " ),
+	TWOPT_UINT("traffic", traffic, "UNIFORM RANDOM=1, NEAREST NEIGHBOR=2 "),
+	TWOPT_STIME("arrival_time", arrival_time, "INTER-ARRIVAL TIME"),
+        TWOPT_STIME("load", load, "percentage of terminal link bandiwdth to inject packets"),
+        TWOPT_END()
+};
+
+const tw_lptype* svr_get_lp_type()
+{
+            return(&svr_lp);
+}
+
+static void svr_add_lp_type()
+{
+  lp_type_register("server", svr_get_lp_type());
+}
+
+static void issue_event(
+    svr_state * ns,
+    tw_lp * lp)
+{
+    (void)ns;
+    tw_event *e;
+    svr_msg *m;
+    tw_stime kickoff_time;
+
+    /* each server sends a dummy event to itself that will kick off the real
+     * simulation
+     */
+
+    int this_packet_size = 0;
+    double this_link_bandwidth = 0.0;
+
+    configuration_get_value_int(&config, "PARAMS", "packet_size", NULL, &this_packet_size);
+    if(!this_packet_size) {
+        this_packet_size = 0;
+        fprintf(stderr, "Packet size not specified, setting to %d\n", this_packet_size);
+        exit(0);
+    }
+
+    configuration_get_value_double(&config, "PARAMS", "link_bandwidth", NULL, &this_link_bandwidth);
+    if(!this_link_bandwidth) {
+        this_link_bandwidth = 4.7;
+        fprintf(stderr, "Bandwidth of channels not specified, setting to %lf\n", this_link_bandwidth);
+    }
+
+    if(arrival_time!=0)
+    {
+        MEAN_INTERVAL = arrival_time;
+    }
+    if(load != 0)
+    {
+        MEAN_INTERVAL = bytes_to_ns(this_packet_size, load*this_link_bandwidth);
+    }
+
+    /* skew each kickoff event slightly to help avoid event ties later on */
+//    kickoff_time = 1.1 * g_tw_lookahead + tw_rand_exponential(lp->rng, arrival_time);
+    kickoff_time = g_tw_lookahead + tw_rand_exponential(lp->rng, MEAN_INTERVAL);
+
+    e = tw_event_new(lp->gid, kickoff_time, lp);
+    m = tw_event_data(e);
+    m->svr_event_type = KICKOFF;
+    tw_event_send(e);
+}
+
+static void svr_init(
+    svr_state * ns,
+    tw_lp * lp)
+{
+    ns->start_ts = 0.0;
+
+    issue_event(ns, lp);
+    return;
+}
+
+static void handle_kickoff_rev_event(
+            svr_state * ns,
+            tw_bf * b,
+            svr_msg * m,
+            tw_lp * lp)
+{
+    (void)b;
+    (void)m;
+	ns->msg_sent_count--;
+	model_net_event_rc(net_id, lp, PAYLOAD_SZ);
+    tw_rand_reverse_unif(lp->rng);
+}	
+static void handle_kickoff_event(
+	    svr_state * ns,
+	    tw_bf * b,
+	    svr_msg * m,
+	    tw_lp * lp)
+{
+    (void)b;
+    (void)m;
+//    char* anno;
+    char anno[MAX_NAME_LENGTH];
+    tw_lpid local_dest = -1, global_dest = -1;
+   
+    svr_msg * m_local = malloc(sizeof(svr_msg));
+    svr_msg * m_remote = malloc(sizeof(svr_msg));
+
+    m_local->svr_event_type = LOCAL;
+    m_local->src = lp->gid;
+
+    memcpy(m_remote, m_local, sizeof(svr_msg));
+    m_remote->svr_event_type = REMOTE;
+
+    ns->start_ts = tw_now(lp);
+
+   codes_mapping_get_lp_info(lp->gid, group_name, &group_index, lp_type_name, &lp_type_index, anno, &rep_id, &offset);
+   /* in case of uniform random traffic, send to a random destination. */
+   if(traffic == UNIFORM)
+   {
+   	local_dest = tw_rand_integer(lp->rng, 0, num_nodes - 1);
+   }
+
+   assert(local_dest < LLU(num_nodes));
+
+   global_dest = codes_mapping_get_lpid_from_relative(local_dest, group_name, lp_type_name, NULL, 0);
+
+   // If Destination is self, then generate new destination
+   if((int)global_dest == (int)lp->gid)
+   {
+       local_dest = (local_dest+1) % (num_nodes-1);
+       global_dest = codes_mapping_get_lpid_from_relative(local_dest, group_name, lp_type_name, NULL, 0);
+   }
+
+   int div1 = floor((int)lp->gid/11);
+   int mult1 = div1 * 4;
+   int mod1 = (int)lp->gid % 11;
+   int sum1 = mult1 + mod1;
+//   if((int)lp->gid == 3)
+   if((int)global_dest == (int)lp->gid)
+       printf("global_src:%d, local_src:%d, global_dest:%d, local_dest:%d\n",(int)lp->gid, sum1, (int)global_dest,(int)local_dest);
+
+   ns->msg_sent_count++;
+
+   model_net_event(net_id, "test", global_dest, PAYLOAD_SZ, 0.0, sizeof(svr_msg), (const void*)m_remote, sizeof(svr_msg), (const void*)m_local, lp);
+
+   //printf("LP:%d localID:%d Here\n",(int)lp->gid, (int)local_dest);
+   issue_event(ns, lp);
+   //printf("Just Checking net_id:%d\n",net_id);
+   return;
+}
+
+static void handle_remote_rev_event(
+            svr_state * ns,     
+            tw_bf * b,
+            svr_msg * m,
+            tw_lp * lp)
+{
+        (void)b;
+        (void)m;
+        (void)lp;
+        ns->msg_recvd_count--;
+}
+
+static void handle_remote_event(
+	    svr_state * ns,
+	    tw_bf * b,
+	    svr_msg * m,
+	    tw_lp * lp)
+{
+    (void)b;
+    (void)m;
+    (void)lp;
+	ns->msg_recvd_count++;
+}
+
+static void handle_local_rev_event(
+                svr_state * ns,
+                tw_bf * b,
+                svr_msg * m,
+                tw_lp * lp)
+{
+    (void)b;
+    (void)m;
+    (void)lp;
+	ns->local_recvd_count--;
+}
+
+static void handle_local_event(
+                svr_state * ns,
+                tw_bf * b,
+                svr_msg * m,
+                tw_lp * lp)
+{
+    (void)b;
+    (void)m;
+    (void)lp;
+    ns->local_recvd_count++;
+}
+
+static void svr_finalize(
+    svr_state * ns,
+    tw_lp * lp)
+{
+    ns->end_ts = tw_now(lp);
+
+//    printf("server %llu recvd %d bytes in %f seconds, %f MiB/s sent_count %d recvd_count %d local_count %d \n", (unsigned long long)lp->gid, PAYLOAD_SZ*ns->msg_recvd_count, ns_to_s(ns->end_ts-ns->start_ts),
+//        ((double)(PAYLOAD_SZ*ns->msg_sent_count)/(double)(1024*1024)/ns_to_s(ns->end_ts-ns->start_ts)), ns->msg_sent_count, ns->msg_recvd_count, ns->local_recvd_count);
+    return;
+}
+
+static void svr_rev_event(
+    svr_state * ns,
+    tw_bf * b,
+    svr_msg * m,
+    tw_lp * lp)
+{
+    switch (m->svr_event_type)
+    {
+	case REMOTE:
+		handle_remote_rev_event(ns, b, m, lp);
+		break;
+	case LOCAL:
+		handle_local_rev_event(ns, b, m, lp);
+		break;
+	case KICKOFF:
+		handle_kickoff_rev_event(ns, b, m, lp);
+		break;
+	default:
+		assert(0);
+		break;
+    }
+}
+
+static void svr_event(
+    svr_state * ns,
+    tw_bf * b,
+    svr_msg * m,
+    tw_lp * lp)