More updates to network models, recording router busy times in stats, using RC...

More updates to network models, recording router busy times in stats, using RC stack for reverse computation instead of copying back data to message

codes/net/dragonfly.h

@@ -48,7 +48,7 @@ struct terminal_message
   short new_vc;
   short saved_vc;
   /* last hop of the message, can be a terminal, local router or global router */
-  short last_hop;
+  int last_hop;
    /* For routing */
    int intm_group_id;
    int chunk_id;
@@ -61,7 +61,7 @@ struct terminal_message
    int local_event_size_bytes;
 
   // For buffer message
-   short vc_index;
+   int vc_index;
    int sender_radix;
    int output_chan;
    model_net_event_return event_rc;
@@ -69,15 +69,17 @@ struct terminal_message
     uint64_t pull_size;
 
    /* for reverse computation */   
-   short path_type;
+   int path_type;
    tw_stime saved_available_time;
+   
    tw_stime saved_avg_time;
-   tw_stime saved_start_time;
-   tw_stime saved_collective_init_time;  
+   tw_stime saved_rcv_time;
+   
+   tw_stime saved_busy_time; 
+   tw_stime saved_total_time;
    tw_stime saved_hist_start_time;
    tw_stime msg_start_time;
 
-   short saved_completed_chunks;
    int saved_hist_num;
    int saved_occupancy;
 

src/models/network-workloads/conf/modelnet-mpi-test-dragonfly.conf

@@ -18,12 +18,12 @@ PARAMS
    # modelnet_scheduler="round-robin";
    num_vcs="1";
    num_routers="4";
-   local_vc_size="32768";
-   global_vc_size="65536";
-   cn_vc_size="32768";
+   local_vc_size="8192";
+   global_vc_size="16384";
+   cn_vc_size="8192";
    local_bandwidth="5.25";
    global_bandwidth="4.7";
    cn_bandwidth="5.25";
-   message_size="536";
-   routing="adaptive";
+   message_size="552";
+   routing="minimal";
 }

src/models/network-workloads/conf/modelnet-mpi-test.conf

@@ -2,7 +2,7 @@ LPGROUPS
 {
    MODELNET_GRP
    {
-      repetitions="8";
+      repetitions="27";
       modelnet_simplenet="1";
       nw-lp="1";
    }

src/models/network-workloads/model-net-mpi-replay.c

@@ -14,7 +14,8 @@
 #include "codes/rc-stack.h"
 #include "codes/quicklist.h"
 
-#define TRACK -1
+/* turning on track lp will generate a lot of output messages */
+#define TRACK_LP -1
 #define TRACE -1
 #define MAX_WAIT_REQS 512
 
@@ -179,6 +180,7 @@ struct nw_message
        double saved_wait_time;
        double saved_delay;
        int saved_num_bytes;
+       struct codes_workload_op * saved_op;
    } rc;
 };
 
@@ -270,8 +272,6 @@ static int clear_completed_reqs(nw_state * s,
             }
        }
     }
-    if(lp->gid == TRACK)
-      printf("\n Pushed num %d ", matched);
     return matched;
 }
 static void add_completed_reqs(nw_state * s, 
@@ -484,7 +484,7 @@ static int rm_matching_rcv(nw_state * ns,
 
     qlist_for_each(ent, &ns->pending_recvs_queue){
         qi = qlist_entry(ent, mpi_msgs_queue, ql);
-        if((qi->num_bytes >= qitem->num_bytes)
+        if((qi->num_bytes == qitem->num_bytes)
                 && ((qi->tag == qitem->tag) || qi->tag == -1)
                 && ((qi->source_rank == qitem->source_rank) || qi->source_rank == -1))
         {
@@ -504,9 +504,6 @@ static int rm_matching_rcv(nw_state * ns,
         
         qlist_del(&qi->ql);
         
-        if(lp->gid == TRACK)
-            printf("\n matched recv req id %ld ", qi->req_id);
-        
         rc_stack_push(lp, qi, free, ns->processed_ops);
         return index;
     }
@@ -525,7 +522,7 @@ static int rm_matching_send(nw_state * ns,
     int index = 0;
     qlist_for_each(ent, &ns->arrival_queue){
         qi = qlist_entry(ent, mpi_msgs_queue, ql);
-        if((qi->num_bytes <= qitem->num_bytes) 
+        if((qi->num_bytes == qitem->num_bytes) 
                 && (qi->tag == qitem->tag || qitem->tag == -1)
                 && ((qi->source_rank == qitem->source_rank) || qitem->source_rank == -1))
         {
@@ -545,8 +542,6 @@ static int rm_matching_send(nw_state * ns,
 
         qlist_del(&qi->ql);
 
-        if(lp->gid == TRACK)
-            printf("\n matched matching send recv req id %ld ", qitem->req_id);
         return index;
     }
     return -1;
@@ -580,13 +575,8 @@ static void codes_exec_comp_delay(
 	tw_stime ts;
 	nw_message* msg;
 
-        if (disable_delay) {
-            ts = 0.0; // no compute time sim
-        }
-        else {
-            s->compute_time += s_to_ns(mpi_op->u.delay.seconds);
-            ts = s_to_ns(mpi_op->u.delay.seconds);
-        }
+    s->compute_time += s_to_ns(mpi_op->u.delay.seconds);
+    ts = s_to_ns(mpi_op->u.delay.seconds);
 
 	ts += g_tw_lookahead + 0.1 + tw_rand_exponential(lp->rng, noise);
 	
@@ -605,7 +595,9 @@ static void codes_exec_mpi_recv_rc(
         nw_message* m, 
         tw_lp* lp)
 {
-	num_bytes_recvd -= m->rc.saved_num_bytes;
+    struct codes_workload_op * mpi_op = m->rc.saved_op;
+
+	num_bytes_recvd -= mpi_op->u.recv.num_bytes;
 	ns->recv_time = m->rc.saved_recv_time;
 	if(m->fwd.found_match >= 0)
 	  {
@@ -666,7 +658,6 @@ static void codes_exec_mpi_recv(
    receive operations. */
 
 	m->rc.saved_recv_time = s->recv_time;
-    m->rc.saved_num_bytes = mpi_op->u.recv.num_bytes;
 	num_bytes_recvd += mpi_op->u.recv.num_bytes;
 
     mpi_msgs_queue * recv_op = (mpi_msgs_queue*) malloc(sizeof(mpi_msgs_queue));
@@ -678,6 +669,10 @@ static void codes_exec_mpi_recv(
     recv_op->tag = mpi_op->u.recv.tag;
     recv_op->req_id = mpi_op->u.recv.req_id;
 
+    if(s->nw_id == TRACK_LP)
+        printf("\n Receive op posted num bytes %d source %d ", recv_op->num_bytes,
+                recv_op->source_rank);
+
 	int found_matching_sends = rm_matching_send(s, bf, m, lp, recv_op);
 
 	/* save the req id inserted in the completed queue for reverse computation. */
@@ -765,8 +760,6 @@ static tw_stime s_to_ns(tw_stime ns)
 static void update_completed_queue_rc(nw_state * s, tw_bf * bf, nw_message * m, tw_lp * lp)
 {
    
-    if(lp->gid == TRACK)
-        printf("\n Reverse computation!!!! ");
     if(bf->c0)
     {
        struct qlist_head * ent = qlist_pop_back(&s->completed_reqs);
@@ -903,6 +896,10 @@ static void update_arrival_queue(nw_state* s, tw_bf * bf, nw_message * m, tw_lp
     arrived_op->num_bytes = m->fwd.num_bytes;
     arrived_op->tag = m->fwd.tag;
 
+    if(s->nw_id == TRACK_LP)
+        printf("\n Send op arrived source rank %ld num bytes %d ", arrived_op->source_rank,
+                arrived_op->num_bytes);
+
     int found_matching_recv = rm_matching_rcv(s, bf, m, lp, arrived_op);
 
     if(found_matching_recv < 0)
@@ -985,8 +982,8 @@ void nw_test_init(nw_state* s, tw_lp* lp)
 void nw_test_event_handler(nw_state* s, tw_bf * bf, nw_message * m, tw_lp * lp)
 {
 	*(int *)bf = (int)0;
-    //rc_stack_gc(lp, s->matched_reqs);
-    //rc_stack_gc(lp, s->processed_ops);
+    rc_stack_gc(lp, s->matched_reqs);
+    rc_stack_gc(lp, s->processed_ops);
 
     switch(m->msg_type)
 	{
@@ -1017,7 +1014,8 @@ void nw_test_event_handler(nw_state* s, tw_bf * bf, nw_message * m, tw_lp * lp)
 
 static void get_next_mpi_operation_rc(nw_state* s, tw_bf * bf, nw_message * m, tw_lp * lp)
 {
-    codes_workload_get_next_rc2(wrkld_id, 0, (int)s->nw_id);
+    struct codes_workload_op * mpi_op = m->rc.saved_op;
+    codes_workload_get_next_rc(wrkld_id, 0, (int)s->nw_id, mpi_op);
 
 	if(m->op_type == CODES_WK_END)
     {
@@ -1028,13 +1026,12 @@ static void get_next_mpi_operation_rc(nw_state* s, tw_bf * bf, nw_message * m, t
 		case CODES_WK_SEND:
 		case CODES_WK_ISEND:
 		{
-            int saved_num_bytes = m->rc.saved_num_bytes;
-			model_net_event_rc(net_id, lp, saved_num_bytes);
+			model_net_event_rc(net_id, lp, m->rc.saved_num_bytes);
 			if(m->op_type == CODES_WK_ISEND)
 				codes_issue_next_event_rc(lp);
 			s->num_sends--;
-            s->num_bytes_sent += saved_num_bytes;
-			num_bytes_sent -= saved_num_bytes;
+            s->num_bytes_sent += m->rc.saved_num_bytes;
+			num_bytes_sent -= m->rc.saved_num_bytes;
 		}
 		break;
 
@@ -1051,10 +1048,6 @@ static void get_next_mpi_operation_rc(nw_state* s, tw_bf * bf, nw_message * m, t
 		{
 			s->num_delays--;
             tw_rand_reverse_unif(lp->rng);
-                        
-            if (!disable_delay) {
-                     s->compute_time -= s_to_ns(m->rc.saved_delay);
-            }
 		}
 		break;
 		case CODES_WK_BCAST:
@@ -1098,23 +1091,24 @@ static void get_next_mpi_operation_rc(nw_state* s, tw_bf * bf, nw_message * m, t
 
 static void get_next_mpi_operation(nw_state* s, tw_bf * bf, nw_message * m, tw_lp * lp)
 {
-		struct codes_workload_op mpi_op;
-        codes_workload_get_next(wrkld_id, 0, (int)s->nw_id, &mpi_op);
+		struct codes_workload_op * mpi_op = malloc(sizeof(struct codes_workload_op));
+        codes_workload_get_next(wrkld_id, 0, (int)s->nw_id, mpi_op);
       
-        m->op_type = mpi_op.op_type;
+        m->op_type = mpi_op->op_type;
+        m->rc.saved_op = mpi_op;
 
-        if(mpi_op.op_type == CODES_WK_END)
+        if(mpi_op->op_type == CODES_WK_END)
         {
             s->elapsed_time = tw_now(lp) - s->start_time;
             return;
         }
-		switch(mpi_op.op_type)
+		switch(mpi_op->op_type)
 		{
 			case CODES_WK_SEND:
 			case CODES_WK_ISEND:
 			 {
 				s->num_sends++;
-				codes_exec_mpi_send(s, bf, m, lp, &mpi_op);
+				codes_exec_mpi_send(s, bf, m, lp, mpi_op);
 			 }
 			break;
 	
@@ -1122,14 +1116,18 @@ static void get_next_mpi_operation(nw_state* s, tw_bf * bf, nw_message * m, tw_l
 			case CODES_WK_IRECV:
 			  {
 				s->num_recvs++;
-				codes_exec_mpi_recv(s, bf, m, lp,  &mpi_op);
+				codes_exec_mpi_recv(s, bf, m, lp,  mpi_op);
 			  }
 			break;
 
 			case CODES_WK_DELAY:
 			  {
+                
 				s->num_delays++;
-				codes_exec_comp_delay(s, lp, &mpi_op);
+                if(disable_delay)
+                    codes_issue_next_event(lp);
+                else
+				    codes_exec_comp_delay(s, lp, mpi_op);
 			  }
 			break;
 
@@ -1144,13 +1142,13 @@ static void get_next_mpi_operation(nw_state* s, tw_bf * bf, nw_message * m, tw_l
 			case CODES_WK_WAITALL:
 			  {
 				s->num_waitall++;
-			    codes_exec_mpi_wait_all(s, bf, m, lp, &mpi_op);
+			    codes_exec_mpi_wait_all(s, bf, m, lp, mpi_op);
               }
 			break;
 			case CODES_WK_WAIT:
 			{
 				s->num_wait++;
-                codes_exec_mpi_wait(s, lp, &mpi_op);
+                codes_exec_mpi_wait(s, lp, mpi_op);
 			}
 			break;
 			case CODES_WK_BCAST:
@@ -1167,7 +1165,7 @@ static void get_next_mpi_operation(nw_state* s, tw_bf * bf, nw_message * m, tw_l
             }
 			break;
 			default:
-				printf("\n Invalid op type %d ", mpi_op.op_type);
+				printf("\n Invalid op type %d ", mpi_op->op_type);
 		}
         return;
 }

src/models/network-workloads/model-net-mpi-wrklds.c

@@ -281,7 +281,7 @@ static void printQueue(tw_lpid lpid, struct mpi_queue_ptrs* mpi_queue, char* msg
 	while(tmp)
 	{
 		if(tmp->mpi_op->op_type == CODES_WK_SEND || tmp->mpi_op->op_type == CODES_WK_ISEND)
-			printf("\n lpid %llu send operation count %d tag %d source %d", 
+			printf("\n lpid %llu send operation num bytes %d tag %d source %d", 
 				    lpid, tmp->mpi_op->u.send.num_bytes, 
 				     tmp->mpi_op->u.send.tag, tmp->mpi_op->u.send.source_rank);
 		else if(tmp->mpi_op->op_type == CODES_WK_IRECV || tmp->mpi_op->op_type == CODES_WK_RECV)
@@ -667,7 +667,7 @@ static int match_receive(
         assert(op1->op_type == CODES_WK_IRECV || op1->op_type == CODES_WK_RECV);
         assert(op2->op_type == CODES_WK_SEND || op2->op_type == CODES_WK_ISEND);
 
-        if((op1->u.recv.num_bytes >= op2->u.send.num_bytes) &&
+        if((op1->u.recv.num_bytes == op2->u.send.num_bytes) &&
                    ((op1->u.recv.tag == op2->u.send.tag) || op1->u.recv.tag == -1) &&
                    ((op1->u.recv.source_rank == op2->u.send.source_rank) || op1->u.recv.source_rank == -1))
                    {
@@ -1215,7 +1215,7 @@ static void get_next_mpi_operation_rc(nw_state* s, tw_bf * bf, nw_message * m, t
 		{
 			s->num_waitall--;
 			codes_exec_mpi_wait_all_rc(s, m, lp, mpi_op);
-		}
+        }
 		break;
 		case CODES_WK_WAITSOME:
 		case CODES_WK_WAITANY:
@@ -1305,7 +1305,7 @@ static void get_next_mpi_operation(nw_state* s, tw_bf * bf, nw_message * m, tw_l
 			{
 				s->num_waitall++;
 				codes_exec_mpi_wait_all(s, lp, m, mpi_op);
-			}
+            }
 			break;
 			default:
 				printf("\n Invalid op type %d ", mpi_op->op_type);
@@ -1323,11 +1323,11 @@ void nw_test_finalize(nw_state* s, tw_lp* lp)
 	{
 		printf("\n LP %llu unmatched irecvs %d unmatched sends %d Total sends %ld receives %ld collectives %ld delays %ld wait alls %ld waits %ld send time %lf wait %lf", 
 			lp->gid, s->pending_recvs_queue->num_elems, s->arrival_queue->num_elems, s->num_sends, s->num_recvs, s->num_cols, s->num_delays, s->num_waitall, s->num_wait, s->send_time, s->wait_time);
-		//if(lp->gid == TRACE)
-		//{
+		if(lp->gid == TRACE)
+		{
 		   printQueue(lp->gid, s->pending_recvs_queue, "irecv ");
-		  printQueue(lp->gid, s->arrival_queue, "isend");
-	        //}
+		   printQueue(lp->gid, s->arrival_queue, "isend");
+	    }
         
             written += sprintf(s->output_buf + written, "\n %lu %lu %ld %ld %ld %ld %lf %lf %lf", lp->gid, s->nw_id, s->num_sends, s->num_recvs, s->num_bytes_sent, 
                 s->num_bytes_recvd, s->send_time, s->elapsed_time - s->compute_time, s->compute_time);

src/models/networks/model-net/dragonfly.c

@@ -32,10 +32,11 @@
 #define COLLECTIVE_COMPUTATION_DELAY 5700
 #define DRAGONFLY_FAN_OUT_DELAY 20.0
 #define WINDOW_LENGTH 0
-#define DFLY_HASH_TABLE_SIZE 65536
+#define DFLY_HASH_TABLE_SIZE 262144
 
 // debugging parameters
-#define TRACK -1
+#define TRACK 2
+#define TRACK_PKT 45543
 #define TRACK_MSG -1
 #define PRINT_ROUTER_TABLE 1
 #define DEBUG 0
@@ -45,6 +46,7 @@
 #define LP_METHOD_NM (model_net_method_names[DRAGONFLY])
 
 long term_ecount, router_ecount, term_rev_ecount, router_rev_ecount;
+long packet_gen = 0, packet_fin = 0;
 
 static double maxd(double a, double b) { return a < b ? b : a; }
 
@@ -68,6 +70,8 @@ int router_magic_num = 0;
 /* terminal magic number */
 int terminal_magic_num = 0;
 
+FILE * dragonfly_log = NULL;
+
 typedef struct terminal_message_list terminal_message_list;
 struct terminal_message_list {
     terminal_message msg;
@@ -112,6 +116,7 @@ struct dragonfly_param
     double local_delay;
     double global_delay;
     double credit_delay;
+    double router_delay;
 };
 
 struct dfly_hash_key
@@ -139,6 +144,9 @@ struct terminal_state
 {
    uint64_t packet_counter;
 
+   int packet_gen;
+   int packet_fin;
+
    // Dragonfly specific parameters
    unsigned int router_id;
    unsigned int terminal_id;
@@ -191,12 +199,15 @@ struct terminal_state
 
    tw_stime   total_time;
    long total_msg_size;
-   long total_hops;
+   double total_hops;
    long finished_msgs;
-   long finished_chunks;
+   double finished_chunks;
    long finished_packets;
 
-   char output_buf[512];
+   tw_stime last_buf_full;
+   tw_stime busy_time;
+
+   char output_buf[1024];
 };
 
 /* terminal event type (1-4) */
@@ -250,20 +261,27 @@ struct router_state
    
    tw_stime* next_output_available_time;
    tw_stime* cur_hist_start_time;
+   tw_stime* last_buf_full;
+   tw_stime* busy_time;
+
    terminal_message_list ***pending_msgs;
    terminal_message_list ***pending_msgs_tail;
    terminal_message_list ***queued_msgs;
    terminal_message_list ***queued_msgs_tail;
    int *in_send_loop;
+   int *queued_count;
+   struct rc_stack * st;
    
    int** vc_occupancy;
-   int* link_traffic;
+   int64_t* link_traffic;
 
    const char * anno;
    const dragonfly_param *params;
 
    int* prev_hist_num;
    int* cur_hist_num;
+   
+   char output_buf[1024];
 };
 
 static short routing = MINIMAL;
@@ -296,11 +314,9 @@ static int dragonfly_rank_hash_compare(
 static int dragonfly_hash_func(void *k, int table_size)
 {
 	struct dfly_hash_key *tmp = (struct dfly_hash_key *)k;
-	uint64_t key = (~tmp->message_id) + (tmp->message_id << 18);
-	key = key * 21;
-	key = ~key ^ (tmp->sender_id >> 4);
-	key = key * tmp->sender_id;
-	return (int)(key % table_size);	
+    uint32_t pc = 0, pb = 0;
+	bj_hashlittle2(tmp, sizeof(*tmp), &pc, &pb);
+    return (int)(pc % (table_size - 1));	
 }
 
 /* convert GiB/s and bytes to ns */
@@ -324,6 +340,11 @@ static int dragonfly_get_msg_sz(void)
 	   return sizeof(terminal_message);
 }
 
+static void free_nothing(void* ptr)
+{
+
+    /* Do nothing */
+}
 static void free_tmp(void * ptr)
 {
     struct dfly_qhash_entry * dfly = ptr; 
@@ -362,7 +383,7 @@ static void create_prepend_to_terminal_message_list(
         terminal_message_list ** thisq,
         terminal_message_list ** thistail,
         int index, 
-        terminal_message *msg) {
+        terminal_message * msg) {
     terminal_message_list* new_entry = (terminal_message_list*)malloc(
         sizeof(terminal_message_list));
     init_terminal_message_list(new_entry, msg);
@@ -397,6 +418,7 @@ static terminal_message_list* return_tail(
         terminal_message_list ** thistail,
         int index) {
     terminal_message_list *tail = thistail[index];
+    assert(tail);
     if(tail->prev != NULL) {
         tail->prev->next = NULL;
         thistail[index] = tail->prev;
@@ -415,7 +437,6 @@ static void copy_terminal_list_entry( terminal_message_list *cur_entry,
     msg->packet_ID = cur_msg->packet_ID;    
     strcpy(msg->category, cur_msg->category);
     msg->final_dest_gid = cur_msg->final_dest_gid;
-    msg->msg_start_time = msg->msg_start_time;
     msg->sender_lp = cur_msg->sender_lp;
     msg->dest_terminal_id = cur_msg->dest_terminal_id;
     msg->src_terminal_id = cur_msg->src_terminal_id;
@@ -507,6 +528,10 @@ static void dragonfly_read_config(const char * anno, dragonfly_param *params){
         fprintf(stderr, "Bandwidth of compute node channels not specified, setting to %lf\n", p->cn_bandwidth);
     }
 
+    p->router_delay = 50;
+    configuration_get_value_double(&config, "PARAMS", "router_delay", anno,
+            &p->router_delay);
+
     char routing_str[MAX_NAME_LENGTH];
     configuration_get_value(&config, "PARAMS", "routing", anno, routing_str,
             MAX_NAME_LENGTH);
@@ -570,6 +595,7 @@ static void dragonfly_report_stats()
    long long total_finished_msgs, final_msg_sz;
    tw_stime avg_time, max_time;
    int total_minimal_packets, total_nonmin_packets;
+   long total_gen, total_fin;
 
    MPI_Reduce( &total_hops, &avg_hops, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
    MPI_Reduce( &N_finished_packets, &total_finished_packets, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
@@ -578,6 +604,9 @@ static void dragonfly_report_stats()
    MPI_Reduce( &total_msg_sz, &final_msg_sz, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
    MPI_Reduce( &dragonfly_total_time, &avg_time, 1,MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
    MPI_Reduce( &dragonfly_max_latency, &max_time, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
+   
+   MPI_Reduce( &packet_gen, &total_gen, 1, MPI_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
+   MPI_Reduce( &packet_fin, &total_fin, 1, MPI_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
    if(routing == ADAPTIVE || routing == PROG_ADAPTIVE)
     {
 	MPI_Reduce(&minimal_count, &total_minimal_packets, 1, MPI_INT, MPI_SUM, 0, MPI_COMM_WORLD);
@@ -589,9 +618,10 @@ static void dragonfly_report_stats()
    {	
       printf(" Average number of hops traversed %f average chunk latency %lf us maximum chunk latency %lf us avg message size %lf bytes finished messages %ld \n", (float)avg_hops/total_finished_chunks, avg_time/(total_finished_chunks*1000), max_time/1000, (float)final_msg_sz/total_finished_msgs, total_finished_msgs);
      if(routing == ADAPTIVE || routing == PROG_ADAPTIVE)
-              printf("\n ADAPTIVE ROUTING STATS: %d percent chunks routed minimally %d percent chunks routed non-minimally completed packets %lld ", total_minimal_packets, total_nonmin_packets, total_finished_chunks);
+              printf("\n ADAPTIVE ROUTING STATS: %d chunks routed minimally %d chunks routed non-minimally completed packets %lld ", total_minimal_packets, total_nonmin_packets, total_finished_chunks);
  
-  }
+  printf("\n Total packets generated %ld finished %ld ", total_gen, total_fin);
+   }
    return;
 }
 
@@ -658,6 +688,9 @@ void
 terminal_init( terminal_state * s, 
 	       tw_lp * lp )
 {
+    s->packet_gen = 0;
+    s->packet_fin = 0;
+
     uint32_t h1 = 0, h2 = 0; 
     bj_hashlittle2(LP_METHOD_NM, strlen(LP_METHOD_NM), &h1, &h2);
     terminal_magic_num = h1 + h2;
@@ -693,6 +726,9 @@ terminal_init( terminal_state * s,
    s->total_time = 0.0;
    s->total_msg_size = 0;
 
+   s->last_buf_full = 0.0;
+   s->busy_time = 0.0;
+
    rc_stack_create(&s->st);
    s->num_vcs = 1;
    s->vc_occupancy = (int*)malloc(s->num_vcs * sizeof(int));
@@ -752,7 +788,7 @@ void router_setup(router_state * r, tw_lp * lp)
    r->global_channel = (int*)malloc(p->num_global_channels * sizeof(int));
    r->next_output_available_time = (tw_stime*)malloc(p->radix * sizeof(tw_stime));
    r->cur_hist_start_time = (tw_stime*)malloc(p->radix * sizeof(tw_stime));
-   r->link_traffic = (int*)malloc(p->radix * sizeof(int));
+   r->link_traffic = (int64_t*)malloc(p->radix * sizeof(int64_t));
    r->cur_hist_num = (int*)malloc(p->radix * sizeof(int));
    r->prev_hist_num = (int*)malloc(p->radix * sizeof(int));
    
@@ -766,16 +802,22 @@ void router_setup(router_state * r, tw_lp * lp)
     (terminal_message_list***)malloc(p->radix * sizeof(terminal_message_list**));
    r->queued_msgs_tail = 
     (terminal_message_list***)malloc(p->radix * sizeof(terminal_message_list**));
-  
+   r->queued_count = (int*)malloc(p->radix * sizeof(int));
+   r->last_buf_full = (tw_stime*)malloc(p->radix * sizeof(tw_stime));
+   r->busy_time = (tw_stime*)malloc(p->radix * sizeof(tw_stime));
+
+   rc_stack_create(&r->st);
    for(int i=0; i < p->radix; i++)
     {
        // Set credit & router occupancy
+    r->last_buf_full[i] = 0.0;
+    r->busy_time[i] = 0.0;
 	r->next_output_available_time[i]=0;
 	r->cur_hist_start_time[i] = 0;
-        r->link_traffic[i]=0;
+    r->link_traffic[i]=0;
 	r->cur_hist_num[i] = 0;
 	r->prev_hist_num[i] = 0;
-        
+    r->queued_count[i] = 0;    
         r->in_send_loop[i] = 0;
         r->vc_occupancy[i] = (int*)malloc(p->num_vcs * sizeof(int));
         r->pending_msgs[i] = (terminal_message_list**)malloc(p->num_vcs * 
@@ -1000,6 +1042,8 @@ void router_credit_send(router_state * s, tw_bf * bf, terminal_message * msg,
 
 void packet_generate_rc(terminal_state * s, tw_bf * bf, terminal_message * msg, tw_lp * lp)
 {
+   s->packet_gen--;
+   packet_gen--;
    tw_rand_reverse_unif(lp->rng);
 
    int num_chunks = msg->packet_size/s->params->chunk_size;
@@ -1021,6 +1065,7 @@ void packet_generate_rc(terminal_state * s, tw_bf * bf, terminal_message * msg,
     }
       if(bf->c11) {
         s->issueIdle = 0;
+        s->last_buf_full = msg->saved_busy_time;
       }
      struct mn_stats* stat;
      stat = model_net_find_stats(msg->category, s->dragonfly_stats_array);
@@ -1032,7 +1077,10 @@ void packet_generate_rc(terminal_state * s, tw_bf * bf, terminal_message * msg,
 /* generates packet at the current dragonfly compute node */
 void packet_generate(terminal_state * s, tw_bf * bf, terminal_message * msg, 
   tw_lp * lp) {
-  
+
+    s->packet_gen++;
+
+    packet_gen++;
   tw_stime ts, nic_ts;
 
   assert(lp->gid != msg->dest_terminal_id);
@@ -1054,14 +1102,20 @@ void packet_generate(terminal_state * s, tw_bf * bf, terminal_message * msg,
   msg->my_g_hop = 0;
   msg->intm_group_id = -1;
 
-  if(msg->packet_ID == TRACK && msg->message_id == TRACK_MSG)
-      printf("\n Packet generated at terminal %lu destination %d ", lp->gid, s->router_id);
+
+  //if(msg->dest_terminal_id == TRACK)
+  if(msg->packet_ID == TRACK_PKT)
+    printf("\n Packet %ld generated at terminal %d dest %ld size %d num chunks %d ", 
+            msg->packet_ID, s->terminal_id, msg->dest_terminal_id,
+            msg->packet_size, num_chunks);
 
   for(i = 0; i < num_chunks; i++)
   {
     terminal_message_list *cur_chunk = (terminal_message_list*)malloc(
       sizeof(terminal_message_list));
+    msg->origin_router_id = s->router_id;
     init_terminal_message_list(cur_chunk, msg);
+  
 
     if(msg->remote_event_size_bytes + msg->local_event_size_bytes > 0) {
       cur_chunk->event_data = (char*)malloc(
@@ -1079,6 +1133,7 @@ void packet_generate(terminal_state * s, tw_bf * bf, terminal_message * msg,
     }
 
     cur_chunk->msg.chunk_id = i;
+    cur_chunk->msg.origin_router_id = s->router_id;
     append_to_terminal_message_list(s->terminal_msgs, s->terminal_msgs_tail,
       0, cur_chunk);
     s->terminal_length += s->params->chunk_size;
@@ -1089,6 +1144,8 @@ void packet_generate(terminal_state * s, tw_bf * bf, terminal_message * msg,
   } else {
     bf->c11 = 1;
     s->issueIdle = 1;
+    msg->saved_busy_time = s->last_buf_full;
+    s->last_buf_full = tw_now(lp);
   }
   
   if(s->in_send_loop == 0) {
@@ -1121,6 +1178,8 @@ void packet_send_rc(terminal_state * s, tw_bf * bf, terminal_message * msg,
 {
       if(bf->c1) {
         s->in_send_loop = 1;
+        s->last_buf_full = msg->saved_busy_time;
+        
         return;
       }
       
@@ -1134,19 +1193,23 @@ void packet_send_rc(terminal_state * s, tw_bf * bf, terminal_message * msg,
       s->packet_counter--;
       s->vc_occupancy[0] -= s->params->chunk_size;
 
-      create_prepend_to_terminal_message_list(s->terminal_msgs,
-          s->terminal_msgs_tail, 0, msg);
+      terminal_message_list* cur_entry = rc_stack_pop(s->st);
+
+//      create_prepend_to_terminal_message_list(s->terminal_msgs,
+//          s->terminal_msgs_tail, 0, cur_entry);
+      prepend_to_terminal_message_list(s->terminal_msgs, 
+              s->terminal_msgs_tail, 0, cur_entry);
       if(bf->c3) {
         tw_rand_reverse_unif(lp->rng);
       }
       if(bf->c4) {
         s->in_send_loop = 1;
       }
-      if(bf->c5)
+      /*if(bf->c5)
       {
           codes_local_latency_reverse(lp);
           s->issueIdle = 1;
-      }
+      }*/
       return;