Merge branch 'wait-time-fix' into 'master'

Adding wait-time fix for dragonfly, dragonfly-custom and slimfly models

See merge request !39

src/networks/model-net/dragonfly-custom.C

@@ -251,7 +251,7 @@ struct terminal_state
    long finished_chunks;
    long finished_packets;
 
-   tw_stime last_buf_full;
+   tw_stime * last_buf_full;
    tw_stime busy_time;
    
    tw_stime max_latency;
@@ -326,7 +326,7 @@ struct router_state
    
    tw_stime* next_output_available_time;
    tw_stime* cur_hist_start_time;
-   tw_stime* last_buf_full;
+   tw_stime** last_buf_full;
 
    tw_stime* busy_time;
    tw_stime* busy_time_sample;
@@ -863,7 +863,6 @@ terminal_custom_init( terminal_state * s,
    s->total_time = 0.0;
    s->total_msg_size = 0;
 
-   s->last_buf_full = 0.0;
    s->busy_time = 0.0;
 
    s->fwd_events = 0;
@@ -872,9 +871,11 @@ terminal_custom_init( terminal_state * s,
    rc_stack_create(&s->st);
    s->num_vcs = 1;
    s->vc_occupancy = (int*)malloc(s->num_vcs * sizeof(int));
+   s->last_buf_full = (tw_stime*)malloc(s->num_vcs * sizeof(int));
 
    for( i = 0; i < s->num_vcs; i++ )
     {
+      s->last_buf_full[i] = 0.0;
       s->vc_occupancy[i]=0;
     }
 
@@ -951,7 +952,7 @@ void router_custom_setup(router_state * r, tw_lp * lp)
    r->queued_msgs_tail = 
     (terminal_custom_message_list***)malloc(p->radix * sizeof(terminal_custom_message_list**));
    r->queued_count = (int*)malloc(p->radix * sizeof(int));
-   r->last_buf_full = (tw_stime*)malloc(p->radix * sizeof(tw_stime));
+   r->last_buf_full = (tw_stime**)malloc(p->radix * sizeof(tw_stime*));
    r->busy_time = (tw_stime*)malloc(p->radix * sizeof(tw_stime));
    r->busy_time_sample = (tw_stime*)malloc(p->radix * sizeof(tw_stime));
 
@@ -963,7 +964,6 @@ void router_custom_setup(router_state * r, tw_lp * lp)
    for(int i=0; i < p->radix; i++)
     {
        // Set credit & router occupancy
-    r->last_buf_full[i] = 0.0;
     r->busy_time[i] = 0.0;
     r->busy_time_sample[i] = 0.0;
 	r->next_output_available_time[i]=0;
@@ -977,6 +977,7 @@ void router_custom_setup(router_state * r, tw_lp * lp)
     r->vc_occupancy[i] = (int*)malloc(p->num_vcs * sizeof(int));
     r->pending_msgs[i] = (terminal_custom_message_list**)malloc(p->num_vcs * 
         sizeof(terminal_custom_message_list*));
+    r->last_buf_full[i] = (tw_stime*)malloc(p->num_vcs * sizeof(tw_stime));
     r->pending_msgs_tail[i] = (terminal_custom_message_list**)malloc(p->num_vcs * 
         sizeof(terminal_custom_message_list*));
     r->queued_msgs[i] = (terminal_custom_message_list**)malloc(p->num_vcs * 
@@ -984,6 +985,7 @@ void router_custom_setup(router_state * r, tw_lp * lp)
     r->queued_msgs_tail[i] = (terminal_custom_message_list**)malloc(p->num_vcs * 
         sizeof(terminal_custom_message_list*));
         for(int j = 0; j < p->num_vcs; j++) {
+            r->last_buf_full[i][j] = 0.0;
             r->vc_occupancy[i][j] = 0;
             r->pending_msgs[i][j] = NULL;
             r->pending_msgs_tail[i][j] = NULL;
@@ -1137,7 +1139,10 @@ static void packet_generate_rc(terminal_state * s, tw_bf * bf, terminal_custom_m
     }
       if(bf->c11) {
         s->issueIdle = 0;
-        s->last_buf_full = msg->saved_busy_time;
+        if(bf->c8)
+        {
+            s->last_buf_full[0] = msg->saved_busy_time;
+        }
       }
      struct mn_stats* stat;
      stat = model_net_find_stats(msg->category, s->dragonfly_stats_array);
@@ -1214,8 +1219,14 @@ static void packet_generate(terminal_state * s, tw_bf * bf, terminal_custom_mess
   } else {
     bf->c11 = 1;
     s->issueIdle = 1;
-    msg->saved_busy_time = s->last_buf_full;
-    s->last_buf_full = tw_now(lp);
+
+      if(s->last_buf_full[0] == 0.0)
+      {
+        bf->c8 = 1;
+        msg->saved_busy_time = s->last_buf_full[0];
+        /* TODO: Assumes a single vc from terminal to router */
+        s->last_buf_full[0] = tw_now(lp);
+      }
   }
   
   if(s->in_send_loop == 0) {
@@ -1248,7 +1259,10 @@ static void packet_send_rc(terminal_state * s, tw_bf * bf, terminal_custom_messa
 {
       if(bf->c1) {
         s->in_send_loop = 1;
-        s->last_buf_full = msg->saved_busy_time;
+        
+        if(bf->c10)
+            s->last_buf_full[0] = msg->saved_busy_time;
+        
         return;
       }
       
@@ -1279,7 +1293,7 @@ static void packet_send_rc(terminal_state * s, tw_bf * bf, terminal_custom_messa
           if(bf->c6)
           {
             s->busy_time = msg->saved_total_time;
-            s->last_buf_full = msg->saved_busy_time;
+            s->last_buf_full[0] = msg->saved_busy_time;
             s->busy_time_sample = msg->saved_sample_time;
           }
       }
@@ -1296,14 +1310,20 @@ static void packet_send(terminal_state * s, tw_bf * bf, terminal_custom_message
 
   terminal_custom_message_list* cur_entry = s->terminal_msgs[0];
 
+  if(s->vc_occupancy[0] + s->params->chunk_size > s->params->cn_vc_size)
+  {
+      if(s->last_buf_full[0] == 0.0)
+      {
+        bf->c10 = 1;
+        msg->saved_busy_time = s->last_buf_full[0];
+        s->last_buf_full[0] = tw_now(lp);
+      }
+  }
+
   if(s->vc_occupancy[0] + s->params->chunk_size > s->params->cn_vc_size 
       || cur_entry == NULL) {
-    /* REMOVE cur_entry check*/
     bf->c1 = 1;
     s->in_send_loop = 0;
-
-    msg->saved_busy_time = s->last_buf_full;
-    s->last_buf_full = tw_now(lp);
     return;
   }
 
@@ -1388,16 +1408,16 @@ static void packet_send(terminal_state * s, tw_bf * bf, terminal_custom_message
     ts += tw_rand_unif(lp->rng);
     model_net_method_idle_event(ts, 0, lp);
    
-    if(s->last_buf_full > 0.0)
+    if(s->last_buf_full[0] > 0.0)
     {
         bf->c6 = 1;
         msg->saved_total_time = s->busy_time;
-        msg->saved_busy_time = s->last_buf_full;
+        msg->saved_busy_time = s->last_buf_full[0];
         msg->saved_sample_time = s->busy_time_sample;
 
-        s->busy_time += (tw_now(lp) - s->last_buf_full);
-        s->busy_time_sample += (tw_now(lp) - s->last_buf_full);
-        s->last_buf_full = 0.0;
+        s->busy_time += (tw_now(lp) - s->last_buf_full[0]);
+        s->busy_time_sample += (tw_now(lp) - s->last_buf_full[0]);
+        s->last_buf_full[0] = 0.0;
     }
   }
   return;
@@ -2734,7 +2754,10 @@ static void router_packet_receive_rc(router_state * s,
         }
       }
       if(bf->c4) {
-      s->last_buf_full[output_port] = msg->saved_busy_time;
+          if(bf->c22)
+          {
+            s->last_buf_full[output_port][output_chan] = msg->saved_busy_time;
+          }
       delete_terminal_custom_message_list(return_tail(s->queued_msgs[output_port], 
           s->queued_msgs_tail[output_port], output_chan));
       s->queued_count[output_port] -= s->params->chunk_size; 
@@ -2943,12 +2966,17 @@ router_packet_receive( router_state * s,
     append_to_terminal_custom_message_list( s->queued_msgs[output_port], 
       s->queued_msgs_tail[output_port], output_chan, cur_chunk);
     s->queued_count[output_port] += s->params->chunk_size;
-    msg->saved_busy_time = s->last_buf_full[output_port];
+
     /* a check for pending msgs is non-empty then we dont set anything. If
      * that is empty then we check if last_buf_full is set or not. If already
      * set then we don't overwrite it. If two packets arrive next to each other
      * then the first person should be setting it. */
-    s->last_buf_full[output_port] = tw_now(lp);
+    if(s->pending_msgs[output_port][output_chan] == NULL && s->last_buf_full[output_port][output_chan] == 0.0)
+          {
+            bf->c22 = 1;
+            msg->saved_busy_time = s->last_buf_full[output_port][output_chan];
+            s->last_buf_full[output_port][output_chan] = tw_now(lp);
+          }
   }
 
   msg->saved_vc = output_port;
@@ -3163,7 +3191,7 @@ static void router_buf_update_rc(router_state * s,
       {
         s->busy_time[indx] = msg->saved_rcv_time;
         s->busy_time_sample[indx] = msg->saved_sample_time;
-        s->last_buf_full[indx] = msg->saved_busy_time;
+        s->last_buf_full[indx][output_chan] = msg->saved_busy_time;
       }
       if(bf->c1) {
         terminal_custom_message_list* head = return_tail(s->pending_msgs[indx],
@@ -3186,15 +3214,15 @@ static void router_buf_update(router_state * s, tw_bf * bf, terminal_custom_mess
   int output_chan = msg->output_chan;
   s->vc_occupancy[indx][output_chan] -= s->params->chunk_size;
   
-  if(s->last_buf_full[indx])
+  if(s->last_buf_full[indx][output_chan] > 0.0)
   {
     bf->c3 = 1;
     msg->saved_rcv_time = s->busy_time[indx];
-    msg->saved_busy_time = s->last_buf_full[indx];
+    msg->saved_busy_time = s->last_buf_full[indx][output_chan];
     msg->saved_sample_time = s->busy_time_sample[indx];
-    s->busy_time[indx] += (tw_now(lp) - s->last_buf_full[indx]);
-    s->busy_time_sample[indx] += (tw_now(lp) - s->last_buf_full[indx]);
-    s->last_buf_full[indx] = 0.0;
+    s->busy_time[indx] += (tw_now(lp) - s->last_buf_full[indx][output_chan]);
+    s->busy_time_sample[indx] += (tw_now(lp) - s->last_buf_full[indx][output_chan]);
+    s->last_buf_full[indx][output_chan] = 0.0;
   }
   if(s->queued_msgs[indx][output_chan] != NULL) {
     bf->c1 = 1;

src/networks/model-net/dragonfly.c

@@ -250,7 +250,7 @@ struct terminal_state
    long finished_chunks;
    long finished_packets;
 
-   tw_stime last_buf_full;
+   tw_stime * last_buf_full;
    tw_stime busy_time;
    char output_buf[4096];
    /* For LP suspend functionality */
@@ -333,7 +333,7 @@ struct router_state
    
    tw_stime* next_output_available_time;
    tw_stime* cur_hist_start_time;
-   tw_stime* last_buf_full;
+   tw_stime** last_buf_full;
 
    tw_stime* busy_time;
    tw_stime* busy_time_sample;
@@ -791,7 +791,7 @@ terminal_init( terminal_state * s,
    s->total_time = 0.0;
    s->total_msg_size = 0;
 
-   s->last_buf_full = 0.0;
+   s->last_buf_full = (tw_stime*)malloc(s->num_vcs * sizeof(int));
    s->busy_time = 0.0;
 
    s->fwd_events = 0;
@@ -803,6 +803,7 @@ terminal_init( terminal_state * s,
 
    for( i = 0; i < s->num_vcs; i++ )
     {
+      s->last_buf_full[i] = 0.0;
       s->vc_occupancy[i]=0;
     }
 
@@ -891,7 +892,7 @@ static void router_setup(router_state * r, tw_lp * lp)
    r->queued_msgs_tail = 
     (terminal_message_list***)malloc(p->radix * sizeof(terminal_message_list**));
    r->queued_count = (int*)malloc(p->radix * sizeof(int));
-   r->last_buf_full = (tw_stime*)malloc(p->radix * sizeof(tw_stime));
+   r->last_buf_full = (tw_stime**)malloc(p->radix * sizeof(tw_stime*));
    r->busy_time = (tw_stime*)malloc(p->radix * sizeof(tw_stime));
    r->busy_time_sample = (tw_stime*)malloc(p->radix * sizeof(tw_stime));
 
@@ -905,7 +906,6 @@ static void router_setup(router_state * r, tw_lp * lp)
    for(int i=0; i < p->radix; i++)
     {
        // Set credit & router occupancy
-    r->last_buf_full[i] = 0.0;
     r->busy_time[i] = 0.0;
     r->busy_time_sample[i] = 0.0;
 	r->next_output_available_time[i]=0;
@@ -917,6 +917,7 @@ static void router_setup(router_state * r, tw_lp * lp)
     r->queued_count[i] = 0;    
     r->in_send_loop[i] = 0;
     r->vc_occupancy[i] = (int*)malloc(p->num_vcs * sizeof(int));
+    r->last_buf_full[i] = (tw_stime*)malloc(p->num_vcs * sizeof(tw_stime));
     r->pending_msgs[i] = (terminal_message_list**)malloc(p->num_vcs * 
         sizeof(terminal_message_list*));
     r->pending_msgs_tail[i] = (terminal_message_list**)malloc(p->num_vcs * 
@@ -926,6 +927,7 @@ static void router_setup(router_state * r, tw_lp * lp)
     r->queued_msgs_tail[i] = (terminal_message_list**)malloc(p->num_vcs * 
         sizeof(terminal_message_list*));
         for(int j = 0; j < p->num_vcs; j++) {
+            r->last_buf_full[i][j] = 0.0;
             r->vc_occupancy[i][j] = 0;
             r->pending_msgs[i][j] = NULL;
             r->pending_msgs_tail[i][j] = NULL;
@@ -1165,7 +1167,10 @@ static void packet_generate_rc(terminal_state * s, tw_bf * bf, terminal_message
     }
       if(bf->c11) {
         s->issueIdle = 0;
-        s->last_buf_full = msg->saved_busy_time;
+        if(bf->c8)
+        {
+            s->last_buf_full[0] = msg->saved_busy_time;
+        }
       }
      struct mn_stats* stat;
      stat = model_net_find_stats(msg->category, s->dragonfly_stats_array);
@@ -1242,8 +1247,13 @@ static void packet_generate(terminal_state * s, tw_bf * bf, terminal_message * m
   } else {
     bf->c11 = 1;
     s->issueIdle = 1;
-    msg->saved_busy_time = s->last_buf_full;
-    s->last_buf_full = tw_now(lp);
+      if(s->last_buf_full[0] == 0.0)
+      {
+        bf->c8 = 1;
+        msg->saved_busy_time = s->last_buf_full[0];
+        /* TODO: Assumes a single vc from terminal to router */
+        s->last_buf_full[0] = tw_now(lp);
+      }
   }
   
   if(s->in_send_loop == 0) {
@@ -1276,7 +1286,8 @@ static void packet_send_rc(terminal_state * s, tw_bf * bf, terminal_message * ms
 {
       if(bf->c1) {
         s->in_send_loop = 1;
-        s->last_buf_full = msg->saved_busy_time;
+        if(bf->c10)
+            s->last_buf_full[0] = msg->saved_busy_time;
         return;
       }
       
@@ -1307,7 +1318,7 @@ static void packet_send_rc(terminal_state * s, tw_bf * bf, terminal_message * ms
           if(bf->c6)
           {
             s->busy_time = msg->saved_total_time;
-            s->last_buf_full = msg->saved_busy_time;
+            s->last_buf_full[0] = msg->saved_busy_time;
             s->busy_time_sample = msg->saved_sample_time;
             s->busy_time_ross_sample = msg->saved_busy_time_ross;
           }
@@ -1325,13 +1336,19 @@ static void packet_send(terminal_state * s, tw_bf * bf, terminal_message * msg,
 
   terminal_message_list* cur_entry = s->terminal_msgs[0];
 
+  if(s->vc_occupancy[0] + s->params->chunk_size > s->params->cn_vc_size)
+  {
+      if(s->last_buf_full[0] == 0.0)
+      {
+        bf->c10 = 1;
+        msg->saved_busy_time = s->last_buf_full[0];
+        s->last_buf_full[0] = tw_now(lp);
+      }
+  }
   if(s->vc_occupancy[0] + s->params->chunk_size > s->params->cn_vc_size 
       || cur_entry == NULL) {
     bf->c1 = 1;
     s->in_send_loop = 0;
-
-    msg->saved_busy_time = s->last_buf_full;
-    s->last_buf_full = tw_now(lp);
     return;
   }
 
@@ -1418,18 +1435,17 @@ static void packet_send(terminal_state * s, tw_bf * bf, terminal_message * msg,
     ts += tw_rand_unif(lp->rng);
     model_net_method_idle_event(ts, 0, lp);
    
-    if(s->last_buf_full > 0.0)
+    if(s->last_buf_full[0] > 0.0)
     {
         bf->c6 = 1;
         msg->saved_total_time = s->busy_time;
-        msg->saved_busy_time = s->last_buf_full;
         msg->saved_sample_time = s->busy_time_sample;
         msg->saved_busy_time_ross = s->busy_time_ross_sample;
 
-        s->busy_time += (tw_now(lp) - s->last_buf_full);
-        s->busy_time_sample += (tw_now(lp) - s->last_buf_full);
-        s->busy_time_ross_sample += (tw_now(lp) - s->last_buf_full);
-        s->last_buf_full = 0.0;
+        s->busy_time += (tw_now(lp) - s->last_buf_full[0]);
+        s->busy_time_sample += (tw_now(lp) - s->last_buf_full[0]);
+        s->busy_time_ross_sample += (tw_now(lp) - s->last_buf_full[0]);
+        s->last_buf_full[0] = 0.0;
     }
   }
   return;
@@ -2797,7 +2813,10 @@ static void router_packet_receive_rc(router_state * s,
         }
       }
       if(bf->c4) {
-      s->last_buf_full[output_port] = msg->saved_busy_time;
+          if(bf->c22)
+          {
+            s->last_buf_full[output_port][output_chan] = msg->saved_busy_time;
+          }
       delete_terminal_message_list(return_tail(s->queued_msgs[output_port], 
           s->queued_msgs_tail[output_port], output_chan));
       s->queued_count[output_port] -= s->params->chunk_size; 
@@ -2917,8 +2936,12 @@ router_packet_receive( router_state * s,
     append_to_terminal_message_list( s->queued_msgs[output_port], 
       s->queued_msgs_tail[output_port], output_chan, cur_chunk);
     s->queued_count[output_port] += s->params->chunk_size;
-    msg->saved_busy_time = s->last_buf_full[output_port];
-    s->last_buf_full[output_port] = tw_now(lp);
+    if(s->pending_msgs[output_port][output_chan] == NULL && s->last_buf_full[output_port][output_chan] == 0.0)
+          {
+            bf->c22 = 1;
+            msg->saved_busy_time = s->last_buf_full[output_port][output_chan];
+            s->last_buf_full[output_port][output_chan] = tw_now(lp);
+          }
   }
 
   msg->saved_vc = output_port;
@@ -3155,7 +3178,7 @@ static void router_buf_update_rc(router_state * s,
         s->busy_time[indx] = msg->saved_rcv_time;
         s->busy_time_sample[indx] = msg->saved_sample_time;
         s->busy_time_ross_sample[indx] = msg->saved_busy_time_ross;
-        s->last_buf_full[indx] = msg->saved_busy_time;
+        s->last_buf_full[indx][output_chan] = msg->saved_busy_time;
       }
       if(bf->c1) {
         terminal_message_list* head = return_tail(s->pending_msgs[indx],
@@ -3178,17 +3201,17 @@ static void router_buf_update(router_state * s, tw_bf * bf, terminal_message * m
   int output_chan = msg->output_chan;
   s->vc_occupancy[indx][output_chan] -= s->params->chunk_size;
   
-  if(s->last_buf_full[indx])
+  if(s->last_buf_full[indx][output_chan])
   {
     bf->c3 = 1;
     msg->saved_rcv_time = s->busy_time[indx];
-    msg->saved_busy_time = s->last_buf_full[indx];
+    msg->saved_busy_time = s->last_buf_full[indx][output_chan];
     msg->saved_sample_time = s->busy_time_sample[indx];
     msg->saved_busy_time_ross = s->busy_time_ross_sample[indx];
-    s->busy_time[indx] += (tw_now(lp) - s->last_buf_full[indx]);
-    s->busy_time_sample[indx] += (tw_now(lp) - s->last_buf_full[indx]);
-    s->busy_time_ross_sample[indx] += (tw_now(lp) - s->last_buf_full[indx]);
-    s->last_buf_full[indx] = 0.0;
+    s->busy_time[indx] += (tw_now(lp) - s->last_buf_full[indx][output_chan]);
+    s->busy_time_sample[indx] += (tw_now(lp) - s->last_buf_full[indx][output_chan]);
+    s->busy_time_ross_sample[indx] += (tw_now(lp) - s->last_buf_full[indx][output_chan]);
+    s->last_buf_full[indx][output_chan] = 0.0;
   }
   if(s->queued_msgs[indx][output_chan] != NULL) {
     bf->c1 = 1;

src/networks/model-net/slimfly.c

@@ -207,7 +207,7 @@ struct terminal_state
     long finished_chunks;
     long finished_packets;
 
-    tw_stime last_buf_full;
+    tw_stime * last_buf_full;
     tw_stime busy_time;
 
     char output_buf[512];
@@ -267,7 +267,7 @@ struct router_state
     int *in_send_loop;
     struct rc_stack * st;
 
-    tw_stime* last_buf_full;
+    tw_stime** last_buf_full;
     tw_stime* busy_time;
     tw_stime* busy_time_sample;
 
@@ -784,7 +784,7 @@ void slim_terminal_init( terminal_state * s,
     s->total_time = 0.0;
     s->total_msg_size = 0;
 
-    s->last_buf_full = 0;
+   s->last_buf_full = (tw_stime*)malloc(s->num_vcs * sizeof(int));
     s->busy_time = 0;
 
     rc_stack_create(&s->st);
@@ -793,6 +793,7 @@ void slim_terminal_init( terminal_state * s,
 
     for( i = 0; i < s->num_vcs; i++ )
     {
+        s->last_buf_full[i] = 0.0;
         s->vc_occupancy[i]=0;
     }
 
@@ -856,7 +857,7 @@ void slim_router_setup(router_state * r, tw_lp * lp)
     r->queued_msgs_tail =
         (slim_terminal_message_list***)malloc(p->radix * sizeof(slim_terminal_message_list**));
 
-    r->last_buf_full = (tw_stime*)malloc(p->radix * sizeof(tw_stime));
+   r->last_buf_full = (tw_stime**)malloc(p->radix * sizeof(tw_stime*));
     r->busy_time = (tw_stime*)malloc(p->radix * sizeof(tw_stime));
 
     rc_stack_create(&r->st);
@@ -869,13 +870,13 @@ void slim_router_setup(router_state * r, tw_lp * lp)
         r->cur_hist_num[i] = 0;
         r->prev_hist_num[i] = 0;
 
-        r->last_buf_full[i] = 0.0;
         r->busy_time[i] = 0.0;
 
         r->in_send_loop[i] = 0;
         r->vc_occupancy[i] = (int*)malloc(p->num_vcs * sizeof(int));
-        r->pending_msgs[i] = (slim_terminal_message_list**)malloc(p->num_vcs *
+        r->pending_msgs[i] = (slim_terminal_message_list**)malloc(p->num_vcs * 
                 sizeof(slim_terminal_message_list*));
+        r->last_buf_full[i] = (tw_stime*)malloc(p->num_vcs * sizeof(tw_stime));
         r->pending_msgs_tail[i] = (slim_terminal_message_list**)malloc(p->num_vcs *
                 sizeof(slim_terminal_message_list*));
         r->queued_msgs[i] = (slim_terminal_message_list**)malloc(p->num_vcs *
@@ -883,6 +884,7 @@ void slim_router_setup(router_state * r, tw_lp * lp)
         r->queued_msgs_tail[i] = (slim_terminal_message_list**)malloc(p->num_vcs *
                 sizeof(slim_terminal_message_list*));
         for(int j = 0; j < p->num_vcs; j++) {
+            r->last_buf_full[i][j] = 0.0;
             r->vc_occupancy[i][j] = 0;
             r->pending_msgs[i][j] = NULL;
             r->pending_msgs_tail[i][j] = NULL;
@@ -1206,7 +1208,10 @@ void slim_packet_generate_rc(terminal_state * s, tw_bf * bf, slim_terminal_messa
     if(bf->c11)
     {
         s->issueIdle = 0;
-        s->last_buf_full = msg->saved_busy_time;
+        if(bf->c8)
+        {
+            s->last_buf_full[0] = msg->saved_busy_time;
+        }
     }
     struct mn_stats* stat;
     stat = model_net_find_stats(msg->category, s->slimfly_stats_array);
@@ -1282,8 +1287,13 @@ void slim_packet_generate(terminal_state * s, tw_bf * bf, slim_terminal_message
     {
         bf->c11 = 1;
         s->issueIdle = 1;
-        msg->saved_busy_time = s->last_buf_full;
-        s->last_buf_full = tw_now(lp);
+        if(s->last_buf_full[0] == 0.0)
+        {
+            bf->c8 = 1;
+            msg->saved_busy_time = s->last_buf_full[0];
+            /* TODO: Assumes a single vc from terminal to router */
+            s->last_buf_full[0] = tw_now(lp);
+        }
     }
 
     if(s->in_send_loop == 0)
@@ -1316,7 +1326,10 @@ void slim_packet_send_rc(terminal_state * s, tw_bf * bf, slim_terminal_message *
 {
     if(bf->c1) {
         s->in_send_loop = 1;
-        s->last_buf_full = msg->saved_busy_time;
+        
+        if(bf->c10)
+            s->last_buf_full[0] = msg->saved_busy_time;
+        
         return;
     }
 
@@ -1347,7 +1360,7 @@ void slim_packet_send_rc(terminal_state * s, tw_bf * bf, slim_terminal_message *
         if(bf->c6)
         {
             s->busy_time = msg->saved_total_time;
-            s->last_buf_full = msg->saved_busy_time;
+            s->last_buf_full[0] = msg->saved_busy_time;
         }
     }
     return;
@@ -1363,14 +1376,21 @@ void slim_packet_send(terminal_state * s, tw_bf * bf, slim_terminal_message * ms
     tw_lpid router_id;
 
     slim_terminal_message_list* cur_entry = s->terminal_msgs[0];
+  if(s->vc_occupancy[0] + s->params->chunk_size > s->params->cn_vc_size)
+  {
+      if(s->last_buf_full[0] == 0.0)
+      {
+        bf->c10 = 1;
+        msg->saved_busy_time = s->last_buf_full[0];
+        s->last_buf_full[0] = tw_now(lp);
+      }
+  }
 
     if(s->vc_occupancy[0] + s->params->chunk_size > s->params->cn_vc_size
             || cur_entry == NULL)
     {
         bf->c1 = 1;
         s->in_send_loop = 0;
-        msg->saved_busy_time = s->last_buf_full;
-        s->last_buf_full = tw_now(lp);
         return;
     }
 
@@ -1481,14 +1501,14 @@ void slim_packet_send(terminal_state * s, tw_bf * bf, slim_terminal_message * ms
         ts += tw_rand_unif(lp->rng);
         model_net_method_idle_event(ts, 0, lp);
 
-        if(s->last_buf_full > 0.0)
+        if(s->last_buf_full[0] > 0.0)
         {
             bf->c6 = 1;
             msg->saved_total_time = s->busy_time;
-            msg->saved_busy_time = s->last_buf_full;
+            msg->saved_busy_time = s->last_buf_full[0];
 
-            s->busy_time += (tw_now(lp) - s->last_buf_full);
-            s->last_buf_full = 0.0;
+            s->busy_time += (tw_now(lp) - s->last_buf_full[0]);
+            s->last_buf_full[0] = 0.0;
         }
     }
     return;
@@ -2760,7 +2780,10 @@ void slim_router_packet_receive_rc(router_state * s,
         }
     }
     if(bf->c4) {
-        s->last_buf_full[output_port] = msg->saved_busy_time;
+          if(bf->c22)
+          {
+        s->last_buf_full[output_port][output_chan] = msg->saved_busy_time;
+          }
         slim_delete_terminal_message_list(return_tail(s->queued_msgs[output_port],
                     s->queued_msgs_tail[output_port], output_chan));
     }
@@ -2939,8 +2962,12 @@ slim_router_packet_receive( router_state * s,
         cur_chunk->msg.saved_vc = msg->vc_index;
         cur_chunk->msg.saved_channel = msg->output_chan;
         append_to_terminal_message_list( s->queued_msgs[output_port], s->queued_msgs_tail[output_port], output_chan, cur_chunk);
-        msg->saved_busy_time = s->last_buf_full[output_port];
-        s->last_buf_full[output_port] = tw_now(lp);
+        if(s->pending_msgs[output_port][output_chan] == NULL && s->last_buf_full[output_port][output_chan] == 0.0)
+          {
+            bf->c22 = 1;
+            msg->saved_busy_time = s->last_buf_full[output_port][output_chan];
+            s->last_buf_full[output_port][output_chan] = tw_now(lp);
+          }
     }
 
     msg->saved_vc = output_port;
@@ -3177,7 +3204,7 @@ void slim_router_buf_update_rc(router_state * s,
     if(bf->c3)
     {
         s->busy_time[indx] = msg->saved_rcv_time;
-        s->last_buf_full[indx] = msg->saved_busy_time;
+        s->last_buf_full[indx][output_chan] = msg->saved_busy_time;
     }
     if(bf->c1) {
         slim_terminal_message_list* head = return_tail(s->pending_msgs[indx],
@@ -3203,9 +3230,9 @@ void slim_router_buf_update(router_state * s, tw_bf * bf, slim_terminal_message
     {
         bf->c3 = 1;
         msg->saved_rcv_time = s->busy_time[indx];
-        msg->saved_busy_time = s->last_buf_full[indx];
-        s->busy_time[indx] += (tw_now(lp) - s->last_buf_full[indx]);
-        s->last_buf_full[indx] = 0.0;
+        msg->saved_busy_time = s->last_buf_full[indx][output_chan];
+        s->busy_time[indx] += (tw_now(lp) - s->last_buf_full[indx][output_chan]);
+        s->last_buf_full[indx][output_chan] = 0.0;
     }
 
 #if ROUTER_OCCUPANCY_LOG