fattree.c 77.9 KB
Newer Older
1
2
#include <ross.h>

Noah Wolfe's avatar
Noah Wolfe committed
3
#include "codes/jenkins-hash.h"
4
5
6
7
8
9
10
#include "codes/codes_mapping.h"
#include "codes/codes.h"
#include "codes/model-net.h"
#include "codes/model-net-method.h"
#include "codes/model-net-lp.h"
#include "codes/net/fattree.h"
#include "sys/file.h"
11
#include "codes/quickhash.h"
Noah Wolfe's avatar
Noah Wolfe committed
12
#include "codes/rc-stack.h"
13
14
15
16
17
//#include "codes/map_messages.h"

#define CREDIT_SIZE 8
#define MEAN_PROCESS 1.0

18
19
#define FTREE_HASH_TABLE_SIZE 262144

20
// debugging parameters
Noah Wolfe's avatar
Noah Wolfe committed
21
22
//#define TRACK_PKT -1
#define TRACK_PKT 2820
23
#define FATTREE_HELLO 0
24
#define FATTREE_DEBUG 0
25
26
#define FATTREE_CONNECTIONS 0
#define FATTREE_MSG 0
Noah Wolfe's avatar
Noah Wolfe committed
27
#define DEBUG_RC 1
28

29
30
31
//Data Collection Output Files
#define PARAMS_LOG 1

32
33
34
#define LP_CONFIG_NM (model_net_lp_config_names[FATTREE])
#define LP_METHOD_NM (model_net_method_names[FATTREE])

35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
#if DEBUG_RC
  //Reverse Compute Debug Variables
  long long packet_event_f = 0;
  long long packet_event_r = 0;
  long long t_generate_f = 0;
  long long t_generate_r = 0;
  long long t_send_f = 0;
  long long t_send_r = 0;
  long long t_arrive_f = 0;
  long long t_arrive_r = 0;
  long long t_buffer_f = 0;
  long long t_buffer_r = 0;
  long long s_send_f = 0;
  long long s_send_r = 0;
  long long s_arrive_f = 0;
  long long s_arrive_r = 0;
  long long s_buffer_f = 0;
  long long s_buffer_r = 0;
#endif

55
56
long fattree_packet_gen = 0, fattree_packet_fin = 0;

57
58
59
static double maxd(double a, double b) { return a < b ? b : a; }

// arrival rate
Noah Wolfe's avatar
Noah Wolfe committed
60
//static double MEAN_INTERVAL=200.0;
61
62
63
64
65
66

typedef struct fattree_param fattree_param;
/* annotation-specific parameters (unannotated entry occurs at the 
 * last index) */
static uint64_t                  num_params = 0;
static fattree_param           * all_params = NULL;
Noah Wolfe's avatar
Noah Wolfe committed
67
static const config_anno_map_t * anno_map   = NULL;
68
69
70
71
72
73
74

/* global variables for codes mapping */
static char lp_group_name[MAX_NAME_LENGTH];
static char def_group_name[MAX_NAME_LENGTH];
static int def_gname_set = 0;
static int mapping_grp_id, mapping_type_id, mapping_rep_id, mapping_offset;

75
76
77
/* switch magic number */
int switch_magic_num = 0;

78
/* terminal magic number */
79
int fattree_terminal_magic_num = 0;
80

81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
typedef struct fattree_message_list fattree_message_list;
struct fattree_message_list {
    fattree_message msg;
    char* event_data;
    fattree_message_list *next;
    fattree_message_list *prev;
};

void init_fattree_message_list(fattree_message_list *this, 
  fattree_message *inmsg) {
    this->msg = *inmsg;
    this->event_data = NULL;
    this->next = NULL;
    this->prev = NULL;
}

void delete_fattree_message_list(fattree_message_list *this) {
    if(this->event_data != NULL) free(this->event_data);
    free(this);
}

struct fattree_param
{
  int ft_type;
  // configuration parameters
  int num_levels;
  int *num_switches; //switches at various levels
  int *switch_radix; //radix of switches are various levels
  double link_bandwidth;/* bandwidth of a wire connecting switches */
  double cn_bandwidth;/* bandwidth of the compute node channels 
                        connected to switch */
  int vc_size; /* buffer size of the link channels */
  int cn_vc_size; /* buffer size of the compute node channels */
114
  int chunk_size; /* full-sized packets are broken into smaller chunks.*/
115
116
117
118
119
120
121
122
123
124
125
  int packet_size; 
  int num_terminals;
  int l1_set_size;
  int l1_term_size;
  double cn_delay;
  double head_delay;
  double credit_delay;
  double router_delay;
  double soft_delay;
};

126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
struct ftree_hash_key
{
    uint64_t message_id;
    tw_lpid sender_id;
};

struct ftree_qhash_entry
{
   struct ftree_hash_key key;
   char * remote_event_data;
   int num_chunks;
   int remote_event_size;
   struct qhash_head hash_link;
};

141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
/* convert GiB/s and bytes to ns */
 static tw_stime bytes_to_ns(uint64_t bytes, double GB_p_s)
 {
     tw_stime time;
 
     /* bytes to GB */
     time = ((double)bytes)/(1024.0*1024.0*1024.0);
     /* GiB to s */
     time = time / GB_p_s;
     /* s to ns */
     time = time * 1000.0 * 1000.0 * 1000.0;
 
     return(time);
 }

156
157
158
159
160
161
162
163
164
/* handles terminal and switch events like packet generate/send/receive/buffer */
typedef enum event_t event_t;
typedef struct ft_terminal_state ft_terminal_state;
typedef struct switch_state switch_state;

/* fattree compute node data structure */
struct ft_terminal_state
{
  unsigned long long packet_counter;
165
  int packet_gen;
166
  int packet_fin;
167
168
169
170
171
172
173
174
175
176
  // Fattree specific parameters
  unsigned int terminal_id;
  unsigned int switch_id;
  tw_lpid switch_lp;

  // Each terminal will have an input and output channel with the switch
  int vc_occupancy; // NUM_VC
  tw_stime terminal_available_time;
  tw_stime next_credit_available_time;
  
177
178
  struct mn_stats fattree_stats_array[CATEGORY_MAX];

179
180
  fattree_message_list **terminal_msgs;
  fattree_message_list **terminal_msgs_tail;
181
  int terminal_length;
182
  int in_send_loop;
183
184
185
  int issueIdle;

   struct rc_stack * st;
186
187
188

  char * anno;
  fattree_param *params;
189
190
191
192
193
194
195
196
197
198
199

  struct qhash_table *rank_tbl;
  uint64_t rank_tbl_pop;

  tw_stime   total_time;
  uint64_t total_msg_size;
  double total_hops;
  long finished_msgs;
  long finished_chunks;
  long finished_packets;

200
  tw_stime last_buf_full;
201
  tw_stime busy_time;
202
  char output_buf[4096];
203
  char output_buf3[4096];
204

205
206
207
208
209
  /* For sampling */
  long fin_chunks_sample;
  long data_size_sample;
  double fin_hops_sample;
  tw_stime fin_chunks_time;
210
 tw_stime busy_time_sample;
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
};

/* terminal event type (1-4) */
enum event_t
{
  T_GENERATE=1,
  T_ARRIVE,
  T_SEND,
  T_BUFFER,
  S_SEND,
  S_ARRIVE,
  S_BUFFER,
};

enum last_hop
{
   LINK,
   TERMINAL
};

struct switch_state
{
  unsigned int switch_id;
  int switch_level;
  int radix;
  int num_cons;
  int num_lcons;
  int con_per_lneigh;
  int con_per_uneigh;;
  int start_lneigh;
  int end_lneigh;
  int start_uneigh;
  int unused;

  tw_stime* next_output_available_time;
  tw_stime* next_credit_available_time;
247
248
249
250
251
  tw_stime* last_buf_full;

  tw_stime* busy_time;
  tw_stime* busy_time_sample;

252
253
254
255
256
257
258
259
260
261
  fattree_message_list **pending_msgs;
  fattree_message_list **pending_msgs_tail;
  fattree_message_list **queued_msgs;
  fattree_message_list **queued_msgs_tail;
  int *queued_length;
  int *in_send_loop;
  int* vc_occupancy;
  int64_t* link_traffic;
  tw_lpid *port_connections;

262
263
264
  char output_buf[4096];
  char output_buf2[4096];

Noah Wolfe's avatar
Noah Wolfe committed
265
266
  struct rc_stack * st;

267
268
269
270
  char * anno;
  fattree_param *params;
};

271
272
static tw_stime         fattree_total_time = 0;
static tw_stime         fattree_max_latency = 0;
Noah Wolfe's avatar
Noah Wolfe committed
273
//static tw_stime         max_collective = 0;
274
275
276
277
278
279
280
281


static long long       total_hops = 0;
static long long       N_finished_packets = 0;
static long long       total_msg_sz = 0;
static long long       N_finished_msgs = 0;
static long long       N_finished_chunks = 0;

282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
static int fattree_rank_hash_compare(
void *key, struct qhash_head *link)
{
    struct ftree_hash_key *message_key = (struct ftree_hash_key *)key;
    struct ftree_qhash_entry *tmp = NULL;

    tmp = qhash_entry(link, struct ftree_qhash_entry, hash_link);

    if (tmp->key.message_id == message_key->message_id
            && tmp->key.sender_id == message_key->sender_id)
        return 1;

    return 0;
}
static int fattree_hash_func(void *k, int table_size)
{
    struct ftree_hash_key *tmp = (struct ftree_hash_key *)k;
    //uint32_t pc = 0, pb = 0;
    //bj_hashlittle2(tmp, sizeof(*tmp), &pc, &pb);
    uint64_t key = (~tmp->message_id) + (tmp->message_id << 18);
    key = key * 21;
    key = ~key ^ (tmp->sender_id >> 4);
    key = key * tmp->sender_id;
    return (int)(key & (table_size - 1));
    //return (int)(pc % (table_size - 1));
}

309
310
311
312
313
314
315
static void free_tmp(void * ptr)
{
    struct ftree_qhash_entry * ftree = ptr; 
    free(ftree->remote_event_data);
    free(ftree);
}

316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
static void append_to_fattree_message_list(  
        fattree_message_list ** thisq,
        fattree_message_list ** thistail,
        int index, 
        fattree_message_list *msg) {
    if(thisq[index] == NULL) {
        thisq[index] = msg;
    } else {
        thistail[index]->next = msg;
        msg->prev = thistail[index];
    } 
    thistail[index] = msg;
}

static void prepend_to_fattree_message_list(  
        fattree_message_list ** thisq,
        fattree_message_list ** thistail,
        int index, 
        fattree_message_list *msg) {
    if(thisq[index] == NULL) {
        thistail[index] = msg;
    } else {
        thisq[index]->prev = msg;
        msg->next = thisq[index];
    } 
    thisq[index] = msg;
}

static fattree_message_list* return_head(
        fattree_message_list ** thisq,
        fattree_message_list ** thistail,
        int index) {
    fattree_message_list *head = thisq[index];
    if(head != NULL) {
        thisq[index] = head->next;
        if(head->next != NULL) {
            head->next->prev = NULL;
            head->next = NULL;
        } else {
            thistail[index] = NULL;
        }
    }
    return head;
}

static fattree_message_list* return_tail(
        fattree_message_list ** thisq,
        fattree_message_list ** thistail,
        int index) {
    fattree_message_list *tail = thistail[index];
    if(tail->prev != NULL) {
        tail->prev->next = NULL;
        thistail[index] = tail->prev;
        tail->prev = NULL;
    } else {
        thistail[index] = NULL;
        thisq[index] = NULL;
    }
    return tail;
}

//decl
void switch_credit_send(switch_state * s, tw_bf * bf, fattree_message * msg,
    tw_lp * lp, int sq);
int ft_get_output_port( switch_state * s, tw_bf * bf, fattree_message * msg,
    tw_lp * lp, int *out_off );
int get_base_port(switch_state *s, int from_term, int index);


/* returns the fattree switch lp type for lp registration */
Noah Wolfe's avatar
Noah Wolfe committed
386
//static const tw_lptype* fattree_get_switch_lp_type(void);
387

388
/* returns the fattree message size */
389
390
391
392
393
static int fattree_get_msg_sz(void)
{
  return sizeof(fattree_message);
}

Noah Wolfe's avatar
Noah Wolfe committed
394
static void fattree_read_config(const char * anno, fattree_param *p){
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
  int i;

  p->ft_type = 0;
  configuration_get_value_int(&config, "PARAMS", "ft_type", anno, 
      &p->ft_type);

  configuration_get_value_int(&config, "PARAMS", "num_levels", anno, 
      &p->num_levels);
  if(p->num_levels <= 0) {
    tw_error(TW_LOC, "Too few num_levels, Aborting\n");
  }
  if(p->num_levels > 3) {
    tw_error(TW_LOC, "Too many num_levels, only upto 3 supported Aborting\n");
  }

  p->num_switches = (int *) malloc (p->num_levels * sizeof(int));
  p->switch_radix = (int*) malloc (p->num_levels * sizeof(int));

  char switch_counts_str[MAX_NAME_LENGTH];
  int rc = configuration_get_value(&config, "PARAMS", "switch_count", anno,
      switch_counts_str, MAX_NAME_LENGTH);
  if (rc == 0){
    tw_error(TW_LOC, "couldn't read PARAMS:switch_count");
  }
  char* token;
  token = strtok(switch_counts_str, ",");
  i = 0;
  while(token != NULL)
  {
    sscanf(token, "%d", &p->num_switches[i]);
    if(p->num_switches[i] <= 0)
    {
      tw_error(TW_LOC, "Invalid switch count  specified "
          "(%d at pos %d), exiting... ", p->num_switches[i], i);
    }
    i++;
    token = strtok(NULL,",");
  }

  //if(i != p->num_levels) {
  //  tw_error(TW_LOC, "Not enough switch counts, Aborting\n");
  //}
  
  char switch_radix_str[MAX_NAME_LENGTH];
  rc = configuration_get_value(&config, "PARAMS", "switch_radix", anno,
      switch_radix_str, MAX_NAME_LENGTH);
  if (rc == 0){
    tw_error(TW_LOC, "couldn't read PARAMS:switch_radix");
  }
  token = strtok(switch_radix_str, ",");
  i = 0;
  while(token != NULL)
  {
    sscanf(token, "%d", &p->switch_radix[i]);
    if(p->switch_radix[i] <= 0)
    {
      tw_error(TW_LOC, "Invalid switch radix  specified "
          "(%d at pos %d), exiting... ", p->switch_radix[i], i);
    }
    i++;
    token = strtok(NULL,",");
  }

  if(p->num_levels == 2) {
    p->num_switches[1] = p->num_switches[0]/2;
    p->switch_radix[1] = p->switch_radix[0];
  } else {
    p->num_switches[1] = p->num_switches[0];
    p->num_switches[2] = p->num_switches[0]/2;
    p->switch_radix[1] = p->switch_radix[2] = p->switch_radix[0];
  }
  //if(i != p->num_levels) {
  //  tw_error(TW_LOC, "Not enough switch radix, Aborting\n");
  //}

  i = 1;
  for(i = 1; i < p->num_levels - 1; i++) {
    if(p->num_switches[i - 1] * p->switch_radix[i - 1] >
       p->num_switches[i] * p->switch_radix[i]) {
      tw_error(TW_LOC, "Not enough switches/radix at level %d for full "
          "bisection bandwidth\n", i);
    }
  }

  if(p->num_switches[i - 1] * p->switch_radix[i - 1] > 2 * p->num_switches[i] *
      p->switch_radix[i]) {
    tw_error(TW_LOC, "Not enough switches/radix at level %d (top) for full "
        "bisection bandwidth\n", i);
  }

  configuration_get_value_int(&config, "PARAMS", "packet_size", anno,
      &p->packet_size);
  if(!p->packet_size) {
    p->packet_size = 512;
    fprintf(stderr, "Packet size is not specified, setting to %d\n", 
        p->packet_size);
  }
    
  p->router_delay = 50;
  configuration_get_value_double(&config, "PARAMS", "router_delay", anno,
      &p->router_delay);
    
  p->soft_delay = 1000;
  configuration_get_value_double(&config, "PARAMS", "soft_delay", anno,
      &p->soft_delay);

  configuration_get_value_int(&config, "PARAMS", "vc_size", anno, &p->vc_size);
  if(!p->vc_size) {
    p->vc_size = 8*p->packet_size;
    fprintf(stderr, "Buffer size of global channels not specified, setting to "
        "%d\n", p->vc_size);
  }

  configuration_get_value_int(&config, "PARAMS", "cn_vc_size", anno, 
      &p->cn_vc_size);
  if(!p->cn_vc_size) {
    p->cn_vc_size = 8*p->packet_size;
    fprintf(stderr, "Buffer size of compute node channels not specified, " 
        "setting to %d\n", p->cn_vc_size);
  }

516
517
518
519
520
521
  rc = configuration_get_value_int(&config, "PARAMS", "chunk_size", anno, &p->chunk_size);
    if(rc) {
        p->chunk_size = 512;
        fprintf(stderr, "Chunk size for packets is specified, setting to %d\n", p->chunk_size);
    }

522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
  configuration_get_value_double(&config, "PARAMS", "link_bandwidth", anno, 
      &p->link_bandwidth);
  if(!p->link_bandwidth) {
    p->link_bandwidth = 5;
    fprintf(stderr, "Bandwidth of links is specified, setting to %lf\n", 
        p->link_bandwidth);
  }

  configuration_get_value_double(&config, "PARAMS", "cn_bandwidth", anno, 
      &p->cn_bandwidth);
  if(!p->cn_bandwidth) {
    p->cn_bandwidth = 5;
    fprintf(stderr, "Bandwidth of compute node channels not specified, " 
        "setting to %lf\n", p->cn_bandwidth);
  }

  p->l1_set_size = p->switch_radix[0]/2;
 
  p->l1_term_size = (p->l1_set_size * (p->switch_radix[0] / 2));

  p->cn_delay = (1.0 / p->cn_bandwidth);
  p->head_delay = (1.0 / p->link_bandwidth);
  p->credit_delay = (1.0 / p->link_bandwidth) * 8; //assume 8 bytes packet
}

static void fattree_configure(){
  anno_map = codes_mapping_get_lp_anno_map(LP_CONFIG_NM);
  assert(anno_map);
  num_params = anno_map->num_annos + (anno_map->has_unanno_lp > 0);
  all_params = malloc(num_params * sizeof(*all_params));

Noah Wolfe's avatar
Noah Wolfe committed
553
554
  for (int i = 0; i < anno_map->num_annos; i++){
    const char * anno = anno_map->annotations[i].ptr;
555
556
557
558
559
560
561
562
563
564
    fattree_read_config(anno, &all_params[i]);
  }
  if (anno_map->has_unanno_lp > 0){
    fattree_read_config(NULL, &all_params[anno_map->num_annos]);
  }
}

/* initialize a fattree compute node terminal */
void ft_terminal_init( ft_terminal_state * s, tw_lp * lp )
{
565
    s->packet_gen = 0;
566
567
    s->packet_fin = 0;

568
    uint32_t h1 = 0, h2 = 0; 
569
570
    bj_hashlittle2(LP_METHOD_NM, strlen(LP_METHOD_NM), &h1, &h2);
    fattree_terminal_magic_num = h1 + h2;
571

572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
    char anno[MAX_NAME_LENGTH];

    if(def_gname_set == 0) {
      def_gname_set = 1;
      codes_mapping_get_lp_info(0, def_group_name, &mapping_grp_id, NULL,
          &mapping_type_id, anno, &mapping_rep_id, &mapping_offset);
    }

    // Assign the global switch ID
    codes_mapping_get_lp_info(lp->gid, lp_group_name, &mapping_grp_id, NULL,
            &mapping_type_id, anno, &mapping_rep_id, &mapping_offset);
    if (anno[0] == '\0'){
        s->anno = NULL;
        s->params = &all_params[num_params-1];
    } else {
        s->anno = strdup(anno);
        int id = configuration_get_annotation_index(anno, anno_map);
        s->params = &all_params[id];
    }

   int num_lps = codes_mapping_get_lp_count(lp_group_name, 1, LP_CONFIG_NM,
           s->anno, 0);

   if(num_lps != (s->params->switch_radix[0]/2)) {
     tw_error(TW_LOC, "Number of NICs per repetition has to be equal to "
         "half the radix of leaf level switches %d vs %d\n", num_lps,
          s->params->switch_radix[0]/2);
   }
   s->terminal_id = (mapping_rep_id * num_lps) + mapping_offset;  
   s->switch_id = s->terminal_id / (s->params->switch_radix[0] / 2);
   codes_mapping_get_lp_id(lp_group_name, "fattree_switch", NULL, 1,
           s->switch_id, 0, &s->switch_lp);
   s->terminal_available_time = 0.0;
   s->packet_counter = 0;
   s->terminal_msgs = 
     (fattree_message_list**)malloc(1*sizeof(fattree_message_list*));
   s->terminal_msgs_tail = 
     (fattree_message_list**)malloc(1*sizeof(fattree_message_list*));

611
612
613
   s->last_buf_full = 0;
   s->busy_time = 0;

614
615
616
617
#if FATTREE_HELLO
   printf("I am terminal %d (%ld), connected to switch %d\n", s->terminal_id,
       lp->gid, s->switch_id);
#endif
Noah Wolfe's avatar
Noah Wolfe committed
618
619
620

   rc_stack_create(&s->st);

621
622
623
624
625
626
627
   s->vc_occupancy = 0;
   s->terminal_msgs[0] = NULL;
   s->terminal_msgs_tail[0] = NULL;
   s->terminal_length = 0;
   s->in_send_loop = 0;
   s->issueIdle = 0;

628
629
630
631
   s->rank_tbl = qhash_init(fattree_rank_hash_compare, fattree_hash_func, FTREE_HASH_TABLE_SIZE);
   if(!s->rank_tbl)
     tw_error(TW_LOC, "\n Hash table not initialized! ");

632
633
634
635
636
637
638
639
640
   s->params->num_terminals = codes_mapping_get_lp_count(lp_group_name, 0, 
      LP_CONFIG_NM, s->anno, 0);
   return;
}

/* sets up the switch */
void switch_init(switch_state * r, tw_lp * lp)
{
  char anno[MAX_NAME_LENGTH];
Noah Wolfe's avatar
Noah Wolfe committed
641
  int num_terminals = -1;
642
643
644
645
646
647
648

  if(def_gname_set == 0) {
    def_gname_set = 1;
    codes_mapping_get_lp_info(0, def_group_name, &mapping_grp_id, NULL,
        &mapping_type_id, anno, &mapping_rep_id, &mapping_offset);
    num_terminals = codes_mapping_get_lp_count(def_group_name, 0, 
      LP_CONFIG_NM, anno, 0);
Noah Wolfe's avatar
Noah Wolfe committed
649
650
//    num_lps = codes_mapping_get_lp_count(def_group_name, 1, LP_CONFIG_NM,
//           anno, 0);
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
  }

  codes_mapping_get_lp_info(lp->gid, lp_group_name, &mapping_grp_id, NULL,
      &mapping_type_id, anno, &mapping_rep_id, &mapping_offset);

  if (anno[0] == '\0'){
    r->anno = NULL;
    r->params = &all_params[num_params-1];
  } else {
    r->anno = strdup(anno);
    int id = configuration_get_annotation_index(anno, anno_map);
    r->params = &all_params[id];
  }

  // shorthand
  fattree_param *p = r->params;
  if(mapping_offset == p->num_levels - 1) {
    if(mapping_rep_id >= p->num_switches[mapping_offset]) {
      r->unused = 1;
      return;
    }
  }

  r->unused = 0;

  r->switch_id = mapping_rep_id + mapping_offset * p->num_switches[0];

  if(num_terminals != -1) {
    p->num_terminals = num_terminals;
  }

  r->switch_level = mapping_offset;

  r->radix = p->switch_radix[r->switch_level];

  r->next_output_available_time = (tw_stime*) malloc (r->radix * 
      sizeof(tw_stime));
  r->next_credit_available_time = (tw_stime*) malloc (r->radix * 
      sizeof(tw_stime));
  r->vc_occupancy = (int*) malloc (r->radix * sizeof(int));
  r->in_send_loop = (int*) malloc (r->radix * sizeof(int));
  r->link_traffic = (int64_t*) malloc (r->radix * sizeof(int64_t));
  r->port_connections = (tw_lpid*) malloc (r->radix * sizeof(tw_lpid));
  r->pending_msgs = 
    (fattree_message_list**)malloc(r->radix * sizeof(fattree_message_list*));
  r->pending_msgs_tail = 
    (fattree_message_list**)malloc(r->radix * sizeof(fattree_message_list*));
  r->queued_msgs = 
    (fattree_message_list**)malloc(r->radix * sizeof(fattree_message_list*));
  r->queued_msgs_tail = 
    (fattree_message_list**)malloc(r->radix * sizeof(fattree_message_list*));
  r->queued_length = (int*)malloc(r->radix * sizeof(int));

704
705
706
707
708
  
  r->last_buf_full = (tw_stime*)malloc(r->radix * sizeof(tw_stime));
  r->busy_time = (tw_stime*)malloc(r->radix * sizeof(tw_stime));
  r->busy_time_sample = (tw_stime*)malloc(r->radix * sizeof(tw_stime));

Noah Wolfe's avatar
Noah Wolfe committed
709
710
  rc_stack_create(&r->st);

711
712
713
  for(int i = 0; i < r->radix; i++)
  {
    // Set credit & switch occupancy
714
715
716
    r->last_buf_full[i] = 0.0;
    r->busy_time[i] = 0.0;
    r->busy_time_sample[i] = 0.0;
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
    r->next_output_available_time[i] = 0;
    r->next_credit_available_time[i] = 0;
    r->vc_occupancy[i] = 0;
    r->in_send_loop[i] = 0;
    r->link_traffic[i] = 0;
    r->pending_msgs[i] = NULL;
    r->pending_msgs_tail[i] = NULL;
    r->queued_msgs[i] = NULL;
    r->queued_msgs_tail[i] = NULL;
    r->queued_length[i] = 0;
  }

  //set lps connected to each port
  r->num_cons = 0;
  r->num_lcons = 0;
#if FATTREE_HELLO
  printf("I am switch %d (%d), level %d, radix %d\n", r->switch_id,
    lp->gid, r->switch_level, r->radix);
#endif
  //if at level 0, first half ports go to terminals
  if(r->switch_level == 0) {
    int start_terminal = r->switch_id * (p->switch_radix[0] / 2);
    int end_terminal = start_terminal + (p->switch_radix[0] / 2);
    for(int term = start_terminal; term < end_terminal; term++) {
      tw_lpid nextTerm;
      int rep = term / (p->switch_radix[0] / 2);
      int off = term % (p->switch_radix[0] / 2);
      codes_mapping_get_lp_id(def_group_name, LP_CONFIG_NM, NULL, 1,
          rep, off, &nextTerm);
      r->port_connections[r->num_cons++] = nextTerm;
      r->num_lcons++;
#if FATTREE_DEBUG
      printf("I am switch %d, connect to terminal %d (%d) at port %d\n",
          r->switch_id, term, nextTerm, r->num_cons - 1);
#endif
    }
    r->start_lneigh = start_terminal;
    r->end_lneigh = end_terminal;
    r->con_per_lneigh = 1;
    assert(r->num_lcons == (r->radix / 2));
    int l1_set;
    if(p->num_levels == 2) {
      l1_set = 0;
    } else {
      l1_set = r->switch_id / p->l1_set_size;
    }
    int l1_base = l1_set * p->l1_set_size;
    r->start_uneigh = p->num_switches[0] + l1_base;
    r->con_per_uneigh = 1;
    for(int l1 = 0; l1 < p->l1_set_size; l1++) {
      tw_lpid nextTerm;
      codes_mapping_get_lp_id(lp_group_name, "fattree_switch", NULL, 1,
          l1_base, 1, &nextTerm);
      for(int con = 0; con < r->con_per_uneigh; con++) {
        r->port_connections[r->num_cons++] = nextTerm;
#if FATTREE_DEBUG
      printf("I am switch %d, connect to upper switch %d L1 (%d) at port %d\n",
          r->switch_id, l1_base, nextTerm, r->num_cons - 1);
#endif
      }
      l1_base++;
    }
  } else if (r->switch_level == 1) {
    int l0_set_size, l0_base;
    if(p->num_levels == 2) {
      l0_set_size = p->num_switches[0];
      l0_base = 0;
      r->start_lneigh = 0;
      r->end_lneigh = p->num_switches[0];
    } else {
      l0_set_size = p->l1_set_size;
      l0_base = ((r->switch_id - p->num_switches[0]) / p->l1_set_size) *
        l0_set_size;
      r->start_lneigh = l0_base;
      r->end_lneigh = l0_base + l0_set_size;
    }
    r->con_per_lneigh = 1;
    for(int l0 = 0; l0 < l0_set_size; l0++) {
      tw_lpid nextTerm;
      codes_mapping_get_lp_id(def_group_name, "fattree_switch", NULL, 1,
          l0_base, 0, &nextTerm);
      for(int con = 0; con < r->con_per_lneigh; con++) {
        r->port_connections[r->num_cons++] = nextTerm;
        r->num_lcons++;
#if FATTREE_DEBUG
        printf("I am switch %d, connect to switch %d L0 (%d) at port %d\n",
            r->switch_id, l0_base, nextTerm, r->num_cons - 1);
#endif
      }
      l0_base++;
    }
    if(p->num_levels == 3) {
809
      int l2_base = 0;
810
      if(p->ft_type == 0) {
811
//        int l2_base = 0;
812
813
814
        /* not true anymore */
        r->start_uneigh = p->num_switches[0] + l2_base;
        r->con_per_uneigh = 1;
Noah Wolfe's avatar
Noah Wolfe committed
815
        if(((int)r->switch_id - p->num_switches[0]) % p->l1_set_size >=
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
            p->l1_set_size/2) {
          l2_base += (p->num_switches[2]/2);
        }
        for(int l2 = 0; l2 < p->num_switches[2]/2; l2++) {
          tw_lpid nextTerm;
          codes_mapping_get_lp_id(lp_group_name, "fattree_switch", NULL, 1,
              l2_base, 2, &nextTerm);
          for(int con = 0; con < r->con_per_uneigh; con++) {
            r->port_connections[r->num_cons++] = nextTerm;
#if FATTREE_DEBUG
            printf("I am switch %d, connect to upper switch %d L2 (%d) at port %d\n",
                r->switch_id, l2_base, nextTerm, r->num_cons - 1);
#endif
          }
          l2_base++;
        }
      } else {
        int l2 = ((r->switch_id - p->num_switches[0]) % p->l1_set_size);
        /* not true anymore */
        r->start_uneigh = p->num_switches[0] + l2;
        r->con_per_uneigh = 2;
        for(; l2 < p->num_switches[2]; l2 += p->l1_set_size) {
          tw_lpid nextTerm;
          codes_mapping_get_lp_id(lp_group_name, "fattree_switch", NULL, 1,
              l2, 2, &nextTerm);
          for(int con = 0; con < r->con_per_uneigh; con++) {
            r->port_connections[r->num_cons++] = nextTerm;
#if FATTREE_DEBUG
            printf("I am switch %d, connect to upper switch %d L2 (%d) at port %d\n",
845
                r->switch_id, l2, nextTerm, r->num_cons - 1);
846
847
848
849
850
851
852
853
854
855
856
857
#endif
          }
        }
      }
    }
  } else {
    if(p->ft_type == 0) {
      r->con_per_lneigh = 1;
      /* not true anymore */
      r->start_lneigh = p->num_switches[0];
      r->end_lneigh = r->start_lneigh + p->num_switches[1];
      int l1 = 0;
Noah Wolfe's avatar
Noah Wolfe committed
858
      if((int)r->switch_id - p->num_switches[0] - p->num_switches[1] >=
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
          (p->num_switches[2]/2)) {
        l1 += (p->l1_set_size/2);
      }
      int count = 0;
      for(; l1 < p->num_switches[1]; l1++) {
        tw_lpid nextTerm;
        codes_mapping_get_lp_id(lp_group_name, "fattree_switch", NULL, 1,
            l1, 1, &nextTerm);
        for(int con = 0; con < r->con_per_lneigh; con++) {
          r->port_connections[r->num_cons++] = nextTerm;
          r->num_lcons++;
#if FATTREE_DEBUG
          printf("I am switch %d, connect to  switch %d L1 (%d) at port %d\n",
              r->switch_id, l1, nextTerm, r->num_cons - 1);
#endif
        }
        count++;
        if(count == (p->l1_set_size/2)) {
          l1 += (p->l1_set_size/2);
          count = 0;
        }
      }
    } else {
      r->con_per_lneigh = 2;
      /* not true anymore */
      r->start_lneigh = p->num_switches[0];
      r->end_lneigh = r->start_lneigh + p->num_switches[1];
      int l1 = (r->switch_id - p->num_switches[0] - p->num_switches[1]) % p->l1_set_size;
      for(; l1 < p->num_switches[1]; l1 += p->l1_set_size) {
        tw_lpid nextTerm;
        codes_mapping_get_lp_id(lp_group_name, "fattree_switch", NULL, 1,
            l1, 1, &nextTerm);
        for(int con = 0; con < r->con_per_lneigh; con++) {
          r->port_connections[r->num_cons++] = nextTerm;
          r->num_lcons++;
#if FATTREE_DEBUG
          printf("I am switch %d, connect to  switch %d L1 (%d) at port %d\n",
              r->switch_id, l1, nextTerm, r->num_cons - 1);
#endif
        }
      }
    }
  }
  return;
}	

/* empty for now.. */
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
static void fattree_report_stats() 
{
#if DEBUG_RC
	long long t_packet_event_f = 0;
	long long t_packet_event_r = 0;
	long long tt_generate_f = 0;
	long long tt_generate_r = 0;
	long long tt_send_f = 0;
	long long tt_send_r = 0;
	long long tt_arrive_f = 0;
	long long tt_arrive_r = 0;
	long long tt_buffer_f = 0;
	long long tt_buffer_r = 0;
	long long ts_send_f = 0;
	long long ts_send_r = 0;
	long long ts_arrive_f = 0;
	long long ts_arrive_r = 0;
	long long ts_buffer_f = 0;
	long long ts_buffer_r = 0;
	
	MPI_Reduce( &packet_event_f, &t_packet_event_f, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
	MPI_Reduce( &packet_event_r, &t_packet_event_r, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
	MPI_Reduce( &t_generate_f, &tt_generate_f, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
	MPI_Reduce( &t_generate_r, &tt_generate_r, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
	MPI_Reduce( &t_send_f, &tt_send_f, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
	MPI_Reduce( &t_send_r, &tt_send_r, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
	MPI_Reduce( &t_arrive_f, &tt_arrive_f, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
	MPI_Reduce( &t_arrive_r, &tt_arrive_r, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
	MPI_Reduce( &t_buffer_f, &tt_buffer_f, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
	MPI_Reduce( &t_buffer_r, &tt_buffer_r, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
	MPI_Reduce( &s_send_f, &ts_send_f, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
	MPI_Reduce( &s_send_r, &ts_send_r, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
	MPI_Reduce( &s_arrive_f, &ts_arrive_f, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
	MPI_Reduce( &s_arrive_r, &ts_arrive_r, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
	MPI_Reduce( &s_buffer_f, &ts_buffer_f, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
	MPI_Reduce( &s_buffer_r, &ts_buffer_r, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_WORLD);

   if(!g_tw_mynode)
   {
	printf("Reverse Computation Counters:\n");
	printf("packet_event_f:%llu\n",t_packet_event_f);
	printf("packet_event_r:%llu\n",t_packet_event_r);
	printf("t_gen_f:%llu\n",tt_generate_f);
	printf("t_gen_r:%llu\n",tt_generate_r);
	printf("t_gen_net:%llu\n",tt_generate_f-tt_generate_r);
	printf("t_send_f:%llu\n",tt_send_f);
	printf("t_send_r:%llu\n",tt_send_r);
	printf("t_send_net:%llu\n",tt_send_f-tt_send_r);
	printf("t_arrive_f:%llu\n",tt_arrive_f);
	printf("t_arrive_r:%llu\n",tt_arrive_r);
	printf("t_arrive_net:%llu\n",tt_arrive_f-tt_arrive_r);
	printf("t_buf_f:%llu\n",tt_buffer_f);
	printf("t_buf_r:%llu\n",tt_buffer_r);
	printf("t_buf_net:%llu\n",tt_buffer_f-tt_buffer_r);
	printf("s_send_f:%llu\n",ts_send_f);
	printf("s_send_r:%llu\n",ts_send_r);
	printf("s_send_net:%llu\n",ts_send_f-ts_send_r);
	printf("s_arrive_f:%llu\n",ts_arrive_f);
	printf("s_arrive_r:%llu\n",ts_arrive_r);
	printf("s_arrive_net:%llu\n",ts_arrive_f-ts_arrive_r);
	printf("s_buffer_f:%llu\n",ts_buffer_f);
	printf("s_buffer_r:%llu\n",ts_buffer_r);
	printf("s_buffer_net:%llu\n",ts_buffer_f-ts_buffer_r);
   }
#endif
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994

   long long avg_hops, total_finished_packets, total_finished_chunks;
   long long total_finished_msgs, final_msg_sz;
   tw_stime avg_time, max_time;
   long total_gen, total_fin;

   MPI_Reduce( &total_hops, &avg_hops, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
   MPI_Reduce( &N_finished_packets, &total_finished_packets, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
   MPI_Reduce( &N_finished_msgs, &total_finished_msgs, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
   MPI_Reduce( &N_finished_chunks, &total_finished_chunks, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
   MPI_Reduce( &total_msg_sz, &final_msg_sz, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
   MPI_Reduce( &fattree_total_time, &avg_time, 1,MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
   MPI_Reduce( &fattree_max_latency, &max_time, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
   
   MPI_Reduce( &fattree_packet_gen, &total_gen, 1, MPI_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
   MPI_Reduce( &fattree_packet_fin, &total_fin, 1, MPI_LONG, MPI_SUM, 0, MPI_COMM_WORLD);

   /* print statistics */
   if(!g_tw_mynode)
   {	
      printf(" Average number of hops traversed %f average chunk latency %lf us maximum chunk latency %lf us avg message size %lf bytes finished messages %lld finished chunks %lld \n", 
              (float)avg_hops/total_finished_chunks, avg_time/(total_finished_chunks*1000), max_time/1000, (float)final_msg_sz/total_finished_msgs, total_finished_msgs, total_finished_chunks);
      printf(" Total packets generated %ld finished %ld \n", total_gen, total_fin);
   }
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021

   if(!g_tw_mynode)
   {	
      printf(" Average number of hops traversed %f average chunk latency %lf us maximum chunk latency %lf us avg message size %lf bytes finished messages %lld \n", (float)avg_hops/total_finished_chunks, avg_time/(total_finished_chunks*1000), max_time/1000, (float)final_msg_sz/total_finished_msgs, total_finished_msgs);

#if PARAMS_LOG
//    throughput_avg = throughput_avg / (float)slim_total_terminals_noah;
//    throughput_avg2 = throughput_avg2 / (float)slim_total_terminals_noah;

	//Open file to append simulation results
	char temp_filename[1024];
	char temp_filename_header[1024];
	sprintf(temp_filename,"%s/sim_log.txt",modelnet_stats_dir);
	sprintf(temp_filename_header,"%s/sim_log_header.txt",modelnet_stats_dir);
	FILE *fattree_results_log=fopen(temp_filename, "a");
	FILE *fattree_results_log_header=fopen(temp_filename_header, "a");
	if(fattree_results_log == NULL)
		printf("\n Failed to open results log file %s \n",temp_filename);
	if(fattree_results_log_header == NULL)
		printf("\n Failed to open results log header file %s \n",temp_filename_header);
	printf("Printing Simulation Parameters/Results Log File\n");
	fprintf(fattree_results_log_header,"<Avg Hops/Total Packets>, <Avg Time/Total Packets>, <Max Latency>, <Total Finished Chunks>");
	fprintf(fattree_results_log,"%10.3lf, %15.3lf, %11.3lf, %16.3lld, ", (float)avg_hops/total_finished_packets, avg_time/(total_finished_packets),max_time,total_finished_chunks);
	fclose(fattree_results_log_header);
	fclose(fattree_results_log);
#endif 
  }
1022
}
1023
1024

/* fattree packet event */
Noah Wolfe's avatar
Noah Wolfe committed
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
static tw_stime fattree_packet_event(
        model_net_request const * req,
        uint64_t message_offset,
        uint64_t packet_size,
        tw_stime offset,
        mn_sched_params const * sched_params,
        void const * remote_event,
        void const * self_event,
        tw_lp *sender,
        int is_last_pckt)
/*	model_net_request* req, 
	char* category, 
    tw_lpid final_dest_lp, 
	uint64_t packet_size, 
	int is_pull, 
    uint64_t pull_size, 
	tw_stime offset, 
	const mn_sched_params *sched_params, 
    int remote_event_size, 
	const void* remote_event, 
	int self_event_size, 
    const void* self_event, 
	tw_lpid src_lp, 
	tw_lp *sender, 
	int is_last_pckt) 
*/{
1051
1052
1053
#if DEBUG_RC
  packet_event_f++;
#endif
1054

Noah Wolfe's avatar
Noah Wolfe committed
1055
1056
  (void)message_offset;
  (void)sched_params;
1057
1058
1059
1060
1061
1062
1063
1064
  tw_event * e_new;
  tw_stime xfer_to_nic_time;
  fattree_message * msg;
  char* tmp_ptr;

  xfer_to_nic_time = codes_local_latency(sender);
  e_new = model_net_method_event_new(sender->gid, xfer_to_nic_time + offset,
      sender, FATTREE, (void**)&msg, (void**)&tmp_ptr);
Noah Wolfe's avatar
Noah Wolfe committed
1065
1066
1067
1068
1069
  strcpy(msg->category, req->category);
  msg->final_dest_gid = req->final_dest_lp;
  msg->total_size = req->msg_size;
  msg->sender_lp = req->src_lp;
  msg->sender_mn_lp = sender->gid;
1070
  msg->packet_size = packet_size;
Noah Wolfe's avatar
Noah Wolfe committed
1071
  msg->travel_start_time = tw_now(sender);
1072
1073
1074
  msg->remote_event_size_bytes = 0;
  msg->local_event_size_bytes = 0;
  msg->type = T_GENERATE;
Noah Wolfe's avatar
Noah Wolfe committed
1075
1076
1077
1078
  msg->dest_terminal_id = req->dest_mn_lp;
  msg->message_id = req->msg_id;
  msg->is_pull = req->is_pull;
  msg->pull_size = req->pull_size;
1079
  msg->magic = fattree_terminal_magic_num; 
Noah Wolfe's avatar
Noah Wolfe committed
1080
  msg->msg_start_time = req->msg_start_time;
1081
1082
1083
1084

  /* Its the last packet so pass in remote and local event information*/
  if(is_last_pckt) 
  {
1085
    if(req->remote_event_size > 0)
1086
    {
1087
1088
1089
      msg->remote_event_size_bytes = req->remote_event_size;
      memcpy(tmp_ptr, remote_event, req->remote_event_size);
      tmp_ptr += req->remote_event_size;
1090
    }
1091
    if(req->self_event_size > 0)
1092
    {
1093
1094
1095
      msg->local_event_size_bytes = req->self_event_size;
      memcpy(tmp_ptr, self_event, req->self_event_size);
      tmp_ptr += req->self_event_size;
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
    }
  }
  //printf("[%d] Send to %d\n", sender->gid, sender->gid);
  tw_event_send(e_new);
  return xfer_to_nic_time;
}

/* fattree packet event reverse handler */
static void fattree_packet_event_rc(tw_lp *sender)
{
1106
1107
1108
#if DEBUG_RC
  packet_event_r++;
#endif
1109
1110
1111
1112
  codes_local_latency_reverse(sender);
  return;
}

1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
void ft_packet_generate_rc(ft_terminal_state * s, tw_bf * bf, fattree_message * msg, tw_lp * lp)
{
#if DEBUG_RC
    t_generate_r++;
#endif
    fattree_packet_gen--;
    s->packet_gen--;

    tw_rand_reverse_unif(lp->rng);

    int num_chunks = msg->packet_size/s->params->chunk_size;
    if(msg->packet_size % s->params->chunk_size)
	num_chunks++;

    if(!num_chunks)
	num_chunks = 1;

    int i;
    for(i = 0; i < num_chunks; i++) {
	delete_fattree_message_list(return_tail(s->terminal_msgs, 
	    s->terminal_msgs_tail, 0));
	s->terminal_length -= s->params->chunk_size;
    }
    if(bf->c11) {
	s->issueIdle = 0;
	s->last_buf_full = msg->saved_busy_time;
    }
    if(bf->c5) {
	codes_local_latency_reverse(lp);
	s->in_send_loop = 0;
    }
    
    struct mn_stats* stat;
    stat = model_net_find_stats(msg->category, s->fattree_stats_array);
    stat->send_count--;
    stat->send_bytes -= msg->packet_size;
    stat->send_time -= (1/s->params->cn_bandwidth) * msg->packet_size;
}

1152
1153
1154
/* generates packet at the current fattree compute node */
void ft_packet_generate(ft_terminal_state * s, tw_bf * bf, fattree_message * msg,
    tw_lp * lp) {
1155
1156
1157
  bf->c11 = 0;
  bf->c5 = 0;

1158
1159
1160
  fattree_packet_gen++;
  s->packet_gen++;
  
1161
1162
1163
  fattree_param *p = s->params;
  tw_stime ts, nic_ts;

1164
1165
1166
1167
1168
1169
1170
1171
1172
  int total_event_size;
  uint64_t num_chunks = msg->packet_size / p->chunk_size;
  if(msg->packet_size % s->params->chunk_size)
    num_chunks++;
  if(!num_chunks)
    num_chunks = 1;
 
  nic_ts = g_tw_lookahead + (num_chunks * s->params->cn_delay) + tw_rand_unif(lp->rng);

1173
1174
  msg->my_N_hop = 0;

1175
  msg->packet_ID = lp->gid + g_tw_nlp * s->packet_counter;
1176
1177
1178
//  msg->dest_terminal_id = msg->final_dest_gid;
  if(msg->packet_ID == LLU(TRACK_PKT))
    printf("\n Packet %llu generated at terminal %d terminal_gid %llu dest_terminal_id %llu final_dest_gid %llu size %llu num chunks %llu \n",
Noah Wolfe's avatar
Noah Wolfe committed
1179
       msg->packet_ID, s->terminal_id, LLU(lp->gid), LLU(msg->dest_terminal_id), LLU(msg->final_dest_gid),
1180
1181
1182
1183
1184
       LLU(msg->packet_size), LLU(num_chunks));

  for(uint64_t i = 0; i < num_chunks; i++)
  {
    fattree_message_list * cur_chunk = (fattree_message_list *)malloc( 
1185
      sizeof(fattree_message_list));
1186
1187
    msg->origin_switch_id = s->switch_id;
    init_fattree_message_list(cur_chunk, msg);
1188
  
1189
1190
    if(msg->remote_event_size_bytes + msg->local_event_size_bytes > 0) {
      cur_chunk->event_data = (char*)malloc(
1191
        msg->remote_event_size_bytes + msg->local_event_size_bytes);
1192
    }
1193

1194
1195
1196
1197
1198
1199
1200
    void *m_data_src = model_net_method_get_edata(FATTREE, msg);
    if(msg->remote_event_size_bytes){
      memcpy(cur_chunk->event_data, m_data_src, msg->remote_event_size_bytes);
    }
    if(msg->local_event_size_bytes){
      m_data_src = (char*)m_data_src + msg->remote_event_size_bytes;
      memcpy((char*)cur_chunk->event_data + msg->remote_event_size_bytes, 
1201
        m_data_src, msg->local_event_size_bytes);
1202
    }
1203

1204
1205
1206
1207
1208
1209
    cur_chunk->msg.chunk_id = i;
    cur_chunk->msg.origin_switch_id = s->switch_id;
    append_to_fattree_message_list(s->terminal_msgs, s->terminal_msgs_tail, 
      0, cur_chunk);
    s->terminal_length += s->params->chunk_size;
  }
Noah Wolfe's avatar
Noah Wolfe committed
1210
1211
//  if(s->terminal_id == 1)	
//    printf("gene time:%5.6lf lp_id:%3llu terminal_length:%5d \n",tw_now(lp),LLU(lp->gid),s->terminal_length);
1212
1213
1214
1215
1216
1217

  if(s->terminal_length < 2 * s->params->cn_vc_size) {
    model_net_method_idle_event(nic_ts, 0, lp);
  } else {
    bf->c11 = 1;
    s->issueIdle = 1;
1218
1219
    msg->saved_busy_time = s->last_buf_full;
    s->last_buf_full = tw_now(lp);
1220
1221
1222
1223
1224
  }

  if(s->in_send_loop == 0) {
    fattree_message *m;
    bf->c5 = 1;
1225
    ts = codes_local_latency(lp);
1226
1227
1228
    tw_event* e = model_net_method_event_new(lp->gid, ts, lp, FATTREE, 
      (void**)&m, NULL);
    m->type = T_SEND;
1229
    m->magic = fattree_terminal_magic_num;
1230
1231
1232
1233
1234
1235
    s->in_send_loop = 1;
    tw_event_send(e);
    //printf("[%d] send loop triggered with ts %lf band %lf\n", 
    // lp->gid, ts, s->params->cn_bandwidth);
  }

1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
  total_event_size = model_net_get_msg_sz(FATTREE) +
    msg->remote_event_size_bytes+ msg->local_event_size_bytes;
  mn_stats* stat;
  stat = model_net_find_stats(msg->category, s->fattree_stats_array);
  stat->send_count++;
  stat->send_bytes += msg->packet_size;
  stat->send_time += (1/p->cn_bandwidth) * msg->packet_size;
  if(stat->max_event_size < total_event_size)
    stat->max_event_size = total_event_size;

1246
1247
1248
  return;
}

1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
void ft_packet_send_rc(ft_terminal_state * s, tw_bf *bf, fattree_message * msg, tw_lp * lp)
{
#if DEBUG_RC
    t_send_r++;
#endif
    if(bf->c1) {
      s->in_send_loop = 1;
      s->last_buf_full = msg->saved_busy_time;
      return;
    }

    tw_rand_reverse_unif(lp->rng);
    s->terminal_available_time = msg->saved_available_time;
    if(bf->c2) {
      codes_local_latency_reverse(lp);
    }
    s->packet_counter--;
    s->vc_occupancy -= s->params->packet_size;
    
    fattree_message_list* cur_entry = rc_stack_pop(s->st);	

    prepend_to_fattree_message_list(s->terminal_msgs,
	s->terminal_msgs_tail, 0, cur_entry);
    s->terminal_length += s->params->packet_size;
    if(s->terminal_id == 0)	
       printf("time:%lf terminal_length:%d \n",tw_now(lp),s->terminal_length);

    if(bf->c3) {
      tw_rand_reverse_unif(lp->rng);
    }
    if(bf->c4) {
      s->in_send_loop = 1;
    }
    /*if(bf->c5) {
      codes_local_latency_reverse(lp);
      s->issueIdle = 1;
    }*/
}
1287
1288
1289
1290
/* sends the packet from the compute node to the attached switch */
void ft_packet_send(ft_terminal_state * s, tw_bf * bf, fattree_message * msg,
    tw_lp * lp) {

1291
1292
1293
1294
1295
1296
  bf->c1 = 0;
  bf->c2 = 0;
  bf->c3 = 0;
  bf->c4 = 0;
  bf->c5 = 0;

1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
  tw_stime ts;
  tw_event *e;
  fattree_message *m;
 
  fattree_message_list* cur_entry = s->terminal_msgs[0];

  if(s->vc_occupancy + s->params->packet_size > s->params->cn_vc_size ||
    cur_entry == NULL) {
    bf->c1 = 1;
    s->in_send_loop = 0;
1307
1308
1309
    
    msg->saved_busy_time = s->last_buf_full;
    s->last_buf_full = tw_now(lp);
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
    return;
  }

  //  Each packet is broken into chunks and then sent over the channel
  msg->saved_available_time = s->terminal_available_time;
  ts = g_tw_lookahead + s->params->cn_delay * cur_entry->msg.packet_size 
    + g_tw_lookahead * tw_rand_unif(lp->rng);
  s->terminal_available_time = maxd(s->terminal_available_time, tw_now(lp));
  s->terminal_available_time += ts;

  // we are sending an event to the switch, so no method_event here
  ts = s->terminal_available_time - tw_now(lp);
1322

1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
  e = tw_event_new(s->switch_lp, ts, lp);
  m = tw_event_data(e);
  memcpy(m, &cur_entry->msg, sizeof(fattree_message));
  if (m->remote_event_size_bytes){
    memcpy(model_net_method_get_edata(FATTREE, m), cur_entry->event_data,
        m->remote_event_size_bytes);
  }

  m->type = S_ARRIVE;
  m->src_terminal_id = lp->gid;
  m->intm_id = s->terminal_id;
  m->vc_index = 0;
  m->vc_off = 0; //we only have one connection to the terminal NIC
  m->local_event_size_bytes = 0;
  m->last_hop = TERMINAL;
  //printf("[%d] pack send Send to %d\n", lp->gid, s->switch_lp);
  tw_event_send(e);
      
  /* local completion message */
  if(cur_entry->msg.local_event_size_bytes > 0)
  {
    bf->c2 = 1;
    double tsT = codes_local_latency(lp); 
    tw_event *e_new = tw_event_new(cur_entry->msg.sender_lp, tsT, lp);
    fattree_message *m_new = tw_event_data(e_new);
    void *local_event = (char*)cur_entry->event_data + 
                cur_entry->msg.remote_event_size_bytes;
    memcpy(m_new, local_event, cur_entry->msg.local_event_size_bytes);
    tw_event_send(e_new);
  }
   
  s->packet_counter++;
  s->vc_occupancy += s->params->packet_size;
  cur_entry = return_head(s->terminal_msgs, s->terminal_msgs_tail, 0); 
Noah Wolfe's avatar
Noah Wolfe committed
1357
  //delete_fattree_message_list(cur_entry);
1358
  rc_stack_push(lp, cur_entry, free, s->st);
Noah Wolfe's avatar
Noah Wolfe committed
1359
1360
1361
1362
  s->terminal_length -= s->params->chunk_size;
 
//  if(s->terminal_id == 1) 
//    printf("send time:%5.6lf lp_id:%3llu terminal_length:%5d \n",tw_now(lp),LLU(lp->gid),s->terminal_length);
1363
1364
1365
1366
1367
1368

  cur_entry = s->terminal_msgs[0];

  if(cur_entry != NULL &&
    s->vc_occupancy + s->params->packet_size <= s->params->cn_vc_size) {
    bf->c3 = 1;
Noah Wolfe's avatar
Noah Wolfe committed
1369
    fattree_message *m_new;
1370
    ts = ts + g_tw_lookahead * tw_rand_unif(lp->rng);
Noah Wolfe's avatar
Noah Wolfe committed
1371
1372
1373
1374
    e = model_net_method_event_new(lp->gid, ts, lp, FATTREE, 
      (void**)&m_new, NULL);
    m_new->type = T_SEND;
    m_new->magic = fattree_terminal_magic_num;
1375
1376
1377
1378
1379
1380
    tw_event_send(e);
  } else {
    bf->c4 = 1;
    s->in_send_loop = 0;
  }
  
1381
/*  if(s->issueIdle) {
1382
1383
1384
1385
    bf->c5 = 1;
    s->issueIdle = 0;
    model_net_method_idle_event(codes_local_latency(lp), 0, lp);
  }
1386
*/
1387
1388
1389
1390
1391
1392
1393
1394
  return;
}

/* Packet arrives at the switch and a credit is sent back to the sending 
 * terminal/switch */
void switch_packet_receive( switch_state * s, tw_bf * bf, 
    fattree_message * msg, tw_lp * lp ) {

1395
1396
1397
1398
  bf->c1 = 0;
  bf->c2 = 0;
  bf->c3 = 0;

Noah Wolfe's avatar
Noah Wolfe committed
1399
1400
//  tw_event *e;
//  fattree_message *m;
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
  tw_stime ts;

  //printf("[%d] Switch %d recv packet %d\n", lp->gid, msg->vc_index);
  int output_port = -1, out_off = 0;

  output_port = ft_get_output_port(s, bf, msg, lp, &out_off);
  assert(output_port < s->radix);

  int max_vc_size = s->params->vc_size;
  int to_terminal = 0;

  //If going to terminal, use a different max
  if(s->switch_level == 0 && output_port < s->num_lcons) {
    max_vc_size = s->params->cn_vc_size;
    to_terminal = 1;
  }
1417
1418
1419
1420
1421
 
  int dest_term_local_id = codes_mapping_get_lp_relative_id(msg->dest_terminal_id, 0, 0);

if(msg->packet_ID == LLU(TRACK_PKT))
    printf("\n Packet %llu arrived at switch %d switch_gid %llu dest_terminal_id %llu dest_term_local_id %llu final_dest_gid %llu output_port %d to_terminal %d\n",
Noah Wolfe's avatar
Noah Wolfe committed
1422
       msg->packet_ID, s->switch_id, LLU(lp->gid), LLU(msg->dest_terminal_id), LLU(dest_term_local_id), LLU(msg->final_dest_gid),
1423
1424
       output_port, to_terminal);
 
1425
1426
1427
  fattree_message_list * cur_chunk = (fattree_message_list *)malloc( 
      sizeof(fattree_message_list));
  init_fattree_message_list(cur_chunk, msg);
1428
1429
1430
1431
1432
1433
  if(msg->remote_event_size_bytes > 0) 
  {
       void *m_data_src = model_net_method_get_edata(FATTREE, msg);

       cur_chunk->event_data = (char*)malloc(msg->remote_event_size_bytes);
       memcpy(cur_chunk->event_data, m_data_src, 
1434
1435
1436
1437
1438
        msg->remote_event_size_bytes);
  }

  cur_chunk->msg.vc_index = output_port;
  cur_chunk->msg.vc_off = out_off;
1439
  cur_chunk->msg.my_N_hop++;
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465

  if(s->vc_occupancy[output_port] + s->params->packet_size <= max_vc_size) {
    bf->c1 = 1;
    switch_credit_send(s, bf, msg, lp, -1);
    append_to_fattree_message_list( s->pending_msgs, s->pending_msgs_tail, 
      output_port, cur_chunk);
    s->vc_occupancy[output_port] += s->params->packet_size;
    if(s->in_send_loop[output_port] == 0) {
      bf->c2 = 1;
      fattree_message *m;
      ts = codes_local_latency(lp); 
      tw_event *e = tw_event_new(lp->gid, ts, lp);
      m = tw_event_data(e);
      m->type = S_SEND;
      m->vc_index = output_port;
      //printf("[%d] pack recv Send to %d\n", lp->gid, lp->gid);
      tw_event_send(e);
      s->in_send_loop[output_port] = 1;
    }
  } else {
    bf->c3 = 1;
    cur_chunk->msg.saved_vc = msg->vc_index;
    cur_chunk->msg.saved_off = msg->vc_off;
    append_to_fattree_message_list( s->queued_msgs, s->queued_msgs_tail, 
      output_port, cur_chunk);
    s->queued_length[output_port] += s->params->packet_size;
1466
1467
    msg->saved_busy_time = s->last_buf_full[output_port];
    s->last_buf_full[output_port] = tw_now(lp);
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
  }
  //for reverse
  msg->saved_vc = output_port;

  return;
}

/* routes the current packet to the next stop */
void switch_packet_send( switch_state * s, tw_bf * bf, fattree_message * msg,
    tw_lp * lp) {

1479
1480
1481
1482
  bf->c1 = 0;
  bf->c3 = 0;
  bf->c4 = 0;

1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
  tw_stime ts;
  tw_event *e;
  fattree_message *m;

  int output_port = msg->vc_index;
  fattree_message_list *cur_entry = s->pending_msgs[output_port];

  if(cur_entry == NULL) {
    bf->c1 = 1;
    s->in_send_loop[output_port] = 0;
    return;
  }
  
  tw_lpid next_stop = s->port_connections[output_port];
  int to_terminal = 0;
  tw_stime delay = s->params->head_delay;

  // dest can be a switch or a terminal, so we must check
  if(s->switch_level == 0 && output_port < s->num_lcons) {
    to_terminal = 1;
    delay = s->params->cn_delay;
  }

  ts = s->params->router_delay;
  double bytetime = delay * cur_entry->msg.packet_size;
  ts = g_tw_lookahead + g_tw_lookahead * tw_rand_unif( lp->rng) + bytetime + ts;

  msg->saved_available_time = s->next_output_available_time[output_port];
  s->next_output_available_time[output_port] = 
    maxd(s->next_output_available_time[output_port], tw_now(lp));
  s->next_output_available_time[output_port] += ts;

  ts = s->next_output_available_time[output_port] - tw_now(lp);
  void * m_data;
  if (to_terminal) {
    e = model_net_method_event_new(next_stop, ts, lp,
        FATTREE, (void**)&m, &m_data);
  } else {
      e = tw_event_new(next_stop, ts, lp);
      m = tw_event_data(e);
      m_data = model_net_method_get_edata(FATTREE, m);
  }

  memcpy(m, &cur_entry->msg, sizeof(fattree_message));
  if (m->remote_event_size_bytes){
      memcpy(m_data, cur_entry->event_data, m->remote_event_size_bytes);
  }

  m->last_hop = LINK;
  m->intm_lp_id = lp->gid;
  m->intm_id = s->switch_id;
  s->link_traffic[output_port] += cur_entry->msg.packet_size;

  /* Determine the event type. If the packet has arrived at the final destination
     switch then it should arrive at the destination terminal next. */
  if(to_terminal) {
    m->type = T_ARRIVE;
  } else {
    /* The packet has to be sent to another switch */
    m->type = S_ARRIVE;
  }
  tw_event_send(e);
  
  cur_entry = return_head(s->pending_msgs, s->pending_msgs_tail, 
    output_port);
Noah Wolfe's avatar
Noah Wolfe committed
1548
1549
1550
  //delete_fattree_message_list(cur_entry);
  rc_stack_push(lp, cur_entry, free, s->st);    

1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
  if(bytetime > s->params->router_delay) {
    s->next_output_available_time[output_port] -= s->params->router_delay;
    ts -= s->params->router_delay;
  } else {
    s->next_output_available_time[output_port] -= bytetime;
    ts -= bytetime;
  }

  cur_entry = s->pending_msgs[output_port];
  if(cur_entry != NULL) {
    bf->c3 = 1;
Noah Wolfe's avatar
Noah Wolfe committed
1562
    fattree_message *m_new;
1563
    ts = ts + g_tw_lookahead * tw_rand_unif(lp->rng);
Noah Wolfe's avatar
Noah Wolfe committed
1564
1565
1566
1567
    e = tw_event_new(lp->gid, ts, lp);
    m_new = tw_event_data(e);
    m_new->type = S_SEND;
    m_new->vc_index = output_port;
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
    //printf("[%d] switch send loop Send to %d\n", lp->gid, lp->gid);
    tw_event_send(e);
  } else {
    bf->c4 = 1;
    s->in_send_loop[output_port] = 0;
  }
  return;
}

/* When a packet is sent from the current switch and a buffer slot 
 * becomes available, a credit is sent back to schedule another packet 
 * event */
void switch_credit_send(switch_state * s, tw_bf * bf, fattree_message * msg,
    tw_lp * lp, int sq) {
Noah Wolfe's avatar
Noah Wolfe committed
1582
  (void)bf;
1583
1584
1585
1586
1587
1588
1589
  tw_event * buf_e;
  tw_stime ts;
  fattree_message * buf_msg;

  int dest = 0, type = S_BUFFER;
  int is_terminal = 0;

Noah Wolfe's avatar
Noah Wolfe committed
1590
//  fattree_param *p = s->params;
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
  // Notify sender terminal about available buffer space
  if(msg->last_hop == TERMINAL) {
    dest = msg->src_terminal_id;
    type = T_BUFFER;
    is_terminal = 1;
  } else if(msg->last_hop == LINK) {
    dest = msg->intm_lp_id;
  } 

  // Assume it takes 0.1 ns of serialization latency for processing the 
  // credits in the queue
  //int output_port = msg->vc_off; //src used this offset, so I have to
  //if(sq == 1) {
  //  output_port = msg->saved_off;
  //}
  //output_port += get_base_port(s, is_terminal, msg->intm_id);

  ts = g_tw_lookahead + s->params->credit_delay + g_tw_lookahead * tw_rand_unif(lp->rng);
	
  if (is_terminal) {
    buf_e = model_net_method_event_new(dest, ts, lp, FATTREE, 
      (void**)&buf_msg, NULL);
1613
	buf_msg->magic = fattree_terminal_magic_num;
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
  } else {
    buf_e = tw_event_new(dest, ts , lp);
    buf_msg = tw_event_data(buf_e);
  }

  buf_msg->type = type;

  if(sq == 1) {
    buf_msg->vc_index = msg->saved_vc;
  } else {
    buf_msg->vc_index = msg->vc_index; //the port src used to send me this data
  }

  //printf("[%d] credit send Send to %d\n", lp->gid, dest);
  tw_event_send(buf_e);
  return;
}

1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
void ft_terminal_buf_update_rc(ft_terminal_state * s, tw_bf * bf, fattree_message * msg, tw_lp * lp)
{
#if DEBUG_RC
    t_buffer_r++;
#endif
    s->vc_occupancy += s->params->packet_size;
    codes_local_latency_reverse(lp);
    if(bf->c3)
    {
      s->busy_time = msg->saved_total_time;
      s->last_buf_full = msg->saved_busy_time;
      s->busy_time_sample = msg->saved_sample_time;
    }
    if(bf->c1) {
      s->in_send_loop = 0;
    }
}
1649
1650
1651
/* update the compute node-switch channel buffer */
void ft_terminal_buf_update(ft_terminal_state * s, tw_bf * bf,
    fattree_message * msg, tw_lp * lp) {
1652
1653
1654
1655

  bf->c1 = 0;

  tw_stime ts = codes_local_latency(lp);
1656
  s->vc_occupancy -= s->params->packet_size;
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670

  /* Update the terminal buffer time */
  if(s->last_buf_full > 0)
  {
    bf->c3 = 1;
    msg->saved_total_time = s->busy_time;
    msg->saved_busy_time = s->last_buf_full;
    msg->saved_sample_time = s->busy_time_sample;

    s->busy_time += (tw_now(lp) - s->last_buf_full);
    s->busy_time_sample += (tw_now(lp) - s->last_buf_full);
    s->last_buf_full = 0.0;
  }

1671
1672
1673
1674
1675
1676
  if(s->in_send_loop == 0 && s->terminal_msgs[0] != NULL) {
    fattree_message *m;
    bf->c1 = 1;
    tw_event* e = model_net_method_event_new(lp->gid, ts, lp, FATTREE, 
        (void**)&m, NULL);
    m->type = T_SEND;
1677
	m->type = fattree_terminal_magic_num;
1678
1679
1680
1681
    s->in_send_loop = 1;
    //printf("[%d] term buf Send to %d\n", lp->gid, lp->gid);
    tw_event_send(e);
  }
1682
1683
1684
1685
1686
  else if(s->in_send_loop == 0 && s->terminal_msgs[0] == NULL)
  {
     bf->c2 = 1;
     model_net_method_idle_event(ts, 0, lp);
  }
1687
1688
1689
1690
1691
  return;
}

void switch_buf_update(switch_state * s, tw_bf * bf, fattree_message * msg, 
  tw_lp * lp) {
1692
1693
1694
1695
1696

  bf->c1 = 0;
  bf->c2 = 0;
  bf->c3 = 0;

1697
1698
1699
  int indx = msg->vc_index;
  s->vc_occupancy[indx] -= s->params->packet_size;

1700
  if(s->last_buf_full[indx])
1701
  {
1702
1703
1704
1705
1706
1707
1708
    bf->c3 = 1;
    msg->saved_rcv_time = s->busy_time[indx];
    msg->saved_busy_time = s->last_buf_full[indx];
    msg->saved_sample_time = s->busy_time_sample[indx];
    s->busy_time[indx] += (tw_now(lp) - s->last_buf_full[indx]);
    s->busy_time_sample[indx] += (tw_now(lp) - s->last_buf_full[indx]);
    s->last_buf_full[indx] = 0.0;
1709
1710
  }

1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731