express-mesh.C 76.6 KB
Newer Older
Nikhil's avatar
Nikhil committed
1 2 3 4 5 6 7 8
#include <ross.h>

#include "codes/jenkins-hash.h"
#include "codes/codes_mapping.h"
#include "codes/codes.h"
#include "codes/model-net.h"
#include "codes/model-net-method.h"
#include "codes/model-net-lp.h"
9 10

//CHANGE: use the network file created
Nikhil's avatar
Nikhil committed
11
#include "codes/net/express-mesh.h"
12

Nikhil's avatar
Nikhil committed
13 14 15 16 17 18 19
#include "codes/net/common-net.h"
#include "sys/file.h"
#include "codes/quickhash.h"
#include "codes/rc-stack.h"
#include <vector>

#define CREDIT_SZ 8
20 21
#define HASH_TABLE_SIZE 262144
#define MULT_FACTOR 2
Nikhil's avatar
Nikhil committed
22 23 24 25

#define DEBUG 0
#define MAX_STATS 65536

26 27 28 29 30 31 32 33 34 35
//CHANGE: define them for the local network
#define LOCAL_NETWORK_NAME EXPRESS_MESH
#define LOCAL_NETWORK_ROUTER_NAME EXPRESS_MESH_ROUTER
#define LOCAL_MSG_STRUCT em_message
#define LOCAL_MSG_NAME_FROM_UNION em_msg

#define LP_CONFIG_NM_TERM (model_net_lp_config_names[LOCAL_NETWORK_NAME])
#define LP_METHOD_NM_TERM (model_net_method_names[LOCAL_NETWORK_NAME])
#define LP_CONFIG_NM_ROUT (model_net_lp_config_names[LOCAL_NETWORK_ROUTER_NAME])
#define LP_METHOD_NM_ROUT (model_net_method_names[LOCAL_NETWORK_ROUTER_NAME])
Nikhil's avatar
Nikhil committed
36 37 38 39 40

static long packet_gen = 0, packet_fin = 0;

static double maxd(double a, double b) { return a < b ? b : a; }

41
typedef struct local_param local_param;
Nikhil's avatar
Nikhil committed
42
static uint64_t                  num_params = 0;
43
static local_param         * all_params = NULL;
Nikhil's avatar
Nikhil committed
44 45 46 47 48 49 50 51 52 53 54 55 56 57 58
static const config_anno_map_t * anno_map   = NULL;

/* global variables for codes mapping */
static char lp_group_name[MAX_NAME_LENGTH];
static int mapping_grp_id, mapping_type_id, mapping_rep_id, mapping_offset;

/* router magic number */
static int router_magic_num = 0;

/* terminal magic number */
static int terminal_magic_num = 0;

static int sample_bytes_written = 0;
static int sample_rtr_bytes_written = 0;

59 60
static char local_cn_sample_file[MAX_NAME_LENGTH];
static char local_rtr_sample_file[MAX_NAME_LENGTH];
Nikhil's avatar
Nikhil committed
61

62 63 64
static void init_message_list(message_list *thism,
    LOCAL_MSG_STRUCT *inmsg) {
  thism->LOCAL_MSG_NAME_FROM_UNION = *inmsg;
Nikhil's avatar
Nikhil committed
65 66 67 68 69 70 71 72
  thism->event_data = NULL;
  thism->next = NULL;
  thism->prev = NULL;
  thism->in_alt_q = 0;
  thism->altq_next = NULL;
  thism->altq_prev = NULL;
}

73
struct local_param
Nikhil's avatar
Nikhil committed
74 75 76 77 78 79 80 81
{
  double link_bandwidth;/* bandwidth of each link */
  double cn_bandwidth;/* injection bandwidth */
  int num_cn; // number of nodes per router
  int num_vcs; /* number of virtual channels */
  int vc_size; /* buffer size of the router-router channels */
  int cn_vc_size; /* buffer size of the compute node channels */
  int chunk_size; /* full-sized packets are broken into smaller chunks.*/
82 83 84
  int router_delay; /* delay at each router */
  int routing; /* type of routing */

Nikhil's avatar
Nikhil committed
85
  //derived param
86 87 88 89 90 91 92 93 94 95 96 97 98
  int radix; /* radix of the routers */
  int total_routers; /* how many routers in the system */
  int total_terminals; /* how many terminals in the system */
  double cn_delay; /* bandwidth based time for 1 byte */
  double link_delay; /* bandwidth based time for 1 byte */
  double credit_delay; /* how long for credit to arrive - all bytes */

  //CHANGE: add network specific data here
  int n_dims; // Dimensions in the base torus layout
  int *dim_length;
  int gap; // Gap at which nodes are connected (0 for log)
  int * factor; /* used in torus coordinate calculation */
  int *cons_per_dim, *offset_per_dim;
Nikhil's avatar
Nikhil committed
99 100
};

101
struct local_router_sample
Nikhil's avatar
Nikhil committed
102 103 104 105 106 107 108 109 110
{
  tw_lpid router_id;
  tw_stime* busy_time;
  int64_t* link_traffic_sample;
  tw_stime end_time;
  long fwd_events;
  long rev_events;
};

111
struct local_cn_sample
Nikhil's avatar
Nikhil committed
112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127
{
  tw_lpid terminal_id;
  long fin_chunks_sample;
  long data_size_sample;
  double fin_hops_sample;
  tw_stime fin_chunks_time;
  tw_stime busy_time_sample;
  tw_stime end_time;
  long fwd_events;
  long rev_events;
};

/* handles terminal and router events like packet generate/send/receive/buffer */
typedef struct terminal_state terminal_state;
typedef struct router_state router_state;

128
/* compute node data (think NIC) structure */
Nikhil's avatar
Nikhil committed
129 130 131
struct terminal_state
{
  unsigned int terminal_id; //what is my local id
132 133
  const char * anno;
  const local_param *params;
Nikhil's avatar
Nikhil committed
134

135 136
  //which router I am connected to
  unsigned int router_id;
Nikhil's avatar
Nikhil committed
137 138
  tw_lpid router_gid;

139
  // Each terminal will have input/output channel(s) with the router
Nikhil's avatar
Nikhil committed
140 141 142
  int** vc_occupancy; // NUM_VC
  tw_stime terminal_available_time;

143
  //available messages to be sent
Nikhil's avatar
Nikhil committed
144 145 146 147 148 149 150 151
  message_list ***terminal_msgs;
  message_list ***terminal_msgs_tail;
  int terminal_length;
  int in_send_loop;
  int issueIdle;

  //packet aggregation
  struct qhash_table *rank_tbl;
152
  //transient storage for reverse computation
Nikhil's avatar
Nikhil committed
153 154
  struct rc_stack * st;

155 156 157 158 159
  //stats collection
  uint64_t packet_counter;
  int packet_gen;
  int packet_fin;
  struct mn_stats local_stats_array[CATEGORY_MAX];
Nikhil's avatar
Nikhil committed
160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176
  tw_stime   total_time;
  uint64_t total_msg_size;
  double total_hops;
  long finished_msgs;
  long finished_chunks;
  long finished_packets;

  //sampling
  tw_stime last_buf_full;
  tw_stime busy_time;
  char output_buf[4096];
  long fin_chunks_sample;
  long data_size_sample;
  double fin_hops_sample;
  tw_stime fin_chunks_time;
  tw_stime busy_time_sample;
  char sample_buf[4096];
177
  struct local_cn_sample * sample_stat;
Nikhil's avatar
Nikhil committed
178 179 180 181 182 183 184
  int op_arr_size;
  int max_arr_size;
  /* for logging forward and reverse events */
  long fwd_events;
  long rev_events;
};

185 186
//CHANGE: may need to change if more functionality is desired
/* event types */
Nikhil's avatar
Nikhil committed
187 188 189 190 191 192 193 194 195 196
enum event_t
{
  T_GENERATE=1,
  T_ARRIVE,
  T_SEND,
  T_BUFFER,
  R_SEND,
  R_ARRIVE,
  R_BUFFER,
};
Nikhil's avatar
Nikhil committed
197
typedef enum event_t event_t;
Nikhil's avatar
Nikhil committed
198

199
//CHANGE: may need to change if more functionality is desired
Nikhil's avatar
Nikhil committed
200 201 202 203 204 205 206
/* whether the last hop of a packet was global, local or a terminal */
enum last_hop
{
  ROUTER=1,
  TERMINAL
};

207
//CHANGE: may need to change if more functionality is desired
Nikhil's avatar
Nikhil committed
208 209 210 211 212 213 214 215 216 217
enum ROUTING_ALGO
{
  STATIC = 0,
  ADAPTIVE,
};

struct router_state
{
  //who am I
  unsigned int router_id;
218 219
  const char * anno;
  const local_param *params;
Nikhil's avatar
Nikhil committed
220

221 222
  //CHANGE: may need to be changed if linear storage is not desired
  //array/linked list based storage of info about ports/vcs
Nikhil's avatar
Nikhil committed
223 224 225 226 227 228
  tw_lpid* link_connections;
  tw_stime* next_output_available_time;
  message_list ***pending_msgs;
  message_list ***pending_msgs_tail;
  message_list ***queued_msgs;
  message_list ***queued_msgs_tail;
229
  int** vc_occupancy;
Nikhil's avatar
Nikhil committed
230 231 232
  int *in_send_loop;
  int *queued_count;

233 234
  //for reverse computation
  struct rc_stack * st;
Nikhil's avatar
Nikhil committed
235 236

  //sampling and stats
237
  int64_t* link_traffic;
Nikhil's avatar
Nikhil committed
238 239 240
  tw_stime* last_buf_full;
  tw_stime* busy_time;
  tw_stime* busy_time_sample;
241
  struct local_router_sample * rsamples;
Nikhil's avatar
Nikhil committed
242 243
  int op_arr_size;
  int max_arr_size;
244
  long fwd_events, rev_events;
Nikhil's avatar
Nikhil committed
245
  int64_t * link_traffic_sample;
246 247 248 249 250
  char output_buf[4096];
  char output_buf2[4096];

  //CHANGE: add network specific data here
  int* dim_position;
Nikhil's avatar
Nikhil committed
251 252 253 254 255 256 257 258
};

struct VC_Entry {
  int vc;
  message_list* entry;
};

//global stats
259 260
static tw_stime         local_total_time = 0;
static tw_stime         local_max_latency = 0;
Nikhil's avatar
Nikhil committed
261 262 263 264 265 266 267

static long long       total_hops = 0;
static long long       N_finished_packets = 0;
static long long       total_msg_sz = 0;
static long long       N_finished_msgs = 0;
static long long       N_finished_chunks = 0;

268 269
/* returns the message size */
static int local_get_msg_sz(void)
Nikhil's avatar
Nikhil committed
270
{
271
  return sizeof(LOCAL_MSG_STRUCT);
Nikhil's avatar
Nikhil committed
272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300
}

/* helper functions - convert between flat ids and torus n-dimensional ids */
static void to_dim_id(
    int flat_id,
    int ndims,
    const int *dim_lens,
    int *out_dim_ids)
{
  for (int i = 0; i < ndims; i++) {
    out_dim_ids[i] = flat_id % dim_lens[i];
    flat_id /= dim_lens[i];
  }
}

static int to_flat_id(
    int ndims,
    const int *dim_lens,
    const int *dim_ids)
{
  int flat_id = dim_ids[0];
  int mult = dim_lens[0];
  for (int i = 1; i < ndims; i++) {
    flat_id += dim_ids[i] * mult;
    mult *= dim_lens[i];
  }
  return flat_id;
}

301 302 303 304 305
//CHANGE: network specific params have to be read here
static void local_read_config(const char * anno, local_param *params){
  local_param *p = params;

  // general params - do not change unless you intent to modify them
Nikhil's avatar
Nikhil committed
306
  int rc = configuration_get_value_double(&config, "PARAMS", "link_bandwidth",
Nikhil's avatar
Nikhil committed
307 308 309
      anno, &p->link_bandwidth);
  if(rc) {
    p->link_bandwidth = 5.25;
310
    fprintf(stderr, "Bandwidth of links  not specified, setting to %lf\n",
Nikhil's avatar
Nikhil committed
311 312 313
        p->link_bandwidth);
  }

314
  rc = configuration_get_value_double(&config, "PARAMS", "cn_bandwidth",
Nikhil's avatar
Nikhil committed
315 316 317 318 319 320 321
      anno, &p->cn_bandwidth);
  if(rc) {
    p->cn_bandwidth = 5.25;
    fprintf(stderr, "Bandwidth of compute node channels not specified, setting "
        "to %lf\n", p->cn_bandwidth);
  }

322
  rc = configuration_get_value_int(&config, "PARAMS", "num_cn", anno,
Nikhil's avatar
Nikhil committed
323 324 325 326 327
      &p->num_cn);
  if(rc) {
    tw_error(TW_LOC, "Nodes per router (num_cn) not specified\n");
  }

328
  rc = configuration_get_value_int(&config, "PARAMS", "num_vcs", anno,
Nikhil's avatar
Nikhil committed
329 330 331 332
      &p->num_vcs);
  if(rc) {
    p->num_vcs = 1;
  }
333 334

  rc = configuration_get_value_int(&config, "PARAMS", "chunk_size", anno,
Nikhil's avatar
Nikhil committed
335 336 337
      &p->chunk_size);
  if(rc) {
    p->chunk_size = 512;
338
    fprintf(stderr, "Chunk size for packets is not specified, setting to %d\n",
Nikhil's avatar
Nikhil committed
339 340 341
      p->chunk_size);
  }

342
  rc = configuration_get_value_int(&config, "PARAMS", "vc_size", anno,
Nikhil's avatar
Nikhil committed
343 344 345
      &p->vc_size);
  if(rc) {
    p->vc_size = 32768;
346
    fprintf(stderr, "Buffer size of link channels not specified, setting to %d\n",
Nikhil's avatar
Nikhil committed
347 348 349
      p->vc_size);
  }

350
  rc = configuration_get_value_int(&config, "PARAMS", "cn_vc_size", anno,
Nikhil's avatar
Nikhil committed
351 352 353 354 355 356 357 358 359 360 361
      &p->cn_vc_size);
  if(rc) {
    p->cn_vc_size = 65536;
    fprintf(stderr, "Buffer size of compute node channels not specified, "
        "setting to %d\n", p->cn_vc_size);
  }

  p->router_delay = 50;
  configuration_get_value_int(&config, "PARAMS", "router_delay", anno,
      &p->router_delay);

362 363 364 365 366 367
  configuration_get_value(&config, "PARAMS", "cn_sample_file", anno,
      local_cn_sample_file, MAX_NAME_LENGTH);
  configuration_get_value(&config, "PARAMS", "rt_sample_file", anno,
      local_rtr_sample_file, MAX_NAME_LENGTH);

  //CHANGE: add network specific parameters here
Nikhil's avatar
Nikhil committed
368
  rc = configuration_get_value_int(&config, "PARAMS", "n_dims", anno,
369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399
      &p->n_dims);
  if(rc) {
    tw_error(TW_LOC, "Number of dimensions not specified\n");
  }

  rc = configuration_get_value_int(&config, "PARAMS", "gap", anno, &p->gap);
  if(rc) {
    tw_error(TW_LOC, "Gap not specified\n");
  }

  char dim_length_str[MAX_NAME_LENGTH];
  rc = configuration_get_value(&config, "PARAMS", "dim_length", anno,
      dim_length_str, MAX_NAME_LENGTH);
  if (rc == 0){
    tw_error(TW_LOC, "couldn't read PARAMS:dim_length");
  }
  char* token;
  p->dim_length= (int*)malloc(p->n_dims * sizeof(*p->dim_length));
  token = strtok(dim_length_str, ",");
  int i = 0;
  while(token != NULL)
  {
    sscanf(token, "%d", &p->dim_length[i]);
    if(p->dim_length[i] <= 0)
    {
      tw_error(TW_LOC, "Invalid torus dimension specified "
          "(%d at pos %d), exiting... ", p->dim_length[i], i);
    }
    i++;
    token = strtok(NULL,",");
  }
Nikhil's avatar
Nikhil committed
400 401 402 403 404 405 406 407 408 409 410 411 412 413 414

  char routing_str[MAX_NAME_LENGTH];
  configuration_get_value(&config, "PARAMS", "routing", anno, routing_str,
      MAX_NAME_LENGTH);
  if(strcmp(routing_str, "static") == 0)
    p->routing = STATIC;
  else if (strcmp(routing_str, "adaptive") == 0) {
    p->routing = ADAPTIVE;
    if(p->num_vcs < 2) {
      p->num_vcs = 2;
    }
  }
  else
  {
    p->routing = STATIC;
415
    fprintf(stderr,
Nikhil's avatar
Nikhil committed
416 417 418
        "No routing protocol specified, setting to static routing\n");
  }

419
  //CHANGE: derived parameters often are computed based on network specifics
Nikhil's avatar
Nikhil committed
420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437
  p->radix = 0;
  p->total_routers = 1;
  p->cons_per_dim = (int *)malloc(p->n_dims * sizeof(int));
  p->offset_per_dim = (int *)malloc(p->n_dims * sizeof(int));
  for(int i = 0; i < p->n_dims; i++) {
    p->cons_per_dim[i] = 2 + (p->dim_length[i] - 3)/p->gap;
    p->radix += p->cons_per_dim[i];
    p->total_routers *= p->dim_length[i];
    if(i == 0) {
      p->offset_per_dim[i] = p->num_cn;
    } else {
      p->offset_per_dim[i] = p->offset_per_dim[i - 1] + p->cons_per_dim[i - 1];
    }
  }
  p->radix += p->num_cn;
  p->total_terminals = p->total_routers * p->num_cn;

  int rank;
438
  MPI_Comm_rank(MPI_COMM_CODES, &rank);
Nikhil's avatar
Nikhil committed
439 440 441 442 443
  if(!rank) {
    printf("\n Total nodes %d routers %d radix %d \n",
        p->total_terminals, p->total_routers, p->radix);
  }

444
  //general derived parameters
Nikhil's avatar
Nikhil committed
445 446 447
  p->cn_delay = bytes_to_ns(1, p->cn_bandwidth);
  p->link_delay = bytes_to_ns(1, p->link_bandwidth);
  p->credit_delay = bytes_to_ns(CREDIT_SZ, p->link_bandwidth);
448 449 450 451 452 453 454 455

  uint32_t h1 = 0, h2 = 0;
  bj_hashlittle2(LP_METHOD_NM_TERM, strlen(LP_METHOD_NM_TERM), &h1, &h2);
  terminal_magic_num = h1 + h2;

  bj_hashlittle2(LP_METHOD_NM_ROUT, strlen(LP_METHOD_NM_ROUT), &h1, &h2);
  router_magic_num = h1 + h2;

Nikhil's avatar
Nikhil committed
456 457
}

458
static void local_configure(){
Nikhil's avatar
Nikhil committed
459 460 461
  anno_map = codes_mapping_get_lp_anno_map(LP_CONFIG_NM_TERM);
  assert(anno_map);
  num_params = anno_map->num_annos + (anno_map->has_unanno_lp > 0);
462
  all_params = (local_param *)malloc(num_params * sizeof(*all_params));
Nikhil's avatar
Nikhil committed
463 464 465

  for (int i = 0; i < anno_map->num_annos; i++){
    const char * anno = anno_map->annotations[i].ptr;
466
    local_read_config(anno, &all_params[i]);
Nikhil's avatar
Nikhil committed
467 468
  }
  if (anno_map->has_unanno_lp > 0){
469
    local_read_config(NULL, &all_params[anno_map->num_annos]);
Nikhil's avatar
Nikhil committed
470 471 472
  }
}

473 474
/* report statistics like average and maximum packet latency, average number of hops traversed */
static void local_report_stats()
Nikhil's avatar
Nikhil committed
475 476 477 478 479 480 481
{
  long long avg_hops, total_finished_packets, total_finished_chunks;
  long long total_finished_msgs, final_msg_sz;
  tw_stime avg_time, max_time;
  int total_minimal_packets, total_nonmin_packets;
  long total_gen, total_fin;

482
  MPI_Reduce( &total_hops, &avg_hops, 1, MPI_LONG_LONG, MPI_SUM, 0,
483
      MPI_COMM_CODES);
484
  MPI_Reduce( &N_finished_packets, &total_finished_packets, 1, MPI_LONG_LONG,
485
      MPI_SUM, 0, MPI_COMM_CODES);
486
  MPI_Reduce( &N_finished_msgs, &total_finished_msgs, 1, MPI_LONG_LONG, MPI_SUM,
487
      0, MPI_COMM_CODES);
488
  MPI_Reduce( &N_finished_chunks, &total_finished_chunks, 1, MPI_LONG_LONG,
489
      MPI_SUM, 0, MPI_COMM_CODES);
490
  MPI_Reduce( &total_msg_sz, &final_msg_sz, 1, MPI_LONG_LONG, MPI_SUM, 0,
491
      MPI_COMM_CODES);
492
  MPI_Reduce( &local_total_time, &avg_time, 1,MPI_DOUBLE, MPI_SUM, 0,
493
      MPI_COMM_CODES);
494
  MPI_Reduce( &local_max_latency, &max_time, 1, MPI_DOUBLE, MPI_MAX, 0,
495
      MPI_COMM_CODES);
Nikhil's avatar
Nikhil committed
496

497 498
  MPI_Reduce( &packet_gen, &total_gen, 1, MPI_LONG, MPI_SUM, 0, MPI_COMM_CODES);
  MPI_Reduce( &packet_fin, &total_fin, 1, MPI_LONG, MPI_SUM, 0, MPI_COMM_CODES);
Nikhil's avatar
Nikhil committed
499 500 501

  /* print statistics */
  if(!g_tw_mynode)
502
  {
Nikhil's avatar
Nikhil committed
503 504
    printf(" Average number of hops traversed %f average chunk latency %lf us "
      "maximum chunk latency %lf us avg message size %lf bytes finished "
505 506 507 508
      "messages %lld finished chunks %lld \n",
      (float)avg_hops/total_finished_chunks,
      avg_time/(total_finished_chunks*1000), max_time/1000,
      (float)final_msg_sz/total_finished_msgs, total_finished_msgs,
Nikhil's avatar
Nikhil committed
509 510 511 512 513 514
      total_finished_chunks);
    printf("\n Total packets generated %ld finished %ld \n", total_gen, total_fin);
  }
  return;
}

515 516
/* initialize a compute node terminal */
static void terminal_init( terminal_state * s, tw_lp * lp )
Nikhil's avatar
Nikhil committed
517
{
518 519 520
  int i;
  char anno[MAX_NAME_LENGTH];

Nikhil's avatar
Nikhil committed
521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542
  s->packet_gen = 0;
  s->packet_fin = 0;

  codes_mapping_get_lp_info(lp->gid, lp_group_name, &mapping_grp_id, NULL,
      &mapping_type_id, anno, &mapping_rep_id, &mapping_offset);

  if (anno[0] == '\0') {
    s->anno = NULL;
    s->params = &all_params[num_params-1];
  } else {
    s->anno = strdup(anno);
    int id = configuration_get_annotation_index(anno, anno_map);
    s->params = &all_params[id];
  }

  int num_lps = codes_mapping_get_lp_count(lp_group_name, 0, LP_CONFIG_NM_TERM,
      s->anno, 0);

  if(num_lps != s->params->total_terminals) {
    tw_error(TW_LOC, "Number of terminals LP does not match number of nodes\n");
  }

543
  s->terminal_id = codes_mapping_get_lp_relative_id(lp->gid, 0, 0);
Nikhil's avatar
Nikhil committed
544 545

  s->router_id = (int)s->terminal_id / s->params->num_cn;
546
  s->router_gid = codes_mapping_get_lpid_from_relative(s->router_id, NULL,
Nikhil's avatar
Nikhil committed
547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570
      LP_CONFIG_NM_ROUT, s->anno, 1);

  s->terminal_available_time = 0.0;
  s->packet_counter = 0;
  s->finished_msgs = 0;
  s->finished_chunks = 0;
  s->finished_packets = 0;
  s->total_time = 0.0;
  s->total_msg_size = 0;

  s->last_buf_full = 0.0;
  s->busy_time = 0.0;

  s->fwd_events = 0;
  s->rev_events = 0;

  rc_stack_create(&s->st);
  s->vc_occupancy = (int **)malloc(sizeof(int*));

  s->vc_occupancy[0] = (int*)malloc(s->params->num_vcs * sizeof(int));
  for(i = 0; i < s->params->num_vcs; i++ ) {
    s->vc_occupancy[0][i] = 0;
  }

571
  s->rank_tbl = qhash_init(mn_rank_hash_compare, mn_hash_func, HASH_TABLE_SIZE);
Nikhil's avatar
Nikhil committed
572 573 574 575 576 577

  if(!s->rank_tbl)
    tw_error(TW_LOC, "\n Hash table not initialized! ");

  s->terminal_msgs = (message_list ***)malloc(sizeof(message_list**));
  s->terminal_msgs_tail = (message_list ***)malloc(sizeof(message_list**));
578
  s->terminal_msgs[0] =
Nikhil's avatar
Nikhil committed
579
    (message_list **)malloc(s->params->num_vcs * sizeof(message_list*));
580
  s->terminal_msgs_tail[0] =
Nikhil's avatar
Nikhil committed
581 582 583 584 585 586 587 588 589 590 591 592
    (message_list **)malloc(s->params->num_vcs * sizeof(message_list*));
  for(i = 0; i < s->params->num_vcs; i++ ) {
    s->terminal_msgs[0][i] = NULL;
    s->terminal_msgs_tail[0][i] = NULL;
  }
  s->terminal_length = 0;
  s->in_send_loop = 0;
  s->issueIdle = 0;

  return;
}

593 594 595 596
//CHANGE: fill this function to set up router's connectivity and network
//specific data structures
static void create_router_connections(router_state * r, tw_lp * lp) {
  local_param *p = (local_param *)r->params;
Nikhil's avatar
Nikhil committed
597 598 599

  r->dim_position = (int *)malloc(p->n_dims * sizeof(int));

600
  to_dim_id(r->router_id, r->params->n_dims, r->params->dim_length,
Nikhil's avatar
Nikhil committed
601 602 603 604 605 606 607 608 609 610
      r->dim_position);

  //set up connections
  int curr_con = 0;
  int first = r->router_id * p->num_cn;
  for(; curr_con < p->num_cn; curr_con++) {
    r->link_connections[curr_con] = codes_mapping_get_lpid_from_relative(
      first, NULL, LP_CONFIG_NM_TERM, r->anno, 1);
    first++;
  }
611

Nikhil's avatar
Nikhil committed
612 613 614 615 616 617 618 619
  int temp_dim_pos[p->n_dims];
  for(int i = 0; i < p->n_dims; i++)
    temp_dim_pos[i] = r->dim_position[i];

  for(int curr_dim = 0; curr_dim < p->n_dims; curr_dim++) {
    curr_con = p->offset_per_dim[curr_dim];
    for(int loc = 0; loc < p->dim_length[curr_dim]; loc++) {
      if(loc != r->dim_position[curr_dim]) {
620
        if(loc == r->dim_position[curr_dim] - 1 ||
Nikhil's avatar
Nikhil committed
621
           loc == r->dim_position[curr_dim] + 1 ||
622 623 624
           (loc < r->dim_position[curr_dim] &&
           ((r->dim_position[curr_dim] - 1 - loc) % p->gap == 0)) ||
           (loc > r->dim_position[curr_dim] &&
Nikhil's avatar
Nikhil committed
625 626 627 628 629 630 631 632 633 634 635
           ((loc - r->dim_position[curr_dim] - 1) % p->gap == 0))) {
          temp_dim_pos[curr_dim] = loc;
          int neigh_id = to_flat_id(p->n_dims, p->dim_length, temp_dim_pos);
          temp_dim_pos[curr_dim] = r->dim_position[curr_dim];
          r->link_connections[curr_con] = codes_mapping_get_lpid_from_relative(
              neigh_id, NULL, LP_CONFIG_NM_ROUT, r->anno, 1);
          curr_con++;
        }
      }
    }
  }
636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657
}

static void router_setup(router_state * r, tw_lp * lp)
{
  char anno[MAX_NAME_LENGTH];
  codes_mapping_get_lp_info(lp->gid, lp_group_name, &mapping_grp_id, NULL,
      &mapping_type_id, anno, &mapping_rep_id, &mapping_offset);

  if (anno[0] == '\0'){
    r->anno = NULL;
    r->params = &all_params[num_params-1];
  } else{
    r->anno = strdup(anno);
    int id = configuration_get_annotation_index(anno, anno_map);
    r->params = &all_params[id];
  }

  local_param *p = (local_param *)r->params;

  r->link_connections = (tw_lpid *)malloc(p->radix * sizeof(tw_lpid));

  r->router_id = codes_mapping_get_lp_relative_id(lp->gid, 0, 0);
Nikhil's avatar
Nikhil committed
658 659 660 661 662 663 664 665 666 667

  r->fwd_events = 0;
  r->rev_events = 0;

  r->next_output_available_time = (tw_stime*)malloc(p->radix * sizeof(tw_stime));
  r->link_traffic = (int64_t*)malloc(p->radix * sizeof(int64_t));
  r->link_traffic_sample = (int64_t*)malloc(p->radix * sizeof(int64_t));

  r->vc_occupancy = (int**)malloc(p->radix * sizeof(int*));
  r->in_send_loop = (int*)malloc(p->radix * sizeof(int));
668
  r->pending_msgs =
Nikhil's avatar
Nikhil committed
669
    (message_list ***)malloc(p->radix * sizeof(message_list**));
670
  r->pending_msgs_tail =
Nikhil's avatar
Nikhil committed
671
    (message_list ***)malloc(p->radix * sizeof(message_list**));
672
  r->queued_msgs =
Nikhil's avatar
Nikhil committed
673
    (message_list ***)malloc(p->radix * sizeof(message_list**));
674
  r->queued_msgs_tail =
Nikhil's avatar
Nikhil committed
675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690
    (message_list ***)malloc(p->radix * sizeof(message_list**));
  r->queued_count = (int*)malloc(p->radix * sizeof(int));
  r->last_buf_full = (tw_stime*)malloc(p->radix * sizeof(tw_stime));
  r->busy_time = (tw_stime*)malloc(p->radix * sizeof(tw_stime));
  r->busy_time_sample = (tw_stime*)malloc(p->radix * sizeof(tw_stime));

  rc_stack_create(&r->st);
  for(int i = 0; i < p->radix; i++)
  {
    // Set credit & router occupancy
    r->last_buf_full[i] = 0.0;
    r->busy_time[i] = 0.0;
    r->busy_time_sample[i] = 0.0;
    r->next_output_available_time[i] = 0;
    r->link_traffic[i] = 0;
    r->link_traffic_sample[i] = 0;
691
    r->queued_count[i] = 0;
Nikhil's avatar
Nikhil committed
692 693
    r->in_send_loop[i] = 0;
    r->vc_occupancy[i] = (int *)malloc(p->num_vcs * sizeof(int));
694
    r->pending_msgs[i] = (message_list **)malloc(p->num_vcs *
Nikhil's avatar
Nikhil committed
695
        sizeof(message_list*));
696
    r->pending_msgs_tail[i] = (message_list **)malloc(p->num_vcs *
Nikhil's avatar
Nikhil committed
697
        sizeof(message_list*));
698
    r->queued_msgs[i] = (message_list **)malloc(p->num_vcs *
Nikhil's avatar
Nikhil committed
699
        sizeof(message_list*));
700
    r->queued_msgs_tail[i] = (message_list **)malloc(p->num_vcs *
Nikhil's avatar
Nikhil committed
701 702 703 704 705 706 707 708 709 710
        sizeof(message_list*));
    for(int j = 0; j < p->num_vcs; j++) {
      r->vc_occupancy[i][j] = 0;
      r->pending_msgs[i][j] = NULL;
      r->pending_msgs_tail[i][j] = NULL;
      r->queued_msgs[i][j] = NULL;
      r->queued_msgs_tail[i][j] = NULL;
    }
  }

711 712 713
  create_router_connections(r, lp);
  return;
}
Nikhil's avatar
Nikhil committed
714

715 716
/* packet event , generates a packet on the compute node */
static tw_stime local_packet_event(
Nikhil's avatar
Nikhil committed
717 718 719 720 721 722 723 724 725 726 727 728 729 730
    model_net_request const * req,
    uint64_t message_offset,
    uint64_t packet_size,
    tw_stime offset,
    mn_sched_params const * sched_params,
    void const * remote_event,
    void const * self_event,
    tw_lp *sender,
    int is_last_pckt)
{
  (void)message_offset;
  (void)sched_params;
  tw_event * e_new;
  tw_stime xfer_to_nic_time;
731
  LOCAL_MSG_STRUCT * msg;
Nikhil's avatar
Nikhil committed
732 733
  char* tmp_ptr;

734
  xfer_to_nic_time = codes_local_latency(sender);
Nikhil's avatar
Nikhil committed
735
  e_new = model_net_method_event_new(sender->gid, xfer_to_nic_time+offset,
736
      sender, LOCAL_NETWORK_NAME, (void**)&msg, (void**)&tmp_ptr);
Nikhil's avatar
Nikhil committed
737 738 739 740 741 742 743 744 745 746 747 748 749 750 751
  strcpy(msg->category, req->category);
  msg->final_dest_gid = req->final_dest_lp;
  msg->total_size = req->msg_size;
  msg->sender_lp = req->src_lp;
  msg->sender_mn_lp = sender->gid;
  msg->packet_size = packet_size;
  msg->travel_start_time = tw_now(sender);
  msg->remote_event_size_bytes = 0;
  msg->local_event_size_bytes = 0;
  msg->type = T_GENERATE;
  msg->dest_terminal_id = req->dest_mn_lp;
  msg->dest_terminal = codes_mapping_get_lp_relative_id(msg->dest_terminal_id, 0, 0);
  msg->message_id = req->msg_id;
  msg->is_pull = req->is_pull;
  msg->pull_size = req->pull_size;
752
  msg->magic = terminal_magic_num;
Nikhil's avatar
Nikhil committed
753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773
  msg->msg_start_time = req->msg_start_time;

  if(is_last_pckt) /* Its the last packet so pass in remote and local event information*/
  {
    if(req->remote_event_size > 0)
    {
      msg->remote_event_size_bytes = req->remote_event_size;
      memcpy(tmp_ptr, remote_event, req->remote_event_size);
      tmp_ptr += req->remote_event_size;
    }
    if(req->self_event_size > 0)
    {
      msg->local_event_size_bytes = req->self_event_size;
      memcpy(tmp_ptr, self_event, req->self_event_size);
      tmp_ptr += req->self_event_size;
    }
  }
  tw_event_send(e_new);
  return xfer_to_nic_time;
}

774 775
/* packet event reverse handler */
static void local_packet_event_rc(tw_lp *sender)
Nikhil's avatar
Nikhil committed
776 777 778 779 780
{
  codes_local_latency_reverse(sender);
  return;
}

781 782
/* generates packet at the current compute node */
static void packet_generate(terminal_state * s, tw_bf * bf, LOCAL_MSG_STRUCT * msg,
Nikhil's avatar
Nikhil committed
783 784 785 786 787 788 789
    tw_lp * lp) {
  packet_gen++;
  s->packet_gen++;

  tw_stime ts, nic_ts;

  assert(lp->gid != msg->dest_terminal_id);
790
  const local_param *p = s->params;
Nikhil's avatar
Nikhil committed
791 792 793

  int total_event_size;
  uint64_t num_chunks = msg->packet_size / p->chunk_size;
794
  if (msg->packet_size % s->params->chunk_size)
Nikhil's avatar
Nikhil committed
795 796 797 798 799
    num_chunks++;

  if(!num_chunks)
    num_chunks = 1;

800
  nic_ts = g_tw_lookahead + (msg->packet_size * s->params->cn_delay) +
Nikhil's avatar
Nikhil committed
801 802 803 804 805 806 807 808 809
    tw_rand_unif(lp->rng);

  msg->packet_ID = lp->gid + g_tw_nlp * s->packet_counter;
  for(int i = 0; i < p->n_dims; i++) {
    msg->hops[i] = 0;
  }
  msg->my_N_hop = 0;

  /* TODO: how do we get a better vc selection mechanism */
810
  //CHANGE: use only 1 VC for NIC to router transfer; VC selection mechanism on the NIC can be added here
Nikhil's avatar
Nikhil committed
811 812 813 814 815 816 817 818 819 820 821 822 823 824
  int use_vc = 0;
  msg->saved_channel = use_vc;

  for(uint64_t i = 0; i < num_chunks; i++)
  {
    message_list *cur_chunk = (message_list*)malloc(
        sizeof(message_list));
    init_message_list(cur_chunk, msg);

    if(msg->remote_event_size_bytes + msg->local_event_size_bytes > 0) {
      cur_chunk->event_data = (char*)malloc(
          msg->remote_event_size_bytes + msg->local_event_size_bytes);
    }

825
    void * m_data_src = model_net_method_get_edata(LOCAL_NETWORK_NAME, msg);
Nikhil's avatar
Nikhil committed
826 827 828
    if (msg->remote_event_size_bytes){
      memcpy(cur_chunk->event_data, m_data_src, msg->remote_event_size_bytes);
    }
829
    if (msg->local_event_size_bytes){
Nikhil's avatar
Nikhil committed
830
      m_data_src = (char*)m_data_src + msg->remote_event_size_bytes;
831
      memcpy((char*)cur_chunk->event_data + msg->remote_event_size_bytes,
Nikhil's avatar
Nikhil committed
832 833 834
          m_data_src, msg->local_event_size_bytes);
    }

835 836
    cur_chunk->LOCAL_MSG_NAME_FROM_UNION.chunk_id = i;
    cur_chunk->port = 0; cur_chunk->index = use_vc;
Nikhil's avatar
Nikhil committed
837 838 839 840 841
    append_to_message_list(s->terminal_msgs[0], s->terminal_msgs_tail[0],
        use_vc, cur_chunk);
    s->terminal_length += s->params->chunk_size;
  }

842
  if(s->terminal_length < s->params->num_vcs * s->params->cn_vc_size) {
Nikhil's avatar
Nikhil committed
843 844 845 846 847 848 849 850 851 852 853
    model_net_method_idle_event(nic_ts, 0, lp);
  } else {
    bf->c11 = 1;
    s->issueIdle = 1;
    msg->saved_busy_time = s->last_buf_full;
    s->last_buf_full = tw_now(lp);
  }

  if(s->in_send_loop == 0) {
    bf->c5 = 1;
    ts = codes_local_latency(lp);
854 855
    LOCAL_MSG_STRUCT *m;
    tw_event* e = model_net_method_event_new(lp->gid, ts, lp, LOCAL_NETWORK_NAME,
Nikhil's avatar
Nikhil committed
856 857 858 859 860 861 862
        (void**)&m, NULL);
    m->type = T_SEND;
    m->magic = terminal_magic_num;
    s->in_send_loop = 1;
    tw_event_send(e);
  }

863
  total_event_size = model_net_get_msg_sz(LOCAL_NETWORK_NAME) +
Nikhil's avatar
Nikhil committed
864 865
    msg->remote_event_size_bytes + msg->local_event_size_bytes;
  mn_stats* stat;
866
  stat = model_net_find_stats(msg->category, s->local_stats_array);
Nikhil's avatar
Nikhil committed
867 868 869 870 871 872 873 874 875
  stat->send_count++;
  stat->send_bytes += msg->packet_size;
  stat->send_time += p->cn_delay * msg->packet_size;
  if(stat->max_event_size < total_event_size)
    stat->max_event_size = total_event_size;

  return;
}

876
static void packet_generate_rc(terminal_state * s, tw_bf * bf, LOCAL_MSG_STRUCT * msg, tw_lp * lp)
Nikhil's avatar
Nikhil committed
877 878 879 880 881 882 883 884 885 886 887 888 889 890 891
{
  s->packet_gen--;
  packet_gen--;

  tw_rand_reverse_unif(lp->rng);

  int num_chunks = msg->packet_size/s->params->chunk_size;
  if(msg->packet_size % s->params->chunk_size)
    num_chunks++;

  if(!num_chunks)
    num_chunks = 1;

  int i;
  for(i = 0; i < num_chunks; i++) {
892
    delete_message_list(return_tail(s->terminal_msgs[0],
Nikhil's avatar
Nikhil committed
893 894 895 896 897 898 899 900 901 902 903 904
          s->terminal_msgs_tail[0], msg->saved_channel));
    s->terminal_length -= s->params->chunk_size;
  }
  if(bf->c11) {
    s->issueIdle = 0;
    s->last_buf_full = msg->saved_busy_time;
  }
  if(bf->c5) {
    codes_local_latency_reverse(lp);
    s->in_send_loop = 0;
  }
  struct mn_stats* stat;
905
  stat = model_net_find_stats(msg->category, s->local_stats_array);
Nikhil's avatar
Nikhil committed
906 907 908 909 910 911
  stat->send_count--;
  stat->send_bytes -= msg->packet_size;
  stat->send_time -= s->params->cn_delay * msg->packet_size;
}


912 913
/* sends the packet from the current compute node to the attached router */
static void packet_send(terminal_state * s, tw_bf * bf, LOCAL_MSG_STRUCT * msg,
Nikhil's avatar
Nikhil committed
914 915 916 917
    tw_lp * lp) {

  tw_stime ts;
  tw_event *e;
918
  LOCAL_MSG_STRUCT *m;
Nikhil's avatar
Nikhil committed
919 920 921
  tw_lpid router_id;

  std::vector<VC_Entry> entries;
922

Nikhil's avatar
Nikhil committed
923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940
  for(int i = 0; i < s->params->num_vcs; i++) {
    if(s->terminal_msgs[0][i] != NULL &&
      s->vc_occupancy[0][i] + s->params->chunk_size <= s->params->cn_vc_size) {
      VC_Entry tmp;
      tmp.vc = i; tmp.entry = s->terminal_msgs[0][i];
      entries.push_back(tmp);
    }
  }

  if(entries.size() == 0) {
    bf->c1 = 1;
    s->in_send_loop = 0;

    msg->saved_busy_time = s->last_buf_full;
    s->last_buf_full = tw_now(lp);
    return;
  }

941
  //CHANGE: randomly pick the VC to send on (by default only VC 0 is filed); can be changed.
Nikhil's avatar
Nikhil committed
942 943 944 945 946
  int pick = tw_rand_integer(lp->rng, 0, entries.size() - 1);
  message_list* cur_entry = entries[pick].entry;
  int use_vc = entries[pick].vc;
  msg->saved_channel = use_vc;

947 948
  uint64_t num_chunks = cur_entry->LOCAL_MSG_NAME_FROM_UNION.packet_size/s->params->chunk_size;
  if(cur_entry->LOCAL_MSG_NAME_FROM_UNION.packet_size % s->params->chunk_size)
Nikhil's avatar
Nikhil committed
949 950 951 952 953 954
    num_chunks++;

  if(!num_chunks)
    num_chunks = 1;

  tw_stime delay;
955 956 957
  if((cur_entry->LOCAL_MSG_NAME_FROM_UNION.packet_size % s->params->chunk_size)
      && (cur_entry->LOCAL_MSG_NAME_FROM_UNION.chunk_id == num_chunks - 1))
    delay = (cur_entry->LOCAL_MSG_NAME_FROM_UNION.packet_size % s->params->chunk_size) *
Nikhil's avatar
Nikhil committed
958
      s->params->cn_delay;
959
  else
Nikhil's avatar
Nikhil committed
960 961 962 963 964 965
    delay = s->params->chunk_size * s->params->cn_delay;

  msg->saved_available_time = s->terminal_available_time;
  ts = g_tw_lookahead + delay + tw_rand_unif(lp->rng);
  s->terminal_available_time = maxd(s->terminal_available_time, tw_now(lp));
  s->terminal_available_time += ts;
966

Nikhil's avatar
Nikhil committed
967 968
  ts = s->terminal_available_time - tw_now(lp);
  void * remote_event;
969
  e = model_net_method_event_new(s->router_gid, ts, lp, LOCAL_NETWORK_ROUTER_NAME,
Nikhil's avatar
Nikhil committed
970
      (void**)&m, &remote_event);
971
  memcpy(m, &cur_entry->LOCAL_MSG_NAME_FROM_UNION, sizeof(LOCAL_MSG_STRUCT));
Nikhil's avatar
Nikhil committed
972 973 974 975 976 977 978 979 980 981 982 983 984
  if (m->remote_event_size_bytes){
    memcpy(remote_event, cur_entry->event_data, m->remote_event_size_bytes);
  }

  m->type = R_ARRIVE;
  m->src_terminal_id = lp->gid;
  m->vc_index = 0;
  m->output_chan = use_vc;
  m->last_hop = TERMINAL;
  m->magic = router_magic_num;
  m->local_event_size_bytes = 0;
  tw_event_send(e);

985 986
  if(cur_entry->LOCAL_MSG_NAME_FROM_UNION.chunk_id == num_chunks - 1 &&
      (cur_entry->LOCAL_MSG_NAME_FROM_UNION.local_event_size_bytes > 0)) {
Nikhil's avatar
Nikhil committed
987
    bf->c2 = 1;
988 989
    tw_stime local_ts = codes_local_latency(lp);
    tw_event *e_new = tw_event_new(cur_entry->LOCAL_MSG_NAME_FROM_UNION.sender_lp, local_ts, lp);
Nikhil's avatar
Nikhil committed
990
    void * m_new = tw_event_data(e_new);
991 992 993
    void *local_event = (char*)cur_entry->event_data +
      cur_entry->LOCAL_MSG_NAME_FROM_UNION.remote_event_size_bytes;
    memcpy(m_new, local_event, cur_entry->LOCAL_MSG_NAME_FROM_UNION.local_event_size_bytes);
Nikhil's avatar
Nikhil committed
994 995 996 997
    tw_event_send(e_new);
  }
  s->packet_counter++;
  s->vc_occupancy[0][use_vc] += s->params->chunk_size;
998
  cur_entry = return_head(s->terminal_msgs[0], s->terminal_msgs_tail[0], use_vc);
Nikhil's avatar
Nikhil committed
999 1000 1001
  rc_stack_push(lp, cur_entry, delete_message_list, s->st);
  s->terminal_length -= s->params->chunk_size;

1002
  LOCAL_MSG_STRUCT *m_new;
Nikhil's avatar
Nikhil committed
1003
  ts += tw_rand_unif(lp->rng);
1004
  e = model_net_method_event_new(lp->gid, ts, lp, LOCAL_NETWORK_NAME,
Nikhil's avatar
Nikhil committed
1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030
      (void**)&m_new, NULL);
  m_new->type = T_SEND;
  m_new->magic = terminal_magic_num;
  tw_event_send(e);

  if(s->issueIdle) {
    bf->c5 = 1;
    s->issueIdle = 0;
    ts += tw_rand_unif(lp->rng);
    model_net_method_idle_event(ts, 0, lp);

    if(s->last_buf_full > 0.0)
    {
      bf->c6 = 1;
      msg->saved_total_time = s->busy_time;
      msg->saved_busy_time = s->last_buf_full;
      msg->saved_sample_time = s->busy_time_sample;

      s->busy_time += (tw_now(lp) - s->last_buf_full);
      s->busy_time_sample += (tw_now(lp) - s->last_buf_full);
      s->last_buf_full = 0.0;
    }
  }
  return;
}

1031
static void packet_send_rc(terminal_state * s, tw_bf * bf, LOCAL_MSG_STRUCT * msg,
Nikhil's avatar
Nikhil committed
1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053
    tw_lp * lp)
{
  if(bf->c1) {
    s->in_send_loop = 1;
    s->last_buf_full = msg->saved_busy_time;
    return;
  }

  tw_rand_reverse_unif(lp->rng);
  tw_rand_reverse_unif(lp->rng);
  s->terminal_available_time = msg->saved_available_time;
  if(bf->c2) {
    codes_local_latency_reverse(lp);
  }

  int use_vc = msg->saved_channel;

  s->packet_counter--;
  s->vc_occupancy[0][use_vc] -= s->params->chunk_size;

  message_list* cur_entry = (message_list *)rc_stack_pop(s->st);
  cur_entry->port = 0; cur_entry->index = use_vc;
1054
  prepend_to_message_list(s->terminal_msgs[0], s->terminal_msgs_tail[0],
Nikhil's avatar
Nikhil committed
1055 1056
      use_vc, cur_entry);
  s->terminal_length += s->params->chunk_size;
1057

Nikhil's avatar
Nikhil committed
1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072
  tw_rand_reverse_unif(lp->rng);
  if(bf->c5)
  {
    tw_rand_reverse_unif(lp->rng);
    s->issueIdle = 1;
    if(bf->c6)
    {
      s->busy_time = msg->saved_total_time;
      s->last_buf_full = msg->saved_busy_time;
      s->busy_time_sample = msg->saved_sample_time;
    }
  }
  return;
}

1073
static void send_remote_event(terminal_state * s, LOCAL_MSG_STRUCT * msg,
Nikhil's avatar
Nikhil committed
1074 1075
  tw_lp * lp, tw_bf * bf, char * event_data, int remote_event_size)
{
1076 1077
  void * tmp_ptr = model_net_method_get_edata(LOCAL_NETWORK_NAME, msg);
  //tw_stime ts = g_tw_lookahead + s->params->cn_delay *
Nikhil's avatar
Nikhil committed
1078 1079 1080 1081 1082 1083 1084 1085 1086 1087
  //    msg->remote_event_size_bytes + tw_rand_unif(lp->rng);;
  tw_stime ts = g_tw_lookahead + tw_rand_unif(lp->rng);
  if (msg->is_pull){
    bf->c4 = 1;
    struct codes_mctx mc_dst =
      codes_mctx_set_global_direct(msg->sender_mn_lp);
    struct codes_mctx mc_src =
      codes_mctx_set_global_direct(lp->gid);
    int net_id = model_net_get_id(LP_METHOD_NM_TERM);

1088
    model_net_set_msg_param(MN_MSG_PARAM_START_TIME,
Nikhil's avatar
Nikhil committed
1089 1090 1091 1092 1093 1094 1095 1096 1097
        MN_MSG_PARAM_START_TIME_VAL, &(msg->msg_start_time));

    msg->event_rc = model_net_event_mctx(net_id, &mc_src, &mc_dst, msg->category,
        msg->sender_lp, msg->pull_size, ts,
        remote_event_size, tmp_ptr, 0, NULL, lp);
  } else {
    tw_event * e = tw_event_new(msg->final_dest_gid, ts, lp);
    void * m_remote = tw_event_data(e);
    memcpy(m_remote, event_data, remote_event_size);
1098
    tw_event_send(e);
Nikhil's avatar
Nikhil committed
1099 1100 1101 1102 1103
  }
  return;
}

/* packet arrives at the destination terminal */
1104
static void packet_arrive(terminal_state * s, tw_bf * bf, LOCAL_MSG_STRUCT * msg,
Nikhil's avatar
Nikhil committed
1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118
    tw_lp * lp) {

  assert(lp->gid == msg->dest_terminal_id);

  //total chunks expected in this message
  uint64_t total_chunks = msg->total_size / s->params->chunk_size;
  if(msg->total_size % s->params->chunk_size)
    total_chunks++;
  if(!total_chunks)
    total_chunks = 1;

  /* send credit back to router */
  tw_stime ts = g_tw_lookahead + s->params->credit_delay + tw_rand_unif(lp->rng);
  tw_event * buf_e;
1119
  LOCAL_MSG_STRUCT * buf_msg;
Nikhil's avatar
Nikhil committed
1120
  buf_e = model_net_method_event_new(msg->intm_lp_id, ts, lp,
1121
      LOCAL_NETWORK_ROUTER_NAME, (void**)&buf_msg, NULL);
Nikhil's avatar
Nikhil committed
1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137
  buf_msg->magic = router_magic_num;
  buf_msg->vc_index = msg->vc_index;
  buf_msg->output_chan = msg->output_chan;
  buf_msg->type = R_BUFFER;
  tw_event_send(buf_e);

  //save stats
  /* Total overall finished chunks in simulation */
  N_finished_chunks++;
  /* Finished chunks on a LP basis */
  s->finished_chunks++;
  /* Finished chunks per sample */
  s->fin_chunks_sample++;

  assert(lp->gid != msg->src_terminal_id);

1138
  // chunks part of this packet
Nikhil's avatar
Nikhil committed
1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156
  uint64_t num_chunks = msg->packet_size / s->params->chunk_size;
  if (msg->packet_size % s->params->chunk_size)
    num_chunks++;
  if(!num_chunks)
    num_chunks = 1;

  if(msg->chunk_id == num_chunks - 1)
  {
    bf->c31 = 1;
    s->packet_fin++;
    packet_fin++;
  }

  /* save the sample time */
  msg->saved_sample_time = s->fin_chunks_time;
  s->fin_chunks_time += (tw_now(lp) - msg->travel_start_time);
<