dragonfly-custom.C

/*
 * Copyright (C) 2013 University of Chicago.
 * See COPYRIGHT notice in top-level directory.
 *
 */

#include <ross.h>

#include "codes/jenkins-hash.h"
#include "codes/codes_mapping.h"
#include "codes/codes.h"
#include "codes/model-net.h"
#include "codes/model-net-method.h"
#include "codes/model-net-lp.h"
#include "codes/net/dragonfly-custom.h"
#include "sys/file.h"
#include "codes/quickhash.h"
#include "codes/rc-stack.h"
#include <vector>
#include <map>
#include <set>

#ifdef ENABLE_CORTEX
#include <cortex/cortex.h>
#include <cortex/topology.h>
#endif

#define DUMP_CONNECTIONS 0
#define CREDIT_SIZE 8
#define DFLY_HASH_TABLE_SIZE 4999
// debugging parameters
#define DEBUG_LP 892
#define T_ID 10
#define TRACK -1
#define TRACK_PKT 0
#define TRACK_MSG -1
#define DEBUG 0
#define MAX_STATS 65536
#define SHOW_ADAP_STATS 1

#define LP_CONFIG_NM_TERM (model_net_lp_config_names[DRAGONFLY_CUSTOM])
#define LP_METHOD_NM_TERM (model_net_method_names[DRAGONFLY_CUSTOM])
#define LP_CONFIG_NM_ROUT (model_net_lp_config_names[DRAGONFLY_CUSTOM_ROUTER])
#define LP_METHOD_NM_ROUT (model_net_method_names[DRAGONFLY_CUSTOM_ROUTER])

static int BIAS_MIN = 1;
static int DF_DALLY = 0;
static int adaptive_threshold = 1024;

static long num_local_packets_sr = 0;
static long num_local_packets_sg = 0;
static long num_remote_packets = 0;
using namespace std;
struct Link {
  int offset, type;
};
struct bLink {
  int offset, dest;
};
/* Each entry in the vector is for a router id
 * against each router id, there is a map of links (key of the map is the dest
 * router id)
 * link has information on type (green or black) and offset (number of links
 * between that particular source and dest router ID)*/
vector< map< int, vector<Link> > > intraGroupLinks;
/* contains mapping between source router and destination group via link (link
 * has dest ID)*/
vector< map< int, vector<bLink> > > interGroupLinks;
/*MM: Maintains a list of routers connecting the source and destination groups */
vector< vector< vector<int> > > connectionList;

struct IntraGroupLink {
  int src, dest, type;
};

struct InterGroupLink {
  int src, dest;
};

#ifdef ENABLE_CORTEX
/* This structure is defined at the end of the file */
extern "C" {
extern cortex_topology dragonfly_custom_cortex_topology;
}
#endif

static int debug_slot_count = 0;
static long term_ecount, router_ecount, term_rev_ecount, router_rev_ecount;
static long packet_gen = 0, packet_fin = 0;

static double maxd(double a, double b) { return a < b ? b : a; }

/* minimal and non-minimal packet counts for adaptive routing*/
static int minimal_count=0, nonmin_count=0;
static int num_routers_per_mgrp = 0;

typedef struct dragonfly_param dragonfly_param;
/* annotation-specific parameters (unannotated entry occurs at the 
 * last index) */
static uint64_t                  num_params = 0;
static dragonfly_param         * all_params = NULL;
static const config_anno_map_t * anno_map   = NULL;

/* global variables for codes mapping */
static char lp_group_name[MAX_NAME_LENGTH];
static int mapping_grp_id, mapping_type_id, mapping_rep_id, mapping_offset;

/* router magic number */
static int router_magic_num = 0;

/* terminal magic number */
static int terminal_magic_num = 0;

/* Hops within a group */
static int num_intra_nonmin_hops = 4;
static int num_intra_min_hops = 2;

static FILE * dragonfly_log = NULL;

static int sample_bytes_written = 0;
static int sample_rtr_bytes_written = 0;

static char cn_sample_file[MAX_NAME_LENGTH];
static char router_sample_file[MAX_NAME_LENGTH];

//don't do overhead here - job of MPI layer
static tw_stime mpi_soft_overhead = 0;

typedef struct terminal_custom_message_list terminal_custom_message_list;
struct terminal_custom_message_list {
    terminal_custom_message msg;
    char* event_data;
    terminal_custom_message_list *next;
    terminal_custom_message_list *prev;
};

static void init_terminal_custom_message_list(terminal_custom_message_list *thisO, 
    terminal_custom_message *inmsg) {
    thisO->msg = *inmsg;
    thisO->event_data = NULL;
    thisO->next = NULL;
    thisO->prev = NULL;
}

static void delete_terminal_custom_message_list(void *thisO) {
    terminal_custom_message_list* toDel = (terminal_custom_message_list*)thisO;
    if(toDel->event_data != NULL) free(toDel->event_data);
    free(toDel);
}

struct dragonfly_param
{
    // configuration parameters
    int num_routers; /*Number of routers in a group*/
    double local_bandwidth;/* bandwidth of the router-router channels within a group */
    double global_bandwidth;/* bandwidth of the inter-group router connections */
    double cn_bandwidth;/* bandwidth of the compute node channels connected to routers */
    int num_vcs; /* number of virtual channels */
    int local_vc_size; /* buffer size of the router-router channels */
    int global_vc_size; /* buffer size of the global channels */
    int cn_vc_size; /* buffer size of the compute node channels */
    int chunk_size; /* full-sized packets are broken into smaller chunks.*/
    // derived parameters
    int num_cn;
    int intra_grp_radix;
    int num_col_chans;
    int num_row_chans;
    int num_router_rows;
    int num_router_cols;
    int num_groups;
    int radix;
    int total_routers;
    int total_terminals;
    int num_global_channels;
    double cn_delay;
    double local_delay;
    double global_delay;
    double credit_delay;
    double router_delay;
};

struct dfly_hash_key
{
    uint64_t message_id;
    tw_lpid sender_id;
};

struct dfly_router_sample
{
    tw_lpid router_id;
    tw_stime* busy_time;
    int64_t* link_traffic_sample;
    tw_stime end_time;
    long fwd_events;
    long rev_events;
};

struct dfly_cn_sample
{
   tw_lpid terminal_id;
   long fin_chunks_sample;
   long data_size_sample;
   double fin_hops_sample;
   tw_stime fin_chunks_time;
   tw_stime busy_time_sample;
   tw_stime end_time;
   long fwd_events;
   long rev_events;
};

struct dfly_qhash_entry
{
   struct dfly_hash_key key;
   char * remote_event_data;
   int num_chunks;
   int remote_event_size;
   struct qhash_head hash_link;
};

/* handles terminal and router events like packet generate/send/receive/buffer */
typedef struct terminal_state terminal_state;
typedef struct router_state router_state;

/* dragonfly compute node data structure */
struct terminal_state
{
   uint64_t packet_counter;

   int packet_gen;
   int packet_fin;

   // Dragonfly specific parameters
   unsigned int router_id;
   unsigned int terminal_id;

   // Each terminal will have an input and output channel with the router
   int* vc_occupancy; // NUM_VC
   int num_vcs;
   tw_stime terminal_available_time;
   terminal_custom_message_list **terminal_msgs;
   terminal_custom_message_list **terminal_msgs_tail;
   int in_send_loop;
   struct mn_stats dragonfly_stats_array[CATEGORY_MAX];

   struct rc_stack * st;
   int issueIdle;
   int terminal_length;

   const char * anno;
   const dragonfly_param *params;

   struct qhash_table *rank_tbl;
   uint64_t rank_tbl_pop;

   tw_stime   total_time;
   uint64_t total_msg_size;
   double total_hops;
   long finished_msgs;
   long finished_chunks;
   long finished_packets;

   tw_stime * last_buf_full;
   tw_stime busy_time;
   
   tw_stime max_latency;
   tw_stime min_latency;

   char output_buf[4096];
   /* For LP suspend functionality */
   int error_ct;

   /* For sampling */
   long fin_chunks_sample;
   long data_size_sample;
   double fin_hops_sample;
   tw_stime fin_chunks_time;
   tw_stime busy_time_sample;

   char sample_buf[4096];
   struct dfly_cn_sample * sample_stat;
   int op_arr_size;
   int max_arr_size;
  
   /* for logging forward and reverse events */
   long fwd_events;
   long rev_events;
};

/* terminal event type (1-4) */
typedef enum event_t
{
  T_GENERATE=1,
  T_ARRIVE,
  T_SEND,
  T_BUFFER,
  R_SEND,
  R_ARRIVE,
  R_BUFFER,
} event_t;

/* whether the last hop of a packet was global, local or a terminal */
enum last_hop
{
   GLOBAL=1,
   LOCAL,
   TERMINAL,
   ROOT
};

/* three forms of routing algorithms available, adaptive routing is not
 * accurate and fully functional in the current version as the formulas
 * for detecting load on global channels are not very accurate */
enum ROUTING_ALGO
{
    MINIMAL = 1,
    NON_MINIMAL,
    ADAPTIVE,
    PROG_ADAPTIVE
};

enum LINK_TYPE
{
    GREEN,
    BLACK,
};
struct router_state
{
   unsigned int router_id;
   int group_id;
   int op_arr_size;
   int max_arr_size;

   int* global_channel; 
   
   tw_stime* next_output_available_time;
   tw_stime* cur_hist_start_time;
   tw_stime** last_buf_full;

   tw_stime* busy_time;
   tw_stime* busy_time_sample;

   terminal_custom_message_list ***pending_msgs;
   terminal_custom_message_list ***pending_msgs_tail;
   terminal_custom_message_list ***queued_msgs;
   terminal_custom_message_list ***queued_msgs_tail;
   int *in_send_loop;
   int *queued_count;
   struct rc_stack * st;

   int* last_sent_chan;
   int** vc_occupancy;
   int64_t* link_traffic;
   int64_t * link_traffic_sample;

   const char * anno;
   const dragonfly_param *params;

   int* prev_hist_num;
   int* cur_hist_num;
   
   char output_buf[4096];
   char output_buf2[4096];

   struct dfly_router_sample * rsamples;
   
   long fwd_events;
   long rev_events;
};

static short routing = MINIMAL;

static tw_stime         dragonfly_total_time = 0;
static tw_stime         dragonfly_max_latency = 0;


static long long       total_hops = 0;
static long long       N_finished_packets = 0;
static long long       total_msg_sz = 0;
static long long       N_finished_msgs = 0;
static long long       N_finished_chunks = 0;

static int dragonfly_rank_hash_compare(
        void *key, struct qhash_head *link)
{
    struct dfly_hash_key *message_key = (struct dfly_hash_key *)key;
    struct dfly_qhash_entry *tmp = NULL;

    tmp = qhash_entry(link, struct dfly_qhash_entry, hash_link);
    
    if (tmp->key.message_id == message_key->message_id
            && tmp->key.sender_id == message_key->sender_id)
        return 1;

    return 0;
}
static int dragonfly_hash_func(void *k, int table_size)
{
    struct dfly_hash_key *tmp = (struct dfly_hash_key *)k;
    uint32_t pc = 0, pb = 0;	
    bj_hashlittle2(tmp, sizeof(*tmp), &pc, &pb);
    return (int)(pc % (table_size - 1));
    /*uint64_t key = (~tmp->message_id) + (tmp->message_id << 18);
    key = key * 21;
    key = ~key ^ (tmp->sender_id >> 4);
    key = key * tmp->sender_id; 
    return (int)(key & (table_size - 1));*/
}

/* convert GiB/s and bytes to ns */
static tw_stime bytes_to_ns(uint64_t bytes, double GB_p_s)
{
    tw_stime time;

    /* bytes to GB */
    time = ((double)bytes)/(1024.0*1024.0*1024.0);
    /* GiB to s */
    time = time / GB_p_s;
    /* s to ns */
    time = time * 1000.0 * 1000.0 * 1000.0;

    return(time);
}

/* returns the dragonfly message size */
int dragonfly_custom_get_msg_sz(void)
{
	   return sizeof(terminal_custom_message);
}

static void free_tmp(void * ptr)
{
    struct dfly_qhash_entry * dfly = (dfly_qhash_entry *)ptr; 
    if(dfly->remote_event_data)
        free(dfly->remote_event_data);
   
    if(dfly)
        free(dfly);
}

static void append_to_terminal_custom_message_list(  
        terminal_custom_message_list ** thisq,
        terminal_custom_message_list ** thistail,
        int index, 
        terminal_custom_message_list *msg) {
    if(thisq[index] == NULL) {
        thisq[index] = msg;
    } else {
        thistail[index]->next = msg;
        msg->prev = thistail[index];
    } 
    thistail[index] = msg;
}

static void prepend_to_terminal_custom_message_list(  
        terminal_custom_message_list ** thisq,
        terminal_custom_message_list ** thistail,
        int index, 
        terminal_custom_message_list *msg) {
    if(thisq[index] == NULL) {
        thistail[index] = msg;
    } else {
        thisq[index]->prev = msg;
        msg->next = thisq[index];
    } 
    thisq[index] = msg;
}

static terminal_custom_message_list* return_head(
        terminal_custom_message_list ** thisq,
        terminal_custom_message_list ** thistail,
        int index) {
    terminal_custom_message_list *head = thisq[index];
    if(head != NULL) {
        thisq[index] = head->next;
        if(head->next != NULL) {
            head->next->prev = NULL;
            head->next = NULL;
        } else {
            thistail[index] = NULL;
        }
    }
    return head;
}

static terminal_custom_message_list* return_tail(
        terminal_custom_message_list ** thisq,
        terminal_custom_message_list ** thistail,
        int index) {
    terminal_custom_message_list *tail = thistail[index];
    assert(tail);
    if(tail->prev != NULL) {
        tail->prev->next = NULL;
        thistail[index] = tail->prev;
        tail->prev = NULL;
    } else {
        thistail[index] = NULL;
        thisq[index] = NULL;
    }
    return tail;
}

static void dragonfly_read_config(const char * anno, dragonfly_param *params){
    /*Adding init for router magic number*/
    uint32_t h1 = 0, h2 = 0; 
    bj_hashlittle2(LP_METHOD_NM_ROUT, strlen(LP_METHOD_NM_ROUT), &h1, &h2);
    router_magic_num = h1 + h2;
    
    bj_hashlittle2(LP_METHOD_NM_TERM, strlen(LP_METHOD_NM_TERM), &h1, &h2);
    terminal_magic_num = h1 + h2;
    
    // shorthand
    dragonfly_param *p = params;
    int myRank;
    MPI_Comm_rank(MPI_COMM_CODES, &myRank);

    int rc = configuration_get_value_int(&config, "PARAMS", "local_vc_size", anno, &p->local_vc_size);
    if(rc) {
        p->local_vc_size = 1024;
        fprintf(stderr, "Buffer size of local channels not specified, setting to %d\n", p->local_vc_size);
    }

    rc = configuration_get_value_int(&config, "PARAMS", "adaptive_threshold", anno, &adaptive_threshold);
    if(rc) {
    	adaptive_threshold = p->local_vc_size / 8;
        printf("\n Setting adaptive threshold to %d ", adaptive_threshold);
	}
    else
    {
        printf("\n Setting adaptive threshold to %d ", adaptive_threshold);
    }

    rc = configuration_get_value_int(&config, "PARAMS", "global_vc_size", anno, &p->global_vc_size);
    if(rc) {
        p->global_vc_size = 2048;
        fprintf(stderr, "Buffer size of global channels not specified, setting to %d\n", p->global_vc_size);
    }

    rc = configuration_get_value_int(&config, "PARAMS", "df-dally-vc", anno, &DF_DALLY);
    if(rc) {
        DF_DALLY = 0;
    }
    
    rc = configuration_get_value_int(&config, "PARAMS", "minimal-bias", anno, &BIAS_MIN);
    if(rc) {
        BIAS_MIN = 0;
    }
    else
	printf("\n Setting minimal bias");

    rc = configuration_get_value_int(&config, "PARAMS", "cn_vc_size", anno, &p->cn_vc_size);
    if(rc) {
        p->cn_vc_size = 1024;
        fprintf(stderr, "Buffer size of compute node channels not specified, setting to %d\n", p->cn_vc_size);
    }

    rc = configuration_get_value_int(&config, "PARAMS", "chunk_size", anno, &p->chunk_size);
    if(rc) {
        p->chunk_size = 512;
        fprintf(stderr, "Chunk size for packets is specified, setting to %d\n", p->chunk_size);
    }

    rc = configuration_get_value_double(&config, "PARAMS", "local_bandwidth", anno, &p->local_bandwidth);
    if(rc) {
        p->local_bandwidth = 5.25;
        fprintf(stderr, "Bandwidth of local channels not specified, setting to %lf\n", p->local_bandwidth);
    }

    rc = configuration_get_value_double(&config, "PARAMS", "global_bandwidth", anno, &p->global_bandwidth);
    if(rc) {
        p->global_bandwidth = 4.7;
        fprintf(stderr, "Bandwidth of global channels not specified, setting to %lf\n", p->global_bandwidth);
    }

    rc = configuration_get_value_double(&config, "PARAMS", "cn_bandwidth", anno, &p->cn_bandwidth);
    if(rc) {
        p->cn_bandwidth = 5.25;
        fprintf(stderr, "Bandwidth of compute node channels not specified, setting to %lf\n", p->cn_bandwidth);
    }

    rc = configuration_get_value_double(&config, "PARAMS", "router_delay", anno,
            &p->router_delay);
    if(rc) {
      p->router_delay = 100;
    }

    configuration_get_value(&config, "PARAMS", "cn_sample_file", anno, cn_sample_file,
            MAX_NAME_LENGTH);
    configuration_get_value(&config, "PARAMS", "rt_sample_file", anno, router_sample_file,
            MAX_NAME_LENGTH);
    
    char routing_str[MAX_NAME_LENGTH];
    configuration_get_value(&config, "PARAMS", "routing", anno, routing_str,
            MAX_NAME_LENGTH);
    if(strcmp(routing_str, "minimal") == 0)
        routing = MINIMAL;
    else if(strcmp(routing_str, "nonminimal")==0 || 
            strcmp(routing_str,"non-minimal")==0)
        routing = NON_MINIMAL;
    else if (strcmp(routing_str, "adaptive") == 0)
        routing = ADAPTIVE;
    else if (strcmp(routing_str, "prog-adaptive") == 0)
	      routing = PROG_ADAPTIVE;
    else
    {
        fprintf(stderr, 
                "No routing protocol specified, setting to minimal routing\n");
        routing = -1;
    }

    // rc = configuration_get_value_int(&config, "PARAMS", "num_vcs_override", anno, &p->num_vcs);
    // if(rc) {
    //     if(routing == PROG_ADAPTIVE)
    //         p->num_vcs = 10;
    //     else
    //         p->num_vcs = 8;
    // }
    // else {
    //     printf("Overriding num_vcs: p->num_vcs=%d\n",p->num_vcs);
    // }
   
if(DF_DALLY == 0) 
{
    if(routing == PROG_ADAPTIVE)
        p->num_vcs = 10;
    else
        p->num_vcs = 8;
}
else
{
    if(routing == PROG_ADAPTIVE)
        p->num_vcs = 5;
    else
        p->num_vcs = 4;
}
    rc = configuration_get_value_int(&config, "PARAMS", "num_groups", anno, &p->num_groups);
    if(rc) {
      printf("Number of groups not specified. Aborting");
      MPI_Abort(MPI_COMM_CODES, 1);
    }
    rc = configuration_get_value_int(&config, "PARAMS", "num_col_chans", anno, &p->num_col_chans);
    if(rc) {
//        printf("\n Number of links connecting chassis not specified, setting to default value 3 ");
        p->num_col_chans = 3;
    }
    rc = configuration_get_value_int(&config, "PARAMS", "num_row_chans", anno, &p->num_row_chans);
    if(rc) {
//        printf("\n Number of links connecting chassis not specified, setting to default value 3 ");
        p->num_row_chans = 1;
    }
    rc = configuration_get_value_int(&config, "PARAMS", "num_router_rows", anno, &p->num_router_rows);
    if(rc) {
        printf("\n Number of router rows not specified, setting to 6 ");
        p->num_router_rows = 6;
    }
    rc = configuration_get_value_int(&config, "PARAMS", "num_router_cols", anno, &p->num_router_cols);
    if(rc) {
        printf("\n Number of router columns not specified, setting to 16 ");
        p->num_router_cols = 16;
    }
    p->intra_grp_radix = (p->num_router_cols * p->num_row_chans);
    if(p->num_router_rows > 1)
        p->intra_grp_radix += (p->num_router_rows * p->num_col_chans);

    p->num_routers = p->num_router_rows * p->num_router_cols;
    
    rc = configuration_get_value_int(&config, "PARAMS", "num_cns_per_router", anno, &p->num_cn);
    if(rc) {
        printf("\n Number of cns per router not specified, setting to %d ", p->num_routers/2);
        p->num_cn = p->num_routers/2;
    }

    rc = configuration_get_value_int(&config, "PARAMS", "num_global_channels", anno, &p->num_global_channels);
    if(rc) {
        printf("\n Number of global channels per router not specified, setting to 10 ");
        p->num_global_channels = 10;
    }
    p->radix = p->intra_grp_radix + p->num_global_channels + p->num_cn;
    p->total_routers = p->num_groups * p->num_routers;
    p->total_terminals = p->total_routers * p->num_cn;
    
    // read intra group connections, store from a router's perspective
    // all links to the same router form a vector
    char intraFile[MAX_NAME_LENGTH];
    configuration_get_value(&config, "PARAMS", "intra-group-connections", 
        anno, intraFile, MAX_NAME_LENGTH);
    if(strlen(intraFile) <= 0) {
      tw_error(TW_LOC, "Intra group connections file not specified. Aborting");
    }
    FILE *groupFile = fopen(intraFile, "rb");
    if(!groupFile)
        tw_error(TW_LOC, "intra-group file not found ");

    if(!myRank)
      printf("Reading intra-group connectivity file: %s\n", intraFile);

    {
      vector< int > offsets;
      offsets.resize(p->num_routers, 0);
      intraGroupLinks.resize(p->num_routers);
      IntraGroupLink newLink;

      while(fread(&newLink, sizeof(IntraGroupLink), 1, groupFile) != 0) {
        Link tmpLink;
        tmpLink.type = newLink.type;
        tmpLink.offset = offsets[newLink.src]++;
        intraGroupLinks[newLink.src][newLink.dest].push_back(tmpLink);
      }
    }

    fclose(groupFile);

    // read inter group connections, store from a router's perspective
    // also create a group level table that tells all the connecting routers
    char interFile[MAX_NAME_LENGTH];
    configuration_get_value(&config, "PARAMS", "inter-group-connections", 
        anno, interFile, MAX_NAME_LENGTH);
    if(strlen(interFile) <= 0) {
      tw_error(TW_LOC, "Inter group connections file not specified. Aborting");
    }
    FILE *systemFile = fopen(interFile, "rb");
    if(!myRank)
    {
      printf("Reading inter-group connectivity file: %s\n", interFile);
      printf("\n Total routers %d total groups %d ", p->total_routers, p->num_groups);
    }

    {
      vector< int > offsets;
      offsets.resize(p->total_routers, 0);
      interGroupLinks.resize(p->total_routers);
      connectionList.resize(p->num_groups);
      for(int g = 0; g < connectionList.size(); g++) {
        connectionList[g].resize(p->num_groups);
      }
      
      InterGroupLink newLink;

      while(fread(&newLink, sizeof(InterGroupLink), 1, systemFile) != 0) {
        bLink tmpLink;
        tmpLink.dest = newLink.dest;
        int srcG = newLink.src / p->num_routers;
        int destG = newLink.dest / p->num_routers;
        tmpLink.offset = offsets[newLink.src]++;
        interGroupLinks[newLink.src][destG].push_back(tmpLink);
        int r;
        for(r = 0; r < connectionList[srcG][destG].size(); r++) {
          if(connectionList[srcG][destG][r] == newLink.src) break;
        }
        if(r == connectionList[srcG][destG].size()) {
          connectionList[srcG][destG].push_back(newLink.src);
        }
      }
    }

    fclose(systemFile);

#if DUMP_CONNECTIONS == 1
    printf("Dumping intra-group connections\n");
    for(int a = 0; a < intraGroupLinks.size(); a++) {
      printf("Connections for router %d\n", a);
      map< int, vector<Link> >  &curMap = intraGroupLinks[a];
      map< int, vector<Link> >::iterator it = curMap.begin();
      for(; it != curMap.end(); it++) {
        printf(" ( %d - ", it->first);
        for(int l = 0; l < it->second.size(); l++) {
          // offset is number of local connections
          // type is black or green according to Cray architecture 
          printf("%d,%d ", it->second[l].offset, it->second[l].type);
        }
        printf(")");
      }
      printf("\n");
    }
#endif
#if DUMP_CONNECTIONS == 1
    printf("Dumping inter-group connections\n");
    for(int a = 0; a < interGroupLinks.size(); a++) {
      printf("Connections for router %d\n", a);
      map< int, vector<bLink> >  &curMap = interGroupLinks[a];
      map< int, vector<bLink> >::iterator it = curMap.begin();
      for(; it != curMap.end(); it++) {
        // dest group ID 
        printf(" ( %d - ", it->first);
        for(int l = 0; l < it->second.size(); l++) {
            // dest is dest router ID
            // offset is number of global connections
          printf("%d,%d ", it->second[l].offset, it->second[l].dest);
        }
        printf(")");
      }
      printf("\n");
    }
#endif

#if DUMP_CONNECTIONS == 1
    printf("Dumping source aries for global connections\n");
    for(int g = 0; g < p->num_groups; g++) {
      for(int g1 = 0; g1 < p->num_groups; g1++) {
        printf(" ( ");
        for(int l = 0; l < connectionList[g][g1].size(); l++) {
          printf("%d ", connectionList[g][g1][l]);
        }
        printf(")");
      }
      printf("\n");
    }
#endif
    if(!myRank) {
        printf("\n Total nodes %d routers %d groups %d routers per group %d radix %d\n",
                p->num_cn * p->total_routers, p->total_routers, p->num_groups,
                p->num_routers, p->radix);
    }

    p->cn_delay = bytes_to_ns(p->chunk_size, p->cn_bandwidth);
    p->local_delay = bytes_to_ns(p->chunk_size, p->local_bandwidth);
    p->global_delay = bytes_to_ns(p->chunk_size, p->global_bandwidth);
    p->credit_delay = bytes_to_ns(CREDIT_SIZE, p->local_bandwidth); //assume 8 bytes packet
}

void dragonfly_custom_configure(){
    anno_map = codes_mapping_get_lp_anno_map(LP_CONFIG_NM_TERM);
    assert(anno_map);
    num_params = anno_map->num_annos + (anno_map->has_unanno_lp > 0);
    all_params = (dragonfly_param *)malloc(num_params * sizeof(*all_params));

    for (int i = 0; i < anno_map->num_annos; i++){
        const char * anno = anno_map->annotations[i].ptr;
        dragonfly_read_config(anno, &all_params[i]);
    }
    if (anno_map->has_unanno_lp > 0){
        dragonfly_read_config(NULL, &all_params[anno_map->num_annos]);
    }
#ifdef ENABLE_CORTEX
	model_net_topology = dragonfly_custom_cortex_topology;
#endif
}

/* report dragonfly statistics like average and maximum packet latency, average number of hops traversed */
void dragonfly_custom_report_stats()
{
   long long avg_hops, total_finished_packets, total_finished_chunks;
   long long total_finished_msgs, final_msg_sz;
   tw_stime avg_time, max_time;
   int total_minimal_packets, total_nonmin_packets;
   long total_gen, total_fin;
   long total_local_packets_sr, total_local_packets_sg, total_remote_packets;

   MPI_Reduce( &total_hops, &avg_hops, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_CODES);
   MPI_Reduce( &N_finished_packets, &total_finished_packets, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_CODES);
   MPI_Reduce( &N_finished_msgs, &total_finished_msgs, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_CODES);
   MPI_Reduce( &N_finished_chunks, &total_finished_chunks, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_CODES);
   MPI_Reduce( &total_msg_sz, &final_msg_sz, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_CODES);
   MPI_Reduce( &dragonfly_total_time, &avg_time, 1,MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_CODES);
   MPI_Reduce( &dragonfly_max_latency, &max_time, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_CODES);
   
   MPI_Reduce( &packet_gen, &total_gen, 1, MPI_LONG, MPI_SUM, 0, MPI_COMM_CODES);
   MPI_Reduce(&packet_fin, &total_fin, 1, MPI_LONG, MPI_SUM, 0, MPI_COMM_CODES);
    MPI_Reduce( &num_local_packets_sr, &total_local_packets_sr, 1, MPI_LONG, MPI_SUM, 0, MPI_COMM_CODES);
    MPI_Reduce( &num_local_packets_sg, &total_local_packets_sg, 1, MPI_LONG, MPI_SUM, 0, MPI_COMM_CODES);
   MPI_Reduce( &num_remote_packets, &total_remote_packets, 1, MPI_LONG, MPI_SUM, 0, MPI_COMM_CODES);
   if(routing == ADAPTIVE || routing == PROG_ADAPTIVE || SHOW_ADAP_STATS)
    {
	MPI_Reduce(&minimal_count, &total_minimal_packets, 1, MPI_INT, MPI_SUM, 0, MPI_COMM_CODES);
 	MPI_Reduce(&nonmin_count, &total_nonmin_packets, 1, MPI_INT, MPI_SUM, 0, MPI_COMM_CODES);
    }

   /* print statistics */
   if(!g_tw_mynode)
   {	
      printf(" Average number of hops traversed %f average chunk latency %lf us maximum chunk latency %lf us avg message size %lf bytes finished messages %lld finished chunks %lld \n", 
              (float)avg_hops/total_finished_chunks, avg_time/(total_finished_chunks*1000), max_time/1000, (float)final_msg_sz/total_finished_msgs, total_finished_msgs, total_finished_chunks);
     if(routing == ADAPTIVE || routing == PROG_ADAPTIVE || SHOW_ADAP_STATS)
              printf("\n ADAPTIVE ROUTING STATS: %d chunks routed minimally %d chunks routed non-minimally completed packets %lld \n", 
                      total_minimal_packets, total_nonmin_packets, total_finished_chunks);
 
      printf("\n Total packets generated %ld finished %ld Locally routed- same router %ld different-router %ld Remote (inter-group) %ld \n", total_gen, total_fin, total_local_packets_sr, total_local_packets_sg, total_remote_packets);
   }
   return;
}


/* initialize a dragonfly compute node terminal */
void 
terminal_custom_init( terminal_state * s, 
	       tw_lp * lp )
{
    s->packet_gen = 0;
    s->packet_fin = 0;

    int i;
    char anno[MAX_NAME_LENGTH];

    // Assign the global router ID
    // TODO: be annotation-aware
    codes_mapping_get_lp_info(lp->gid, lp_group_name, &mapping_grp_id, NULL,
            &mapping_type_id, anno, &mapping_rep_id, &mapping_offset);
    if (anno[0] == '\0'){
        s->anno = NULL;
        s->params = &all_params[num_params-1];
    }
    else{
        s->anno = strdup(anno);
        int id = configuration_get_annotation_index(anno, anno_map);
        s->params = &all_params[id];
    }

   int num_lps = codes_mapping_get_lp_count(lp_group_name, 1, LP_CONFIG_NM_TERM,
           s->anno, 0);

   s->terminal_id = codes_mapping_get_lp_relative_id(lp->gid, 0, 0);
   s->router_id=(int)s->terminal_id / (s->params->num_cn);
   s->terminal_available_time = 0.0;
   s->packet_counter = 0;
   s->min_latency = INT_MAX;
   s->max_latency = 0;   

   s->finished_msgs = 0;
   s->finished_chunks = 0;
   s->finished_packets = 0;
   s->total_time = 0.0;
   s->total_msg_size = 0;

   s->busy_time = 0.0;

   s->fwd_events = 0;
   s->rev_events = 0;

   rc_stack_create(&s->st);
   s->num_vcs = 1;
   s->vc_occupancy = (int*)malloc(s->num_vcs * sizeof(int));
   s->last_buf_full = (tw_stime*)malloc(s->num_vcs * sizeof(tw_stime));

   for( i = 0; i < s->num_vcs; i++ )
    {
      s->last_buf_full[i] = 0.0;
      s->vc_occupancy[i]=0;
    }


   s->rank_tbl = NULL;
   s->terminal_msgs = 
       (terminal_custom_message_list**)malloc(s->num_vcs*sizeof(terminal_custom_message_list*));
   s->terminal_msgs_tail = 
       (terminal_custom_message_list**)malloc(s->num_vcs*sizeof(terminal_custom_message_list*));
   s->terminal_msgs[0] = NULL;
   s->terminal_msgs_tail[0] = NULL;
   s->terminal_length = 0;
   s->in_send_loop = 0;
   s->issueIdle = 0;

   return;
}

/* sets up the router virtual channels, global channels, 
 * local channels, compute node channels */
void router_custom_setup(router_state * r, tw_lp * lp)
{
    
    char anno[MAX_NAME_LENGTH];
    codes_mapping_get_lp_info(lp->gid, lp_group_name, &mapping_grp_id, NULL,
            &mapping_type_id, anno, &mapping_rep_id, &mapping_offset);

    if (anno[0] == '\0'){
        r->anno = NULL;
        r->params = &all_params[num_params-1];
    } else{
        r->anno = strdup(anno);
        int id = configuration_get_annotation_index(anno, anno_map);
        r->params = &all_params[id];
    }

    // shorthand
    const dragonfly_param *p = r->params;

    num_routers_per_mgrp = codes_mapping_get_lp_count (lp_group_name, 1, "modelnet_dragonfly_custom_router",
            NULL, 0);
    int num_grp_reps = codes_mapping_get_group_reps(lp_group_name);
    if(p->total_routers != num_grp_reps * num_routers_per_mgrp)
        tw_error(TW_LOC, "\n Config error: num_routers specified %d total routers computed in the network %d "
                "does not match with repetitions * dragonfly_router %d  ",
                p->num_routers, p->total_routers, num_grp_reps * num_routers_per_mgrp);

   r->router_id = codes_mapping_get_lp_relative_id(lp->gid, 0, 0);
   r->group_id=r->router_id/p->num_routers;
    
   //printf("\n Local router id %d global id %d ", r->router_id, lp->gid);

   r->fwd_events = 0;
   r->rev_events = 0;


   r->global_channel = (int*)malloc(p->num_global_channels * sizeof(int));
   r->next_output_available_time = (tw_stime*)malloc(p->radix * sizeof(tw_stime));
   r->cur_hist_start_time = (tw_stime*)malloc(p->radix * sizeof(tw_stime));
   r->link_traffic = (int64_t*)malloc(p->radix * sizeof(int64_t));
   r->link_traffic_sample = (int64_t*)malloc(p->radix * sizeof(int64_t));
   r->cur_hist_num = (int*)malloc(p->radix * sizeof(int));
   r->prev_hist_num = (int*)malloc(p->radix * sizeof(int));
  
   r->last_sent_chan = (int*) malloc(p->num_router_rows * sizeof(int));
   r->vc_occupancy = (int**)malloc(p->radix * sizeof(int*));
   r->in_send_loop = (int*)malloc(p->radix * sizeof(int));
   r->pending_msgs = 
    (terminal_custom_message_list***)malloc(p->radix * sizeof(terminal_custom_message_list**));
   r->pending_msgs_tail = 
    (terminal_custom_message_list***)malloc(p->radix * sizeof(terminal_custom_message_list**));
   r->queued_msgs = 
    (terminal_custom_message_list***)malloc(p->radix * sizeof(terminal_custom_message_list**));
   r->queued_msgs_tail = 
    (terminal_custom_message_list***)malloc(p->radix * sizeof(terminal_custom_message_list**));
   r->queued_count = (int*)malloc(p->radix * sizeof(int));
   r->last_buf_full = (tw_stime**)malloc(p->radix * sizeof(tw_stime*));
   r->busy_time = (tw_stime*)malloc(p->radix * sizeof(tw_stime));
   r->busy_time_sample = (tw_stime*)malloc(p->radix * sizeof(tw_stime));

   rc_stack_create(&r->st);

   for(int i = 0; i < p->num_router_rows; i++)
       r->last_sent_chan[i] = 0;

   for(int i=0; i < p->radix; i++)
    {
       // Set credit & router occupancy
    r->busy_time[i] = 0.0;
    r->busy_time_sample[i] = 0.0;
	r->next_output_available_time[i]=0;
	r->cur_hist_start_time[i] = 0;
    r->link_traffic[i]=0;
    r->link_traffic_sample[i] = 0;
	r->cur_hist_num[i] = 0;
	r->prev_hist_num[i] = 0;
    r->queued_count[i] = 0;    
    r->in_send_loop[i] = 0;
    r->vc_occupancy[i] = (int*)malloc(p->num_vcs * sizeof(int));
    r->pending_msgs[i] = (terminal_custom_message_list**)malloc(p->num_vcs * 
        sizeof(terminal_custom_message_list*));
    r->last_buf_full[i] = (tw_stime*)malloc(p->num_vcs * sizeof(tw_stime));
    r->pending_msgs_tail[i] = (terminal_custom_message_list**)malloc(p->num_vcs * 
        sizeof(terminal_custom_message_list*));
    r->queued_msgs[i] = (terminal_custom_message_list**)malloc(p->num_vcs * 
        sizeof(terminal_custom_message_list*));
    r->queued_msgs_tail[i] = (terminal_custom_message_list**)malloc(p->num_vcs * 
        sizeof(terminal_custom_message_list*));
        for(int j = 0; j < p->num_vcs; j++) {
            r->last_buf_full[i][j] = 0.0;
            r->vc_occupancy[i][j] = 0;
            r->pending_msgs[i][j] = NULL;
            r->pending_msgs_tail[i][j] = NULL;
            r->queued_msgs[i][j] = NULL;
            r->queued_msgs_tail[i][j] = NULL;
        }
    }
   return;
}	


/* dragonfly packet event , generates a dragonfly packet on the compute node */
static tw_stime dragonfly_custom_packet_event(
        model_net_request const * req,
        uint64_t message_offset,
        uint64_t packet_size,
        tw_stime offset,
        mn_sched_params const * sched_params,
        void const * remote_event,
        void const * self_event,
        tw_lp *sender,
        int is_last_pckt)
{
    (void)message_offset;
    (void)sched_params;
    tw_event * e_new;
    tw_stime xfer_to_nic_time;
    terminal_custom_message * msg;
    char* tmp_ptr;

    xfer_to_nic_time = codes_local_latency(sender); 
    //e_new = tw_event_new(sender->gid, xfer_to_nic_time+offset, sender);
    //msg = tw_event_data(e_new);
    e_new = model_net_method_event_new(sender->gid, xfer_to_nic_time+offset,
            sender, DRAGONFLY_CUSTOM, (void**)&msg, (void**)&tmp_ptr);
    strcpy(msg->category, req->category);
    msg->final_dest_gid = req->final_dest_lp;
    msg->total_size = req->msg_size;
    msg->sender_lp=req->src_lp;
    msg->sender_mn_lp = sender->gid;
    msg->packet_size = packet_size;
    msg->travel_start_time = tw_now(sender);
    msg->remote_event_size_bytes = 0;
    msg->local_event_size_bytes = 0;
    msg->type = T_GENERATE;
    msg->dest_terminal_id = req->dest_mn_lp;
    msg->message_id = req->msg_id;
    msg->is_pull = req->is_pull;
    msg->pull_size = req->pull_size;
    msg->magic = terminal_magic_num; 
    msg->msg_start_time = req->msg_start_time;

    if(is_last_pckt) /* Its the last packet so pass in remote and local event information*/
      {
        if(req->remote_event_size > 0)
         {
            msg->remote_event_size_bytes = req->remote_event_size;
            memcpy(tmp_ptr, remote_event, req->remote_event_size);
            tmp_ptr += req->remote_event_size;
        }
        if(req->self_event_size > 0)
        {
            msg->local_event_size_bytes = req->self_event_size;
            memcpy(tmp_ptr, self_event, req->self_event_size);
            tmp_ptr += req->self_event_size;
        }
     }
	   //printf("\n dragonfly remote event %d local event %d last packet %d %lf ", msg->remote_event_size_bytes, msg->local_event_size_bytes, is_last_pckt, xfer_to_nic_time);
    tw_event_send(e_new);
    return xfer_to_nic_time;
}

/* dragonfly packet event reverse handler */
static void dragonfly_custom_packet_event_rc(tw_lp *sender)
{
	  codes_local_latency_reverse(sender);
	    return;
}

/*When a packet is sent from the current router and a buffer slot becomes available, a credit is sent back to schedule another packet event*/
static void router_credit_send(router_state * s, terminal_custom_message * msg