fattree.c

#include <ross.h>

#include "codes/jenkins-hash.h"
#include "codes/codes_mapping.h"
#include "codes/codes.h"
#include "codes/model-net.h"
#include "codes/model-net-method.h"
#include "codes/model-net-lp.h"
#include "codes/net/fattree.h"
#include "sys/file.h"
#include "codes/quickhash.h"
#include "codes/rc-stack.h"
#include <ctype.h>
#include <search.h>

#ifdef ENABLE_CORTEX
#include <cortex/cortex.h>
#include <cortex/topology.h>
#endif

#define CREDIT_SIZE 8
#define MEAN_PROCESS 1.0

#define TERMINAL_GUID_PREFIX ((uint64_t)(64) << 32)
#define FTREE_HASH_TABLE_SIZE 4999

// debugging parameters
#define TRACK_PKT -1
//#define TRACK_PKT 2820
#define FATTREE_HELLO 0
#define FATTREE_DEBUG 0
#define FATTREE_CONNECTIONS 0
#define FATTREE_MSG 0
#define DEBUG_RC 0

//Data Collection Output Files
#define PARAMS_LOG 0

#define LP_CONFIG_NM (model_net_lp_config_names[FATTREE])
#define LP_METHOD_NM (model_net_method_names[FATTREE])

#ifdef ENABLE_CORTEX
/* This structure is defined at the end of the file */
extern cortex_topology fattree_cortex_topology;
#endif

#if DEBUG_RC
  //Reverse Compute Debug Variables
  long long packet_event_f = 0;
  long long packet_event_r = 0;
  long long t_generate_f = 0;
  long long t_generate_r = 0;
  long long t_send_f = 0;
  long long t_send_r = 0;
  long long t_arrive_f = 0;
  long long t_arrive_r = 0;
  long long t_buffer_f = 0;
  long long t_buffer_r = 0;
  long long s_send_f = 0;
  long long s_send_r = 0;
  long long s_arrive_f = 0;
  long long s_arrive_r = 0;
  long long s_buffer_f = 0;
  long long s_buffer_r = 0;
#endif

long fattree_packet_gen = 0, fattree_packet_fin = 0;
int dump_topo = 0;

static double maxd(double a, double b) { return a < b ? b : a; }

// arrival rate
//static double MEAN_INTERVAL=200.0;

typedef struct fattree_param fattree_param;
/* annotation-specific parameters (unannotated entry occurs at the
 * last index) */
static uint64_t                  num_params = 0;
static fattree_param           * all_params = NULL;
static const config_anno_map_t * anno_map   = NULL;

/* global variables for codes mapping */
static char lp_group_name[MAX_NAME_LENGTH];
static char def_group_name[MAX_NAME_LENGTH];
static int def_gname_set = 0;
static int mapping_grp_id, mapping_type_id, mapping_rep_id, mapping_offset;

enum ROUTING_ALGO
{
    STATIC=1,
    ADAPTIVE,
};

enum RAIL_SELECTION_ALGO
{
    RAIL_DEDICATED=1,
    RAIL_ADAPTIVE,
};
static char routing_folder[MAX_NAME_LENGTH];
static char dot_file_p[MAX_NAME_LENGTH];

/* switch magic number */
int switch_magic_num = 0;

/* terminal magic number */
int fattree_terminal_magic_num = 0;

/* global variables for DOT file generation */
static FILE *dot_file = NULL;

typedef struct fattree_message_list fattree_message_list;
struct fattree_message_list {
    fattree_message msg;
    char* event_data;
    fattree_message_list *next;
    fattree_message_list *prev;
};

void init_fattree_message_list(fattree_message_list *this,
  fattree_message *inmsg) {
    this->msg = *inmsg;
    this->event_data = NULL;
    this->next = NULL;
    this->prev = NULL;
}

void delete_fattree_message_list(fattree_message_list *this) {
    if(this->event_data != NULL) free(this->event_data);
    free(this);
}

struct fattree_param
{
  int ft_type;
  // configuration parameters
  int num_levels;
  int link_repetitions;
  int Ns; // number of switches per pod
  int Np; //Number of pods
  int *num_switches; //switches at various levels
  int *switch_radix; //radix of switches are various levels
  double link_bandwidth;/* bandwidth of a wire connecting switches */
  double cn_bandwidth;/* bandwidth of the compute node channels
                        connected to switch */
  int vc_size; /* buffer size of the link channels */
  int cn_vc_size; /* buffer size of the compute node channels */
  int chunk_size; /* full-sized packets are broken into smaller chunks.*/
  int packet_size;
  int num_terminals;
  int l1_set_size, l0_set_size;
  int l1_term_size, l0_term_size;
  int cn_per_switch;
  double tapering_ratio;
  double cn_delay;
  double head_delay;
  double credit_delay;
  double router_delay;
  double soft_delay;
  int routing;
  int rail_select;
  int rail_size_limit;
  int num_rails;
  int ports_per_nic;
};

struct ftree_hash_key
{
    uint64_t message_id;
    tw_lpid sender_id;
};

struct ftree_qhash_entry
{
   struct ftree_hash_key key;
   char * remote_event_data;
   int num_chunks;
   int remote_event_size;
   struct qhash_head hash_link;
};

/* handles terminal and switch events like packet generate/send/receive/buffer */
typedef enum event_t event_t;
typedef struct ft_terminal_state ft_terminal_state;
typedef struct switch_state switch_state;

/* fattree compute node data structure */
struct ft_terminal_state
{
  unsigned long long packet_counter;
  int packet_gen;
  int packet_fin;
  // Fattree specific parameters
  unsigned int terminal_id;
  unsigned int switch_id;
  unsigned int rail_id;
  tw_lpid *switch_lp;

  // Each terminal will have an input and output channel with the switch
  int *vc_occupancy; // NUM_VC
  tw_stime* terminal_available_time;

  struct mn_stats fattree_stats_array[CATEGORY_MAX];

  fattree_message_list **terminal_msgs;
  fattree_message_list **terminal_msgs_tail;
  int *terminal_length;
  int *in_send_loop;
  int *issueIdle;

   struct rc_stack * st;

  char * anno;
  fattree_param *params;

  struct qhash_table *rank_tbl;
  uint64_t rank_tbl_pop;

  tw_stime   total_time;
  uint64_t total_msg_size;
  double total_hops;
  long finished_msgs;
  long finished_chunks;
  long finished_packets;

  tw_stime *last_buf_full;
  tw_stime *busy_time;
  char output_buf[4096];
  char output_buf3[4096];

  /* For sampling */
  long fin_chunks_sample;
  long data_size_sample;
  double fin_hops_sample;
  tw_stime fin_chunks_time;
  tw_stime busy_time_sample;

};

/* terminal event type (1-4) */
enum event_t
{
  T_GENERATE=1,
  T_ARRIVE,
  T_SEND,
  T_BUFFER,
  S_SEND,
  S_ARRIVE,
  S_BUFFER,
};

enum last_hop
{
   LINK,
   TERMINAL
};

struct switch_state
{
  unsigned int switch_id;
  int switch_level;
  unsigned int rail_id;
  int radix;
  int num_cons;
  int num_lcons;
  int con_per_lneigh;
  int con_per_uneigh;;
  int start_lneigh;
  int end_lneigh;
  int start_uneigh;
  int unused;

  tw_stime* next_output_available_time;
  tw_stime* last_buf_full;

  tw_stime* busy_time;
  tw_stime* busy_time_sample;

  fattree_message_list **pending_msgs;
  fattree_message_list **pending_msgs_tail;
  fattree_message_list **queued_msgs;
  fattree_message_list **queued_msgs_tail;
  int *queued_length;
  int *in_send_loop;
  int* vc_occupancy;
  int64_t* link_traffic;
  tw_lpid *port_connections;

  char output_buf[4096];
  char output_buf2[4096];

  struct rc_stack * st;

  char * anno;
  fattree_param *params;
  /* array to store linear forwaring tables in case we use static routing */
  int *lft;
};

static tw_stime         fattree_total_time = 0;
static tw_stime         fattree_max_latency = 0;
//static tw_stime         max_collective = 0;


static long long       total_hops = 0;
static long long       N_finished_packets = 0;
static long long       total_msg_sz = 0;
static long long       N_finished_msgs = 0;
static long long       N_finished_chunks = 0;

static int fattree_rank_hash_compare(
void *key, struct qhash_head *link)
{
    struct ftree_hash_key *message_key = (struct ftree_hash_key *)key;
    struct ftree_qhash_entry *tmp = NULL;

    tmp = qhash_entry(link, struct ftree_qhash_entry, hash_link);

    if (tmp->key.message_id == message_key->message_id
            && tmp->key.sender_id == message_key->sender_id)
        return 1;

    return 0;
}
static int fattree_hash_func(void *k, int table_size)
{
    struct ftree_hash_key *tmp = (struct ftree_hash_key *)k;
    uint64_t key = (~tmp->message_id) + (tmp->message_id << 18);
    key = key * 21;
    key = ~key ^ (tmp->sender_id >> 4);
    key = key * tmp->sender_id;
    return (int)(key & (table_size - 1));
}

static void free_tmp(void * ptr)
{
    struct ftree_qhash_entry * ftree = ptr;
    free(ftree->remote_event_data);
    free(ftree);
}

static void append_to_fattree_message_list(
        fattree_message_list ** thisq,
        fattree_message_list ** thistail,
        int index,
        fattree_message_list *msg) {
    if(thisq[index] == NULL) {
        thisq[index] = msg;
    } else {
        thistail[index]->next = msg;
        msg->prev = thistail[index];
    }
    thistail[index] = msg;
}

static void prepend_to_fattree_message_list(
        fattree_message_list ** thisq,
        fattree_message_list ** thistail,
        int index,
        fattree_message_list *msg) {
    if(thisq[index] == NULL) {
        thistail[index] = msg;
    } else {
        thisq[index]->prev = msg;
        msg->next = thisq[index];
    }
    thisq[index] = msg;
}

static fattree_message_list* return_head(
        fattree_message_list ** thisq,
        fattree_message_list ** thistail,
        int index) {
    fattree_message_list *head = thisq[index];
    if(head != NULL) {
        thisq[index] = head->next;
        if(head->next != NULL) {
            head->next->prev = NULL;
            head->next = NULL;
        } else {
            thistail[index] = NULL;
        }
    }
    return head;
}

static fattree_message_list* return_tail(
        fattree_message_list ** thisq,
        fattree_message_list ** thistail,
        int index) {
    fattree_message_list *tail = thistail[index];
    if(tail->prev != NULL) {
        tail->prev->next = NULL;
        thistail[index] = tail->prev;
        tail->prev = NULL;
    } else {
        thistail[index] = NULL;
        thisq[index] = NULL;
    }
    return tail;
}

//decl
void switch_credit_send(switch_state * s, tw_bf * bf, fattree_message * msg,
    tw_lp * lp, int sq);
int ft_get_output_port( switch_state * s, tw_bf * bf, fattree_message * msg,
    tw_lp * lp, int *out_off );
int get_base_port(switch_state *s, int from_term, int index);


/* returns the fattree switch lp type for lp registration */
//static const tw_lptype* fattree_get_switch_lp_type(void);

/* returns the fattree message size */
static int fattree_get_msg_sz(void)
{
  return sizeof(fattree_message);
}

static inline uint64_t get_term_guid(ft_terminal_state *t)
{
  return TERMINAL_GUID_PREFIX + t->terminal_id;
}

static inline uint64_t get_switch_guid(switch_state *s)
{
  return (((uint64_t)(s->switch_level + 1)) << 32) + s->switch_id;
}

typedef struct {
  uint64_t guid;
  uint64_t port;
} guid_port_combi_t;

static int cmp_guids(const void *g1, const void *g2)
{
  uint64_t guid1 = *((uint64_t *) g1);
  uint64_t guid2 = ((guid_port_combi_t *) g2)->guid;
  if (guid1 == guid2)
    return 0;
  else
    return 1;
}

/* parse external file with give forwarding tables
 * file name: 0x<switch guid>.lft
 * content: '0x<terminal guid> <egress port>' per line, e.g.:
 * `head 0x0000000100000000.lf`
 *    0x0000000100000000 0
 *    0x00000040000000ff 22
 *    0x0000000100000001 19
 */
static int read_static_lft(switch_state *s, tw_lp *lp)
{
  if (!s || !lp)
    return -1;

  char dir_name[512];
  char file_name[512];

  char *io_dir = routing_folder;
  sprintf(dir_name, "%s", io_dir);

  sprintf(file_name, "%s/0x%016"PRIx64".lft", dir_name, get_switch_guid(s));
  FILE *file = NULL;
  if (!(file = fopen(file_name, "r")))
    return -1;

  char line[UINT8_MAX];
  char *p = NULL, *e = NULL;
  uint64_t dest_guid = 0, port = 0;

  int num_added = 0;
  guid_port_combi_t *tmpLFT = calloc(s->params->num_terminals,
        sizeof(guid_port_combi_t));
  s->lft = malloc(s->params->num_terminals * sizeof(int));
  /* init all with -1 so that we find missing routing entries */
  for (int i = 0; i < s->params->num_terminals; i++) s->lft[i] = -1;

  while (fgets(line, sizeof(line), file)) {
    p = line;
    while (isspace(*p))
      p++;

    if (*p == '\0' || *p == '\n' || *p == '#')
      continue;

    dest_guid = strtoull(p, &e, 16);
    if (e == p || (!isspace(*e) && *e != '#' && *e != '\0')) {
      errno = EINVAL;
      return -1;
    }

    p = e;
    while (isspace(*p))
      p++;

    port = strtoull(p, &e, 0);
    if (e == p || (!isspace(*e) && *e != '#' && *e != '\0')) {
      errno = EINVAL;
      return -1;
    }

    if (dest_guid < TERMINAL_GUID_PREFIX){
      // we don't need switches
    } else {
      tmpLFT[num_added].guid = dest_guid;
      tmpLFT[num_added].port = port;
      num_added++;
    }
  }

  /* we want a real LFT with terminal_id as lookup index, so have to
   * convert the read lft, which might contain entries for switches
   * or is permutated, into the correct format
   */
  ft_terminal_state tmpTerm;
  for (int dest_num = 0; dest_num < s->params->num_terminals; dest_num++) {
    tmpTerm.terminal_id = dest_num;
    uint64_t key = get_term_guid(&tmpTerm);
    size_t size = s->params->num_terminals;
    guid_port_combi_t *elem = lfind(&key, tmpLFT, &size,
          sizeof(guid_port_combi_t), cmp_guids);
    assert(elem);
    // opensm uses ports=1...n, so convert back here
    //if(NULL != getenv("OSM_ROUTING"))
       s->lft[dest_num] = elem->port - 1;
    //else
    //   s->lft[dest_num] = elem->port;
  }

#if FATTREE_DEBUG
  printf("I am switch %d (guid=%016"PRIx64") and my LFT is:\n",
        s->switch_id, get_switch_guid(s));
  for (int dest_num = 0; dest_num < s->params->num_terminals; dest_num++)
     printf("\tdest %d -> egress port %d\n", dest_num, s->lft[dest_num]);
#endif

  free(tmpLFT);
  fclose(file);
  return 0;
}

static void dot_write_open_file(FILE **fout)
{
  if(!fout || *fout) return;
  if(!dump_topo) return;

  char dir_name[512];
  char file_name[512];

  char *io_dir = routing_folder;
  sprintf(dir_name, "%s", io_dir);

  char *dot_file_prefix = dot_file_p;
  int rank;
  MPI_Comm_rank(MPI_COMM_CODES, &rank);
  sprintf(file_name, "%s/%s.dot.%d", dir_name, dot_file_prefix, rank);
  *fout = fopen(file_name, "w+");
  if(*fout == NULL) {
    tw_error(TW_LOC, "Error opening dot file for writing");
  }
}

/*static void dot_write_close_file(FILE **fout)
{
  if(!fout || !(*fout)) return;
  if(!dump_topo) return;
  fclose(*fout);
  *fout = NULL;
}*/

/* sw IDs aren't unique, but level+ID info is */
static void dot_write_switch_info(switch_state *s, int sw_gid, FILE *fout)
{
  if(!s || s->unused || !(s->params) || !fout) return;
  if(!dump_topo) return;
  if(s->rail_id) return;

  const char *root_attr = ",root_switch", *empty_attr = "";
  uint64_t switch_guid = get_switch_guid(s);
  fattree_param *p = s->params;

  fprintf(fout, "\t\"S_%d_%d\" [comment=\"0x%016"PRIx64",radix=%d%s\"];\n",
      s->switch_level, s->switch_id, switch_guid, s->radix,
      (p->num_levels == s->switch_level + 1) ? root_attr : empty_attr);
}

/* terminal IDs are unique in this model */
static void dot_write_term_info(ft_terminal_state *t, FILE *fout)
{
  uint64_t term_guid = t->terminal_id;
  if(!dump_topo) return;
  if(t->rail_id) return;

  if(!t || !fout) return;
  /* need to shift the term guid, because opensm doesn't like guid=0...0 */
  //if(NULL != getenv("OSM_ROUTING"))
     term_guid = get_term_guid(t);
  fprintf(fout, "\t\"H_%d\" [comment=\"0x%016"PRIx64"\"];\n",
        t->terminal_id, term_guid);
}

static void dot_write_sw2sw_link(int lsw_lvl, int lsw_id, int lsw_port,
                                 int rsw_lvl, int rsw_id, int rsw_port,
                                 FILE *fout)
{
  int tmp_lsw_port = 0, tmp_rsw_port = 0;

  if(!fout) return;
  if(!dump_topo) return;

  /* add +1 to ports, since ibsim/opensm start with physical port number 1
   * (port 0 is internal sw port)
   */
  //if(NULL != getenv("OSM_ROUTING")) {
     tmp_lsw_port = lsw_port + 1;
     tmp_rsw_port = rsw_port + 1;
  //} else {
  //   tmp_lsw_port = lsw_port;
  //   tmp_rsw_port = rsw_port;
  //}

  /* if original right hand side is unknown we write '?' instead
   * and must fix it later ... nice topology definition ... :-(
   */
  if(-1 == rsw_port)
     fprintf(fout, "\t\"S_%d_%d\" -> \"S_%d_%d\" [comment=\"P%d->P?\"];\n",
           lsw_lvl, lsw_id, rsw_lvl, rsw_id, tmp_lsw_port);
  else
     fprintf(fout, "\t\"S_%d_%d\" -> \"S_%d_%d\" [comment=\"P%d->P%d\"];\n",
           lsw_lvl, lsw_id, rsw_lvl, rsw_id, tmp_lsw_port, tmp_rsw_port);
}

static void dot_write_sw2term_link(int lsw_lvl, int lsw_id, int lsw_port,
                                   int term_id, int dump_reverse, FILE *fout)
{
  if(!fout) return;
  if(!dump_topo) return;

  /* add +1 to ports, since ibsim/opensm start with physical port number 1
   * (port 0 is internal sw port)
   */
  //if(NULL != getenv("OSM_ROUTING"))
     lsw_port++;

  fprintf(fout, "\t\"S_%d_%d\" -> \"H_%d\" [comment=\"P%d->P1\"];\n",
        lsw_lvl, lsw_id, term_id, lsw_port);
  if(dump_reverse)
     fprintf(fout, "\t\"H_%d\" -> \"S_%d_%d\" [comment=\"P1->P%d\"];\n",
           term_id, lsw_lvl, lsw_id, lsw_port);
}

/*static void dot_write_term2sw_link(int term_id, int rsw_lvl, int rsw_id,
                                   int rsw_port, FILE *fout)
{
  if(!fout) return;
  if(!dump_topo) return;

  *//* add +1 to ports, since ibsim/opensm start with physical port number 1
   * (port 0 is internal sw port)
   */
  //if(NULL != getenv("OSM_ROUTING"))
  //   rsw_port++;

  //fprintf(fout, "\t\"H_%d\" -> \"S_%d_%d\" [comment=\"P1->P%d\"];\n",
  //      term_id, rsw_lvl, rsw_id, rsw_port);
//}

void post_switch_init(switch_state *s, tw_lp *lp)
{
  /* read any LFTs which might have been generated by an external routing
   * algorithm, e.g., through the use of opensm\
   */
  if(s->params->routing == STATIC && !dump_topo) {
   if(0 != read_static_lft(s, lp)) {
     tw_error(TW_LOC, "Error while reading the routing table");
   }
  }
}


static void fattree_read_config(const char * anno, fattree_param *p){
  uint32_t h1 = 0, h2 = 0;
  bj_hashlittle2(LP_METHOD_NM, strlen(LP_METHOD_NM), &h1, &h2);
  switch_magic_num = h1 + h2;
    
  bj_hashlittle2(LP_METHOD_NM, strlen(LP_METHOD_NM), &h1, &h2);
  fattree_terminal_magic_num = h1 + h2;
  
  int i;

  p->ft_type = 1;
  configuration_get_value_int(&config, "PARAMS", "ft_type", anno,
      &p->ft_type);
  if(!g_tw_mynode) printf("FT type is %d\n", p->ft_type);

  configuration_get_value_int(&config, "PARAMS", "num_levels", anno,
      &p->num_levels);
  if(p->num_levels <= 0) {
    tw_error(TW_LOC, "Too few num_levels, Aborting\n");
  }
  if(p->num_levels > 3) {
    tw_error(TW_LOC, "Too many num_levels, only upto 3 supported Aborting\n");
  }
  if(!g_tw_mynode) printf("FT num levels is %d\n", p->num_levels);
  
  p->tapering_ratio = 1;
  configuration_get_value_double(&config, "PARAMS", "tapering", anno,
      &p->tapering_ratio);
  if(!g_tw_mynode) printf("FT tapering is %lf\n", p->tapering_ratio);

  p->tapering_ratio += 1;

  p->num_switches = (int *) malloc (p->num_levels * sizeof(int));
  p->switch_radix = (int*) malloc (p->num_levels * sizeof(int));

  char switch_counts_str[MAX_NAME_LENGTH];
  int rc = configuration_get_value(&config, "PARAMS", "switch_count", anno,
      switch_counts_str, MAX_NAME_LENGTH);
  if (rc == 0){
    tw_error(TW_LOC, "couldn't read PARAMS:switch_count");
  }
  char* token;
  token = strtok(switch_counts_str, ",");
  i = 0;
  while(token != NULL)
  {
    sscanf(token, "%d", &p->num_switches[i]);
    if(p->num_switches[i] <= 0)
    {
      tw_error(TW_LOC, "Invalid switch count  specified "
          "(%d at pos %d), exiting... ", p->num_switches[i], i);
    }
    i++;
    token = strtok(NULL,",");
  }

  char switch_radix_str[MAX_NAME_LENGTH];
  rc = configuration_get_value(&config, "PARAMS", "switch_radix", anno,
      switch_radix_str, MAX_NAME_LENGTH);
  if (rc == 0){
    tw_error(TW_LOC, "couldn't read PARAMS:switch_radix");
  }
  token = strtok(switch_radix_str, ",");
  i = 0;
  while(token != NULL)
  {
    sscanf(token, "%d", &p->switch_radix[i]);
    if(p->switch_radix[i] <= 0)
    {
      tw_error(TW_LOC, "Invalid switch radix  specified "
          "(%d at pos %d), exiting... ", p->switch_radix[i], i);
    }
    i++;
    token = strtok(NULL,",");
  }

  if(fmod(p->switch_radix[0],  p->tapering_ratio)) {
    tw_error(TW_LOC, "Tapering ratio does not fit the given radix; tapering + 1 should"
    " evenly divide the L0 switch radix\n");
  }

  p->Ns = p->switch_radix[0]/2;
  if(p->num_switches[0] % p->Ns) {
    if(!g_tw_mynode) printf("Number of switches has to be a multiple of switch radix/2 (%d);" 
        "next largest value is %d\n", p->Ns, (p->num_switches[0]/p->Ns + 1)*p->Ns);
    tw_error(TW_LOC, "Incorrect number of switches");
  }

  int num_terminals = p->num_switches[0] * (p->tapering_ratio - 1)*(p->switch_radix[0]/p->tapering_ratio);
  p->num_terminals = num_terminals;
    
  p->l0_set_size = p->switch_radix[0]/2;
  p->l0_term_size = (p->tapering_ratio - 1)*(p->switch_radix[0]/p->tapering_ratio);

  if(p->num_levels == 2) {
    p->num_switches[1] = p->num_switches[0]/p->tapering_ratio/2;
    p->switch_radix[1] = p->switch_radix[0];
    p->l1_set_size =  p->num_switches[1];
    p->l1_term_size = num_terminals;
  } else {
    p->switch_radix[1] = p->switch_radix[2] = p->switch_radix[0];
	  p->l1_set_size = p->switch_radix[0]/p->tapering_ratio;
    p->l1_term_size = p->l0_set_size * p->l0_term_size;
    p->Np = p->num_switches[0]/p->l0_set_size;
    p->num_switches[1] =  p->Np * p->l1_set_size;
    p->link_repetitions = p->switch_radix[0]/p->Np;
    if(p->l1_set_size % p->link_repetitions) {
      while(p->l1_set_size % p->link_repetitions) {
        p->link_repetitions--;
      }
    } 
    int halfPods = p->l1_set_size/p->link_repetitions;
    p->num_switches[2] = halfPods * p->Ns;
    if(!g_tw_mynode) printf("Pods %d; L2 pods %d; Link repetition:%d; l1_set_size %d\n", 
        p->Np, halfPods, p->link_repetitions, p->l1_set_size);
  }

  if(p->ft_type == 1 && p->num_levels == 3) {
    if(p->Np != p->switch_radix[0] && p->Np !=  p->switch_radix[0]/2) {
      if(!g_tw_mynode) {
        printf("ft_type 1 should be used for a system with %d or %d pods, which "
        "needs %d or %d L0 switches, respectively\n",  p->switch_radix[0], 
         p->switch_radix[0]/2,  p->switch_radix[0] *  p->switch_radix[0]/2,
          p->switch_radix[0]/2 *  p->switch_radix[0]/2);
      }
      tw_error(TW_LOC, "Incorrect number of switches for ft_type 1");
    }
  }

  for(int jj=0;jj<3 && !g_tw_mynode;jj++)
  {
    printf("num_switches[%d]=%d\n",jj,p->num_switches[jj]);
  }

  rc = configuration_get_value_int(&config, "PARAMS", "packet_size", anno,
      &p->packet_size);
  if(rc) {
    p->packet_size = 512;
  }
  if(!g_tw_mynode) printf("FT packet size is %d\n", p->packet_size);

  p->num_rails = 1;
  configuration_get_value_int(&config, "PARAMS", "num_rails", anno, &p->num_rails);
  if(!g_tw_mynode) printf("FT num rails is %d\n", p->num_rails);
  
  if(p->ft_type == 2) {
    p->ports_per_nic = p->num_rails;
  } else {
    p->ports_per_nic = 1;
  }

  p->router_delay = 50;
  configuration_get_value_double(&config, "PARAMS", "router_delay", anno,
      &p->router_delay);
  if(!g_tw_mynode) printf("FT router delay is %.1lf\n", p->router_delay);

  p->soft_delay = 1000;
  configuration_get_value_double(&config, "PARAMS", "soft_delay", anno,
      &p->soft_delay);

  rc = configuration_get_value_int(&config, "PARAMS", "vc_size", anno, &p->vc_size);
  if(rc) {
    p->vc_size = 8 * p->packet_size;
  }
  if(!g_tw_mynode) printf("FT buffer size links is %d\n", p->vc_size);

  rc = configuration_get_value_int(&config, "PARAMS", "cn_vc_size", anno,
      &p->cn_vc_size);
  if(rc) {
    p->cn_vc_size = 8 * p->packet_size;
  }
  if(!g_tw_mynode) printf("FT buffer size injection is %d\n", p->cn_vc_size);

  rc = configuration_get_value_int(&config, "PARAMS", "chunk_size", anno, &p->chunk_size);
  if(rc) {
    p->chunk_size = 512;
  }

  char routing_str[MAX_NAME_LENGTH];
  configuration_get_value(&config, "PARAMS", "routing", anno, routing_str,
      MAX_NAME_LENGTH);
  if(strcmp(routing_str, "static") == 0)
    p->routing = STATIC;
  else if(strcmp(routing_str, "adaptive")==0)
    p->routing = ADAPTIVE;
  else
  {
    p->routing = ADAPTIVE;
  }
  if(!g_tw_mynode) printf("FT routing is %d\n", p->routing);
  
  char rail_select_str[MAX_NAME_LENGTH];
  configuration_get_value(&config, "PARAMS", "rail_select", anno, rail_select_str,
      MAX_NAME_LENGTH);
  if(strcmp(rail_select_str, "dedicated") == 0)
    p->rail_select = RAIL_DEDICATED;
  else if(strcmp(rail_select_str, "adaptive")==0)
    p->rail_select = RAIL_ADAPTIVE;
  else {
    p->rail_select = RAIL_DEDICATED;
  }
  if(!g_tw_mynode) printf("FT rail selection is %d\n", p->rail_select);

  p->rail_size_limit = 32768;
  configuration_get_value_int(&config, "PARAMS", "rail_select_limit", anno,
      &p->rail_size_limit);
  if(!g_tw_mynode) printf("FT rail select limit  is %d\n", p->rail_size_limit);
  
  configuration_get_value_int(&config, "PARAMS", "dump_topo", anno,
      &dump_topo);

  routing_folder[0] = '\0';
  rc = configuration_get_value(&config, "PARAMS", "routing_folder", anno, routing_folder,
      MAX_NAME_LENGTH);

  if(routing_folder[0] == '\0') {
    if(dump_topo || p->routing == STATIC) {
      tw_error(TW_LOC, "routing_folder has to be provided with dump_topo || static routing");
    }
  }
 
  dot_file_p[0] = '\0'; 
  rc = configuration_get_value(&config, "PARAMS", "dot_file", anno, dot_file_p,
      MAX_NAME_LENGTH);
  
  if(dot_file_p[0] == '\0') {
    if(dump_topo) {
      tw_error(TW_LOC, "dot_file has to be provided with dump_topo");
    }
  }
  
  rc = configuration_get_value_double(&config, "PARAMS", "link_bandwidth", anno,
      &p->link_bandwidth);
  if(rc) {
    p->link_bandwidth = 5;
  }

  rc = configuration_get_value_double(&config, "PARAMS", "cn_bandwidth", anno,
      &p->cn_bandwidth);
  if(!p->cn_bandwidth) {
    p->cn_bandwidth = 5;
  }
  
  if(!g_tw_mynode) printf("FT bandwidths link %.1lf node %.1lf\n", 
      p->link_bandwidth, p->cn_bandwidth);

#if FATTREE_DEBUG
  printf("l1_set_size:%d l1_term_size:%d\n",p->l1_set_size,p->l1_term_size);
#endif

  p->cn_delay = (1.0 / p->cn_bandwidth);
  p->head_delay = (1.0 / p->link_bandwidth);
  p->credit_delay = (1.0 / p->link_bandwidth) * 8; //assume 8 bytes packet
}

static void fattree_configure(){
  anno_map = codes_mapping_get_lp_anno_map(LP_CONFIG_NM);
  assert(anno_map);
  num_params = anno_map->num_annos + (anno_map->has_unanno_lp > 0);
  all_params = malloc(num_params * sizeof(*all_params));

  for (int i = 0; i < anno_map->num_annos; i++){
    const char * anno = anno_map->annotations[i].ptr;
    fattree_read_config(anno, &all_params[i]);
  }
  if (anno_map->has_unanno_lp > 0){
    fattree_read_config(NULL, &all_params[anno_map->num_annos]);
  }
#ifdef ENABLE_CORTEX
  model_net_topology = fattree_cortex_topology;
#endif
}

/* initialize a fattree compute node terminal */
void ft_terminal_init( ft_terminal_state * s, tw_lp * lp )
{
    s->packet_gen = 0;
    s->packet_fin = 0;

    char anno[MAX_NAME_LENGTH];

    if(def_gname_set == 0) {
      def_gname_set = 1;
      codes_mapping_get_lp_info(0, def_group_name, &mapping_grp_id, NULL,
          &mapping_type_id, anno, &mapping_rep_id, &mapping_offset);
    }

    // Assign the global switch ID
    codes_mapping_get_lp_info(lp->gid, lp_group_name, &mapping_grp_id, NULL,
            &mapping_type_id, anno, &mapping_rep_id, &mapping_offset);
    if (anno[0] == '\0'){
        s->anno = NULL;
        s->params = &all_params[num_params-1];
    } else {
        s->anno = strdup(anno);
        int id = configuration_get_annotation_index(anno, anno_map);
        s->params = &all_params[id];
    }

   int num_lps = codes_mapping_get_lp_count(lp_group_name, 1, LP_CONFIG_NM,
           s->anno, 0);

   if(s->params->ft_type != 2) {
    num_lps /= s->params->num_rails;
   }

   if(num_lps != s->params->l0_term_size) {
     tw_error(TW_LOC, "Number of NICs per repetition per rail has to be equal to "
         "%d, not the given value of %d\n", s->params->l0_term_size, num_lps);
   }
  
   if(s->params->ft_type != 2) {
    s->terminal_id = (mapping_rep_id * num_lps) + (mapping_offset/s->params->num_rails);
    s->rail_id = (mapping_offset % s->params->num_rails);
   } else {
    s->terminal_id = (mapping_rep_id * num_lps) + mapping_offset;
    s->rail_id = 0;
   }
   s->switch_id = s->terminal_id / s->params->l0_term_size;
   s->switch_lp = (tw_lpid*)malloc(s->params->ports_per_nic * sizeof(tw_lpid));
   if(s->params->ft_type != 2) {
     codes_mapping_get_lp_id(lp_group_name, "fattree_switch", NULL, 1,
         s->switch_id, 0 + s->params->num_levels * s->rail_id, &s->switch_lp[0]);
   } else {
     for(int i = 0; i < s->params->ports_per_nic; i++) {
       codes_mapping_get_lp_id(lp_group_name, "fattree_switch", NULL, 1,
           s->switch_id, 0 + s->params->num_levels * i, &s->switch_lp[i]);
     }
   }
   s->terminal_available_time = (tw_stime*) malloc(s->params->ports_per_nic * sizeof(tw_stime));
   s->vc_occupancy = (int*) malloc(s->params->ports_per_nic * sizeof(int));
   s->terminal_length = (int*) malloc(s->params->ports_per_nic * sizeof(int));
   s->in_send_loop = (int*) malloc(s->params->ports_per_nic * sizeof(int));
   s->issueIdle = (int*) malloc(s->params->ports_per_nic * sizeof(int));
   s->terminal_msgs =
     (fattree_message_list**)malloc(s->params->ports_per_nic*sizeof(fattree_message_list*));
   s->terminal_msgs_tail =
     (fattree_message_list**)malloc(s->params->ports_per_nic*sizeof(fattree_message_list*));
   s->last_buf_full = (tw_stime*) malloc(s->params->ports_per_nic * sizeof(tw_stime));
   s->busy_time = (tw_stime*) malloc(s->params->ports_per_nic * sizeof(tw_stime));
   for(int i = 0; i < s->params->ports_per_nic; i++) {
     s->terminal_available_time[i] = 0.0;
     s->vc_occupancy[i] = 0;
     s->terminal_msgs[i] = NULL;
     s->terminal_msgs_tail[i] = NULL;
     s->terminal_length[i] = 0;
     s->in_send_loop[i] = 0;
     s->issueIdle[i] = 0;
     s->last_buf_full[i] = 0;
     s->busy_time[i] = 0;
   }
   s->packet_counter = 0;
   s->finished_msgs = 0;
   s->finished_chunks = 0;
   s->finished_packets = 0;
   s->total_time = 0.0;
   s->total_msg_size = 0;

#if FATTREE_DEBUG
   printf("I am terminal %d (%llu), connected to switch %d in rail id %d\n", s->terminal_id,
       LLU(lp->gid), s->switch_id, s->rail_id);
#endif

   rc_stack_create(&s->st);

   s->rank_tbl = NULL;
   //if(!s->rank_tbl)
   //  tw_error(TW_LOC, "\n Hash table not initialized! ");

   /* dump partial topology into DOT format
    * skip term2sw link part, because we are missing the remote switch port
    * in this stage of the code
    */
   if(!dot_file && !s->rail_id)
     dot_write_open_file(&dot_file);
   dot_write_term_info(s, dot_file);

   /* ensure the DOT file is out before reading it with external tools */
   if(dot_file && !s->rail_id)
     fflush(dot_file);

   return;
}

/* sets up the switch */
void switch_init(switch_state * r, tw_lp * lp)
{

  char anno[MAX_NAME_LENGTH];

  if(def_gname_set == 0) {
    def_gname_set = 1;
    codes_mapping_get_lp_info(0, def_group_name, &mapping_grp_id, NULL,
        &mapping_type_id, anno, &mapping_rep_id, &mapping_offset);
  }

  codes_mapping_get_lp_info(lp->gid, lp_group_name, &mapping_grp_id, NULL,
      &mapping_type_id, anno, &mapping_rep_id, &mapping_offset);

  if (anno[0] == '\0'){
    r->anno = NULL;
    r->params = &all_params[num_params-1];
  } else {
    r->anno = strdup(anno);
    int id = configuration_get_annotation_index(anno, anno_map);
    r->params = &all_params[id];
  }

  // shorthand
  fattree_param *p = r->params;
  if(mapping_rep_id >= p->num_switches[mapping_offset % p->num_levels]) {
    r->unused = 1;
    return;
  }

  r->unused = 0;

  if((mapping_offset % p->num_levels) == 0) {
    r->switch_id = mapping_rep_id;
  } else if((mapping_offset % p->num_levels) ==  1) {
    r->switch_id = mapping_rep_id + p->num_switches[0];
  } else {
    r->switch_id = mapping_rep_id + p->num_switches[0] + p->num_switches[1];
  }
  r->rail_id = mapping_offset / p->num_levels;

  r->switch_level = mapping_offset % p->num_levels;

  r->radix = p->switch_radix[r->switch_level];

  r->next_output_available_time = (tw_stime*) malloc (r->radix *
      sizeof(tw_stime));
  r->vc_occupancy = (int*) malloc (r->radix * sizeof(int));
  r->in_send_loop = (int*) malloc (r->radix * sizeof(int));
  r->link_traffic = (int64_t*) malloc (r->radix * sizeof(int64_t));
  r->port_connections = (tw_lpid*) malloc (r->radix * sizeof(tw_lpid));
  r->pending_msgs =
    (fattree_message_list**)malloc(r->radix * sizeof(fattree_message_list*));
  r->pending_msgs_tail =
    (fattree_message_list**)malloc(r->radix * sizeof(fattree_message_list*));
  r->queued_msgs =
    (fattree_message_list**)malloc(r->radix * sizeof(fattree_message_list*));
  r->queued_msgs_tail =
    (fattree_message_list**)malloc(r->radix * sizeof(fattree_message_list*));
  r->queued_length = (int*)malloc(r->radix * sizeof(int));
  r->lft = NULL;

  r->last_buf_full = (tw_stime*)malloc(r->radix * sizeof(tw_stime));
  r->busy_time = (tw_stime*)malloc(r->radix * sizeof(tw_stime));
  r->busy_time_sample = (tw_stime*)malloc(r->radix * sizeof(tw_stime));

  rc_stack_create(&r->st);

  for(int i = 0; i < r->radix; i++)
  {
    // Set credit & switch occupancy
    r->last_buf_full[i] = 0.0;
    r->busy_time[i] = 0.0;
    r->busy_time_sample[i] = 0.0;
    r->next_output_available_time[i] = 0;
    r->vc_occupancy[i] = 0;
    r->in_send_loop[i] = 0;
    r->link_traffic[i] = 0;
    r->pending_msgs[i] = NULL;
    r->pending_msgs_tail[i] = NULL;
    r->queued_msgs[i] = NULL;
    r->queued_msgs_tail[i] = NULL;
    r->queued_length[i] = 0;
  }

  /* dump partial topology info into DOT format (switch radix, guid, ...) */
  if(!dot_file && !r->rail_id)
    dot_write_open_file(&dot_file);
  dot_write_switch_info(r, lp->gid, dot_file);

#if FATTREE_CONNECTIONS || FATTREE_DEBUG
    tw_lpid next_switch_lid;
	int written = 0;
    int written_2 = 0;
#endif

  //set lps connected to each port
  r->num_cons = 0;
  r->num_lcons = 0;
#if FATTREE_DEBUG
  printf("I am switch %d (%llu), level %d, radix %d, rail id %d\n", r->switch_id,
    LLU(lp->gid), r->switch_level, r->radix, r->rail_id);
#endif
  //if at level 0, first half ports go to terminals
  if(r->switch_level == 0) {
    int term_rails, term_railid;
    if(p->ft_type != 2) {
      term_rails = p->num_rails;
      term_railid = r->rail_id;
    } else {
      term_rails = 1;
      term_railid = 0;
    }
    int start_terminal = r->switch_id * p->l0_term_size;
    int end_terminal = start_terminal + p->l0_term_size;
    for(int term = start_terminal; term < end_terminal; term++) {
      tw_lpid nextTerm;
      int rep = term / p->l0_term_size;
      int off = (term % p->l0_term_size) * term_rails + term_railid;
      codes_mapping_get_lp_id(def_group_name, LP_CONFIG_NM, NULL, 1,
          rep, off, &nextTerm);
      r->port_connections[r->num_cons++] = nextTerm;
#if FATTREE_CONNECTIONS
	    written += sprintf(r->output_buf + written, "%u, %llu, ", r->switch_id+p->num_terminals,LLU(codes_mapping_get_lp_relative_id(nextTerm,0,0)/term_rails));
#endif
      r->num_lcons++;
#if FATTREE_DEBUG
      printf("L0->term I am switch %d, connect to terminal %d (%llu) at port %d, rail id %d yes collecting\n",
          r->switch_id, term, LLU(nextTerm), r->num_cons - 1, r->rail_id);
#endif
      /* write sw2term links and (reverse link, too) into DOT file */
      if(!r->rail_id) {
        dot_write_sw2term_link(r->switch_level, r->switch_id, r->num_cons - 1, term, 1, dot_file);
      }
    }
    r->start_lneigh = start_terminal;
    r->end_lneigh = end_terminal;
    r->con_per_lneigh = 1;
    assert(r->num_lcons == p->l0_term_size);
    int l1_set;
    if(p->num_levels == 2) {
      l1_set = 0;
      r->con_per_uneigh = 2;
    } else {
      l1_set = r->switch_id / p->l0_set_size;
      r->con_per_uneigh = 1;
    }
    int l1_base = l1_set * p->l1_set_size;
    r->start_uneigh = p->num_switches[0] + l1_base;
    for(int l1 = 0; l1 < p->l1_set_size; l1++) {
      tw_lpid nextTerm;
      assert(l1_base < p->num_switches[1]);
      codes_mapping_get_lp_id(lp_group_name, "fattree_switch", NULL, 1,
          l1_base, 1 + r->rail_id * p->num_levels, &nextTerm);
      for(int con = 0; con < r->con_per_uneigh; con++) {
        r->port_connections[r->num_cons++] = nextTerm;
#if FATTREE_CONNECTIONS
        codes_mapping_get_lp_info(nextTerm, lp_group_name, &mapping_grp_id, NULL,
            &mapping_type_id, anno, &mapping_rep_id, &mapping_offset);
        next_switch_lid = mapping_rep_id +  p->num_switches[0];
	      written += sprintf(r->output_buf + written, "%u, %llu, ", r->switch_id+p->num_terminals,LLU(next_switch_lid)+p->num_terminals);
#endif
#if FATTREE_DEBUG
    printf("L0->L1 I am switch %d, connect to upper switch %d L1 (%llu) rel_id:%llu at port %d, rail id %d yes collecting\n",
          r->switch_id, l1_base, LLU(nextTerm), LLU(next_switch_lid), r->num_cons - 1, r->rail_id);
#endif
        /* write all inter switch links to DOT file (initialized before) */
        if(!r->rail_id) {
          dot_write_sw2sw_link(r->switch_level, r->switch_id, r->num_cons - 1,
              r->switch_level + 1, p->num_switches[0] + l1_base, -1, dot_file);
        }
      }
      l1_base++;
    }
  } else if (r->switch_level == 1) {
    int l0_base;
    if(p->num_levels == 2) {
      l0_base = 0;
      r->start_lneigh = 0;
      r->end_lneigh = p->num_switches[0];
      r->con_per_lneigh = 2;
    } else {
      l0_base = ((r->switch_id - p->num_switches[0]) / p->l1_set_size) *
        p->l0_set_size;
      r->start_lneigh = l0_base;
      r->end_lneigh = l0_base + p->l0_set_size;
      r->con_per_lneigh = 1;
    }
    for(int l0 = 0; l0 < p->l0_set_size; l0++) {
      tw_lpid nextTerm;
      codes_mapping_get_lp_id(def_group_name, "fattree_switch", NULL, 1,
          l0_base, 0 + r->rail_id * p->num_levels, &nextTerm);
      for(int con = 0; con < r->con_per_lneigh; con++) {
        r->port_connections[r->num_cons++] = nextTerm;
#if FATTREE_CONNECTIONS
        codes_mapping_get_lp_info(nextTerm, lp_group_name, &mapping_grp_id, NULL,
            &mapping_type_id, anno, &mapping_rep_id, &mapping_offset);
        next_switch_lid = mapping_rep_id;
        written_2 += sprintf(r->output_buf2 + written_2, "%u, %llu, ", r->switch_id+p->num_terminals,LLU(next_switch_lid)+p->num_terminals);
#endif
        r->num_lcons++;
#if FATTREE_DEBUG
        printf("L1->L0 I am switch %d, connect to switch %d L0 (%llu) rel_id:%llu at port %d, rail id %d not collecting\n",
            r->switch_id, l0_base, LLU(nextTerm), LLU(next_switch_lid), r->num_cons - 1, r->rail_id);
#endif
        /* write all inter switch links to DOT file (initialized before) */
        if(!r->rail_id) {
          dot_write_sw2sw_link(r->switch_level, r->switch_id, r->num_cons - 1,
              r->switch_level - 1, l0_base, -1, dot_file);
        }
      }
      l0_base++;
    }
    if(p->num_levels == 3) {
      int l2_base = 0;
      int rep = p->link_repetitions;
      int l2 = ((r->switch_id - p->num_switches[0]) % p->l1_set_size)/rep * p->Ns;
      r->con_per_uneigh = 1;
      for(int off = 0; off < p->Ns; off++) {
        tw_lpid nextTerm;
        assert(l2 + off < p->num_switches[2]);
        codes_mapping_get_lp_id(lp_group_name, "fattree_switch", NULL, 1,
            l2 + off, 2 + r->rail_id * p->num_levels, &nextTerm);
        for(int con = 0; con < r->con_per_uneigh; con++) {
          r->port_connections[r->num_cons++] = nextTerm;
#if FATTREE_CONNECTIONS
          codes_mapping_get_lp_info(nextTerm, lp_group_name, &mapping_grp_id, NULL,
              &mapping_type_id, anno, &mapping_rep_id, &mapping_offset);
          next_switch_lid = mapping_rep_id +  p->num_switches[0] + p->num_switches[1];
          written += sprintf(r->output_buf + written, "%u, %llu, ", r->switch_id+p->num_terminals,LLU(next_switch_lid)+p->num_terminals);
#endif
#if FATTREE_DEBUG
          printf("L1->L2:t=0 I am switch %d, connect to upper switch %d L2 (%llu) rel_id:%llu at port %d, rail id %d yes collecting\n",
              r->switch_id, l2+off, LLU(nextTerm), LLU(next_switch_lid), r->num_cons - 1, r->rail_id);
#endif
          /* write all inter switch links to DOT file (initialized before) */