Commit 56e896e4 authored by Nikhil Jane's avatar Nikhil Jane Committed by Nikhil

Adding Nikhil's fat tree model from the uiuc_tracer branch of the codes-net repo

parent 45fdd016
#ifndef FATTREE_H
#define FATTREE_H
#include <ross.h>
typedef struct fattree_message fattree_message;
/* this message is used for both dragonfly compute nodes and routers */
struct fattree_message
{
/* packet ID of the flit */
unsigned long long packet_ID;
/* event type of the flit */
short type;
/* category: comes from codes */
char category[CATEGORY_NAME_MAX];
/* final destination LP ID, this comes from codes can be a server or any other LP type*/
tw_lpid final_dest_gid;
/*sending LP ID from CODES, can be a server or any other LP type */
tw_lpid sender_lp;
/* destination terminal ID of the message */
int dest_num;
/* source terminal ID of the fattree */
unsigned int src_terminal_id;
/* Intermediate LP ID from which this message is coming */
unsigned int intm_lp_id;
short saved_vc;
short saved_off;
int last_hop;
int intm_id; //to find which port I connect to sender with
// For buffer message
short vc_index;
short vc_off;
/* for reverse computation */
tw_stime saved_available_time;
tw_stime saved_credit_time;
uint64_t packet_size;
/* meta data to aggregate packets into a message at receiver */
uint64_t msg_size;
uint64_t src_nic;
uint64_t uniq_id;
uint64_t saved_size;
int remote_event_size_bytes;
int local_event_size_bytes;
};
#endif /* end of include guard: FATTREE_H */
#include <ross.h>
#include "codes/codes_mapping.h"
#include "codes/codes.h"
#include "codes/model-net.h"
#include "codes/model-net-method.h"
#include "codes/model-net-lp.h"
#include "codes/net/fattree.h"
#include "sys/file.h"
//#include "codes/map_messages.h"
#define CREDIT_SIZE 8
#define MEAN_PROCESS 1.0
// debugging parameters
#define FATTREE_HELLO 0
#define FATTREE_CONNECTIONS 0
#define FATTREE_MSG 0
#define LP_CONFIG_NM (model_net_lp_config_names[FATTREE])
#define LP_METHOD_NM (model_net_method_names[FATTREE])
static double maxd(double a, double b) { return a < b ? b : a; }
// arrival rate
static double MEAN_INTERVAL=200.0;
typedef struct fattree_param fattree_param;
/* annotation-specific parameters (unannotated entry occurs at the
* last index) */
static uint64_t num_params = 0;
static fattree_param * all_params = NULL;
static config_anno_map_t * anno_map = NULL;
/* global variables for codes mapping */
static char lp_group_name[MAX_NAME_LENGTH];
static char def_group_name[MAX_NAME_LENGTH];
static int def_gname_set = 0;
static int mapping_grp_id, mapping_type_id, mapping_rep_id, mapping_offset;
typedef struct fattree_message_list fattree_message_list;
struct fattree_message_list {
fattree_message msg;
char* event_data;
fattree_message_list *next;
fattree_message_list *prev;
};
void init_fattree_message_list(fattree_message_list *this,
fattree_message *inmsg) {
this->msg = *inmsg;
this->event_data = NULL;
this->next = NULL;
this->prev = NULL;
}
void delete_fattree_message_list(fattree_message_list *this) {
if(this->event_data != NULL) free(this->event_data);
free(this);
}
struct fattree_param
{
int ft_type;
// configuration parameters
int num_levels;
int *num_switches; //switches at various levels
int *switch_radix; //radix of switches are various levels
double link_bandwidth;/* bandwidth of a wire connecting switches */
double cn_bandwidth;/* bandwidth of the compute node channels
connected to switch */
int vc_size; /* buffer size of the link channels */
int cn_vc_size; /* buffer size of the compute node channels */
int packet_size;
int num_terminals;
int l1_set_size;
int l1_term_size;
double cn_delay;
double head_delay;
double credit_delay;
double router_delay;
double soft_delay;
};
/* handles terminal and switch events like packet generate/send/receive/buffer */
typedef enum event_t event_t;
typedef struct ft_terminal_state ft_terminal_state;
typedef struct switch_state switch_state;
/* fattree compute node data structure */
struct ft_terminal_state
{
unsigned long long packet_counter;
// Fattree specific parameters
unsigned int terminal_id;
unsigned int switch_id;
tw_lpid switch_lp;
// Each terminal will have an input and output channel with the switch
int vc_occupancy; // NUM_VC
tw_stime terminal_available_time;
tw_stime next_credit_available_time;
fattree_message_list **terminal_msgs;
fattree_message_list **terminal_msgs_tail;
int terminal_length;
int in_send_loop;
int issueIdle;
char * anno;
fattree_param *params;
};
/* terminal event type (1-4) */
enum event_t
{
T_GENERATE=1,
T_ARRIVE,
T_SEND,
T_BUFFER,
S_SEND,
S_ARRIVE,
S_BUFFER,
};
enum last_hop
{
LINK,
TERMINAL
};
struct switch_state
{
unsigned int switch_id;
int switch_level;
int radix;
int num_cons;
int num_lcons;
int con_per_lneigh;
int con_per_uneigh;;
int start_lneigh;
int end_lneigh;
int start_uneigh;
int unused;
tw_stime* next_output_available_time;
tw_stime* next_credit_available_time;
fattree_message_list **pending_msgs;
fattree_message_list **pending_msgs_tail;
fattree_message_list **queued_msgs;
fattree_message_list **queued_msgs_tail;
int *queued_length;
int *in_send_loop;
int* vc_occupancy;
int64_t* link_traffic;
tw_lpid *port_connections;
char * anno;
fattree_param *params;
};
static void append_to_fattree_message_list(
fattree_message_list ** thisq,
fattree_message_list ** thistail,
int index,
fattree_message_list *msg) {
if(thisq[index] == NULL) {
thisq[index] = msg;
} else {
thistail[index]->next = msg;
msg->prev = thistail[index];
}
thistail[index] = msg;
}
static void prepend_to_fattree_message_list(
fattree_message_list ** thisq,
fattree_message_list ** thistail,
int index,
fattree_message_list *msg) {
if(thisq[index] == NULL) {
thistail[index] = msg;
} else {
thisq[index]->prev = msg;
msg->next = thisq[index];
}
thisq[index] = msg;
}
static void create_prepend_to_fattree_message_list(
fattree_message_list ** thisq,
fattree_message_list ** thistail,
int index,
fattree_message *msg) {
fattree_message_list* new_entry = (fattree_message_list*)malloc(
sizeof(fattree_message_list));
init_fattree_message_list(new_entry, msg);
if(msg->remote_event_size_bytes) {
void *m_data = model_net_method_get_edata(FATTREE, msg);
new_entry->event_data = (void*)malloc(msg->remote_event_size_bytes);
memcpy(new_entry->event_data, m_data, msg->remote_event_size_bytes);
}
prepend_to_fattree_message_list( thisq, thistail, index, new_entry);
}
static fattree_message_list* return_head(
fattree_message_list ** thisq,
fattree_message_list ** thistail,
int index) {
fattree_message_list *head = thisq[index];
if(head != NULL) {
thisq[index] = head->next;
if(head->next != NULL) {
head->next->prev = NULL;
head->next = NULL;
} else {
thistail[index] = NULL;
}
}
return head;
}
static fattree_message_list* return_tail(
fattree_message_list ** thisq,
fattree_message_list ** thistail,
int index) {
fattree_message_list *tail = thistail[index];
if(tail->prev != NULL) {
tail->prev->next = NULL;
thistail[index] = tail->prev;
tail->prev = NULL;
} else {
thistail[index] = NULL;
thisq[index] = NULL;
}
return tail;
}
static void copy_fattree_list_entry( fattree_message_list *cur_entry,
fattree_message *msg) {
fattree_message *cur_msg = &cur_entry->msg;
msg->packet_ID = cur_msg->packet_ID;
strcpy(msg->category, cur_msg->category);
msg->final_dest_gid = cur_msg->final_dest_gid;
msg->sender_lp = cur_msg->sender_lp;
msg->dest_num = cur_msg->dest_num;
msg->src_terminal_id = cur_msg->src_terminal_id;
msg->intm_lp_id = cur_msg->intm_lp_id;
msg->saved_vc = cur_msg->saved_vc;
msg->saved_off = cur_msg->saved_off;
msg->last_hop = cur_msg->last_hop;
msg->intm_id = cur_msg->intm_id;
msg->vc_index = cur_msg->vc_index;
msg->vc_off = cur_msg->vc_off;
msg->packet_size = cur_msg->packet_size;
msg->msg_size = cur_msg->msg_size;
msg->src_nic = cur_msg->src_nic;
msg->uniq_id = cur_msg->uniq_id;
msg->saved_size = cur_msg->saved_size;
msg->local_event_size_bytes = cur_msg->local_event_size_bytes;
msg->remote_event_size_bytes = cur_msg->remote_event_size_bytes;
if(msg->local_event_size_bytes + msg->remote_event_size_bytes > 0) {
void *m_data = model_net_method_get_edata(FATTREE, msg);
memcpy(m_data, cur_entry->event_data,
msg->local_event_size_bytes + msg->remote_event_size_bytes);
}
}
//decl
void switch_credit_send(switch_state * s, tw_bf * bf, fattree_message * msg,
tw_lp * lp, int sq);
int ft_get_output_port( switch_state * s, tw_bf * bf, fattree_message * msg,
tw_lp * lp, int *out_off );
int get_base_port(switch_state *s, int from_term, int index);
/* returns the fattree switch lp type for lp registration */
static const tw_lptype* fattree_get_switch_lp_type(void);
/* returns the dragonfly message size */
static int fattree_get_msg_sz(void)
{
return sizeof(fattree_message);
}
static void fattree_read_config(char * anno, fattree_param *p){
int i;
p->ft_type = 0;
configuration_get_value_int(&config, "PARAMS", "ft_type", anno,
&p->ft_type);
configuration_get_value_int(&config, "PARAMS", "num_levels", anno,
&p->num_levels);
if(p->num_levels <= 0) {
tw_error(TW_LOC, "Too few num_levels, Aborting\n");
}
if(p->num_levels > 3) {
tw_error(TW_LOC, "Too many num_levels, only upto 3 supported Aborting\n");
}
p->num_switches = (int *) malloc (p->num_levels * sizeof(int));
p->switch_radix = (int*) malloc (p->num_levels * sizeof(int));
char switch_counts_str[MAX_NAME_LENGTH];
int rc = configuration_get_value(&config, "PARAMS", "switch_count", anno,
switch_counts_str, MAX_NAME_LENGTH);
if (rc == 0){
tw_error(TW_LOC, "couldn't read PARAMS:switch_count");
}
char* token;
token = strtok(switch_counts_str, ",");
i = 0;
while(token != NULL)
{
sscanf(token, "%d", &p->num_switches[i]);
if(p->num_switches[i] <= 0)
{
tw_error(TW_LOC, "Invalid switch count specified "
"(%d at pos %d), exiting... ", p->num_switches[i], i);
}
i++;
token = strtok(NULL,",");
}
//if(i != p->num_levels) {
// tw_error(TW_LOC, "Not enough switch counts, Aborting\n");
//}
char switch_radix_str[MAX_NAME_LENGTH];
rc = configuration_get_value(&config, "PARAMS", "switch_radix", anno,
switch_radix_str, MAX_NAME_LENGTH);
if (rc == 0){
tw_error(TW_LOC, "couldn't read PARAMS:switch_radix");
}
token = strtok(switch_radix_str, ",");
i = 0;
while(token != NULL)
{
sscanf(token, "%d", &p->switch_radix[i]);
if(p->switch_radix[i] <= 0)
{
tw_error(TW_LOC, "Invalid switch radix specified "
"(%d at pos %d), exiting... ", p->switch_radix[i], i);
}
i++;
token = strtok(NULL,",");
}
if(p->num_levels == 2) {
p->num_switches[1] = p->num_switches[0]/2;
p->switch_radix[1] = p->switch_radix[0];
} else {
p->num_switches[1] = p->num_switches[0];
p->num_switches[2] = p->num_switches[0]/2;
p->switch_radix[1] = p->switch_radix[2] = p->switch_radix[0];
}
//if(i != p->num_levels) {
// tw_error(TW_LOC, "Not enough switch radix, Aborting\n");
//}
i = 1;
for(i = 1; i < p->num_levels - 1; i++) {
if(p->num_switches[i - 1] * p->switch_radix[i - 1] >
p->num_switches[i] * p->switch_radix[i]) {
tw_error(TW_LOC, "Not enough switches/radix at level %d for full "
"bisection bandwidth\n", i);
}
}
if(p->num_switches[i - 1] * p->switch_radix[i - 1] > 2 * p->num_switches[i] *
p->switch_radix[i]) {
tw_error(TW_LOC, "Not enough switches/radix at level %d (top) for full "
"bisection bandwidth\n", i);
}
configuration_get_value_int(&config, "PARAMS", "packet_size", anno,
&p->packet_size);
if(!p->packet_size) {
p->packet_size = 512;
fprintf(stderr, "Packet size is not specified, setting to %d\n",
p->packet_size);
}
p->router_delay = 50;
configuration_get_value_double(&config, "PARAMS", "router_delay", anno,
&p->router_delay);
p->soft_delay = 1000;
configuration_get_value_double(&config, "PARAMS", "soft_delay", anno,
&p->soft_delay);
configuration_get_value_int(&config, "PARAMS", "vc_size", anno, &p->vc_size);
if(!p->vc_size) {
p->vc_size = 8*p->packet_size;
fprintf(stderr, "Buffer size of global channels not specified, setting to "
"%d\n", p->vc_size);
}
configuration_get_value_int(&config, "PARAMS", "cn_vc_size", anno,
&p->cn_vc_size);
if(!p->cn_vc_size) {
p->cn_vc_size = 8*p->packet_size;
fprintf(stderr, "Buffer size of compute node channels not specified, "
"setting to %d\n", p->cn_vc_size);
}
configuration_get_value_double(&config, "PARAMS", "link_bandwidth", anno,
&p->link_bandwidth);
if(!p->link_bandwidth) {
p->link_bandwidth = 5;
fprintf(stderr, "Bandwidth of links is specified, setting to %lf\n",
p->link_bandwidth);
}
configuration_get_value_double(&config, "PARAMS", "cn_bandwidth", anno,
&p->cn_bandwidth);
if(!p->cn_bandwidth) {
p->cn_bandwidth = 5;
fprintf(stderr, "Bandwidth of compute node channels not specified, "
"setting to %lf\n", p->cn_bandwidth);
}
p->l1_set_size = p->switch_radix[0]/2;
p->l1_term_size = (p->l1_set_size * (p->switch_radix[0] / 2));
p->cn_delay = (1.0 / p->cn_bandwidth);
p->head_delay = (1.0 / p->link_bandwidth);
p->credit_delay = (1.0 / p->link_bandwidth) * 8; //assume 8 bytes packet
}
static void fattree_configure(){
anno_map = codes_mapping_get_lp_anno_map(LP_CONFIG_NM);
assert(anno_map);
num_params = anno_map->num_annos + (anno_map->has_unanno_lp > 0);
all_params = malloc(num_params * sizeof(*all_params));
for (uint64_t i = 0; i < anno_map->num_annos; i++){
char * anno = anno_map->annotations[i].ptr;
fattree_read_config(anno, &all_params[i]);
}
if (anno_map->has_unanno_lp > 0){
fattree_read_config(NULL, &all_params[anno_map->num_annos]);
}
}
/* initialize a fattree compute node terminal */
void ft_terminal_init( ft_terminal_state * s, tw_lp * lp )
{
int i;
char anno[MAX_NAME_LENGTH];
if(def_gname_set == 0) {
def_gname_set = 1;
codes_mapping_get_lp_info(0, def_group_name, &mapping_grp_id, NULL,
&mapping_type_id, anno, &mapping_rep_id, &mapping_offset);
}
// Assign the global switch ID
codes_mapping_get_lp_info(lp->gid, lp_group_name, &mapping_grp_id, NULL,
&mapping_type_id, anno, &mapping_rep_id, &mapping_offset);
if (anno[0] == '\0'){
s->anno = NULL;
s->params = &all_params[num_params-1];
} else {
s->anno = strdup(anno);
int id = configuration_get_annotation_index(anno, anno_map);
s->params = &all_params[id];
}
int num_lps = codes_mapping_get_lp_count(lp_group_name, 1, LP_CONFIG_NM,
s->anno, 0);
if(num_lps != (s->params->switch_radix[0]/2)) {
tw_error(TW_LOC, "Number of NICs per repetition has to be equal to "
"half the radix of leaf level switches %d vs %d\n", num_lps,
s->params->switch_radix[0]/2);
}
s->terminal_id = (mapping_rep_id * num_lps) + mapping_offset;
s->switch_id = s->terminal_id / (s->params->switch_radix[0] / 2);
codes_mapping_get_lp_id(lp_group_name, "fattree_switch", NULL, 1,
s->switch_id, 0, &s->switch_lp);
s->terminal_available_time = 0.0;
s->packet_counter = 0;
s->terminal_msgs =
(fattree_message_list**)malloc(1*sizeof(fattree_message_list*));
s->terminal_msgs_tail =
(fattree_message_list**)malloc(1*sizeof(fattree_message_list*));
#if FATTREE_HELLO
printf("I am terminal %d (%ld), connected to switch %d\n", s->terminal_id,
lp->gid, s->switch_id);
#endif
s->vc_occupancy = 0;
s->terminal_msgs[0] = NULL;
s->terminal_msgs_tail[0] = NULL;
s->terminal_length = 0;
s->in_send_loop = 0;
s->issueIdle = 0;
s->params->num_terminals = codes_mapping_get_lp_count(lp_group_name, 0,
LP_CONFIG_NM, s->anno, 0);
return;
}
/* sets up the switch */
void switch_init(switch_state * r, tw_lp * lp)
{
char anno[MAX_NAME_LENGTH];
int num_terminals = -1, num_lps;
if(def_gname_set == 0) {
def_gname_set = 1;
codes_mapping_get_lp_info(0, def_group_name, &mapping_grp_id, NULL,
&mapping_type_id, anno, &mapping_rep_id, &mapping_offset);
num_terminals = codes_mapping_get_lp_count(def_group_name, 0,
LP_CONFIG_NM, anno, 0);
num_lps = codes_mapping_get_lp_count(def_group_name, 1, LP_CONFIG_NM,
anno, 0);
}
codes_mapping_get_lp_info(lp->gid, lp_group_name, &mapping_grp_id, NULL,
&mapping_type_id, anno, &mapping_rep_id, &mapping_offset);
if (anno[0] == '\0'){
r->anno = NULL;
r->params = &all_params[num_params-1];
} else {
r->anno = strdup(anno);
int id = configuration_get_annotation_index(anno, anno_map);
r->params = &all_params[id];
}
// shorthand
fattree_param *p = r->params;
if(mapping_offset == p->num_levels - 1) {
if(mapping_rep_id >= p->num_switches[mapping_offset]) {
r->unused = 1;
return;
}
}
r->unused = 0;
r->switch_id = mapping_rep_id + mapping_offset * p->num_switches[0];
if(num_terminals != -1) {
p->num_terminals = num_terminals;
}
r->switch_level = mapping_offset;
r->radix = p->switch_radix[r->switch_level];
r->next_output_available_time = (tw_stime*) malloc (r->radix *
sizeof(tw_stime));
r->next_credit_available_time = (tw_stime*) malloc (r->radix *
sizeof(tw_stime));
r->vc_occupancy = (int*) malloc (r->radix * sizeof(int));
r->in_send_loop = (int*) malloc (r->radix * sizeof(int));
r->link_traffic = (int64_t*) malloc (r->radix * sizeof(int64_t));
r->port_connections = (tw_lpid*) malloc (r->radix * sizeof(tw_lpid));
r->pending_msgs =
(fattree_message_list**)malloc(r->radix * sizeof(fattree_message_list*));
r->pending_msgs_tail =
(fattree_message_list**)malloc(r->radix * sizeof(fattree_message_list*));
r->queued_msgs =
(fattree_message_list**)malloc(r->radix * sizeof(fattree_message_list*));
r->queued_msgs_tail =
(fattree_message_list**)malloc(r->radix * sizeof(fattree_message_list*));
r->queued_length = (int*)malloc(r->radix * sizeof(int));
for(int i = 0; i < r->radix; i++)
{
// Set credit & switch occupancy
r->next_output_available_time[i] = 0;
r->next_credit_available_time[i] = 0;
r->vc_occupancy[i] = 0;
r->in_send_loop[i] = 0;
r->link_traffic[i] = 0;
r->pending_msgs[i] = NULL;
r->pending_msgs_tail[i] = NULL;
r->queued_msgs[i] = NULL;
r->queued_msgs_tail[i] = NULL;
r->queued_length[i] = 0;
}
//set lps connected to each port
r->num_cons = 0;
r->num_lcons = 0;
#if FATTREE_HELLO
printf("I am switch %d (%d), level %d, radix %d\n", r->switch_id,
lp->gid, r->switch_level, r->radix);
#endif
//if at level 0, first half ports go to terminals
if(r->switch_level == 0) {
int start_terminal = r->switch_id * (p->switch_radix[0] / 2);
int end_terminal = start_terminal + (p->switch_radix[0] / 2);
for(int term = start_terminal; term < end_terminal; term++) {
tw_lpid nextTerm;
int rep = term / (p->switch_radix[0] / 2);
int off = term % (p->switch_radix[0] / 2);
codes_mapping_get_lp_id(def_group_name, LP_CONFIG_NM, NULL, 1,
rep, off, &nextTerm);
r->port_connections[r->num_cons++] = nextTerm;
r->num_lcons++;
#if FATTREE_DEBUG
printf("I am switch %d, connect to terminal %d (%d) at port %d\n",
r->switch_id, term, nextTerm, r->num_cons - 1);
#endif
}
r->start_lneigh = start_terminal;
r->end_lneigh = end_terminal;
r->con_per_lneigh = 1;
assert(r->num_lcons == (r->radix / 2));
int l1_set;
if(p->num_levels == 2) {
l1_set = 0;
} else {
l1_set = r->switch_id / p->l1_set_size;
}
int l1_base = l1_set * p->l1_set_size;
r->start_uneigh = p->num_switches[0] + l1_base;
r->con_per_uneigh = 1;
for(int l1 = 0; l1 < p->l1_set_size; l1++) {
tw_lpid nextTerm;
codes_mapping_get_lp_id(lp_group_name, "fattree_switch", NULL, 1,
l1_base, 1, &nextTerm);
for(int con = 0; con < r->con_per_uneigh; con++) {
r->port_connections[r->num_cons++] = nextTerm;
#if FATTREE_DEBUG
printf("I am switch %d, connect to upper switch %d L1 (%d) at port %d\n",
r->switch_id, l1_base, nextTerm, r->num_cons - 1);
#endif
}
l1_base++;
}
} else if (r->switch_level == 1) {
int l0_set_size, l0_base;
if(p->num_levels == 2) {
l0_set_size = p->num_switches[0];
l0_base = 0;
r->start_lneigh = 0;
r->end_lneigh = p->num_switches[0];
} else {
l0_set_size = p->l1_set_size;
l0_base = ((r->switch_id - p->num_switches[0]) / p->l1_set_size) *