Commit e602f6bc authored by Neil McGlohon's avatar Neil McGlohon
Browse files

Dragonfly Plus Model: Reorg, Indent, Styling Fix

Long term development of large files often leads to many
developers contributing bits and pieces. This leaves the
source code vulnerable to fragmented code style and
inconsistend indentation, spacing, etc.

The orignal Dragonfly model was no exception. Dragonfly Plus
is so similar to the original dragonfly model that it made more
sense to clone the entire model and then adjust its behavior
to match Dragonfly Plus specifications. This meant, however that
whatever styling issues existed in the original Dragonfly will
also exist in the subsequent version.

Developers are often hesitant to adjust styling of lines that
they did not write, lest they be git blamed for the code itself.
But since I copied and pasted the original dragonfly custom source
code into a new file, I'm already git blamed for everything. So I
can reformat the entire file without modifying ownership of any
lines of code since I'm already the technical owner of them...
parent 478fa3c1
/*
* Copyright (C) 2017 Rensselaer Polytechnic Institute.
* See COPYRIGHT notice in top-level directory.
*
*/
#include <ross.h>
#define DEBUG_LP 892
#include "codes/jenkins-hash.h"
#include "codes/codes_mapping.h"
#include "codes/codes.h"
#include "codes/model-net.h"
#include "codes/model-net-method.h"
#include "codes/model-net-lp.h"
#include "codes/net/dragonfly-plus.h"
#include "sys/file.h"
#include "codes/quickhash.h"
#include "codes/rc-stack.h"
#include <vector>
#include <map>
#include <set>
#ifdef ENABLE_CORTEX
#include <cortex/cortex.h>
#include <cortex/topology.h>
#endif
#define DUMP_CONNECTIONS 1
#define CREDIT_SIZE 8
#define DFLY_HASH_TABLE_SIZE 4999
// debugging parameters
#define TRACK -1
#define TRACK_PKT -1
#define TRACK_MSG -1
#define DEBUG 0
#define MAX_STATS 65536
#define LP_CONFIG_NM_TERM (model_net_lp_config_names[DRAGONFLY_PLUS])
#define LP_METHOD_NM_TERM (model_net_method_names[DRAGONFLY_PLUS])
#define LP_CONFIG_NM_ROUT (model_net_lp_config_names[DRAGONFLY_PLUS_ROUTER])
#define LP_METHOD_NM_ROUT (model_net_method_names[DRAGONFLY_PLUS_ROUTER])
using namespace std;
struct Link {
int offset;
};
struct bLink {
int offset, dest;
};
/* Each entry in the vector is for a router id
* against each router id, there is a map of links (key of the map is the dest
* router id)
* link has information on type (green or black) and offset (number of links
* between that particular source and dest router ID)*/
static vector< map< int, vector<Link> > > intraGroupLinks;
/* contains mapping between source router and destination group via link (link
* has dest ID)*/
static vector< map< int, vector<bLink> > > interGroupLinks;
/*MM: Maintains a list of routers connecting the source and destination groups */
static vector< vector< vector<int> > > connectionList;
struct IntraGroupLink {
int src, dest;
};
struct InterGroupLink {
int src, dest;
};
#ifdef ENABLE_CORTEX
/* This structure is defined at the end of the file */
extern "C" {
extern cortex_topology dragonfly_plus_cortex_topology;
}
#endif
static int debug_slot_count = 0;
static long term_ecount, router_ecount, term_rev_ecount, router_rev_ecount;
static long packet_gen = 0, packet_fin = 0;
static double maxd(double a, double b) { return a < b ? b : a; }
/* minimal and non-minimal packet counts for adaptive routing*/
static int minimal_count=0, nonmin_count=0;
static int num_routers_per_mgrp = 0;
typedef struct dragonfly_plus_param dragonfly_plus_param;
/* annotation-specific parameters (unannotated entry occurs at the
* last index) */
static uint64_t num_params = 0;
static dragonfly_plus_param * all_params = NULL;
static const config_anno_map_t * anno_map = NULL;
/* global variables for codes mapping */
static char lp_group_name[MAX_NAME_LENGTH];
static int mapping_grp_id, mapping_type_id, mapping_rep_id, mapping_offset;
/* router magic number */
static int router_magic_num = 0;
/* terminal magic number */
static int terminal_magic_num = 0;
/* Hops within a group */
static int num_intra_nonmin_hops = 4;
static int num_intra_min_hops = 2;
static FILE * dragonfly_log = NULL;
static int sample_bytes_written = 0;
static int sample_rtr_bytes_written = 0;
static char cn_sample_file[MAX_NAME_LENGTH];
static char router_sample_file[MAX_NAME_LENGTH];
//don't do overhead here - job of MPI layer
static tw_stime mpi_soft_overhead = 0;
typedef struct terminal_plus_message_list terminal_plus_message_list;
struct terminal_plus_message_list {
terminal_plus_message msg;
char* event_data;
terminal_plus_message_list *next;
terminal_plus_message_list *prev;
};
static void init_terminal_plus_message_list(terminal_plus_message_list *thisO,
terminal_plus_message *inmsg) {
thisO->msg = *inmsg;
thisO->event_data = NULL;
thisO->next = NULL;
thisO->prev = NULL;
}
static void delete_terminal_plus_message_list(void *thisO) {
terminal_plus_message_list* toDel = (terminal_plus_message_list*)thisO;
if(toDel->event_data != NULL) free(toDel->event_data);
free(toDel);
}
struct dragonfly_plus_param
{
// configuration parameters
int num_routers; /*Number of routers in a group*/
double local_bandwidth;/* bandwidth of the router-router channels within a group */
double global_bandwidth;/* bandwidth of the inter-group router connections */
double cn_bandwidth;/* bandwidth of the compute node channels connected to routers */
int num_vcs; /* number of virtual channels */
int local_vc_size; /* buffer size of the router-router channels */
int global_vc_size; /* buffer size of the global channels */
int cn_vc_size; /* buffer size of the compute node channels */
int chunk_size; /* full-sized packets are broken into smaller chunks.*/
// derived parameters
int num_cn;
int intra_grp_radix;
// MM: won't need these four variables, remove
int num_col_chans;
int num_row_chans;
int num_router_rows;
int num_router_cols;
//dfp params start
int num_level_chans; //number of channels between levels of the group(?)
int num_router_spine; //number of spine routers (top level)
int num_router_leaf; //number of leaf routers (bottom level)
//dfp params end
int num_groups;
int radix;
int total_routers;
int total_terminals;
int num_global_channels;
double cn_delay;
double local_delay;
double global_delay;
double credit_delay;
double router_delay;
};
struct dfly_hash_key
{
uint64_t message_id;
tw_lpid sender_id;
};
struct dfly_router_sample
{
tw_lpid router_id;
tw_stime* busy_time;
int64_t* link_traffic_sample;
tw_stime end_time;
long fwd_events;
long rev_events;
};
struct dfly_cn_sample
{
tw_lpid terminal_id;
long fin_chunks_sample;
long data_size_sample;
double fin_hops_sample;
tw_stime fin_chunks_time;
tw_stime busy_time_sample;
tw_stime end_time;
long fwd_events;
long rev_events;
};
struct dfly_qhash_entry
{
struct dfly_hash_key key;
char * remote_event_data;
int num_chunks;
int remote_event_size;
struct qhash_head hash_link;
};
/* handles terminal and router events like packet generate/send/receive/buffer */
typedef struct terminal_state terminal_state;
typedef struct router_state router_state;
/* dragonfly compute node data structure */
struct terminal_state
{
uint64_t packet_counter;
int packet_gen;
int packet_fin;
// Dragonfly specific parameters
unsigned int router_id;
unsigned int terminal_id;
// Each terminal will have an input and output channel with the router
int* vc_occupancy; // NUM_VC
int num_vcs;
tw_stime terminal_available_time;
terminal_plus_message_list **terminal_msgs;
terminal_plus_message_list **terminal_msgs_tail;
int in_send_loop;
struct mn_stats dragonfly_stats_array[CATEGORY_MAX];
struct rc_stack * st;
int issueIdle;
int terminal_length;
const char * anno;
const dragonfly_plus_param *params;
struct qhash_table *rank_tbl;
uint64_t rank_tbl_pop;
tw_stime total_time;
uint64_t total_msg_size;
double total_hops;
long finished_msgs;
long finished_chunks;
long finished_packets;
tw_stime * last_buf_full;
tw_stime busy_time;
tw_stime max_latency;
tw_stime min_latency;
char output_buf[4096];
/* For LP suspend functionality */
int error_ct;
/* For sampling */
long fin_chunks_sample;
long data_size_sample;
double fin_hops_sample;
tw_stime fin_chunks_time;
tw_stime busy_time_sample;
char sample_buf[4096];
struct dfly_cn_sample * sample_stat;
int op_arr_size;
int max_arr_size;
/* for logging forward and reverse events */
long fwd_events;
long rev_events;
};
/* terminal event type (1-4) */
typedef enum event_t
{
T_GENERATE=1,
T_ARRIVE,
T_SEND,
T_BUFFER,
R_SEND,
R_ARRIVE,
R_BUFFER,
} event_t;
/* whether the last hop of a packet was global, local or a terminal */
enum last_hop
{
GLOBAL=1,
LOCAL,
TERMINAL,
ROOT
};
/* three forms of routing algorithms available, adaptive routing is not
* accurate and fully functional in the current version as the formulas
* for detecting load on global channels are not very accurate */
enum ROUTING_ALGO
{
MINIMAL = 1,
NON_MINIMAL,
ADAPTIVE,
PROG_ADAPTIVE
};
enum LINK_TYPE
{
GREEN,
BLACK,
};
enum router_type
{
SPINE = 1,
LEAF
};
struct router_state
{
unsigned int router_id;
int group_id;
int op_arr_size;
int max_arr_size;
router_type dfp_router_type; //Enum to specify whether this router is a spine or a leaf
int* global_channel;
tw_stime* next_output_available_time;
tw_stime** last_buf_full;
tw_stime* busy_time;
tw_stime* busy_time_sample;
terminal_plus_message_list ***pending_msgs;
terminal_plus_message_list ***pending_msgs_tail;
terminal_plus_message_list ***queued_msgs;
terminal_plus_message_list ***queued_msgs_tail;
int *in_send_loop;
int *queued_count;
struct rc_stack * st;
int** vc_occupancy;
int64_t* link_traffic;
int64_t * link_traffic_sample;
const char * anno;
const dragonfly_plus_param *params;
char output_buf[4096];
char output_buf2[4096];
struct dfly_router_sample * rsamples;
long fwd_events;
long rev_events;
};
static short routing = MINIMAL;
static tw_stime dragonfly_total_time = 0;
static tw_stime dragonfly_max_latency = 0;
static long long total_hops = 0;
static long long N_finished_packets = 0;
static long long total_msg_sz = 0;
static long long N_finished_msgs = 0;
static long long N_finished_chunks = 0;
static int dragonfly_rank_hash_compare(
void *key, struct qhash_head *link)
{
struct dfly_hash_key *message_key = (struct dfly_hash_key *)key;
struct dfly_qhash_entry *tmp = NULL;
tmp = qhash_entry(link, struct dfly_qhash_entry, hash_link);
if (tmp->key.message_id == message_key->message_id
&& tmp->key.sender_id == message_key->sender_id)
return 1;
return 0;
}
static int dragonfly_hash_func(void *k, int table_size)
{
struct dfly_hash_key *tmp = (struct dfly_hash_key *)k;
uint32_t pc = 0, pb = 0;
bj_hashlittle2(tmp, sizeof(*tmp), &pc, &pb);
return (int)(pc % (table_size - 1));
/*uint64_t key = (~tmp->message_id) + (tmp->message_id << 18);
key = key * 21;
key = ~key ^ (tmp->sender_id >> 4);
key = key * tmp->sender_id;
return (int)(key & (table_size - 1));*/
}
/* convert GiB/s and bytes to ns */
static tw_stime bytes_to_ns(uint64_t bytes, double GB_p_s)
{
tw_stime time;
/* bytes to GB */
time = ((double)bytes)/(1024.0*1024.0*1024.0);
/* GiB to s */
time = time / GB_p_s;
/* s to ns */
time = time * 1000.0 * 1000.0 * 1000.0;
return(time);
}
/* returns the dragonfly message size */
int dragonfly_plus_get_msg_sz(void)
{
return sizeof(terminal_plus_message);
}
static void free_tmp(void * ptr)
{
struct dfly_qhash_entry * dfly = (dfly_qhash_entry *)ptr;
if(dfly->remote_event_data)
free(dfly->remote_event_data);
if(dfly)
free(dfly);
}
static void append_to_terminal_plus_message_list(
terminal_plus_message_list ** thisq,
terminal_plus_message_list ** thistail,
int index,
terminal_plus_message_list *msg) {
if(thisq[index] == NULL) {
thisq[index] = msg;
} else {
thistail[index]->next = msg;
msg->prev = thistail[index];
}
thistail[index] = msg;
}
static void prepend_to_terminal_plus_message_list(
terminal_plus_message_list ** thisq,
terminal_plus_message_list ** thistail,
int index,
terminal_plus_message_list *msg) {
if(thisq[index] == NULL) {
thistail[index] = msg;
} else {
thisq[index]->prev = msg;
msg->next = thisq[index];
}
thisq[index] = msg;
}
static terminal_plus_message_list* return_head(
terminal_plus_message_list ** thisq,
terminal_plus_message_list ** thistail,
int index) {
terminal_plus_message_list *head = thisq[index];
if(head != NULL) {
thisq[index] = head->next;
if(head->next != NULL) {
head->next->prev = NULL;
head->next = NULL;
} else {
thistail[index] = NULL;
}
}
return head;
}
static terminal_plus_message_list* return_tail(
terminal_plus_message_list ** thisq,
terminal_plus_message_list ** thistail,
int index) {
terminal_plus_message_list *tail = thistail[index];
assert(tail);
if(tail->prev != NULL) {
tail->prev->next = NULL;
thistail[index] = tail->prev;
tail->prev = NULL;
} else {
thistail[index] = NULL;
thisq[index] = NULL;
}
return tail;
}
static void dragonfly_read_config(const char * anno, dragonfly_plus_param *params){
/*Adding init for router magic number*/
uint32_t h1 = 0, h2 = 0;
bj_hashlittle2(LP_METHOD_NM_ROUT, strlen(LP_METHOD_NM_ROUT), &h1, &h2);
router_magic_num = h1 + h2;
bj_hashlittle2(LP_METHOD_NM_TERM, strlen(LP_METHOD_NM_TERM), &h1, &h2);
terminal_magic_num = h1 + h2;
// shorthand
dragonfly_plus_param *p = params;
int myRank;
MPI_Comm_rank(MPI_COMM_CODES, &myRank);
int rc = configuration_get_value_int(&config, "PARAMS", "local_vc_size", anno, &p->local_vc_size);
if(rc) {
p->local_vc_size = 1024;
fprintf(stderr, "Buffer size of local channels not specified, setting to %d\n", p->local_vc_size);
}
rc = configuration_get_value_int(&config, "PARAMS", "global_vc_size", anno, &p->global_vc_size);
if(rc) {
p->global_vc_size = 2048;
fprintf(stderr, "Buffer size of global channels not specified, setting to %d\n", p->global_vc_size);
}
rc = configuration_get_value_int(&config, "PARAMS", "cn_vc_size", anno, &p->cn_vc_size);
if(rc) {
p->cn_vc_size = 1024;
fprintf(stderr, "Buffer size of compute node channels not specified, setting to %d\n", p->cn_vc_size);
}
rc = configuration_get_value_int(&config, "PARAMS", "chunk_size", anno, &p->chunk_size);
if(rc) {
p->chunk_size = 512;
fprintf(stderr, "Chunk size for packets is specified, setting to %d\n", p->chunk_size);
}
rc = configuration_get_value_double(&config, "PARAMS", "local_bandwidth", anno, &p->local_bandwidth);
if(rc) {
p->local_bandwidth = 5.25;
fprintf(stderr, "Bandwidth of local channels not specified, setting to %lf\n", p->local_bandwidth);
}
rc = configuration_get_value_double(&config, "PARAMS", "global_bandwidth", anno, &p->global_bandwidth);
if(rc) {
p->global_bandwidth = 4.7;
fprintf(stderr, "Bandwidth of global channels not specified, setting to %lf\n", p->global_bandwidth);
}
rc = configuration_get_value_double(&config, "PARAMS", "cn_bandwidth", anno, &p->cn_bandwidth);
if(rc) {
p->cn_bandwidth = 5.25;
fprintf(stderr, "Bandwidth of compute node channels not specified, setting to %lf\n", p->cn_bandwidth);
}
rc = configuration_get_value_double(&config, "PARAMS", "router_delay", anno,
&p->router_delay);
if(rc) {
p->router_delay = 100;
}
configuration_get_value(&config, "PARAMS", "cn_sample_file", anno, cn_sample_file,
MAX_NAME_LENGTH);
configuration_get_value(&config, "PARAMS", "rt_sample_file", anno, router_sample_file,
MAX_NAME_LENGTH);
char routing_str[MAX_NAME_LENGTH];
configuration_get_value(&config, "PARAMS", "routing", anno, routing_str,
MAX_NAME_LENGTH);
if(strcmp(routing_str, "minimal") == 0)
routing = MINIMAL;
else if(strcmp(routing_str, "nonminimal")==0 ||
strcmp(routing_str,"non-minimal")==0)
routing = NON_MINIMAL;
else if (strcmp(routing_str, "adaptive") == 0)
routing = ADAPTIVE;
else if (strcmp(routing_str, "prog-adaptive") == 0)
routing = PROG_ADAPTIVE;
else
{
fprintf(stderr,
"No routing protocol specified, setting to minimal routing\n");
routing = -1;
}
/* MM: This should be 2 for dragonfly plus*/
p->num_vcs = 2;
rc = configuration_get_value_int(&config, "PARAMS", "num_groups", anno, &p->num_groups);
if(rc) {
printf("Number of groups not specified. Aborting");
MPI_Abort(MPI_COMM_CODES, 1);
}
/* MM: Don't need the following four config variables. Remove. */
rc = configuration_get_value_int(&config, "PARAMS", "num_col_chans", anno, &p->num_col_chans);
if(rc) {
// printf("\n Number of links connecting chassis not specified, setting to default value 3 ");
p->num_col_chans = 1;
}