Commit a8eeeab9 authored by Misbah Mubarak's avatar Misbah Mubarak

Adding dragonfly network model with codes, model-net-test case run time...

Adding dragonfly network model with codes, model-net-test case run time parameters are updated (see README_MN_TEST.txt for details)
parent f32af872
/*
* * Copyright (C) 2011, University of Chicago
* *
* * See COPYRIGHT notice in top-level directory.
* */
#ifndef INC_dragonfly_h
#define INC_dragonfly_h
#include <ross.h>
#include "codes/codes_mapping.h"
#include "codes/codes.h"
#include "codes/model-net.h"
#include "codes/model-net-method.h"
#define CHUNK_SIZE 32.0
#define CREDIT_SIZE 8
#define MEAN_PROCESS 1.0
#define MAX_NAME_LENGTH 256
// debugging parameters
#define TRACK 235221
#define PRINT_ROUTER_TABLE 1
#define DEBUG 1
// arrival rate
static double MEAN_INTERVAL=200.0;
/* radix of a dragonfly router = number of global channels + number of
* compute node channels + number of local router channels */
static int radix=0;
/* configurable parameters, coming from the codes config file*/
/* number of virtual channels, number of routers comes from the
* config file, number of compute nodes, global channels and group
* is calculated from these configurable parameters */
int num_vcs, num_routers, num_cn, num_global_channels, num_groups;
/* configurable parameters, global channel, local channel and
* compute node bandwidth */
double global_bandwidth, local_bandwidth, cn_bandwidth;
/*configurable parameters, global virtual channel size, local
* virtual channel size and compute node channel size */
int global_vc_size, local_vc_size, cn_vc_size;
/* global variables for codes mapping */
char lp_group_name[MAX_NAME_LENGTH], lp_type_name[MAX_NAME_LENGTH];
int mapping_grp_id, mapping_type_id, mapping_rep_id, mapping_offset;
/* sets up the dragonfly initial network parameters like number of dragonfly groups,
* number of compute nodes, number of global channels, bandwidth etc. */
static void dragonfly_setup(const void* net_params);
/* reports dragonfly statistics like average packet delay, average number of hops
* traversed and maximum packet latency */
static void dragonfly_report_stats();
/* dragonfly packet event method called by modelnet, this method triggers the packet
* generate event of dragonfly and attached remote and local events to the last packet
* of the message */
static void dragonfly_packet_event(char* category, tw_lpid final_dest_lp, int packet_size, int remote_event_size, const void* remote_event, int self_event_size, const void* self_event, tw_lp *sender, int is_last_pckt);
/* returns dragonfly message size */
static int dragonfly_get_msg_sz(void);
/* reverse handler for dragonfly packet event */
static void dragonfly_packet_event_rc(tw_lp *sender);
/* returns the lp type of dragonfly compute node (terminal) */
static const tw_lptype* dragonfly_get_cn_lp_type(void);
/* returns the lp type of dragonfly router */
static const tw_lptype* dragonfly_get_router_lp_type(void);
/* data structure for dragonfly statistics */
struct model_net_method dragonfly_method =
{
.method_name = "dragonfly",
.mn_setup = dragonfly_setup,
.model_net_method_packet_event = dragonfly_packet_event,
.model_net_method_packet_event_rc = dragonfly_packet_event_rc,
.mn_get_lp_type = dragonfly_get_cn_lp_type,
.mn_get_msg_sz = dragonfly_get_msg_sz,
.mn_report_stats = dragonfly_report_stats,
};
/* handles terminal and router events like packet generate/send/receive/buffer */
typedef enum event_t event_t;
typedef struct terminal_state terminal_state;
typedef struct terminal_message terminal_message;
typedef struct router_state router_state;
/* dragonfly compute node data structure */
struct terminal_state
{
unsigned long long packet_counter;
// Dragonfly specific parameters
unsigned int router_id;
unsigned int terminal_id;
// Each terminal will have an input and output channel with the router
int* vc_occupancy; // NUM_VC
int* output_vc_state;
tw_stime terminal_available_time;
tw_stime next_credit_available_time;
// Terminal generate, sends and arrival T_SEND, T_ARRIVAL, T_GENERATE
// Router-Router Intra-group sends and receives RR_LSEND, RR_LARRIVE
// Router-Router Inter-group sends and receives RR_GSEND, RR_GARRIVE
};
/* terminal event type (1-4) */
enum event_t
{
T_GENERATE=1,
T_ARRIVE,
T_SEND,
T_BUFFER,
R_SEND,
R_ARRIVE,
R_BUFFER
};
/* status of a virtual channel can be idle, active, allocated or wait for credit */
enum vc_status
{
VC_IDLE,
VC_ACTIVE,
VC_ALLOC,
VC_CREDIT
};
/* whether the last hop of a packet was global, local or a terminal */
enum last_hop
{
GLOBAL,
LOCAL,
TERMINAL
};
/* three forms of routing algorithms available, adaptive routing is not
* accurate and fully functional in the current version as the formulas
* for detecting load on global channels are not very accurate */
enum ROUTING_ALGO
{
MINIMAL,
NON_MINIMAL,
ADAPTIVE
};
/* this message is used for both dragonfly compute nodes and routers */
struct terminal_message
{
/* flit travel start time*/
tw_stime travel_start_time;
/* packet ID of the flit */
unsigned long long packet_ID;
/* event type of the flit */
short type;
/* category: comes from codes */
char category[MAX_NAME_LENGTH];
/* final destination LP ID, this comes from codes can be a server or any other LP type*/
tw_lpid final_dest_gid;
/*sending LP ID from CODES, can be a server or any other LP type */
tw_lpid sender_lp;
/* destination terminal ID of the dragonfly */
unsigned int dest_terminal_id;
/* source terminal ID of the dragonfly */
unsigned int src_terminal_id;
/* number of hops traversed by the packet */
short my_N_hop;
/* Intermediate LP ID from which this message is coming */
unsigned int intm_lp_id;
short old_vc;
short saved_vc;
/* last hop of the message, can be a terminal, local router or global router */
short last_hop;
// For buffer message
short vc_index;
int input_chan;
int output_chan;
tw_stime saved_available_time;
tw_stime saved_credit_time;
int intm_group_id;
short chunk_id;
int packet_size;
int remote_event_size_bytes;
int local_event_size_bytes;
};
struct router_state
{
unsigned int router_id;
unsigned int group_id;
int* global_channel;
tw_stime* next_output_available_time;
tw_stime* next_credit_available_time;
int* vc_occupancy;
int* output_vc_state;
};
short routing = MINIMAL;
int minimal_count, nonmin_count;
int adaptive_threshold;
int head_delay;
int num_packets;
int num_chunks;
static tw_stime dragonfly_total_time = 0;
static tw_stime dragonfly_max_latency = 0;
int num_vc;
int num_terminal=0, num_router=0;
int total_routers, total_terminals, total_mpi_procs;
static long long total_hops = 0;
static long long N_finished_packets = 0;
#endif
......@@ -34,10 +34,15 @@ struct simplenet_param
struct dragonfly_param
{
char* name;
int num_routers; /*Number of routers in a group*/
int num_nodes; /*Number of compute nodes connected to a router*/
int num_chans; /*Number of global channels connected to each router*/
double local_bandwidth;/* bandwidth of the router-router channels within a group */
double global_bandwidth;/* bandwidth of the inter-group router connections */
double cn_bandwidth;/* bandwidth of the compute node channels connected to routers */
int num_vcs; /* number of virtual channels */
int local_vc_size; /* buffer size of the router-router channels */
int global_vc_size; /* buffer size of the global channels */
int cn_vc_size; /* buffer size of the compute node channels */
short routing; /* minimal or non-minimal routing */
};
struct torus_param
......@@ -49,10 +54,7 @@ struct torus_param
int num_vc; /* number of virtual channels for each torus link */
float mean_process;/* mean process time for each flit */
};
/* NOTE: the following auxilliary functions are probably wrong; just leaving
* these examples from simplenet for reference purposes.
*
* In general we need to figure out how to pass configuration information to
/* In general we need to figure out how to pass configuration information to
* the methods and we need to be able to calculate ross event message size.
*/
/*Initialize the network by specifying the network parameters. The
......
/*
* * Copyright (C) 2011, University of Chicago
* *
* * See COPYRIGHT notice in top-level directory.
* */
#ifndef INC_torus_h
#define INC_torus_h
......@@ -16,21 +22,35 @@
#define MAX_NAME_LENGTH 256
#define TRACE -1
/* Torus network model implementation of codes, implements the modelnet API */
// Total number of nodes in torus, calculate in main
int N_nodes = 1;
/* Link bandwidth for each torus link, configurable from the config file */
double link_bandwidth;
/* buffer size of each torus link, configurable */
int buffer_size;
/* number of virtual channels for each torus link, configurable */
int num_vc;
/* number of torus dimensions, configurable */
int n_dims;
/* length of each torus dimension, configurable */
int * dim_length;
/* factor, used in torus coordinate calculation */
int * factor;
/* half length of each dimension, used in torus coordinates calculation */
int * half_length;
/* codes mapping group name, lp type name */
char grp_name[MAX_NAME_LENGTH], type_name[MAX_NAME_LENGTH];
/* codes mapping group id, lp type id, repetition id and offset */
int grp_id, lp_type_id, rep_id, offset;
/* nodes event enumeration, packet generation, send, receive and buffer event types */
typedef enum nodes_event_t nodes_event_t;
/* state of a torus compute node (come up with a better name instead of compute node?)*/
typedef struct nodes_state nodes_state;
/* torus message--- can be a packet or a flit */
typedef struct nodes_message nodes_message;
/* Issues a torus packet event call */
......@@ -44,11 +64,16 @@ static void torus_packet_event(
const void* self_event,
tw_lp *sender,
int is_last_pckt);
/* torus reverse event handler */
static void torus_packet_event_rc(tw_lp *sender);
/* torus setup function, sets up configurable parameters like torus dimensions,
* length of each dimension, channel bandwidth, buffer size etc. */
static void torus_setup(const void* net_params);
/* returns size of the torus message */
static int torus_get_msg_sz(void);
/* returns torus lp type */
static const tw_lptype* torus_get_lp_type(void);
/* reports torus statistics */
static void torus_report_stats(void);
/* data structure for torus statistics */
......@@ -63,6 +88,7 @@ struct model_net_method torus_method =
.mn_report_stats = torus_report_stats,
};
/* event type of each torus message, can be packet generate, flit arrival, flit send or credit */
enum nodes_event_t
{
GENERATE = 1,
......@@ -71,62 +97,87 @@ enum nodes_event_t
CREDIT,
};
/* state of a torus node */
struct nodes_state
{
/* counts the number of packets sent from this compute node */
unsigned long long packet_counter;
/* availability time of each torus link */
tw_stime** next_link_available_time;
/* availability of each torus credit link */
tw_stime** next_credit_available_time;
/* next flit generate time */
tw_stime** next_flit_generate_time;
/* buffer size for each torus virtual channel */
int** buffer;
/* coordinates of the current torus node */
int* dim_position;
/* neighbor LP ids for this torus node */
int* neighbour_minus_lpID;
int* neighbour_plus_lpID;
int source_dim;
int direction;
};
struct nodes_message
{
/* category: comes from codes message */
char category[CATEGORY_NAME_MAX];
/* time the packet was generated */
tw_stime travel_start_time;
/* for reverse event computation*/
tw_stime saved_available_time;
/* packet ID */
unsigned long long packet_ID;
/* event type of the message */
nodes_event_t type;
/* for reverse computation */
int saved_src_dim;
int saved_src_dir;
/* coordinates of the destination torus nodes */
int* dest;
/* final destination LP ID, comes from codes, can be a server or any other I/O LP type */
tw_lpid final_dest_gid;
/* destination torus node of the message */
tw_lpid dest_lp;
/* LP ID of the sender, comes from codes, can be a server or any other I/O LP type */
tw_lpid sender_lp;
/* number of hops traversed by the packet */
int my_N_hop;
/* source dimension of the message */
int source_dim;
/* source direction of the message */
int source_direction;
/* next torus hop that the packet will traverse */
int next_stop;
/* size of the torus packet */
int packet_size;
/* chunk id of the flit (distinguishes flits) */
short chunk_id;
// for codes local and remote events
/* for codes local and remote events, only carried by the last packet of the message */
int local_event_size_bytes;
int remote_event_size_bytes;
};
/* for calculating torus model statistics, average and maximum travel time of a packet */
tw_stime average_travel_time = 0;
tw_stime total_time = 0;
tw_stime max_latency = 0;
/* indicates delays calculated through the bandwidth calculation of the torus link */
float head_delay=0.0;
float credit_delay = 0.0;
/* number of finished packets on each PE */
static long long N_finished_packets = 0;
/* total number of hops traversed by a message on each PE */
static long long total_hops = 0;
// run time arguments
int num_packets;
/* number of chunks/flits in each torus packet, calculated through the size of each flit (32 bytes by default) */
int num_chunks;
int packet_offset = 0;
#endif
......@@ -3,11 +3,13 @@ lib_LIBRARIES += src/libcodes-net.a
nobase_include_HEADERS = \
codes/model-net.h \
codes/torus.h \
codes/dragonfly.h \
codes/model-net-method.h
src_libcodes_net_a_SOURCES = \
src/models/networks/model-net/model-net-method.h \
src/models/networks/model-net/model-net.c \
src/models/networks/model-net/simplenet-upd.c \
src/models/networks/model-net/torus.c
src/models/networks/model-net/torus.c \
src/models/networks/model-net/dragonfly.c
......@@ -6,3 +6,8 @@ end of the simulation.
2- Mapping:
- Update the mapping code to handle remainder (where number of ROSS LPs are not evenly divisible by the number of PEs).
- Modify the get_lp_info function to return the LP count of the given LP? Can help with local lp count calculation.
- Remove the group id and type id arguments from get_lp_info function.
Make the model-net test case running for two different types of networks?
- Use the I/O activity trace logs from BG/Q and BG/P for I/O modelling.
......@@ -4,9 +4,12 @@ will be executed in sequential mode.
You can also run the test program manually in parallel (conservative or
optimistic) mode as follows:
mpiexec -n 4 tests/modelnet-test --sync=2 (optional --nkp=n) tests/modelnet-test.conf
mpiexec -n 4 tests/modelnet-test --sync=2 --num_servers=n (optional --nkp=n) tests/modelnet-test.conf
<or>
mpiexec -n 4 tests/modelnet-test --sync=3 (optional --nkp=n) tests/modelnet-test.conf
mpiexec -n 4 tests/modelnet-test --sync=3 --num_servers=n (optional --nkp=n) tests/modelnet-test.conf
Note: The num_servers runtime argument should match the number of server repetitions specified in the modelnet-test config file.
(1)- To run the modelnet test with the simplenet network plugin, use tests/modelnet-test.conf (default setting).
(2)- To run the modelnet test with the torus network plugin, use tests/modelnet-test-torus.conf file.
This diff is collapsed.
......@@ -19,6 +19,7 @@ static void torus_setup(const void* net_params)
dim_length[i] = t_param->dim_length[i]; /* TODO, read comma separated values from files */
}
/* torus packet reverse event */
static void torus_packet_event_rc(tw_lp *sender)
{
codes_local_latency_reverse(sender);
......@@ -75,6 +76,7 @@ static void torus_packet_event(char* category, tw_lpid final_dest_lp, int packet
//printf("\n torus remote event %d local event %d last packet %d %lf ", msg->remote_event_size_bytes, msg->local_event_size_bytes, is_last_pckt, xfer_to_nic_time);
tw_event_send(e_new);
}
/*Initialize the torus model, this initialization part is borrowed from Ning's torus model */
static void torus_init( nodes_state * s,
tw_lp * lp )
......@@ -328,8 +330,8 @@ static void credit_send( nodes_state * s,
tw_event_send( buf_e );
}
// send a packet from one torus node to another torus node
// A packet can be up to 256 bytes on BG/L and BG/P and up to 512 bytes on BG/Q
/* send a packet from one torus node to another torus node
A packet can be up to 256 bytes on BG/L and BG/P and up to 512 bytes on BG/Q */
static void packet_send( nodes_state * s,
tw_bf * bf,
nodes_message * msg,
......@@ -460,6 +462,8 @@ static void packet_arrive( nodes_state * s,
}
}
/* reports torus statistics like average packet latency, maximum packet latency and average
* number of torus hops traversed by the packet */
static void torus_report_stats()
{
long long avg_hops, total_finished_packets;
......@@ -475,6 +479,7 @@ static void torus_report_stats()
printf(" Average number of hops traversed %f average message latency %lf us maximum message latency %lf us \n", (float)avg_hops/total_finished_packets, avg_time/(total_finished_packets*1000), max_time/1000);
}
}
/* finalize the torus node and free all event buffers available */
void
final( nodes_state * s, tw_lp * lp )
{
......@@ -490,7 +495,7 @@ static void packet_buffer_process( nodes_state * s, tw_bf * bf, nodes_message *
s->buffer[ msg->source_direction + ( msg->source_dim * 2 ) ][ 0 ]--;
}
/* reverse handler */
/* reverse handler for torus node */
static void node_rc_handler(nodes_state * s, tw_bf * bf, nodes_message * msg, tw_lp * lp)
{
switch(msg->type)
......@@ -547,7 +552,7 @@ static void node_rc_handler(nodes_state * s, tw_bf * bf, nodes_message * msg, tw
}
}
/* forward event handler */
/* forward event handler for torus node event */
static void event_handler(nodes_state * s, tw_bf * bf, nodes_message * msg, tw_lp * lp)
{
*(int *) bf = (int) 0;
......@@ -570,7 +575,7 @@ static void event_handler(nodes_state * s, tw_bf * bf, nodes_message * msg, tw_l
break;
}
}
/* event types */
tw_lptype torus_lp =
{
(init_f) torus_init,
......
LPGROUPS
{
MODELNET_GRP
{
repetitions="72";
server="1";
modelnet_dragonfly="1";
}
DRAGONFLY_ROUTER
{
repetitions="36";
dragonfly_router="1";
}
}
PARAMS
{
packet_size="512";
modelnet="dragonfly";
num_vcs="1";
num_routers="4";
local_vc_size="16384";
global_vc_size="32768";
cn_vc_size="16384";
local_bandwidth="5.25";
global_bandwidth="4.7";
cn_bandwidth="5.25";
message_size="1024";
routing="nonminimal";
}
......@@ -6,7 +6,7 @@
/* SUMMARY:
*
* This is a test harness for the simplenet module. It sets up a number of
* This is a test harness for the modelnet module. It sets up a number of
* servers, each of which is paired up with a simplenet LP to serve as the
* NIC. Each server exchanges a sequence of requests and acks with one peer
* and measures the throughput in terms of payload bytes (ack size) moved
......@@ -24,11 +24,11 @@
#include "codes/configuration.h"
#include "codes/lp-type-lookup.h"
#define NUM_SERVERS 16 /* number of servers */
#define NUM_REQS 1000 /* number of requests sent by each server */
#define PAYLOAD_SZ 2048 /* size of simulated data payload, bytes */
static int net_id = 0;
static int num_servers = 16;
typedef struct svr_msg svr_msg;
typedef struct svr_state svr_state;
......@@ -59,12 +59,6 @@ struct svr_msg
int incremented_flag; /* helper for reverse computation */
};
const tw_optdef app_opt[] = {
TWOPT_GROUP("Simple Network Test Model"),
TWOPT_END()
};
static void svr_init(
svr_state * ns,
tw_lp * lp);
......@@ -138,6 +132,14 @@ static void handle_req_rev_event(
tw_bf * b,
svr_msg * m,
tw_lp * lp);
const tw_optdef app_opt [] =
{
TWOPT_GROUP("Model net test case" ),
TWOPT_UINT("num_servers", num_servers, "num_servers"),
TWOPT_END()
};
int main(
int argc,
char **argv)
......@@ -153,9 +155,9 @@ int main(
tw_opt_add(app_opt);
tw_init(&argc, &argv);
if(argc != 2)
if(argc < 2)
{
printf("\n Usage: mpirun <args> --sync=2/3 mapping_file_name.conf");
printf("\n Usage: mpirun <args> --sync=2/3 mapping_file_name.conf (optional --nkp) num_servers");
MPI_Finalize();
return 0;
}
......@@ -174,13 +176,13 @@ int main(
g_tw_mapping=CUSTOM;
g_tw_custom_initial_mapping=&codes_mapping_init;
g_tw_custom_lp_global_to_local_map=&codes_mapping_to_lp;
g_tw_events_per_pe = 2048 * codes_mapping_get_lps_for_pe();
if(!message_size)
{
message_size = 256;
printf("\n Warning: ross message size not defined, resetting it to %d", message_size);
}
printf("\n Initializing %d lps on %d ", codes_mapping_get_lps_for_pe(), g_tw_mynode);
tw_define_lps(codes_mapping_get_lps_for_pe(), message_size, 0 );
//g_tw_events_per_pe = 2 * NUM_REQS * (codes_mapping_get_lps_for_pe());
......@@ -231,7 +233,7 @@ static void svr_init(
* simulation
*/
printf("\n Initializing servers %d ", (int)lp->gid);
// printf("\n Initializing servers %d ", (int)lp->gid);
/* skew each kickoff event slightly to help avoid event ties later on */
kickoff_time = g_tw_lookahead + tw_rand_unif(lp->rng);
......@@ -347,7 +349,7 @@ static void handle_kickoff_event(
ns->start_ts = tw_now(lp);
/* each server sends a request to the next highest server */
model_net_event(net_id, "test", (lp->gid + 2)%(NUM_SERVERS*2), PAYLOAD_SZ, sizeof(svr_msg), (const void*)m_remote, sizeof(svr_msg), (const void*)m_local, lp);
model_net_event(net_id, "test", (lp->gid + 2)%(num_servers*2), PAYLOAD_SZ, sizeof(svr_msg), (const void*)m_remote, sizeof(svr_msg), (const void*)m_local, lp);
ns->msg_sent_count++;
}
......@@ -431,7 +433,7 @@ static void handle_ack_event(
/* safety check that this request got to the right server */
// printf("\n m->src %d lp->gid %d ", m->src, lp->gid);
assert(m->src == (lp->gid + 2)%(NUM_SERVERS*2));
assert(m->src == (lp->gid + 2)%(num_servers*2));
if(ns->msg_sent_count < NUM_REQS)
{
......@@ -467,7 +469,7 @@ static void handle_req_event(
/* safety check that this request got to the right server */
// printf("\n m->src %d lp->gid %d ", m->src, lp->gid);
assert(lp->gid == (m->src + 2)%(NUM_SERVERS*2));
assert(lp->gid == (m->src + 2)%(num_servers*2));
ns->msg_recvd_count++;
/* send ack back */
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment