Commit f384820a authored by Jonathan Jenkins's avatar Jonathan Jenkins
Browse files

model-net 'pull' (e.g., RDMA) support

- pull itself not yet tested
parent e5cd0779
...@@ -65,7 +65,7 @@ static tw_lpid dragonfly_find_local_device(tw_lp *sender); ...@@ -65,7 +65,7 @@ static tw_lpid dragonfly_find_local_device(tw_lp *sender);
/* dragonfly packet event method called by modelnet, this method triggers the packet /* dragonfly packet event method called by modelnet, this method triggers the packet
* generate event of dragonfly and attached remote and local events to the last packet * generate event of dragonfly and attached remote and local events to the last packet
* of the message */ * of the message */
static tw_stime dragonfly_packet_event(char* category, tw_lpid final_dest_lp, uint64_t packet_size, tw_stime offset, int remote_event_size, const void* remote_event, int self_event_size, const void* self_event, tw_lp *sender, int is_last_pckt); static tw_stime dragonfly_packet_event(char* category, tw_lpid final_dest_lp, uint64_t packet_size, int is_pull, uint64_t pull_size, tw_stime offset, int remote_event_size, const void* remote_event, int self_event_size, const void* self_event, tw_lp *sender, int is_last_pckt);
/* returns dragonfly message size */ /* returns dragonfly message size */
static int dragonfly_get_msg_sz(void); static int dragonfly_get_msg_sz(void);
...@@ -187,6 +187,8 @@ struct terminal_message ...@@ -187,6 +187,8 @@ struct terminal_message
short vc_index; short vc_index;
int input_chan; int input_chan;
int output_chan; int output_chan;
int is_pull;
uint64_t pull_size;
tw_stime saved_available_time; tw_stime saved_available_time;
tw_stime saved_credit_time; tw_stime saved_credit_time;
......
...@@ -18,6 +18,8 @@ struct model_net_method ...@@ -18,6 +18,8 @@ struct model_net_method
char* category, char* category,
tw_lpid final_dest_lp, tw_lpid final_dest_lp,
uint64_t packet_size, uint64_t packet_size,
int is_pull,
uint64_t pull_size, /* only used when is_pull==1 */
tw_stime offset, tw_stime offset,
int remote_event_size, /* 0 means don't deliver remote event */ int remote_event_size, /* 0 means don't deliver remote event */
const void* remote_event, const void* remote_event,
......
...@@ -121,6 +121,9 @@ int model_net_set_params(); ...@@ -121,6 +121,9 @@ int model_net_set_params();
// setup the modelnet parameters // setup the modelnet parameters
int model_net_setup(char* net_name, uint64_t packet_size, const void* net_params); int model_net_setup(char* net_name, uint64_t packet_size, const void* net_params);
/* utility function to get the modelnet ID post-setup */
int model_net_get_id(char *net_name);
/* allocate and transmit a new event that will pass through model_net to /* allocate and transmit a new event that will pass through model_net to
* arrive at its destination: * arrive at its destination:
* *
...@@ -154,6 +157,7 @@ void model_net_event( ...@@ -154,6 +157,7 @@ void model_net_event(
char* category, char* category,
tw_lpid final_dest_lp, tw_lpid final_dest_lp,
uint64_t message_size, uint64_t message_size,
tw_stime offset,
int remote_event_size, int remote_event_size,
const void* remote_event, const void* remote_event,
int self_event_size, int self_event_size,
...@@ -181,6 +185,32 @@ void model_net_event_rc( ...@@ -181,6 +185,32 @@ void model_net_event_rc(
tw_lp *sender, tw_lp *sender,
uint64_t message_size); uint64_t message_size);
/* Issue a 'pull' from the memory of the destination LP, without
* requiring the destination LP to do event processing. This is meant as a
* simulation-based abstraction of RDMA. A control packet will be sent to the
* destination LP, the payload will be sent back to the requesting LP, and the
* requesting LP will be issued it's given completion event.
*
* Parameters are largely the same as model_net_event, with the following
* exceptions:
* - final_dest_lp is the lp to pull data from
* - self_event_size, self_event are applied at the requester upon receipt of
* the payload from the dest
*/
void model_net_pull_event(
int net_id,
char *category,
tw_lpid final_dest_lp,
uint64_t message_size,
tw_stime offset,
int self_event_size,
const void *self_event,
tw_lp *sender);
void model_net_pull_event_rc(
int net_id,
tw_lp *sender);
/* returns pointer to LP information for simplenet module */ /* returns pointer to LP information for simplenet module */
const tw_lptype* model_net_get_lp_type(int net_id); const tw_lptype* model_net_get_lp_type(int net_id);
......
...@@ -63,6 +63,8 @@ static tw_stime torus_packet_event( ...@@ -63,6 +63,8 @@ static tw_stime torus_packet_event(
char* category, char* category,
tw_lpid final_dest_lp, tw_lpid final_dest_lp,
uint64_t packet_size, uint64_t packet_size,
int is_pull,
uint64_t pull_size,
tw_stime offset, tw_stime offset,
int remote_event_size, int remote_event_size,
const void* remote_event, const void* remote_event,
...@@ -170,6 +172,9 @@ struct nodes_message ...@@ -170,6 +172,9 @@ struct nodes_message
/* chunk id of the flit (distinguishes flits) */ /* chunk id of the flit (distinguishes flits) */
short chunk_id; short chunk_id;
int is_pull;
uint64_t pull_size;
/* for codes local and remote events, only carried by the last packet of the message */ /* for codes local and remote events, only carried by the last packet of the message */
int local_event_size_bytes; int local_event_size_bytes;
int remote_event_size_bytes; int remote_event_size_bytes;
......
...@@ -56,7 +56,7 @@ static void dragonfly_report_stats() ...@@ -56,7 +56,7 @@ static void dragonfly_report_stats()
return; return;
} }
/* dragonfly packet event , generates a dragonfly packet on the compute node */ /* dragonfly packet event , generates a dragonfly packet on the compute node */
static tw_stime dragonfly_packet_event(char* category, tw_lpid final_dest_lp, uint64_t packet_size, tw_stime offset, int remote_event_size, const void* remote_event, int self_event_size, const void* self_event, tw_lp *sender, int is_last_pckt) static tw_stime dragonfly_packet_event(char* category, tw_lpid final_dest_lp, uint64_t packet_size, int is_pull, uint64_t pull_size, tw_stime offset, int remote_event_size, const void* remote_event, int self_event_size, const void* self_event, tw_lp *sender, int is_last_pckt)
{ {
tw_event * e_new; tw_event * e_new;
tw_stime xfer_to_nic_time; tw_stime xfer_to_nic_time;
...@@ -83,6 +83,8 @@ static tw_stime dragonfly_packet_event(char* category, tw_lpid final_dest_lp, ui ...@@ -83,6 +83,8 @@ static tw_stime dragonfly_packet_event(char* category, tw_lpid final_dest_lp, ui
msg->remote_event_size_bytes = 0; msg->remote_event_size_bytes = 0;
msg->local_event_size_bytes = 0; msg->local_event_size_bytes = 0;
msg->type = T_GENERATE; msg->type = T_GENERATE;
msg->is_pull = is_pull;
msg->pull_size = pull_size;
if(is_last_pckt) /* Its the last packet so pass in remote and local event information*/ if(is_last_pckt) /* Its the last packet so pass in remote and local event information*/
{ {
...@@ -363,15 +365,23 @@ if( msg->packet_ID == TRACK && msg->chunk_id == num_chunks-1) ...@@ -363,15 +365,23 @@ if( msg->packet_ID == TRACK && msg->chunk_id == num_chunks-1)
// Trigger an event on receiving server // Trigger an event on receiving server
if(msg->remote_event_size_bytes) if(msg->remote_event_size_bytes)
{ {
char* tmp_ptr = (char*)msg;
tmp_ptr += dragonfly_get_msg_sz();
ts = (1/cn_bandwidth) * msg->remote_event_size_bytes; ts = (1/cn_bandwidth) * msg->remote_event_size_bytes;
if (msg->is_pull){
int net_id = model_net_get_id("dragonfly");
model_net_event(net_id, msg->category, msg->sender_lp,
msg->pull_size, ts, msg->remote_event_size_bytes,
tmp_ptr, 0, NULL, lp);
}
else{
e = tw_event_new(msg->final_dest_gid, ts, lp); e = tw_event_new(msg->final_dest_gid, ts, lp);
m = tw_event_data(e); m = tw_event_data(e);
char* tmp_ptr = (char*)msg;
tmp_ptr += dragonfly_get_msg_sz();
memcpy(m, tmp_ptr, msg->remote_event_size_bytes); memcpy(m, tmp_ptr, msg->remote_event_size_bytes);
tw_event_send(e); tw_event_send(e);
} }
} }
}
int credit_delay = (1 / cn_bandwidth) * CREDIT_SIZE; int credit_delay = (1 / cn_bandwidth) * CREDIT_SIZE;
ts = credit_delay + tw_rand_exponential(lp->rng, credit_delay/1000); ts = credit_delay + tw_rand_exponential(lp->rng, credit_delay/1000);
...@@ -896,6 +906,12 @@ void router_rc_event_handler(router_state * s, tw_bf * bf, terminal_message * ms ...@@ -896,6 +906,12 @@ void router_rc_event_handler(router_state * s, tw_bf * bf, terminal_message * ms
tw_rand_reverse_unif(lp->rng); tw_rand_reverse_unif(lp->rng);
int output_port = msg->saved_vc/num_vcs; int output_port = msg->saved_vc/num_vcs;
s->next_credit_available_time[output_port] = msg->saved_available_time; s->next_credit_available_time[output_port] = msg->saved_available_time;
if (msg->chunk_id == num_chunks-1 &&
msg->remote_event_size_bytes &&
msg->is_pull){
int net_id = model_net_get_id("dragonfly");
model_net_event_rc(net_id, lp, msg->pull_size);
}
} }
break; break;
......
...@@ -48,6 +48,8 @@ struct loggp_message ...@@ -48,6 +48,8 @@ struct loggp_message
int event_size_bytes; /* size of simulator event message that will be tunnelled to destination */ int event_size_bytes; /* size of simulator event message that will be tunnelled to destination */
int local_event_size_bytes; /* size of simulator event message that delivered locally upon local completion */ int local_event_size_bytes; /* size of simulator event message that delivered locally upon local completion */
char category[CATEGORY_NAME_MAX]; /* category for communication */ char category[CATEGORY_NAME_MAX]; /* category for communication */
int is_pull;
uint64_t pull_size;
/* for reverse computation */ /* for reverse computation */
tw_stime net_send_next_idle_saved; tw_stime net_send_next_idle_saved;
...@@ -109,6 +111,8 @@ static tw_stime loggp_packet_event( ...@@ -109,6 +111,8 @@ static tw_stime loggp_packet_event(
char* category, char* category,
tw_lpid final_dest_lp, tw_lpid final_dest_lp,
uint64_t packet_size, uint64_t packet_size,
int is_pull,
uint64_t pull_size, /* only used when is_pull==1 */
tw_stime offset, tw_stime offset,
int remote_event_size, int remote_event_size,
const void* remote_event, const void* remote_event,
...@@ -299,6 +303,11 @@ static void handle_msg_ready_rev_event( ...@@ -299,6 +303,11 @@ static void handle_msg_ready_rev_event(
stat->recv_bytes -= m->net_msg_size_bytes; stat->recv_bytes -= m->net_msg_size_bytes;
stat->recv_time -= m->recv_time_saved; stat->recv_time -= m->recv_time_saved;
if (m->event_size_bytes && m->is_pull){
int net_id = model_net_get_id("loggp");
model_net_event_rc(net_id, lp, m->pull_size);
}
return; return;
} }
...@@ -350,13 +359,22 @@ static void handle_msg_ready_event( ...@@ -350,13 +359,22 @@ static void handle_msg_ready_event(
/* schedule event to final destination for when the recv is complete */ /* schedule event to final destination for when the recv is complete */
// printf("\n Remote message to LP %d ", m->final_dest_gid); // printf("\n Remote message to LP %d ", m->final_dest_gid);
e_new = tw_event_new(m->final_dest_gid, recv_queue_time, lp);
m_new = tw_event_data(e_new);
char* tmp_ptr = (char*)m; char* tmp_ptr = (char*)m;
tmp_ptr += loggp_get_msg_sz(); tmp_ptr += loggp_get_msg_sz();
if (m->is_pull){
/* call the model-net event */
int net_id = model_net_get_id("loggp");
model_net_event(net_id, m->category, m->src_gid, m->pull_size,
recv_queue_time, m->event_size_bytes, tmp_ptr, 0, NULL,
lp);
}
else{
e_new = tw_event_new(m->final_dest_gid, recv_queue_time, lp);
m_new = tw_event_data(e_new);
memcpy(m_new, tmp_ptr, m->event_size_bytes); memcpy(m_new, tmp_ptr, m->event_size_bytes);
tw_event_send(e_new); tw_event_send(e_new);
} }
}
return; return;
} }
...@@ -485,6 +503,8 @@ static tw_stime loggp_packet_event( ...@@ -485,6 +503,8 @@ static tw_stime loggp_packet_event(
char* category, char* category,
tw_lpid final_dest_lp, tw_lpid final_dest_lp,
uint64_t packet_size, uint64_t packet_size,
int is_pull,
uint64_t pull_size, /* only used when is_pull==1 */
tw_stime offset, tw_stime offset,
int remote_event_size, int remote_event_size,
const void* remote_event, const void* remote_event,
...@@ -515,6 +535,8 @@ static tw_stime loggp_packet_event( ...@@ -515,6 +535,8 @@ static tw_stime loggp_packet_event(
msg->event_size_bytes = 0; msg->event_size_bytes = 0;
msg->local_event_size_bytes = 0; msg->local_event_size_bytes = 0;
msg->event_type = MSG_START; msg->event_type = MSG_START;
msg->is_pull = is_pull;
msg->pull_size = pull_size;
tmp_ptr = (char*)msg; tmp_ptr = (char*)msg;
tmp_ptr += loggp_get_msg_sz(); tmp_ptr += loggp_get_msg_sz();
......
...@@ -10,6 +10,8 @@ ...@@ -10,6 +10,8 @@
#include "codes/model-net.h" #include "codes/model-net.h"
#include "codes/model-net-method.h" #include "codes/model-net-method.h"
#define PULL_MSG_SIZE 128
#define STR_SIZE 16 #define STR_SIZE 16
#define PROC_TIME 10.0 #define PROC_TIME 10.0
...@@ -48,6 +50,16 @@ int model_net_setup(char* name, ...@@ -48,6 +50,16 @@ int model_net_setup(char* name,
return -1; // indicating error return -1; // indicating error
} }
int model_net_get_id(char *name){
int i;
for(i=0; method_array[i] != NULL; i++) {
if(strcmp(method_array[i]->method_name, name) == 0) {
return i;
}
}
return -1;
}
void model_net_write_stats(tw_lpid lpid, struct mn_stats* stat) void model_net_write_stats(tw_lpid lpid, struct mn_stats* stat)
{ {
int ret; int ret;
...@@ -130,17 +142,19 @@ struct mn_stats* model_net_find_stats(const char* category, mn_stats mn_stats_ar ...@@ -130,17 +142,19 @@ struct mn_stats* model_net_find_stats(const char* category, mn_stats mn_stats_ar
return(&mn_stats_array[i]); return(&mn_stats_array[i]);
} }
void model_net_event( static void model_net_event_impl(
int net_id, int net_id,
char* category, char* category,
tw_lpid final_dest_lp, tw_lpid final_dest_lp,
uint64_t message_size, uint64_t message_size,
int is_pull,
uint64_t pull_msg_size,
tw_stime offset,
int remote_event_size, int remote_event_size,
const void* remote_event, const void* remote_event,
int self_event_size, int self_event_size,
const void* self_event, const void* self_event,
tw_lp *sender) tw_lp *sender) {
{
/* determine packet size for underlying method */ /* determine packet size for underlying method */
uint64_t packet_size = model_net_get_packet_size(net_id); uint64_t packet_size = model_net_get_packet_size(net_id);
uint64_t num_packets = message_size/packet_size; /* Number of packets to be issued by the API */ uint64_t num_packets = message_size/packet_size; /* Number of packets to be issued by the API */
...@@ -178,7 +192,7 @@ void model_net_event( ...@@ -178,7 +192,7 @@ void model_net_event(
* passed along through network hops and delivered to final_dest_lp * passed along through network hops and delivered to final_dest_lp
*/ */
tw_stime offset = (in_sequence) ? mn_msg_offset : 0.0; tw_stime poffset = (in_sequence) ? mn_msg_offset : 0.0;
for( i = 0; i < num_packets; i++ ) for( i = 0; i < num_packets; i++ )
{ {
/*Mark the last packet to the net method API*/ /*Mark the last packet to the net method API*/
...@@ -189,14 +203,49 @@ void model_net_event( ...@@ -189,14 +203,49 @@ void model_net_event(
packet_size = message_size - ((num_packets-1)*packet_size); packet_size = message_size - ((num_packets-1)*packet_size);
} }
/* Number of packets and packet ID is passed to the underlying network to mark the final packet for local event completion*/ /* Number of packets and packet ID is passed to the underlying network to mark the final packet for local event completion*/
offset += method_array[net_id]->model_net_method_packet_event(category, poffset += method_array[net_id]->model_net_method_packet_event(category,
final_dest_lp, packet_size, offset, remote_event_size, remote_event, final_dest_lp, packet_size, is_pull, pull_msg_size, poffset+offset,
self_event_size, self_event, sender, last); remote_event_size, remote_event, self_event_size, self_event, sender, last);
} }
if (in_sequence) mn_msg_offset = offset; if (in_sequence) mn_msg_offset = poffset;
return; return;
} }
void model_net_event(
int net_id,
char* category,
tw_lpid final_dest_lp,
uint64_t message_size,
tw_stime offset,
int remote_event_size,
const void* remote_event,
int self_event_size,
const void* self_event,
tw_lp *sender)
{
model_net_event_impl(net_id, category, final_dest_lp, message_size, 0, 0,
offset, remote_event_size, remote_event, self_event_size,
self_event, sender);
}
void model_net_pull_event(
int net_id,
char *category,
tw_lpid final_dest_lp,
uint64_t message_size,
tw_stime offset,
int self_event_size,
const void *self_event,
tw_lp *sender){
/* NOTE: for a pull, we are filling the *remote* event - it will be remote
* from the destination's POV */
model_net_event_impl(net_id, category, final_dest_lp, PULL_MSG_SIZE, 1,
message_size, offset, self_event_size, self_event, 0, NULL,
sender);
}
int model_net_set_params() int model_net_set_params()
{ {
char mn_name[MAX_NAME_LENGTH]; char mn_name[MAX_NAME_LENGTH];
...@@ -393,7 +442,7 @@ int model_net_set_params() ...@@ -393,7 +442,7 @@ int model_net_set_params()
printf("\n Invalid network argument %s ", mn_name); printf("\n Invalid network argument %s ", mn_name);
return net_id; return net_id;
} }
void model_net_event_rc( static void model_net_event_impl_rc(
int net_id, int net_id,
tw_lp *sender, tw_lp *sender,
uint64_t message_size) uint64_t message_size)
...@@ -418,6 +467,19 @@ void model_net_event_rc( ...@@ -418,6 +467,19 @@ void model_net_event_rc(
return; return;
} }
void model_net_event_rc(
int net_id,
tw_lp *sender,
uint64_t message_size){
model_net_event_impl_rc(net_id,sender,message_size);
}
void model_net_pull_event_rc(
int net_id,
tw_lp *sender) {
model_net_event_impl_rc(net_id, sender, PULL_MSG_SIZE);
}
/* returns the message size, can be either simplenet, dragonfly or torus message size*/ /* returns the message size, can be either simplenet, dragonfly or torus message size*/
static int model_net_get_msg_sz(int net_id) static int model_net_get_msg_sz(int net_id)
{ {
......
...@@ -48,6 +48,8 @@ struct sn_message ...@@ -48,6 +48,8 @@ struct sn_message
int event_size_bytes; /* size of simulator event message that will be tunnelled to destination */ int event_size_bytes; /* size of simulator event message that will be tunnelled to destination */
int local_event_size_bytes; /* size of simulator event message that delivered locally upon local completion */ int local_event_size_bytes; /* size of simulator event message that delivered locally upon local completion */
char category[CATEGORY_NAME_MAX]; /* category for communication */ char category[CATEGORY_NAME_MAX]; /* category for communication */
int is_pull; /* this message represents a pull request from the destination LP to the source */
uint64_t pull_size; /* data size to pull from dest LP */
/* for reverse computation */ /* for reverse computation */
tw_stime net_send_next_idle_saved; tw_stime net_send_next_idle_saved;
...@@ -102,6 +104,8 @@ static tw_stime simplenet_packet_event( ...@@ -102,6 +104,8 @@ static tw_stime simplenet_packet_event(
char* category, char* category,
tw_lpid final_dest_lp, tw_lpid final_dest_lp,
uint64_t packet_size, uint64_t packet_size,
int is_pull,
uint64_t pull_size, /* only used when is_pull==1 */
tw_stime offset, tw_stime offset,
int remote_event_size, int remote_event_size,
const void* remote_event, const void* remote_event,
...@@ -317,6 +321,11 @@ static void handle_msg_ready_rev_event( ...@@ -317,6 +321,11 @@ static void handle_msg_ready_rev_event(
stat->recv_bytes -= m->net_msg_size_bytes; stat->recv_bytes -= m->net_msg_size_bytes;
stat->recv_time = m->recv_time_saved; stat->recv_time = m->recv_time_saved;
if (m->event_size_bytes && m->is_pull){
int net_id = model_net_get_id("simplenet");
model_net_event_rc(net_id, lp, m->pull_size);
}
return; return;
} }
...@@ -357,16 +366,24 @@ static void handle_msg_ready_event( ...@@ -357,16 +366,24 @@ static void handle_msg_ready_event(
/* copy only the part of the message used by higher level */ /* copy only the part of the message used by higher level */
if(m->event_size_bytes) if(m->event_size_bytes)
{ {
char *tmp_ptr = (char*)m;
tmp_ptr += sn_get_msg_sz();
/* schedule event to final destination for when the recv is complete */ /* schedule event to final destination for when the recv is complete */
// printf("\n Remote message to LP %d ", m->final_dest_gid); // printf("\n Remote message to LP %d ", m->final_dest_gid);
if (m->is_pull){
/* call the model-net event */
int net_id = model_net_get_id("simplenet");
model_net_event(net_id, m->category, m->src_gid, m->pull_size,
recv_queue_time, m->event_size_bytes, tmp_ptr, 0, NULL,
lp);
}
else{
e_new = tw_event_new(m->final_dest_gid, recv_queue_time, lp); e_new = tw_event_new(m->final_dest_gid, recv_queue_time, lp);
m_new = tw_event_data(e_new); m_new = tw_event_data(e_new);
char* tmp_ptr = (char*)m;
tmp_ptr += sn_get_msg_sz();
memcpy(m_new, tmp_ptr, m->event_size_bytes); memcpy(m_new, tmp_ptr, m->event_size_bytes);
tw_event_send(e_new); tw_event_send(e_new);
} }
}
return; return;
} }
...@@ -481,6 +498,8 @@ static tw_stime simplenet_packet_event( ...@@ -481,6 +498,8 @@ static tw_stime simplenet_packet_event(
char* category, char* category,
tw_lpid final_dest_lp, tw_lpid final_dest_lp,
uint64_t packet_size, uint64_t packet_size,
int is_pull,
uint64_t pull_size, /* only used when is_pull == 1 */
tw_stime offset, tw_stime offset,
int remote_event_size, int remote_event_size,
const void* remote_event, const void* remote_event,
...@@ -511,6 +530,8 @@ static tw_stime simplenet_packet_event( ...@@ -511,6 +530,8 @@ static tw_stime simplenet_packet_event(
msg->event_size_bytes = 0; msg->event_size_bytes = 0;
msg->local_event_size_bytes = 0; msg->local_event_size_bytes = 0;
msg->event_type = MSG_START; msg->event_type = MSG_START;
msg->is_pull = is_pull;
msg->pull_size = pull_size;
tmp_ptr = (char*)msg; tmp_ptr = (char*)msg;
tmp_ptr += sn_get_msg_sz(); tmp_ptr += sn_get_msg_sz();
......