Commit 6440307d authored by Darius Buntinas's avatar Darius Buntinas
Browse files

[svn-r9034] Added parameter to allow netmods to force ready sends to use...

[svn-r9034] Added parameter to allow netmods to force ready sends to use rendezvous for large messages.  Reviewed by goodell@
parent 4e2ce456
......@@ -275,6 +275,7 @@ MPID_nem_mx_vc_init (MPIDI_VC_t *vc)
mx_get_info(MPID_nem_mx_local_endpoint, MX_COPY_SEND_MAX, NULL, 0, &threshold, sizeof(uint32_t));
vc->eager_max_msg_sz = threshold;
vc->ready_eager_max_msg_sz = threshold;
vc->rndvSend_fn = NULL;
vc->sendNoncontig_fn = MPID_nem_mx_SendNoncontig;
vc->comm_ops = &comm_ops;
......
......@@ -262,6 +262,7 @@ MPID_nem_newmad_vc_init (MPIDI_VC_t *vc)
MPIDI_CHANGE_VC_STATE(vc, ACTIVE);
vc->eager_max_msg_sz = 32768;
vc->ready_eager_max_msg_sz = 32768;
vc->rndvSend_fn = NULL;
vc->sendNoncontig_fn = MPID_nem_newmad_SendNoncontig;
vc->comm_ops = &comm_ops;
......
......@@ -451,7 +451,15 @@ MPID_nem_vc_init (MPIDI_VC_t *vc)
vc_ch->lmt_active_lmt = NULL;
vc_ch->lmt_enqueued = FALSE;
vc->eager_max_msg_sz = MPID_NEM_MPICH2_DATA_LEN - sizeof(MPIDI_CH3_Pkt_t);
if (MPIR_PARAM_SHM_EAGER_MAX_SZ == -1)
vc->eager_max_msg_sz = MPID_NEM_MPICH2_DATA_LEN - sizeof(MPIDI_CH3_Pkt_t);
else
vc->eager_max_msg_sz = MPIR_PARAM_SHM_EAGER_MAX_SZ;
if (MPIR_PARAM_SHM_READY_EAGER_MAX_SZ == -2)
vc->ready_eager_max_msg_sz = vc->eager_max_msg_sz; /* force local ready sends to use LMT */
else
vc->ready_eager_max_msg_sz = MPIR_PARAM_SHM_READY_EAGER_MAX_SZ;
MPIU_DBG_MSG(VC, VERBOSE, "vc using shared memory");
}
......
......@@ -747,7 +747,9 @@ typedef struct MPIDI_VC
/* eager message threshold */
int eager_max_msg_sz;
/* eager message threshold for ready sends. -1 means there's no limit */
int ready_eager_max_msg_sz;
/* noncontiguous send function pointer. Called to send a
noncontiguous message. Caller must initialize
sreq->dev.segment, _first and _size. Contiguous messages are
......@@ -756,7 +758,7 @@ typedef struct MPIDI_VC
void *header, MPIDI_msg_sz_t hdr_sz );
#ifdef ENABLE_COMM_OVERRIDES
MPIDI_Comm_ops_t *comm_ops;
MPIDI_Comm_ops_t *comm_ops;
#endif
/* Rather than have each channel define its own fields for the
......
......@@ -100,27 +100,41 @@ int MPID_Irsend(const void * buf, int count, MPI_Datatype datatype, int rank, in
goto fn_exit;
}
if (dt_contig) {
mpi_errno = MPIDI_CH3_EagerContigIsend( &sreq,
MPIDI_CH3_PKT_READY_SEND,
(char*)buf + dt_true_lb,
data_sz, rank, tag,
comm, context_offset );
}
else {
mpi_errno = MPIDI_CH3_EagerNoncontigSend( &sreq,
MPIDI_CH3_PKT_READY_SEND,
buf, count, datatype,
data_sz, rank, tag,
comm, context_offset );
/* If we're not complete, then add a reference to the datatype */
if (sreq && sreq->dev.OnDataAvail) {
if (vc->ready_eager_max_msg_sz < 0 || data_sz + sizeof(MPIDI_CH3_Pkt_ready_send_t) <= vc->ready_eager_max_msg_sz) {
if (dt_contig) {
mpi_errno = MPIDI_CH3_EagerContigIsend( &sreq,
MPIDI_CH3_PKT_READY_SEND,
(char*)buf + dt_true_lb,
data_sz, rank, tag,
comm, context_offset );
}
else {
mpi_errno = MPIDI_CH3_EagerNoncontigSend( &sreq,
MPIDI_CH3_PKT_READY_SEND,
buf, count, datatype,
data_sz, rank, tag,
comm, context_offset );
/* If we're not complete, then add a reference to the datatype */
if (sreq && sreq->dev.OnDataAvail) {
sreq->dev.datatype_ptr = dt_ptr;
MPID_Datatype_add_ref(dt_ptr);
}
}
} else {
/* Do rendezvous. This will be sent as a regular send not as
a ready send, so the receiver won't know to send an error
if the receive has not been posted */
MPIDI_Request_set_msg_type( sreq, MPIDI_REQUEST_RNDV_MSG );
mpi_errno = vc->rndvSend_fn( &sreq, buf, count, datatype, dt_contig,
data_sz, dt_true_lb, rank, tag, comm,
context_offset );
if (sreq && dt_ptr != NULL) {
sreq->dev.datatype_ptr = dt_ptr;
MPID_Datatype_add_ref(dt_ptr);
}
}
fn_exit:
*request = sreq;
......
......@@ -102,24 +102,38 @@ int MPID_Rsend(const void * buf, int count, MPI_Datatype datatype, int rank, int
goto fn_exit;
}
if (dt_contig)
{
mpi_errno = MPIDI_CH3_EagerContigSend( &sreq,
MPIDI_CH3_PKT_READY_SEND,
(char *)buf + dt_true_lb,
data_sz, rank, tag, comm,
context_offset );
}
else
{
MPIDI_Request_create_sreq(sreq, mpi_errno, goto fn_exit);
if (vc->ready_eager_max_msg_sz < 0 || data_sz + sizeof(MPIDI_CH3_Pkt_ready_send_t) <= vc->ready_eager_max_msg_sz) {
if (dt_contig)
{
mpi_errno = MPIDI_CH3_EagerContigSend( &sreq,
MPIDI_CH3_PKT_READY_SEND,
(char *)buf + dt_true_lb,
data_sz, rank, tag, comm,
context_offset );
}
else
{
MPIDI_Request_create_sreq(sreq, mpi_errno, goto fn_exit);
MPIDI_Request_set_type(sreq, MPIDI_REQUEST_TYPE_SEND);
mpi_errno = MPIDI_CH3_EagerNoncontigSend( &sreq,
MPIDI_CH3_PKT_READY_SEND,
buf, count, datatype,
data_sz, rank, tag,
comm, context_offset );
}
} else {
/* Do rendezvous. This will be sent as a regular send not as
a ready send, so the receiver won't know to send an error
if the receive has not been posted */
MPIDI_Request_create_sreq(sreq, mpi_errno, goto fn_exit);
MPIDI_Request_set_type(sreq, MPIDI_REQUEST_TYPE_SEND);
mpi_errno = MPIDI_CH3_EagerNoncontigSend( &sreq,
MPIDI_CH3_PKT_READY_SEND,
buf, count, datatype,
data_sz, rank, tag,
comm, context_offset );
}
mpi_errno = vc->rndvSend_fn( &sreq, buf, count, datatype, dt_contig,
data_sz, dt_true_lb, rank, tag, comm,
context_offset );
/* Note that we don't increase the ref count on the datatype
because this is a blocking call, and the calling routine
must wait until sreq completes */
}
fn_exit:
*request = sreq;
......
......@@ -740,6 +740,8 @@ int MPIDI_VC_Init( MPIDI_VC_t *vc, MPIDI_PG_t *pg, int rank )
vc->rndvSend_fn = MPIDI_CH3_RndvSend;
vc->rndvRecv_fn = MPIDI_CH3_RecvRndv;
vc->eager_max_msg_sz = MPIDI_CH3_EAGER_MAX_MSG_SIZE;
vc->ready_eager_max_msg_sz = -1; /* no limit */;
vc->sendNoncontig_fn = MPIDI_CH3_SendNoncontig_iov;
#ifdef ENABLE_COMM_OVERRIDES
vc->comm_ops = NULL;
......
......@@ -307,6 +307,26 @@ parameters:
This parameter controls the number of times to retry the
gethostbyname() function before giving up.
- category : nemesis
name : SHM_EAGER_MAX_SZ
type : int
default : -1
description : >-
This parameter controls the message size at which Nemesis
switches from eager to rendezvous mode. If this parameter
is set to -1, then Nemesis will choose an appropriate value.
- category : nemesis
name : SHM_READY_EAGER_MAX_SZ
type : int
default : -2
description : >-
This parameter controls the message size at which Nemesis
switches from eager to rendezvous mode for ready-send
messages. If this parameter is set to -1, then ready messages
will always be sent eagerly. If this parameter is set to -2,
then Nemesis will choose an appropriate value.
##############################################################
# debugging parameters
- category : debugger
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment