Commit c96f0509 authored by Pavan Balaji's avatar Pavan Balaji
Browse files

[svn-r3938] Use MPI_Ssend instead of MPI_Send for Gatherv with large communicators

(more than 32 processes for now). Just using plain MPI_Send was causing
the non-root processes to run ahead and queue up a lot of unexpected
messages on the root processes. This was: (i) causing performance loss,
and (ii) causing some MPICH2 derivatives (such as BG/P) to abort due to
lack of resources. We don't use MPI_Ssend for small communicators as it
adds overhead.

Reviewed by gropp.
parent d15e643a
......@@ -3531,6 +3531,7 @@ int MPID_VCR_Get_lpid(MPID_VCR vcr, int * lpid_ptr);
#define MPIR_GATHER_VSMALL_MSG 1024
#define MPIR_SCATTER_SHORT_MSG 2048 /* for intercommunicator scatter */
#define MPIR_GATHER_SHORT_MSG 2048 /* for intercommunicator scatter */
#define MPIR_GATHERV_MIN_PROCS 32
/* Tags for point to point operations which implement collective operations */
#define MPIR_BARRIER_TAG 1
......@@ -3567,6 +3568,8 @@ int MPIC_Send(void *buf, int count, MPI_Datatype datatype, int dest, int tag,
MPI_Comm comm);
int MPIC_Recv(void *buf, int count, MPI_Datatype datatype, int source, int tag,
MPI_Comm comm, MPI_Status *status);
int MPIC_Ssend(void *buf, int count, MPI_Datatype datatype, int dest, int tag,
MPI_Comm comm);
int MPIC_Sendrecv(void *sendbuf, int sendcount, MPI_Datatype sendtype,
int dest, int sendtag, void *recvbuf, int recvcount,
MPI_Datatype recvtype, int source, int recvtag,
......
......@@ -120,9 +120,16 @@ int MPIR_Gatherv (
}
else if (root != MPI_PROC_NULL) { /* non-root nodes, and in the intercomm. case, non-root nodes on remote side */
if (sendcnt)
mpi_errno = MPIC_Send(sendbuf, sendcnt, sendtype, root,
MPIR_GATHERV_TAG, comm);
if (sendcnt) {
if (comm_size >= MPIR_GATHERV_MIN_PROCS) {
mpi_errno = MPIC_Ssend(sendbuf, sendcnt, sendtype, root,
MPIR_GATHERV_TAG, comm);
}
else {
mpi_errno = MPIC_Send(sendbuf, sendcnt, sendtype, root,
MPIR_GATHERV_TAG, comm);
}
}
}
/* check if multiple threads are calling this collective function */
......
......@@ -94,6 +94,42 @@ int MPIC_Recv(void *buf, int count, MPI_Datatype datatype, int source, int tag,
goto fn_exit;
}
#undef FUNCNAME
#define FUNCNAME MPIC_Ssend
#undef FCNAME
#define FCNAME "MPIC_Ssend"
int MPIC_Ssend(void *buf, int count, MPI_Datatype datatype, int dest, int tag,
MPI_Comm comm)
{
int mpi_errno, context_id;
MPID_Request *request_ptr=NULL;
MPID_Comm *comm_ptr=NULL;
MPIDI_STATE_DECL(MPID_STATE_MPIC_SEND);
MPIDI_PT2PT_FUNC_ENTER_FRONT(MPID_STATE_MPIC_SEND);
MPID_Comm_get_ptr( comm, comm_ptr );
context_id = (comm_ptr->comm_kind == MPID_INTRACOMM) ?
MPID_CONTEXT_INTRA_COLL : MPID_CONTEXT_INTER_COLL;
mpi_errno = MPID_Ssend(buf, count, datatype, dest, tag, comm_ptr,
context_id, &request_ptr);
if (mpi_errno) { MPIU_ERR_POP(mpi_errno); }
if (request_ptr) {
mpi_errno = MPIC_Wait(request_ptr);
if (mpi_errno) { MPIU_ERR_POP(mpi_errno); }
MPID_Request_release(request_ptr);
}
fn_exit:
MPIDI_PT2PT_FUNC_EXIT(MPID_STATE_MPIC_SEND);
return mpi_errno;
fn_fail:
if (request_ptr) {
MPID_Request_release(request_ptr);
}
goto fn_exit;
}
#undef FUNCNAME
#define FUNCNAME MPIC_Sendrecv
#undef FCNAME
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment