Commit c8f360b2 authored by James Dinan's avatar James Dinan
Browse files

[svn-r10436] Refactoring of RMA ops list code

Refactored RMA ops enqueue/dequeue operations into inline functions to reduce
copy/paste code.  Added extra safety checks that will be needed as we renovate
for MPI-3.

Reviewer: goodell
parent 2bea811b
......@@ -103,4 +103,118 @@ typedef struct MPIDI_Win_lock_queue {
/* Routine use to tune RMA optimizations */
void MPIDI_CH3_RMA_SetAccImmed( int flag );
/*** RMA OPS LIST HELPER ROUTINES ***/
/* Return nonzero if the RMA operations list is empty.
*/
#undef FUNCNAME
#define FUNCNAME MPIDI_CH3I_Win_ops_isempty
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
static inline int MPIDI_CH3I_Win_ops_isempty(MPID_Win *win_ptr)
{
return win_ptr->rma_ops_list_head == NULL;
}
/* Allocate a new element on the tail of the RMA operations list.
*
* @param IN win_ptr Window containing the RMA ops list
* @param OUT curr_ptr Pointer to the element to the element that was
* updated.
* @return MPI error class
*/
#undef FUNCNAME
#define FUNCNAME MPIDI_CH3I_Win_ops_alloc_tail
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
static inline int MPIDI_CH3I_Win_ops_alloc_tail(MPID_Win *win_ptr,
MPIDI_RMA_ops **curr_ptr)
{
int mpi_errno = MPI_SUCCESS;
MPIDI_RMA_ops *tmp_ptr;
MPIU_CHKPMEM_DECL(1);
/* This assertion can fail because the tail pointer is not maintainted (see
free_and_next). If this happens, consider refactoring to a previous
element pointer (instead of prevNext) or a better list. */
MPIU_Assert(win_ptr->rma_ops_list_tail != NULL);
/* FIXME: We should use a pool allocator here */
MPIU_CHKPMEM_MALLOC(tmp_ptr, MPIDI_RMA_ops *, sizeof(MPIDI_RMA_ops),
mpi_errno, "RMA operation entry");
tmp_ptr->next = NULL;
tmp_ptr->dataloop = NULL;
if (MPIDI_CH3I_Win_ops_isempty(win_ptr))
win_ptr->rma_ops_list_head = tmp_ptr;
else
win_ptr->rma_ops_list_tail->next = tmp_ptr;
win_ptr->rma_ops_list_tail = tmp_ptr;
*curr_ptr = tmp_ptr;
fn_exit:
MPIU_CHKPMEM_COMMIT();
return mpi_errno;
fn_fail:
MPIU_CHKPMEM_REAP();
goto fn_exit;
}
/* Free an element in the RMA operations list.
*
* NOTE: It is not currently possible (singly-linked list) for this operation
* to maintain the rma_ops_list_tail pointer. If the freed element was at the
* tail, the tail pointer will become stale. One should not rely a correct
* value of rma_ops_list_tail after calling this function -- specifically, one
* should be wary of calling alloc_tail unless the free operation emptied the
* list.
*
* @param IN win_ptr Window containing the RMA ops list
* @param INOUT curr_ptr Pointer to the element to be freed. Will be updated
* to point to the element following the element that
* was freed.
* @param IN prev_next_ptr Pointer to the previous operation's next pointer.
* Used to unlink the element reference by curr_ptr.
*/
#undef FUNCNAME
#define FUNCNAME MPIDI_CH3I_Win_ops_free_and_next
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
static inline void MPIDI_CH3I_Win_ops_free_and_next(MPID_Win *win_ptr,
MPIDI_RMA_ops **curr_ptr,
MPIDI_RMA_ops **prev_next_ptr)
{
MPIDI_RMA_ops *tmp_ptr = *curr_ptr;
MPIU_Assert(tmp_ptr != NULL && *prev_next_ptr == tmp_ptr);
/* Check if we are down to one element in the ops list */
if (win_ptr->rma_ops_list_head->next == NULL) {
MPIU_Assert(tmp_ptr == win_ptr->rma_ops_list_head);
win_ptr->rma_ops_list_tail = NULL;
win_ptr->rma_ops_list_head = NULL;
}
/* Check if this free invalidates the tail pointer. If so, set it to NULL
for safety. */
if (win_ptr->rma_ops_list_tail == *curr_ptr)
win_ptr->rma_ops_list_tail = NULL;
/* Unlink the element */
*curr_ptr = tmp_ptr->next;
*prev_next_ptr = tmp_ptr->next;
/* Check if we allocated a dataloop for this op (see send/recv_rma_msg) */
if (tmp_ptr->dataloop != NULL)
MPIU_Free(tmp_ptr->dataloop);
MPIU_Free( tmp_ptr );
}
#undef FUNCNAME
#undef FCNAME
#endif
......@@ -28,9 +28,7 @@ int MPIDI_Get_accumulate(const void *origin_addr, int origin_count,
int dt_contig ATTRIBUTE((unused));
MPI_Aint dt_true_lb ATTRIBUTE((unused));
MPID_Datatype *dtp;
MPIDI_RMA_ops *new_ptr;
MPIU_CHKLMEM_DECL(2);
MPIU_CHKPMEM_DECL(1);
MPIDI_STATE_DECL(MPID_STATE_MPIDI_GET_ACCUMULATE);
MPIDI_RMA_FUNC_ENTER(MPID_STATE_MPIDI_GET_ACCUMULATE);
......@@ -154,21 +152,17 @@ int MPIDI_Get_accumulate(const void *origin_addr, int origin_count,
}
}
else {
MPIDI_RMA_ops *new_ptr = NULL;
/* Append the operation to the window's RMA ops queue */
MPIU_INSTR_DURATION_START(rmaqueue_alloc);
MPIU_CHKPMEM_MALLOC(new_ptr, MPIDI_RMA_ops *, sizeof(MPIDI_RMA_ops),
mpi_errno, "RMA operation entry");
mpi_errno = MPIDI_CH3I_Win_ops_alloc_tail(win_ptr, &new_ptr);
MPIU_INSTR_DURATION_END(rmaqueue_alloc);
if (win_ptr->rma_ops_list_tail)
win_ptr->rma_ops_list_tail->next = new_ptr;
else
win_ptr->rma_ops_list_head = new_ptr;
win_ptr->rma_ops_list_tail = new_ptr;
if (mpi_errno) { MPIU_ERR_POP(mpi_errno); }
/* TODO: Can we use the MPIDI_RMA_ACC_CONTIG optimization? */
MPIU_INSTR_DURATION_START(rmaqueue_set);
new_ptr->next = NULL;
new_ptr->type = MPIDI_RMA_GET_ACCUMULATE;
/* Cast away const'ness for origin_address as MPIDI_RMA_ops
* contain both PUT and GET like ops */
......@@ -202,14 +196,12 @@ int MPIDI_Get_accumulate(const void *origin_addr, int origin_count,
}
fn_exit:
MPIU_CHKPMEM_COMMIT();
MPIU_CHKLMEM_FREEALL();
MPIDI_RMA_FUNC_EXIT(MPID_STATE_MPIDI_GET_ACCUMULATE);
return mpi_errno;
/* --BEGIN ERROR HANDLING-- */
fn_fail:
MPIU_CHKPMEM_REAP();
goto fn_exit;
/* --END ERROR HANDLING-- */
}
......@@ -225,9 +217,6 @@ int MPIDI_Compare_and_swap(const void *origin_addr, const void *compare_addr,
{
int mpi_errno = MPI_SUCCESS;
int rank;
MPIDI_RMA_ops *new_ptr;
MPIU_CHKPMEM_DECL(1);
MPIDI_STATE_DECL(MPID_STATE_MPIDI_COMPARE_AND_SWAP);
MPIDI_RMA_FUNC_ENTER(MPID_STATE_MPIDI_COMPARE_AND_SWAP);
......@@ -256,19 +245,15 @@ int MPIDI_Compare_and_swap(const void *origin_addr, const void *compare_addr,
goto fn_exit;
}
else {
MPIDI_RMA_ops *new_ptr = NULL;
/* Append this operation to the RMA ops queue */
MPIU_INSTR_DURATION_START(rmaqueue_alloc);
MPIU_CHKPMEM_MALLOC(new_ptr, MPIDI_RMA_ops *, sizeof(MPIDI_RMA_ops),
mpi_errno, "RMA operation entry");
mpi_errno = MPIDI_CH3I_Win_ops_alloc_tail(win_ptr, &new_ptr);
MPIU_INSTR_DURATION_END(rmaqueue_alloc);
if (win_ptr->rma_ops_list_tail)
win_ptr->rma_ops_list_tail->next = new_ptr;
else
win_ptr->rma_ops_list_head = new_ptr;
win_ptr->rma_ops_list_tail = new_ptr;
if (mpi_errno) { MPIU_ERR_POP(mpi_errno); }
MPIU_INSTR_DURATION_START(rmaqueue_set);
new_ptr->next = NULL;
new_ptr->type = MPIDI_RMA_COMPARE_AND_SWAP;
new_ptr->origin_addr = (void *) origin_addr;
new_ptr->origin_count = 1;
......@@ -287,12 +272,10 @@ int MPIDI_Compare_and_swap(const void *origin_addr, const void *compare_addr,
}
fn_exit:
MPIU_CHKPMEM_COMMIT();
MPIDI_RMA_FUNC_EXIT(MPID_STATE_MPIDI_COMPARE_AND_SWAP);
return mpi_errno;
/* --BEGIN ERROR HANDLING-- */
fn_fail:
MPIU_CHKPMEM_REAP();
goto fn_exit;
/* --END ERROR HANDLING-- */
}
......@@ -308,9 +291,6 @@ int MPIDI_Fetch_and_op(const void *origin_addr, void *result_addr,
{
int mpi_errno = MPI_SUCCESS;
int rank;
MPIDI_RMA_ops *new_ptr;
MPIU_CHKPMEM_DECL(1);
MPIDI_STATE_DECL(MPID_STATE_MPIDI_FETCH_AND_OP);
MPIDI_RMA_FUNC_ENTER(MPID_STATE_MPIDI_FETCH_AND_OP);
......@@ -340,19 +320,15 @@ int MPIDI_Fetch_and_op(const void *origin_addr, void *result_addr,
goto fn_exit;
}
else {
MPIDI_RMA_ops *new_ptr = NULL;
/* Append this operation to the RMA ops queue */
MPIU_INSTR_DURATION_START(rmaqueue_alloc);
MPIU_CHKPMEM_MALLOC(new_ptr, MPIDI_RMA_ops *, sizeof(MPIDI_RMA_ops),
mpi_errno, "RMA operation entry");
mpi_errno = MPIDI_CH3I_Win_ops_alloc_tail(win_ptr, &new_ptr);
MPIU_INSTR_DURATION_END(rmaqueue_alloc);
if (win_ptr->rma_ops_list_tail)
win_ptr->rma_ops_list_tail->next = new_ptr;
else
win_ptr->rma_ops_list_head = new_ptr;
win_ptr->rma_ops_list_tail = new_ptr;
if (mpi_errno) { MPIU_ERR_POP(mpi_errno); }
MPIU_INSTR_DURATION_START(rmaqueue_set);
new_ptr->next = NULL;
new_ptr->type = MPIDI_RMA_FETCH_AND_OP;
new_ptr->origin_addr = (void *) origin_addr;
new_ptr->origin_count = 1;
......@@ -369,12 +345,10 @@ int MPIDI_Fetch_and_op(const void *origin_addr, void *result_addr,
}
fn_exit:
MPIU_CHKPMEM_COMMIT();
MPIDI_RMA_FUNC_EXIT(MPID_STATE_MPIDI_FETCH_AND_OP);
return mpi_errno;
/* --BEGIN ERROR HANDLING-- */
fn_fail:
MPIU_CHKPMEM_REAP();
goto fn_exit;
/* --END ERROR HANDLING-- */
}
......@@ -121,11 +121,9 @@ int MPIDI_Put(const void *origin_addr, int origin_count, MPI_Datatype
{
int mpi_errno = MPI_SUCCESS;
int dt_contig ATTRIBUTE((unused)), rank, predefined;
MPIDI_RMA_ops *new_ptr;
MPID_Datatype *dtp;
MPI_Aint dt_true_lb ATTRIBUTE((unused));
MPIDI_msg_sz_t data_sz;
MPIU_CHKPMEM_DECL(1);
MPIDI_STATE_DECL(MPID_STATE_MPIDI_PUT);
MPIDI_RMA_FUNC_ENTER(MPID_STATE_MPIDI_PUT);
......@@ -150,21 +148,16 @@ int MPIDI_Put(const void *origin_addr, int origin_count, MPI_Datatype
}
else
{
MPIDI_RMA_ops *new_ptr = NULL;
/* queue it up */
/* FIXME: For short operations, should we use a (per-thread) pool? */
MPIU_INSTR_DURATION_START(rmaqueue_alloc);
MPIU_CHKPMEM_MALLOC(new_ptr, MPIDI_RMA_ops *, sizeof(MPIDI_RMA_ops),
mpi_errno, "RMA operation entry");
MPIU_INSTR_DURATION_END(rmaqueue_alloc);
if (win_ptr->rma_ops_list_tail)
win_ptr->rma_ops_list_tail->next = new_ptr;
else
win_ptr->rma_ops_list_head = new_ptr;
win_ptr->rma_ops_list_tail = new_ptr;
MPIU_INSTR_DURATION_START(rmaqueue_alloc);
mpi_errno = MPIDI_CH3I_Win_ops_alloc_tail(win_ptr, &new_ptr);
MPIU_INSTR_DURATION_END(rmaqueue_alloc);
if (mpi_errno) { MPIU_ERR_POP(mpi_errno); }
MPIU_INSTR_DURATION_START(rmaqueue_set);
/* FIXME: For contig and very short operations, use a streamlined op */
new_ptr->next = NULL;
new_ptr->type = MPIDI_RMA_PUT;
/* Cast away const'ness for the origin address, as the
* MPIDI_RMA_ops structure is used for both PUT and GET like
......@@ -200,7 +193,6 @@ int MPIDI_Put(const void *origin_addr, int origin_count, MPI_Datatype
/* --BEGIN ERROR HANDLING-- */
fn_fail:
MPIU_CHKPMEM_REAP();
goto fn_exit;
/* --END ERROR HANDLING-- */
}
......@@ -219,9 +211,7 @@ int MPIDI_Get(void *origin_addr, int origin_count, MPI_Datatype
MPIDI_msg_sz_t data_sz;
int dt_contig ATTRIBUTE((unused)), rank, predefined;
MPI_Aint dt_true_lb ATTRIBUTE((unused));
MPIDI_RMA_ops *new_ptr;
MPID_Datatype *dtp;
MPIU_CHKPMEM_DECL(1);
MPIDI_STATE_DECL(MPID_STATE_MPIDI_GET);
MPIDI_RMA_FUNC_ENTER(MPID_STATE_MPIDI_GET);
......@@ -248,20 +238,16 @@ int MPIDI_Get(void *origin_addr, int origin_count, MPI_Datatype
}
else
{
MPIDI_RMA_ops *new_ptr = NULL;
/* queue it up */
MPIU_INSTR_DURATION_START(rmaqueue_alloc);
MPIU_CHKPMEM_MALLOC(new_ptr, MPIDI_RMA_ops *, sizeof(MPIDI_RMA_ops),
mpi_errno, "RMA operation entry");
MPIU_INSTR_DURATION_END(rmaqueue_alloc);
if (win_ptr->rma_ops_list_tail)
win_ptr->rma_ops_list_tail->next = new_ptr;
else
win_ptr->rma_ops_list_head = new_ptr;
win_ptr->rma_ops_list_tail = new_ptr;
MPIU_INSTR_DURATION_START(rmaqueue_alloc);
mpi_errno = MPIDI_CH3I_Win_ops_alloc_tail(win_ptr, &new_ptr);
MPIU_INSTR_DURATION_END(rmaqueue_alloc);
if (mpi_errno) { MPIU_ERR_POP(mpi_errno); }
MPIU_INSTR_DURATION_START(rmaqueue_set);
/* FIXME: For contig and very short operations, use a streamlined op */
new_ptr->next = NULL;
new_ptr->type = MPIDI_RMA_GET;
new_ptr->origin_addr = origin_addr;
new_ptr->origin_count = origin_count;
......@@ -294,7 +280,6 @@ int MPIDI_Get(void *origin_addr, int origin_count, MPI_Datatype
/* --BEGIN ERROR HANDLING-- */
fn_fail:
MPIU_CHKPMEM_REAP();
goto fn_exit;
/* --END ERROR HANDLING-- */
}
......@@ -314,10 +299,8 @@ int MPIDI_Accumulate(const void *origin_addr, int origin_count, MPI_Datatype
MPIDI_msg_sz_t data_sz;
int dt_contig ATTRIBUTE((unused)), rank, origin_predefined, target_predefined;
MPI_Aint dt_true_lb ATTRIBUTE((unused));
MPIDI_RMA_ops *new_ptr;
MPID_Datatype *dtp;
MPIU_CHKLMEM_DECL(2);
MPIU_CHKPMEM_DECL(1);
MPIDI_STATE_DECL(MPID_STATE_MPIDI_ACCUMULATE);
MPIDI_RMA_FUNC_ENTER(MPID_STATE_MPIDI_ACCUMULATE);
......@@ -441,21 +424,17 @@ int MPIDI_Accumulate(const void *origin_addr, int origin_count, MPI_Datatype
}
else
{
MPIDI_RMA_ops *new_ptr = NULL;
/* queue it up */
MPIU_INSTR_DURATION_START(rmaqueue_alloc);
MPIU_CHKPMEM_MALLOC(new_ptr, MPIDI_RMA_ops *, sizeof(MPIDI_RMA_ops),
mpi_errno, "RMA operation entry");
MPIU_INSTR_DURATION_END(rmaqueue_alloc);
if (win_ptr->rma_ops_list_tail)
win_ptr->rma_ops_list_tail->next = new_ptr;
else
win_ptr->rma_ops_list_head = new_ptr;
win_ptr->rma_ops_list_tail = new_ptr;
MPIU_INSTR_DURATION_START(rmaqueue_alloc);
mpi_errno = MPIDI_CH3I_Win_ops_alloc_tail(win_ptr, &new_ptr);
MPIU_INSTR_DURATION_END(rmaqueue_alloc);
if (mpi_errno) { MPIU_ERR_POP(mpi_errno); }
/* If predefined and contiguous, use a simplified element */
if (origin_predefined && target_predefined && enableShortACC) {
MPIU_INSTR_DURATION_START(rmaqueue_set);
new_ptr->next = NULL;
new_ptr->type = MPIDI_RMA_ACC_CONTIG;
/* Only the information needed for the contig/predefined acc */
/* Cast away const'ness for origin_address as
......@@ -473,7 +452,6 @@ int MPIDI_Accumulate(const void *origin_addr, int origin_count, MPI_Datatype
}
MPIU_INSTR_DURATION_START(rmaqueue_set);
new_ptr->next = NULL;
new_ptr->type = MPIDI_RMA_ACCUMULATE;
/* Cast away const'ness for origin_address as MPIDI_RMA_ops
* contain both PUT and GET like ops */
......@@ -508,7 +486,6 @@ int MPIDI_Accumulate(const void *origin_addr, int origin_count, MPI_Datatype
/* --BEGIN ERROR HANDLING-- */
fn_fail:
MPIU_CHKPMEM_REAP();
goto fn_exit;
/* --END ERROR HANDLING-- */
}
......
......@@ -194,7 +194,7 @@ int MPIDI_Win_fence(int assert, MPID_Win *win_ptr)
}
else
{
MPIDI_RMA_ops **prevNextPtr, *tmpptr;
MPIDI_RMA_ops **prevNextPtr;
int nRequest = 0;
int nRequestNew = 0;
MPIU_INSTR_DURATION_START(winfence_rs);
......@@ -273,7 +273,6 @@ int MPIDI_Win_fence(int assert, MPID_Win *win_ptr)
target_win_handle = win_ptr->all_win_handles[curr_ptr->target_rank];
curr_ptr->dataloop = 0;
switch (curr_ptr->type)
{
case (MPIDI_RMA_PUT):
......@@ -313,14 +312,7 @@ int MPIDI_Win_fence(int assert, MPID_Win *win_ptr)
curr_ops_cnt[curr_ptr->target_rank]++;
/* If the request is null, we can remove it immediately */
if (!curr_ptr->request) {
if (curr_ptr->dataloop != NULL) {
MPIU_Free(curr_ptr->dataloop); /* allocated in send_rma_msg or
recv_rma_msg */
}
tmpptr = curr_ptr->next;
*prevNextPtr = tmpptr;
MPIU_Free( curr_ptr );
curr_ptr = tmpptr;
MPIDI_CH3I_Win_ops_free_and_next(win_ptr, &curr_ptr, prevNextPtr);
}
else {
nRequest++;
......@@ -362,9 +354,7 @@ int MPIDI_Win_fence(int assert, MPID_Win *win_ptr)
mpi_errno = MPIDI_CH3I_RMAListComplete(win_ptr);
}
win_ptr->rma_ops_list_head = NULL;
win_ptr->rma_ops_list_tail = NULL;
MPIU_Assert( MPIDI_CH3I_Win_ops_isempty(win_ptr) );
/* wait for all operations from other processes to finish */
if (win_ptr->my_counter)
......@@ -1425,7 +1415,7 @@ int MPIDI_Win_complete(MPID_Win *win_ptr)
int mpi_errno = MPI_SUCCESS;
int comm_size, *nops_to_proc, src, new_total_op_count;
int i, j, dst, total_op_count, *curr_ops_cnt;
MPIDI_RMA_ops *curr_ptr, *tmpptr, **prevNextPtr;
MPIDI_RMA_ops *curr_ptr, **prevNextPtr;
MPID_Comm *comm_ptr;
MPI_Win source_win_handle, target_win_handle;
MPID_Group *win_grp_ptr;
......@@ -1600,14 +1590,7 @@ int MPIDI_Win_complete(MPID_Win *win_ptr)
curr_ops_cnt[curr_ptr->target_rank]++;
/* If the request is null, we can remove it immediately */
if (!curr_ptr->request) {
if (curr_ptr->dataloop != NULL) {
MPIU_Free(curr_ptr->dataloop); /* allocated in send_rma_msg or
recv_rma_msg */
}
tmpptr = curr_ptr->next;
*prevNextPtr = tmpptr;
MPIU_Free( curr_ptr );
curr_ptr = tmpptr;
MPIDI_CH3I_Win_ops_free_and_next(win_ptr, &curr_ptr, prevNextPtr);
}
else {
nRequest++;
......@@ -1664,28 +1647,15 @@ int MPIDI_Win_complete(MPID_Win *win_ptr)
/* In the unlikely event that a request is returned (the message
is not sent yet), add it to the list of pending operations */
if (request) {
/* Its hard to use the automatic allocator here, as those
macros are optimized for a known maximum number of items. */
MPIDI_RMA_ops *new_ptr;
MPIU_INSTR_DURATION_START(rmaqueue_alloc);
new_ptr = (MPIDI_RMA_ops *)MPIU_Malloc(sizeof(MPIDI_RMA_ops) );
MPIU_INSTR_DURATION_END(rmaqueue_alloc);
/* --BEGIN ERROR HANDLING-- */
if (!new_ptr) {
MPIU_CHKMEM_SETERR(mpi_errno,sizeof(MPIDI_RMA_ops),
"RMA operation entry");
goto fn_fail;
}
/* --END ERROR HANDLING-- */
if (win_ptr->rma_ops_list_tail)
win_ptr->rma_ops_list_tail->next = new_ptr;
else
win_ptr->rma_ops_list_head = new_ptr;
win_ptr->rma_ops_list_tail = new_ptr;
MPIDI_RMA_ops *new_ptr = NULL;
MPIU_INSTR_DURATION_START(rmaqueue_alloc);
mpi_errno = MPIDI_CH3I_Win_ops_alloc_tail(win_ptr, &new_ptr);
MPIU_INSTR_DURATION_END(rmaqueue_alloc);
if (mpi_errno) { MPIU_ERR_POP(mpi_errno); }
MPIU_INSTR_DURATION_START(rmaqueue_set);
new_ptr->next = NULL;
new_ptr->request = request;
new_ptr->dataloop = 0;
MPIU_INSTR_DURATION_END(rmaqueue_set);
}
j++;
......@@ -1700,9 +1670,7 @@ int MPIDI_Win_complete(MPID_Win *win_ptr)
mpi_errno = MPIDI_CH3I_RMAListComplete(win_ptr);
}
MPIU_Assert( !win_ptr->rma_ops_list_head );
win_ptr->rma_ops_list_head = NULL;
win_ptr->rma_ops_list_tail = NULL;
MPIU_Assert( MPIDI_CH3I_Win_ops_isempty(win_ptr) );
mpi_errno = MPIR_Group_free_impl(win_grp_ptr);
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
......@@ -1797,8 +1765,6 @@ int MPIDI_Win_test(MPID_Win *win_ptr, int *flag)
int MPIDI_Win_lock(int lock_type, int dest, int assert, MPID_Win *win_ptr)
{
int mpi_errno = MPI_SUCCESS;
MPIDI_RMA_ops *new_ptr;
MPIU_CHKPMEM_DECL(1);
MPIDI_STATE_DECL(MPID_STATE_MPIDI_WIN_LOCK);
MPIDI_RMA_FUNC_ENTER(MPID_STATE_MPIDI_WIN_LOCK);
......@@ -1849,19 +1815,15 @@ int MPIDI_Win_lock(int lock_type, int dest, int assert, MPID_Win *win_ptr)
MPIU_ERR_CHKANDJUMP(mpi_errno != MPI_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**winRMAmessage");
}
else {
MPIDI_RMA_ops *new_ptr = NULL;
/* target is some other process. add the lock request to rma_ops_list */
MPIU_INSTR_DURATION_START(rmaqueue_alloc);
MPIU_CHKPMEM_MALLOC(new_ptr, MPIDI_RMA_ops *, sizeof(MPIDI_RMA_ops),
mpi_errno, "RMA operation entry");
mpi_errno = MPIDI_CH3I_Win_ops_alloc_tail(win_ptr, &new_ptr);
MPIU_INSTR_DURATION_END(rmaqueue_alloc);
MPIU_Assert( win_ptr->rma_ops_list_head == NULL &&
win_ptr->rma_ops_list_tail == NULL );
win_ptr->rma_ops_list_head = new_ptr;
win_ptr->rma_ops_list_tail = new_ptr;
if (mpi_errno) { MPIU_ERR_POP(mpi_errno); }
MPIU_INSTR_DURATION_START(rmaqueue_set);
new_ptr->next = NULL;
new_ptr->type = MPIDI_RMA_LOCK;
new_ptr->target_rank = dest;
new_ptr->lock_type = lock_type;
......@@ -1874,7 +1836,6 @@ int MPIDI_Win_lock(int lock_type, int dest, int assert, MPID_Win *win_ptr)
return mpi_errno;
/* --BEGIN ERROR HANDLING-- */
fn_fail:
MPIU_CHKPMEM_REAP();
goto fn_exit;
/* --END ERROR HANDLING-- */
}
......@@ -1899,7 +1860,7 @@ int MPIDI_Win_unlock(int dest, MPID_Win *win_ptr)
if (dest == win_ptr->myrank) {
/* local lock. release the lock on the window, grant the next one
* in the queue, and return. */
MPIU_Assert(!win_ptr->rma_ops_list_head);
MPIU_Assert(MPIDI_CH3I_Win_ops_isempty(win_ptr));
mpi_errno = MPIDI_CH3I_Release_lock(win_ptr);
if (mpi_errno != MPI_SUCCESS) goto fn_exit;
win_ptr->remote_lock_state = MPIDI_CH3_WIN_LOCK_NONE;
......@@ -1927,9 +1888,7 @@ int MPIDI_Win_unlock(int dest, MPID_Win *win_ptr)
lock, we can do nothing and return. */
if (rma_op && rma_op->type == MPIDI_RMA_LOCK && rma_op->next == NULL &&
win_ptr->remote_lock_state == MPIDI_CH3_WIN_LOCK_NONE) {
MPIU_Free(rma_op);
win_ptr->rma_ops_list_head = NULL;
win_ptr->rma_ops_list_tail = NULL;
MPIDI_CH3I_Win_ops_free_and_next(win_ptr, &win_ptr->rma_ops_list_head, &win_ptr->rma_ops_list_head);
goto fn_exit;
}
......@@ -2027,7 +1986,7 @@ int MPIDI_Win_unlock(int dest, MPID_Win *win_ptr)
}
fn_exit:
MPIU_Assert( !win_ptr->rma_ops_list_head );
MPIU_Assert( MPIDI_CH3I_Win_ops_isempty(win_ptr) );
MPIDI_RMA_FUNC_EXIT(MPID_STATE_MPIDI_WIN_UNLOCK);
return mpi_errno;
/* --BEGIN ERROR HANDLING-- */
......@@ -2063,7 +2022,7 @@ int MPIDI_Win_flush(int rank, MPID_Win *win_ptr)
/* Local flush: ops are performed immediately on the local process */
if (rank == win_ptr->comm_ptr->rank) {
MPIU_Assert(win_ptr->remote_lock_state == MPIDI_CH3_WIN_LOCK_GRANTED);
MPIU_Assert(win_ptr->rma_ops_list_head == NULL && win_ptr->rma_ops_list_tail == NULL);
MPIU_Assert(MPIDI_CH3I_Win_ops_isempty(win_ptr));
goto fn_exit;
}
......@@ -2091,7 +2050,7 @@ int MPIDI_Win_flush(int rank, MPID_Win *win_ptr)
if (win_ptr->remote_lock_state == MPIDI_CH3_WIN_LOCK_NONE) {
/* Ensure that win_lock is waiting at the head of the ops list */
MPIU_ERR_CHKANDJUMP(win_ptr->rma_ops_list_head == NULL ||
MPIU_ERR_CHKANDJUMP(MPIDI_CH3I_Win_ops_isempty(win_ptr) ||
win_ptr->rma_ops_list_head->type != MPIDI_RMA_LOCK,
mpi_errno, MPI_ERR_OTHER, "**rmasync");
mpi_errno = MPIDI_CH3I_Send_lock_msg(win_ptr->rma_ops_list_head->target_rank,
......@@ -2143,7 +2102,7 @@ int MPIDI_Win_flush(int rank, MPID_Win *win_ptr)
}
fn_exit:
MPIU_Assert( win_ptr->rma_ops_list_head == NULL && win_ptr->rma_ops_list_tail == NULL );
MPIU_Assert(MPIDI_CH3I_Win_ops_isempty(win_ptr));
MPIDI_RMA_FUNC_EXIT(MPID_STATE_MPIDI_WIN_FLUSH_ALL);
return mpi_errno;
/* --BEGIN ERROR HANDLING-- */
......@@ -2251,8 +2210,8 @@ static int MPIDI_CH3I_Do_passive_target_rma(MPID_Win *win_ptr,
{
int mpi_errno = MPI_SUCCESS, nops;
MPIDI_RMA_ops *curr_ptr;
MPIDI_RMA_ops **prevNextPtr, *tmpptr;
MPI_Win source_win_handle, target_win_handle;
MPIDI_RMA_ops **prevNextPtr;
MPI_Win source_win_handle, target_win_handle = MPI_WIN_NULL;
int nRequest=0, nRequestNew=0;
MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3I_DO_PASSIVE_TARGET_RMA);
......@@ -2268,11 +2227,11 @@ static int MPIDI_CH3I_Do_passive_target_rma(MPID_Win *win_ptr,
there is no third-party communication issue. */
*wait_for_rma_done_pkt = 0;
}