Commit cdb1b3e4 authored by James Dinan's avatar James Dinan
Browse files

[svn-r10508] Refactoring RMA Ops list to DL

In this patch, I have refactored the RMA ops list again to use the MPL UTList
doubly-linked list and to treat the list as a proper object.  This should set
us up to work with multiple lists, as we will soon have one list per target.
Doubly-linking the list is a big help in terms of maintainability (no more
prevNext pointers) and flexibility (better implementation of request-based
ops and other optimizations).

Reviewer: goodell
parent d9531b7a
......@@ -217,9 +217,8 @@ struct MPIDI_Win_info_args_s {
int *disp_units; /* array of displacement units of all windows */\
MPI_Win *all_win_handles; /* array of handles to the window objects\
of all processes */ \
struct MPIDI_RMA_ops *rma_ops_list_head; /* list of outstanding \
RMA requests */ \
struct MPIDI_RMA_ops *rma_ops_list_tail; \
struct MPIDI_RMA_ops *rma_ops_list; /* list of outstanding RMA \
operations */ \
volatile enum MPIDI_CH3_Lock_states_e remote_lock_state; \
/* Indicates the state of the target \
process' "lock" for passive target \
......
......@@ -6,6 +6,8 @@
#if !defined(MPICH_MPIDRMA_H_INCLUDED)
#define MPICH_MPIDRMA_H_INCLUDED
#include "mpl_utlist.h"
typedef enum MPIDI_RMA_Op_type_e {
MPIDI_RMA_PUT = 23,
MPIDI_RMA_GET = 24,
......@@ -53,6 +55,7 @@ typedef struct MPIDI_RMA_dtype_info { /* for derived datatypes */
/* for keeping track of RMA ops, which will be executed at the next sync call */
typedef struct MPIDI_RMA_ops {
struct MPIDI_RMA_ops *prev; /* pointer to next element in list */
struct MPIDI_RMA_ops *next; /* pointer to next element in list */
/* FIXME: It would be better to setup the packet that will be sent, at
least in most cases (if, as a result of the sync/ops/sync sequence,
......@@ -105,42 +108,77 @@ void MPIDI_CH3_RMA_SetAccImmed( int flag );
/*** RMA OPS LIST HELPER ROUTINES ***/
typedef MPIDI_RMA_ops * MPIDI_RMA_Ops_list_t;
/* Return nonzero if the RMA operations list is empty.
*/
#undef FUNCNAME
#define FUNCNAME MPIDI_CH3I_Win_ops_isempty
#define FUNCNAME MPIDI_CH3I_RMA_Ops_isempty
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
static inline int MPIDI_CH3I_Win_ops_isempty(MPID_Win *win_ptr)
static inline int MPIDI_CH3I_RMA_Ops_isempty(MPIDI_RMA_Ops_list_t *list)
{
return win_ptr->rma_ops_list_head == NULL;
return *list == NULL;
}
/* Return a pointer to the first element in the list.
*/
#undef FUNCNAME
#define FUNCNAME MPIDI_CH3I_RMA_Ops_head
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
static inline MPIDI_RMA_ops *MPIDI_CH3I_RMA_Ops_head(MPIDI_RMA_Ops_list_t *list)
{
return *list;
}
/* Return a pointer to the last element in the list.
*/
#undef FUNCNAME
#define FUNCNAME MPIDI_CH3I_RMA_Ops_tail
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
static inline MPIDI_RMA_ops *MPIDI_CH3I_RMA_Ops_tail(MPIDI_RMA_Ops_list_t *list)
{
return (*list) ? (*list)->prev : NULL;
}
/* Append an element to the tail of the RMA ops list
*
* @param IN list Pointer to the RMA ops list
* @param IN elem Pointer to the element to be appended
*/
#undef FUNCNAME
#define FUNCNAME MPIDI_CH3I_RMA_Ops_append
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
static inline void MPIDI_CH3I_RMA_Ops_append(MPIDI_RMA_Ops_list_t *list,
MPIDI_RMA_ops *elem)
{
MPL_DL_APPEND(*list, elem);
}
/* Allocate a new element on the tail of the RMA operations list.
*
* @param IN win_ptr Window containing the RMA ops list
* @param OUT curr_ptr Pointer to the element to the element that was
* updated.
* @param IN list Pointer to the RMA ops list
* @param OUT new_ptr Pointer to the element that was allocated
* @return MPI error class
*/
#undef FUNCNAME
#define FUNCNAME MPIDI_CH3I_Win_ops_alloc_tail
#define FUNCNAME MPIDI_CH3I_RMA_Ops_alloc_tail
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
static inline int MPIDI_CH3I_Win_ops_alloc_tail(MPID_Win *win_ptr,
MPIDI_RMA_ops **curr_ptr)
static inline int MPIDI_CH3I_RMA_Ops_alloc_tail(MPIDI_RMA_Ops_list_t *list,
MPIDI_RMA_ops **new_elem)
{
int mpi_errno = MPI_SUCCESS;
MPIDI_RMA_ops *tmp_ptr;
MPIU_CHKPMEM_DECL(1);
/* This assertion can fail because the tail pointer is not maintainted (see
free_and_next). If this happens, consider refactoring to a previous
element pointer (instead of a pointer to the prevous next pointer) or a
doubly-linked list. */
MPIU_Assert(MPIDI_CH3I_Win_ops_isempty(win_ptr) || win_ptr->rma_ops_list_tail != NULL);
/* FIXME: We should use a pool allocator here */
MPIU_CHKPMEM_MALLOC(tmp_ptr, MPIDI_RMA_ops *, sizeof(MPIDI_RMA_ops),
mpi_errno, "RMA operation entry");
......@@ -148,66 +186,53 @@ static inline int MPIDI_CH3I_Win_ops_alloc_tail(MPID_Win *win_ptr,
tmp_ptr->next = NULL;
tmp_ptr->dataloop = NULL;
if (MPIDI_CH3I_Win_ops_isempty(win_ptr))
win_ptr->rma_ops_list_head = tmp_ptr;
else
win_ptr->rma_ops_list_tail->next = tmp_ptr;
MPL_DL_APPEND(*list, tmp_ptr);
win_ptr->rma_ops_list_tail = tmp_ptr;
*curr_ptr = tmp_ptr;
*new_elem = tmp_ptr;
fn_exit:
MPIU_CHKPMEM_COMMIT();
return mpi_errno;
fn_fail:
MPIU_CHKPMEM_REAP();
*new_elem = NULL;
goto fn_exit;
}
/* Free an element in the RMA operations list.
/* Unlink an element from the RMA ops list
*
* NOTE: It is not currently possible (singly-linked list) for this operation
* to maintain the rma_ops_list_tail pointer. If the freed element was at the
* tail, the tail pointer will become stale. One should not rely a correct
* value of rma_ops_list_tail after calling this function -- specifically, one
* should be wary of calling alloc_tail unless the free operation emptied the
* list.
*
* @param IN win_ptr Window containing the RMA ops list
* @param INOUT curr_ptr Pointer to the element to be freed. Will be updated
* to point to the element following the element that
* was freed.
* @param IN prev_next_ptr Pointer to the previous operation's next pointer.
* Used to unlink the element reference by curr_ptr.
* @param IN list Pointer to the RMA ops list
* @param IN elem Pointer to the element to be unlinked
*/
#undef FUNCNAME
#define FUNCNAME MPIDI_CH3I_Win_ops_free_and_next
#define FUNCNAME MPIDI_CH3I_RMA_Ops_unlink
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
static inline void MPIDI_CH3I_Win_ops_free_and_next(MPID_Win *win_ptr,
MPIDI_RMA_ops **curr_ptr,
MPIDI_RMA_ops **prev_next_ptr)
static inline void MPIDI_CH3I_RMA_Ops_unlink(MPIDI_RMA_Ops_list_t *list,
MPIDI_RMA_ops *elem)
{
MPIDI_RMA_ops *tmp_ptr = *curr_ptr;
MPL_DL_DELETE(*list, elem);
}
MPIU_Assert(tmp_ptr != NULL && *prev_next_ptr == tmp_ptr);
/* Check if we are down to one element in the ops list */
if (win_ptr->rma_ops_list_head->next == NULL) {
MPIU_Assert(tmp_ptr == win_ptr->rma_ops_list_head);
win_ptr->rma_ops_list_tail = NULL;
win_ptr->rma_ops_list_head = NULL;
}
/* Free an element in the RMA operations list.
*
* @param IN list Pointer to the RMA ops list
* @param IN curr_ptr Pointer to the element to be freed.
*/
#undef FUNCNAME
#define FUNCNAME MPIDI_CH3I_RMA_Ops_free_elem
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
static inline void MPIDI_CH3I_RMA_Ops_free_elem(MPIDI_RMA_Ops_list_t *list,
MPIDI_RMA_ops *curr_ptr)
{
MPIDI_RMA_ops *tmp_ptr = curr_ptr;
/* Check if this free invalidates the tail pointer. If so, set it to NULL
for safety. */
if (win_ptr->rma_ops_list_tail == *curr_ptr)
win_ptr->rma_ops_list_tail = NULL;
MPIU_Assert(curr_ptr != NULL);
/* Unlink the element */
*curr_ptr = tmp_ptr->next;
*prev_next_ptr = tmp_ptr->next;
MPL_DL_DELETE(*list, curr_ptr);
/* Check if we allocated a dataloop for this op (see send/recv_rma_msg) */
if (tmp_ptr->dataloop != NULL)
......@@ -216,29 +241,40 @@ static inline void MPIDI_CH3I_Win_ops_free_and_next(MPID_Win *win_ptr,
}
/* Free the entire RMA operations list.
/* Free an element in the RMA operations list.
*
* @param IN list Pointer to the RMA ops list
* @param INOUT curr_ptr Pointer to the element to be freed. Will be updated
* to point to the element following the element that
* was freed.
*/
#undef FUNCNAME
#define FUNCNAME MPIDI_CH3I_Win_ops_free
#define FUNCNAME MPIDI_CH3I_RMA_Ops_free_and_next
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
static inline void MPIDI_CH3I_Win_ops_free(MPID_Win *win_ptr)
static inline void MPIDI_CH3I_RMA_Ops_free_and_next(MPIDI_RMA_Ops_list_t *list,
MPIDI_RMA_ops **curr_ptr)
{
MPIDI_RMA_ops *curr_ptr = win_ptr->rma_ops_list_head;
MPIDI_RMA_ops *next_ptr = (*curr_ptr)->next;
while (curr_ptr != NULL) {
MPIDI_RMA_ops *tmp_ptr = curr_ptr->next;
MPIDI_CH3I_RMA_Ops_free_elem(list, *curr_ptr);
*curr_ptr = next_ptr;
}
/* Free this ops list entry */
if (curr_ptr->dataloop != NULL)
MPIU_Free(tmp_ptr->dataloop);
MPIU_Free( curr_ptr );
curr_ptr = tmp_ptr;
}
/* Free the entire RMA operations list.
*/
#undef FUNCNAME
#define FUNCNAME MPIDI_CH3I_RMA_Ops_free
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
static inline void MPIDI_CH3I_RMA_Ops_free(MPIDI_RMA_Ops_list_t *list)
{
MPIDI_RMA_ops *curr_ptr, *tmp_ptr;
win_ptr->rma_ops_list_tail = NULL;
win_ptr->rma_ops_list_head = NULL;
MPL_DL_FOREACH_SAFE(*list, curr_ptr, tmp_ptr) {
MPIDI_CH3I_RMA_Ops_free_elem(list, curr_ptr);
}
}
......
......@@ -156,7 +156,7 @@ int MPIDI_Get_accumulate(const void *origin_addr, int origin_count,
/* Append the operation to the window's RMA ops queue */
MPIU_INSTR_DURATION_START(rmaqueue_alloc);
mpi_errno = MPIDI_CH3I_Win_ops_alloc_tail(win_ptr, &new_ptr);
mpi_errno = MPIDI_CH3I_RMA_Ops_alloc_tail(&win_ptr->rma_ops_list, &new_ptr);
MPIU_INSTR_DURATION_END(rmaqueue_alloc);
if (mpi_errno) { MPIU_ERR_POP(mpi_errno); }
......@@ -249,7 +249,7 @@ int MPIDI_Compare_and_swap(const void *origin_addr, const void *compare_addr,
/* Append this operation to the RMA ops queue */
MPIU_INSTR_DURATION_START(rmaqueue_alloc);
mpi_errno = MPIDI_CH3I_Win_ops_alloc_tail(win_ptr, &new_ptr);
mpi_errno = MPIDI_CH3I_RMA_Ops_alloc_tail(&win_ptr->rma_ops_list, &new_ptr);
MPIU_INSTR_DURATION_END(rmaqueue_alloc);
if (mpi_errno) { MPIU_ERR_POP(mpi_errno); }
......@@ -324,7 +324,7 @@ int MPIDI_Fetch_and_op(const void *origin_addr, void *result_addr,
/* Append this operation to the RMA ops queue */
MPIU_INSTR_DURATION_START(rmaqueue_alloc);
mpi_errno = MPIDI_CH3I_Win_ops_alloc_tail(win_ptr, &new_ptr);
mpi_errno = MPIDI_CH3I_RMA_Ops_alloc_tail(&win_ptr->rma_ops_list, &new_ptr);
MPIU_INSTR_DURATION_END(rmaqueue_alloc);
if (mpi_errno) { MPIU_ERR_POP(mpi_errno); }
......
......@@ -152,7 +152,7 @@ int MPIDI_Put(const void *origin_addr, int origin_count, MPI_Datatype
/* queue it up */
MPIU_INSTR_DURATION_START(rmaqueue_alloc);
mpi_errno = MPIDI_CH3I_Win_ops_alloc_tail(win_ptr, &new_ptr);
mpi_errno = MPIDI_CH3I_RMA_Ops_alloc_tail(&win_ptr->rma_ops_list, &new_ptr);
MPIU_INSTR_DURATION_END(rmaqueue_alloc);
if (mpi_errno) { MPIU_ERR_POP(mpi_errno); }
......@@ -242,7 +242,7 @@ int MPIDI_Get(void *origin_addr, int origin_count, MPI_Datatype
/* queue it up */
MPIU_INSTR_DURATION_START(rmaqueue_alloc);
mpi_errno = MPIDI_CH3I_Win_ops_alloc_tail(win_ptr, &new_ptr);
mpi_errno = MPIDI_CH3I_RMA_Ops_alloc_tail(&win_ptr->rma_ops_list, &new_ptr);
MPIU_INSTR_DURATION_END(rmaqueue_alloc);
if (mpi_errno) { MPIU_ERR_POP(mpi_errno); }
......@@ -428,7 +428,7 @@ int MPIDI_Accumulate(const void *origin_addr, int origin_count, MPI_Datatype
/* queue it up */
MPIU_INSTR_DURATION_START(rmaqueue_alloc);
mpi_errno = MPIDI_CH3I_Win_ops_alloc_tail(win_ptr, &new_ptr);
mpi_errno = MPIDI_CH3I_RMA_Ops_alloc_tail(&win_ptr->rma_ops_list, &new_ptr);
MPIU_INSTR_DURATION_END(rmaqueue_alloc);
if (mpi_errno) { MPIU_ERR_POP(mpi_errno); }
......
......@@ -194,7 +194,6 @@ int MPIDI_Win_fence(int assert, MPID_Win *win_ptr)
}
else
{
MPIDI_RMA_ops **prevNextPtr;
int nRequest = 0;
int nRequestNew = 0;
MPIU_INSTR_DURATION_START(winfence_rs);
......@@ -219,7 +218,7 @@ int MPIDI_Win_fence(int assert, MPID_Win *win_ptr)
/* set rma_target_proc[i] to 1 if rank i is a target of RMA
ops from this process */
total_op_count = 0;
curr_ptr = win_ptr->rma_ops_list_head;
curr_ptr = MPIDI_CH3I_RMA_Ops_head(&win_ptr->rma_ops_list);
while (curr_ptr != NULL)
{
total_op_count++;
......@@ -257,8 +256,7 @@ int MPIDI_Win_fence(int assert, MPID_Win *win_ptr)
MPIU_INSTR_DURATION_MAX(winfence_issue,1,total_op_count);
MPIU_INSTR_COUNTER_RESET(winfence_reqs);
i = 0;
curr_ptr = win_ptr->rma_ops_list_head;
prevNextPtr = &win_ptr->rma_ops_list_head;
curr_ptr = MPIDI_CH3I_RMA_Ops_head(&win_ptr->rma_ops_list);
while (curr_ptr != NULL)
{
/* The completion counter at the target is decremented only on
......@@ -312,12 +310,11 @@ int MPIDI_Win_fence(int assert, MPID_Win *win_ptr)
curr_ops_cnt[curr_ptr->target_rank]++;
/* If the request is null, we can remove it immediately */
if (!curr_ptr->request) {
MPIDI_CH3I_Win_ops_free_and_next(win_ptr, &curr_ptr, prevNextPtr);
MPIDI_CH3I_RMA_Ops_free_and_next(&win_ptr->rma_ops_list, &curr_ptr);
}
else {
nRequest++;
MPIU_INSTR_COUNTER_INCR(winfence_reqs,1);
prevNextPtr = &curr_ptr->next;
curr_ptr = curr_ptr->next;
/* The test on the difference is to reduce the number
of times the partial complete routine is called. Without
......@@ -354,7 +351,7 @@ int MPIDI_Win_fence(int assert, MPID_Win *win_ptr)
mpi_errno = MPIDI_CH3I_RMAListComplete(win_ptr);
}
MPIU_Assert( MPIDI_CH3I_Win_ops_isempty(win_ptr) );
MPIU_Assert(MPIDI_CH3I_RMA_Ops_isempty(&win_ptr->rma_ops_list));
/* wait for all operations from other processes to finish */
if (win_ptr->my_counter)
......@@ -1415,7 +1412,7 @@ int MPIDI_Win_complete(MPID_Win *win_ptr)
int mpi_errno = MPI_SUCCESS;
int comm_size, *nops_to_proc, src, new_total_op_count;
int i, j, dst, total_op_count, *curr_ops_cnt;
MPIDI_RMA_ops *curr_ptr, **prevNextPtr;
MPIDI_RMA_ops *curr_ptr;
MPID_Comm *comm_ptr;
MPI_Win source_win_handle, target_win_handle;
MPID_Group *win_grp_ptr;
......@@ -1514,7 +1511,7 @@ int MPIDI_Win_complete(MPID_Win *win_ptr)
for (i=0; i<comm_size; i++) nops_to_proc[i] = 0;
total_op_count = 0;
curr_ptr = win_ptr->rma_ops_list_head;
curr_ptr = MPIDI_CH3I_RMA_Ops_head(&win_ptr->rma_ops_list);
while (curr_ptr != NULL)
{
nops_to_proc[curr_ptr->target_rank]++;
......@@ -1534,8 +1531,7 @@ int MPIDI_Win_complete(MPID_Win *win_ptr)
for (i=0; i<comm_size; i++) curr_ops_cnt[i] = 0;
i = 0;
prevNextPtr = &win_ptr->rma_ops_list_head;
curr_ptr = win_ptr->rma_ops_list_head;
curr_ptr = MPIDI_CH3I_RMA_Ops_head(&win_ptr->rma_ops_list);
while (curr_ptr != NULL)
{
/* The completion counter at the target is decremented only on
......@@ -1590,12 +1586,11 @@ int MPIDI_Win_complete(MPID_Win *win_ptr)
curr_ops_cnt[curr_ptr->target_rank]++;
/* If the request is null, we can remove it immediately */
if (!curr_ptr->request) {
MPIDI_CH3I_Win_ops_free_and_next(win_ptr, &curr_ptr, prevNextPtr);
MPIDI_CH3I_RMA_Ops_free_and_next(&win_ptr->rma_ops_list, &curr_ptr);
}
else {
nRequest++;
MPIU_INSTR_COUNTER_INCR(wincomplete_reqs,1);
prevNextPtr = &curr_ptr->next;
curr_ptr = curr_ptr->next;
if (nRequest > MPIR_PARAM_RMA_NREQUEST_THRESHOLD &&
nRequest - nRequestNew > MPIR_PARAM_RMA_NREQUEST_NEW_THRESHOLD) {
......@@ -1650,7 +1645,7 @@ int MPIDI_Win_complete(MPID_Win *win_ptr)
MPIDI_RMA_ops *new_ptr = NULL;
MPIU_INSTR_DURATION_START(rmaqueue_alloc);
mpi_errno = MPIDI_CH3I_Win_ops_alloc_tail(win_ptr, &new_ptr);
mpi_errno = MPIDI_CH3I_RMA_Ops_alloc_tail(&win_ptr->rma_ops_list, &new_ptr);
MPIU_INSTR_DURATION_END(rmaqueue_alloc);
if (mpi_errno) { MPIU_ERR_POP(mpi_errno); }
......@@ -1670,7 +1665,7 @@ int MPIDI_Win_complete(MPID_Win *win_ptr)
mpi_errno = MPIDI_CH3I_RMAListComplete(win_ptr);
}
MPIU_Assert( MPIDI_CH3I_Win_ops_isempty(win_ptr) );
MPIU_Assert(MPIDI_CH3I_RMA_Ops_isempty(&win_ptr->rma_ops_list));
mpi_errno = MPIR_Group_free_impl(win_grp_ptr);
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
......@@ -1819,7 +1814,7 @@ int MPIDI_Win_lock(int lock_type, int dest, int assert, MPID_Win *win_ptr)
/* target is some other process. add the lock request to rma_ops_list */
MPIU_INSTR_DURATION_START(rmaqueue_alloc);
mpi_errno = MPIDI_CH3I_Win_ops_alloc_tail(win_ptr, &new_ptr);
mpi_errno = MPIDI_CH3I_RMA_Ops_alloc_tail(&win_ptr->rma_ops_list, &new_ptr);
MPIU_INSTR_DURATION_END(rmaqueue_alloc);
if (mpi_errno) { MPIU_ERR_POP(mpi_errno); }
......@@ -1860,7 +1855,7 @@ int MPIDI_Win_unlock(int dest, MPID_Win *win_ptr)
if (dest == win_ptr->myrank) {
/* local lock. release the lock on the window, grant the next one
* in the queue, and return. */
MPIU_Assert(MPIDI_CH3I_Win_ops_isempty(win_ptr));
MPIU_Assert(MPIDI_CH3I_RMA_Ops_isempty(&win_ptr->rma_ops_list));
mpi_errno = MPIDI_CH3I_Release_lock(win_ptr);
if (mpi_errno != MPI_SUCCESS) goto fn_exit;
win_ptr->remote_lock_state = MPIDI_CH3_WIN_LOCK_NONE;
......@@ -1870,7 +1865,7 @@ int MPIDI_Win_unlock(int dest, MPID_Win *win_ptr)
comm_ptr = win_ptr->comm_ptr;
rma_op = win_ptr->rma_ops_list_head;
rma_op = MPIDI_CH3I_RMA_Ops_head(&win_ptr->rma_ops_list);
/* win_lock was not called. return error */
if ( win_ptr->remote_lock_state == MPIDI_CH3_WIN_LOCK_NONE &&
......@@ -1888,7 +1883,7 @@ int MPIDI_Win_unlock(int dest, MPID_Win *win_ptr)
lock, we can do nothing and return. */
if (rma_op && rma_op->type == MPIDI_RMA_LOCK && rma_op->next == NULL &&
win_ptr->remote_lock_state == MPIDI_CH3_WIN_LOCK_NONE) {
MPIDI_CH3I_Win_ops_free(win_ptr);
MPIDI_CH3I_RMA_Ops_free(&win_ptr->rma_ops_list);
goto fn_exit;
}
......@@ -1986,7 +1981,7 @@ int MPIDI_Win_unlock(int dest, MPID_Win *win_ptr)
}
fn_exit:
MPIU_Assert( MPIDI_CH3I_Win_ops_isempty(win_ptr) );
MPIU_Assert(MPIDI_CH3I_RMA_Ops_isempty(&win_ptr->rma_ops_list));
MPIDI_RMA_FUNC_EXIT(MPID_STATE_MPIDI_WIN_UNLOCK);
return mpi_errno;
/* --BEGIN ERROR HANDLING-- */
......@@ -2022,7 +2017,7 @@ int MPIDI_Win_flush(int rank, MPID_Win *win_ptr)
/* Local flush: ops are performed immediately on the local process */
if (rank == win_ptr->comm_ptr->rank) {
MPIU_Assert(win_ptr->remote_lock_state == MPIDI_CH3_WIN_LOCK_GRANTED);
MPIU_Assert(MPIDI_CH3I_Win_ops_isempty(win_ptr));
MPIU_Assert(MPIDI_CH3I_RMA_Ops_isempty(&win_ptr->rma_ops_list));
goto fn_exit;
}
......@@ -2050,11 +2045,11 @@ int MPIDI_Win_flush(int rank, MPID_Win *win_ptr)
if (win_ptr->remote_lock_state == MPIDI_CH3_WIN_LOCK_NONE) {
/* Ensure that win_lock is waiting at the head of the ops list */
MPIU_ERR_CHKANDJUMP(MPIDI_CH3I_Win_ops_isempty(win_ptr) ||
win_ptr->rma_ops_list_head->type != MPIDI_RMA_LOCK,
MPIU_ERR_CHKANDJUMP(MPIDI_CH3I_RMA_Ops_isempty(&win_ptr->rma_ops_list) ||
MPIDI_CH3I_RMA_Ops_head(&win_ptr->rma_ops_list)->type != MPIDI_RMA_LOCK,
mpi_errno, MPI_ERR_OTHER, "**rmasync");
mpi_errno = MPIDI_CH3I_Send_lock_msg(win_ptr->rma_ops_list_head->target_rank,
win_ptr->rma_ops_list_head->lock_type, win_ptr);
mpi_errno = MPIDI_CH3I_Send_lock_msg(MPIDI_CH3I_RMA_Ops_head(&win_ptr->rma_ops_list)->target_rank,
MPIDI_CH3I_RMA_Ops_head(&win_ptr->rma_ops_list)->lock_type, win_ptr);
if (mpi_errno) { MPIU_ERR_POP(mpi_errno); }
}
......@@ -2102,7 +2097,7 @@ int MPIDI_Win_flush(int rank, MPID_Win *win_ptr)
}
fn_exit:
MPIU_Assert(MPIDI_CH3I_Win_ops_isempty(win_ptr));
MPIU_Assert(MPIDI_CH3I_RMA_Ops_isempty(&win_ptr->rma_ops_list));
MPIDI_RMA_FUNC_EXIT(MPID_STATE_MPIDI_WIN_FLUSH);
return mpi_errno;
/* --BEGIN ERROR HANDLING-- */
......@@ -2230,7 +2225,6 @@ static int MPIDI_CH3I_Do_passive_target_rma(MPID_Win *win_ptr,
{
int mpi_errno = MPI_SUCCESS, nops;
MPIDI_RMA_ops *curr_ptr;
MPIDI_RMA_ops **prevNextPtr;
MPI_Win source_win_handle, target_win_handle = MPI_WIN_NULL;
int nRequest=0, nRequestNew=0;
MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3I_DO_PASSIVE_TARGET_RMA);
......@@ -2247,11 +2241,11 @@ static int MPIDI_CH3I_Do_passive_target_rma(MPID_Win *win_ptr,
there is no third-party communication issue. */
*wait_for_rma_done_pkt = 0;
}
else if (MPIDI_CH3I_Win_ops_isempty(win_ptr)) {
else if (MPIDI_CH3I_RMA_Ops_isempty(&win_ptr->rma_ops_list)) {
/* Shared lock -- The ops list is empty. Any issued ops are already
remote complete; done packet is not needed for safe third party
communication. */
MPIU_Assert(MPIDI_CH3I_Win_ops_isempty(win_ptr));
MPIU_Assert(MPIDI_CH3I_RMA_Ops_isempty(&win_ptr->rma_ops_list));
*wait_for_rma_done_pkt = 0;
}
else {
......@@ -2259,25 +2253,25 @@ static int MPIDI_CH3I_Do_passive_target_rma(MPID_Win *win_ptr,
to the end of the list and do it last, in which case an rma done
pkt is not needed. If there is no get, rma done pkt is needed */
if (win_ptr->rma_ops_list_tail->type == MPIDI_RMA_GET ||
win_ptr->rma_ops_list_tail->type == MPIDI_RMA_COMPARE_AND_SWAP ||
win_ptr->rma_ops_list_tail->type == MPIDI_RMA_FETCH_AND_OP ||
win_ptr->rma_ops_list_tail->type == MPIDI_RMA_GET_ACCUMULATE) {
if (MPIDI_CH3I_RMA_Ops_tail(&win_ptr->rma_ops_list)->type == MPIDI_RMA_GET ||
MPIDI_CH3I_RMA_Ops_tail(&win_ptr->rma_ops_list)->type == MPIDI_RMA_COMPARE_AND_SWAP ||
MPIDI_CH3I_RMA_Ops_tail(&win_ptr->rma_ops_list)->type == MPIDI_RMA_FETCH_AND_OP ||
MPIDI_CH3I_RMA_Ops_tail(&win_ptr->rma_ops_list)->type == MPIDI_RMA_GET_ACCUMULATE) {
/* last operation sends a response message. no need to wait
for an additional rma done pkt */
*wait_for_rma_done_pkt = 0;
}
else {
MPIDI_RMA_ops *head = MPIDI_CH3I_RMA_Ops_head(&win_ptr->rma_ops_list);
/* go through the list and move the first get operation
(if there is one) to the end. Note that the first
operation may be a lock, so we can skip it */
if (win_ptr->rma_ops_list_head->type == MPIDI_RMA_LOCK) {
curr_ptr = win_ptr->rma_ops_list_head->next;
prevNextPtr = &win_ptr->rma_ops_list_head->next;
if (head->type == MPIDI_RMA_LOCK) {
curr_ptr = head->next;
} else {
curr_ptr = win_ptr->rma_ops_list_head;
prevNextPtr = &win_ptr->rma_ops_list_head;
curr_ptr = head;
}
*wait_for_rma_done_pkt = 1;
......@@ -2286,26 +2280,24 @@ static int MPIDI_CH3I_Do_passive_target_rma(MPID_Win *win_ptr,
if (curr_ptr->type == MPIDI_RMA_GET) {
/* Found a GET, move it to the end */
*wait_for_rma_done_pkt = 0;
win_ptr->rma_ops_list_tail->next = curr_ptr;
*prevNextPtr = curr_ptr->next;
curr_ptr->next = NULL;
win_ptr->rma_ops_list_tail = curr_ptr;
MPIDI_CH3I_RMA_Ops_unlink(&win_ptr->rma_ops_list, curr_ptr);
MPIDI_CH3I_RMA_Ops_append(&win_ptr->rma_ops_list, curr_ptr);
break;
}
else {
prevNextPtr = &(curr_ptr->next);
curr_ptr = curr_ptr->next;
}
}
}
}
prevNextPtr = &win_ptr->rma_ops_list_head;
curr_ptr = win_ptr->rma_ops_list_head;
curr_ptr = MPIDI_CH3I_RMA_Ops_head(&win_ptr->rma_ops_list);
/* Remove the lock operation if it's still on the head of the list */
if (!MPIDI_CH3I_Win_ops_isempty(win_ptr) && win_ptr->rma_ops_list_head->type == MPIDI_RMA_LOCK)
MPIDI_CH3I_Win_ops_free_and_next(win_ptr, &curr_ptr, prevNextPtr);
if (!MPIDI_CH3I_RMA_Ops_isempty(&win_ptr->rma_ops_list) &&
MPIDI_CH3I_RMA_Ops_head(&win_ptr->rma_ops_list)->type == MPIDI_RMA_LOCK)
MPIDI_CH3I_RMA_Ops_free_and_next(&win_ptr->rma_ops_list, &curr_ptr);
nops = 0;
while (curr_ptr != NULL) {
......@@ -2315,7 +2307,7 @@ static int MPIDI_CH3I_Do_passive_target_rma(MPID_Win *win_ptr,
MPIU_INSTR_DURATION_START(winunlock_issue);
curr_ptr = win_ptr->rma_ops_list_head;
curr_ptr = MPIDI_CH3I_RMA_Ops_head(&win_ptr->rma_ops_list);
if (curr_ptr != NULL) {
target_win_handle = win_ptr->all_win_handles[curr_ptr->target_rank];
......@@ -2378,12 +2370,11 @@ static int MPIDI_CH3I_Do_passive_target_rma(MPID_Win *win_ptr,
/* If the request is null, we can remove it immediately */
if (!curr_ptr->request) {
MPIDI_CH3I_Win_ops_free_and_next(win_ptr, &curr_ptr, prevNextPtr);
MPIDI_CH3I_RMA_Ops_free_and_next(&win_ptr->rma_ops_list, &curr_ptr);
}
else {
nRequest++;
MPIU_INSTR_COUNTER_INCR(winunlock_reqs,1);
prevNextPtr = &curr_ptr->next;
curr_ptr = curr_ptr->next;
if (nRequest > MPIR_PARAM_RMA_NREQUEST_THRESHOLD &&
nRequest - nRequestNew > MPIR_PARAM_RMA_NREQUEST_NEW_THRESHOLD) {
......@@ -2410,7 +2401,7 @@ static int MPIDI_CH3I_Do_passive_target_rma(MPID_Win *win_ptr,
MPIU_ERR_CHKANDJUMP(mpi_errno != MPI_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**winRMAmessage");
}