Commit 0eaf344b authored by Xin Zhao's avatar Xin Zhao Committed by Pavan Balaji
Browse files

Avoid using VC in RMA lock queue structure.



We were adding an unnecessary dependency on VC structure
declarations in the mpidpkt.h file. The required information
in RMA lock queue is only the rank, but not actual VC.
Here we replace VC with rank.
Signed-off-by: Pavan Balaji's avatarPavan Balaji <balaji@anl.gov>
parent cb04acb3
......@@ -102,7 +102,7 @@ typedef struct MPIDI_Win_lock_queue {
struct MPIDI_Win_lock_queue *next;
int lock_type;
MPI_Win source_win_handle;
MPIDI_VC_t *vc;
int origin_rank;
struct MPIDI_PT_single_op *pt_single_op; /* to store info for
* lock-put-unlock optimization */
} MPIDI_Win_lock_queue;
......
......@@ -320,6 +320,7 @@ typedef struct MPIDI_CH3_Pkt_lock {
MPI_Win source_win_handle;
int target_rank; /* Used in unluck/flush response to look up the
* target state at the origin. */
int origin_rank;
} MPIDI_CH3_Pkt_lock_t;
typedef struct MPIDI_CH3_Pkt_lock_granted {
......@@ -338,6 +339,7 @@ typedef struct MPIDI_CH3_Pkt_lock_put_unlock {
MPIDI_CH3_Pkt_flags_t flags;
MPI_Win target_win_handle;
MPI_Win source_win_handle;
int origin_rank;
int lock_type;
void *addr;
int count;
......@@ -349,6 +351,7 @@ typedef struct MPIDI_CH3_Pkt_lock_get_unlock {
MPIDI_CH3_Pkt_flags_t flags;
MPI_Win target_win_handle;
MPI_Win source_win_handle;
int origin_rank;
int lock_type;
void *addr;
int count;
......@@ -361,6 +364,7 @@ typedef struct MPIDI_CH3_Pkt_lock_accum_unlock {
MPIDI_CH3_Pkt_flags_t flags;
MPI_Win target_win_handle;
MPI_Win source_win_handle;
int origin_rank;
int lock_type;
void *addr;
int count;
......
......@@ -37,6 +37,7 @@ static inline int send_lock_msg(int dest, int lock_type, MPID_Win * win_ptr)
lock_pkt->target_win_handle = win_ptr->all_win_handles[dest];
lock_pkt->source_win_handle = win_ptr->handle;
lock_pkt->lock_type = lock_type;
lock_pkt->origin_rank = win_ptr->comm_ptr->rank;
win_ptr->targets[dest].remote_lock_state = MPIDI_CH3_WIN_LOCK_REQUESTED;
win_ptr->targets[dest].remote_lock_mode = lock_type;
......
......@@ -933,9 +933,10 @@ int MPIDI_CH3I_Release_lock(MPID_Win *win_ptr)
they arrive within various packet/request handlers via
MPIDI_CH3_Finish_rma_op_target(). That call cannot be used
here, because it would enter this function recursively. */
MPIDI_VC_t *vc;
MPIDI_Comm_get_vc(win_ptr->comm_ptr, lock_queue->origin_rank, &vc);
mpi_errno =
MPIDI_CH3I_Send_pt_rma_done_pkt(lock_queue->vc, win_ptr,
MPIDI_CH3I_Send_pt_rma_done_pkt(vc, win_ptr,
lock_queue->source_win_handle);
if (mpi_errno) { MPIU_ERR_POP(mpi_errno); }
......@@ -982,8 +983,10 @@ int MPIDI_CH3I_Release_lock(MPID_Win *win_ptr)
else {
/* send lock granted packet. */
MPIDI_VC_t *vc;
MPIDI_Comm_get_vc(win_ptr->comm_ptr, lock_queue->origin_rank, &vc);
mpi_errno =
MPIDI_CH3I_Send_lock_granted_pkt(lock_queue->vc, win_ptr,
MPIDI_CH3I_Send_lock_granted_pkt(vc, win_ptr,
lock_queue->source_win_handle);
/* dequeue entry from lock queue */
......@@ -1077,6 +1080,7 @@ static int do_simple_get(MPID_Win *win_ptr, MPIDI_Win_lock_queue *lock_queue)
MPID_IOV iov[MPID_IOV_LIMIT];
int mpi_errno=MPI_SUCCESS;
MPI_Aint type_size;
MPIDI_VC_t *vc;
MPIDI_STATE_DECL(MPID_STATE_DO_SIMPLE_GET);
MPIDI_FUNC_ENTER(MPID_STATE_DO_SIMPLE_GET);
......@@ -1108,9 +1112,10 @@ static int do_simple_get(MPID_Win *win_ptr, MPIDI_Win_lock_queue *lock_queue)
MPID_Datatype_get_size_macro(lock_queue->pt_single_op->datatype, type_size);
iov[1].MPID_IOV_LEN = lock_queue->pt_single_op->count * type_size;
MPIDI_Comm_get_vc(win_ptr->comm_ptr, lock_queue->origin_rank, &vc);
/* Because this is in a packet handler, it is already within a critical section */
/* MPIU_THREAD_CS_ENTER(CH3COMM,vc); */
mpi_errno = MPIDI_CH3_iSendv(lock_queue->vc, req, iov, 2);
mpi_errno = MPIDI_CH3_iSendv(vc, req, iov, 2);
/* MPIU_THREAD_CS_EXIT(CH3COMM,vc); */
/* --BEGIN ERROR HANDLING-- */
if (mpi_errno != MPI_SUCCESS)
......
......@@ -914,7 +914,7 @@ int MPIDI_CH3_PktHandler_Lock(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
new_ptr->next = NULL;
new_ptr->lock_type = lock_pkt->lock_type;
new_ptr->source_win_handle = lock_pkt->source_win_handle;
new_ptr->vc = vc;
new_ptr->origin_rank = lock_pkt->origin_rank;
new_ptr->pt_single_op = NULL;
}
......@@ -1001,7 +1001,7 @@ int MPIDI_CH3_PktHandler_LockPutUnlock(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
new_ptr->next = NULL;
new_ptr->lock_type = lock_put_unlock_pkt->lock_type;
new_ptr->source_win_handle = lock_put_unlock_pkt->source_win_handle;
new_ptr->vc = vc;
new_ptr->origin_rank = lock_put_unlock_pkt->origin_rank;
new_ptr->pt_single_op->type = MPIDI_RMA_PUT;
new_ptr->pt_single_op->flags = lock_put_unlock_pkt->flags;
......@@ -1149,7 +1149,7 @@ int MPIDI_CH3_PktHandler_LockGetUnlock(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
new_ptr->next = NULL;
new_ptr->lock_type = lock_get_unlock_pkt->lock_type;
new_ptr->source_win_handle = lock_get_unlock_pkt->source_win_handle;
new_ptr->vc = vc;
new_ptr->origin_rank = lock_get_unlock_pkt->origin_rank;
new_ptr->pt_single_op->type = MPIDI_RMA_GET;
new_ptr->pt_single_op->flags = lock_get_unlock_pkt->flags;
......@@ -1240,7 +1240,7 @@ int MPIDI_CH3_PktHandler_LockAccumUnlock(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
new_ptr->next = NULL;
new_ptr->lock_type = lock_accum_unlock_pkt->lock_type;
new_ptr->source_win_handle = lock_accum_unlock_pkt->source_win_handle;
new_ptr->vc = vc;
new_ptr->origin_rank = lock_accum_unlock_pkt->origin_rank;
new_ptr->pt_single_op->type = MPIDI_RMA_ACCUMULATE;
new_ptr->pt_single_op->flags = lock_accum_unlock_pkt->flags;
......
......@@ -2201,6 +2201,7 @@ static int send_lock_put_or_acc(MPID_Win * win_ptr, int target_rank)
lock_put_unlock_pkt->target_win_handle = win_ptr->all_win_handles[rma_op->target_rank];
lock_put_unlock_pkt->source_win_handle = win_ptr->handle;
lock_put_unlock_pkt->lock_type = lock_type;
lock_put_unlock_pkt->origin_rank = win_ptr->comm_ptr->rank;
lock_put_unlock_pkt->addr =
(char *) win_ptr->base_addrs[rma_op->target_rank] +
......@@ -2220,6 +2221,7 @@ static int send_lock_put_or_acc(MPID_Win * win_ptr, int target_rank)
lock_accum_unlock_pkt->target_win_handle = win_ptr->all_win_handles[rma_op->target_rank];
lock_accum_unlock_pkt->source_win_handle = win_ptr->handle;
lock_accum_unlock_pkt->lock_type = lock_type;
lock_accum_unlock_pkt->origin_rank = win_ptr->comm_ptr->rank;
lock_accum_unlock_pkt->addr =
(char *) win_ptr->base_addrs[rma_op->target_rank] +
......@@ -2239,6 +2241,7 @@ static int send_lock_put_or_acc(MPID_Win * win_ptr, int target_rank)
lock_accum_unlock_pkt->target_win_handle = win_ptr->all_win_handles[rma_op->target_rank];
lock_accum_unlock_pkt->source_win_handle = win_ptr->handle;
lock_accum_unlock_pkt->lock_type = lock_type;
lock_accum_unlock_pkt->origin_rank = win_ptr->comm_ptr->rank;
lock_accum_unlock_pkt->addr =
(char *) win_ptr->base_addrs[rma_op->target_rank] +
......@@ -2410,6 +2413,7 @@ static int send_lock_get(MPID_Win * win_ptr, int target_rank)
lock_get_unlock_pkt->target_win_handle = win_ptr->all_win_handles[rma_op->target_rank];
lock_get_unlock_pkt->source_win_handle = win_ptr->handle;
lock_get_unlock_pkt->lock_type = lock_type;
lock_get_unlock_pkt->origin_rank = win_ptr->comm_ptr->rank;
lock_get_unlock_pkt->addr =
(char *) win_ptr->base_addrs[rma_op->target_rank] +
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment