Commit ed20cd37 authored by Xin Zhao's avatar Xin Zhao
Browse files

add original RMA PVARs back.



Add some original RMA PVARs back to the new
RMA infrastructure, including timing of packet
handlers, op allocation and setting, window
creation, etc.
Signed-off-by: Pavan Balaji's avatarPavan Balaji <balaji@anl.gov>
parent b682ec0e
......@@ -14,6 +14,8 @@
#define MPIDI_CH3_PAGESIZE_MASK (~(MPIDI_CH3_PAGESIZE-1))
#define MPIDI_CH3_ROUND_UP_PAGESIZE(x) ((((MPI_Aint)x)+(~MPIDI_CH3_PAGESIZE_MASK)) & MPIDI_CH3_PAGESIZE_MASK)
MPIR_T_PVAR_DOUBLE_TIMER_DECL_EXTERN(RMA, rma_wincreate_allgather);
MPIDI_SHM_Wins_list_t shm_wins_list;
static int MPIDI_CH3I_Win_allocate_shm(MPI_Aint size, int disp_unit, MPID_Info *info, MPID_Comm *comm_ptr,
......@@ -315,6 +317,7 @@ static int MPIDI_CH3I_Win_allocate_shm(MPI_Aint size, int disp_unit, MPID_Info *
node_size = node_comm_ptr->local_size;
node_rank = node_comm_ptr->rank;
MPIR_T_PVAR_TIMER_START(RMA, rma_wincreate_allgather);
/* allocate memory for the base addresses, disp_units, and
completion counters of all processes */
MPIU_CHKPMEM_MALLOC((*win_ptr)->base_addrs, void **,
......@@ -347,6 +350,7 @@ static int MPIDI_CH3I_Win_allocate_shm(MPI_Aint size, int disp_unit, MPID_Info *
mpi_errno = MPIR_Allgather_impl(MPI_IN_PLACE, 0, MPI_DATATYPE_NULL,
tmp_buf, 3 * sizeof(MPI_Aint), MPI_BYTE,
(*win_ptr)->comm_ptr, &errflag);
MPIR_T_PVAR_TIMER_END(RMA, rma_wincreate_allgather);
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
MPIU_ERR_CHKANDJUMP(errflag, mpi_errno, MPI_ERR_OTHER, "**coll_fail");
......
......@@ -21,6 +21,8 @@ extern struct MPIDI_RMA_Target *global_rma_target_pool, *global_rma_target_pool_
extern int num_active_issued_win;
extern int num_passive_win;
MPIR_T_PVAR_DOUBLE_TIMER_DECL_EXTERN(RMA, rma_rmaqueue_alloc);
/* MPIDI_CH3I_Win_op_alloc(): get a new op element from op pool and
* initialize it. If we cannot get one, return NULL. */
#undef FUNCNAME
......@@ -534,7 +536,9 @@ static inline int MPIDI_CH3I_Win_get_op(MPID_Win * win_ptr, MPIDI_RMA_Op_t **e)
int mpi_errno = MPI_SUCCESS;
while (1) {
MPIR_T_PVAR_TIMER_START(RMA, rma_rmaqueue_alloc);
new_ptr = MPIDI_CH3I_Win_op_alloc(win_ptr);
MPIR_T_PVAR_TIMER_END(RMA, rma_rmaqueue_alloc);
if (new_ptr != NULL) break;
mpi_errno = MPIDI_CH3I_RMA_Cleanup_ops_win(win_ptr,
......@@ -542,7 +546,9 @@ static inline int MPIDI_CH3I_Win_get_op(MPID_Win * win_ptr, MPIDI_RMA_Op_t **e)
&remote_completed);
if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
MPIR_T_PVAR_TIMER_START(RMA, rma_rmaqueue_alloc);
new_ptr = MPIDI_CH3I_Win_op_alloc(win_ptr);
MPIR_T_PVAR_TIMER_END(RMA, rma_rmaqueue_alloc);
if (new_ptr != NULL) break;
if (MPIDI_RMA_Pkt_orderings->flush_remote) {
......@@ -550,7 +556,9 @@ static inline int MPIDI_CH3I_Win_get_op(MPID_Win * win_ptr, MPIDI_RMA_Op_t **e)
if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
}
MPIR_T_PVAR_TIMER_START(RMA, rma_rmaqueue_alloc);
new_ptr = MPIDI_CH3I_Win_op_alloc(win_ptr);
MPIR_T_PVAR_TIMER_END(RMA, rma_rmaqueue_alloc);
if (new_ptr != NULL) break;
mpi_errno = MPIDI_CH3I_RMA_Cleanup_ops_aggressive(win_ptr);
......
......@@ -1225,7 +1225,8 @@ void *MPIDI_Alloc_mem(size_t size, MPID_Info *info_ptr);
int MPIDI_Free_mem(void *ptr);
/* Pvars */
void MPIDI_CH3_RMA_Init_Pvars(void);
void MPIDI_CH3_RMA_Init_sync_pvars(void);
void MPIDI_CH3_RMA_Init_pkthandler_pvars(void);
/* internal */
int MPIDI_CH3I_Release_lock(MPID_Win * win_ptr);
......
......@@ -12,6 +12,8 @@
#include "mpid_rma_shm.h"
#include "mpid_rma_issue.h"
MPIR_T_PVAR_DOUBLE_TIMER_DECL_EXTERN(RMA, rma_lockqueue_alloc);
MPIR_T_PVAR_DOUBLE_TIMER_DECL_EXTERN(RMA, rma_winlock_getlocallock);
#undef FUNCNAME
#define FUNCNAME send_lock_msg
......@@ -226,7 +228,9 @@ static inline int enqueue_lock_origin(MPID_Win *win_ptr, MPIDI_CH3_Pkt_t *pkt)
MPIDI_Win_lock_queue *new_ptr = NULL;
int mpi_errno = MPI_SUCCESS;
MPIR_T_PVAR_TIMER_START(RMA, rma_lockqueue_alloc);
new_ptr = (MPIDI_Win_lock_queue *) MPIU_Malloc(sizeof(MPIDI_Win_lock_queue));
MPIR_T_PVAR_TIMER_END(RMA, rma_lockqueue_alloc);
if (!new_ptr) {
MPIU_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %s",
"MPIDI_Win_lock_queue");
......@@ -278,6 +282,8 @@ static inline int acquire_local_lock(MPID_Win * win_ptr, int lock_type)
MPIDI_STATE_DECL(MPID_STATE_ACQUIRE_LOCAL_LOCK);
MPIDI_RMA_FUNC_ENTER(MPID_STATE_ACQUIRE_LOCAL_LOCK);
MPIR_T_PVAR_TIMER_START(RMA, rma_winlock_getlocallock);
if (MPIDI_CH3I_Try_acquire_win_lock(win_ptr, lock_type) == 1) {
mpi_errno = set_lock_sync_counter(win_ptr, win_ptr->comm_ptr->rank);
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
......@@ -296,6 +302,7 @@ static inline int acquire_local_lock(MPID_Win * win_ptr, int lock_type)
}
fn_exit:
MPIR_T_PVAR_TIMER_END(RMA, rma_winlock_getlocallock);
MPIDI_RMA_FUNC_EXIT(MPID_STATE_ACQUIRE_LOCAL_LOCK);
return mpi_errno;
/* --BEGIN ERROR HANDLING-- */
......
......@@ -6,6 +6,8 @@
#include "mpidrma.h"
MPIR_T_PVAR_DOUBLE_TIMER_DECL_EXTERN(RMA, rma_rmaqueue_set);
#define MPIDI_PASSIVE_TARGET_DONE_TAG 348297
#define MPIDI_PASSIVE_TARGET_RMA_TAG 563924
......@@ -98,6 +100,8 @@ int MPIDI_Put(const void *origin_addr, int origin_count, MPI_Datatype
mpi_errno = MPIDI_CH3I_Win_get_op(win_ptr, &new_ptr);
if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
MPIR_T_PVAR_TIMER_START(RMA, rma_rmaqueue_set);
put_pkt = &(new_ptr->pkt.put);
MPIDI_Pkt_init(put_pkt, MPIDI_CH3_PKT_PUT);
put_pkt->addr = (char *) win_ptr->base_addrs[target_rank] +
......@@ -116,10 +120,14 @@ int MPIDI_Put(const void *origin_addr, int origin_count, MPI_Datatype
new_ptr->origin_datatype = origin_datatype;
new_ptr->target_rank = target_rank;
MPIR_T_PVAR_TIMER_END(RMA, rma_rmaqueue_set);
mpi_errno = MPIDI_CH3I_Win_enqueue_op(win_ptr, new_ptr);
if (mpi_errno)
MPIU_ERR_POP(mpi_errno);
MPIR_T_PVAR_TIMER_START(RMA, rma_rmaqueue_set);
/* if source or target datatypes are derived, increment their
* reference counts */
if (!MPIR_DATATYPE_IS_PREDEFINED(origin_datatype)) {
......@@ -160,6 +168,8 @@ int MPIDI_Put(const void *origin_addr, int origin_count, MPI_Datatype
}
}
MPIR_T_PVAR_TIMER_END(RMA, rma_rmaqueue_set);
mpi_errno = MPIDI_CH3I_RMA_Make_progress_target(win_ptr, target_rank, &made_progress);
if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
......@@ -263,6 +273,8 @@ int MPIDI_Get(void *origin_addr, int origin_count, MPI_Datatype
mpi_errno = MPIDI_CH3I_Win_get_op(win_ptr, &new_ptr);
if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
MPIR_T_PVAR_TIMER_START(RMA, rma_rmaqueue_set);
get_pkt = &(new_ptr->pkt.get);
MPIDI_Pkt_init(get_pkt, MPIDI_CH3_PKT_GET);
get_pkt->addr = (char *) win_ptr->base_addrs[target_rank] +
......@@ -280,10 +292,14 @@ int MPIDI_Get(void *origin_addr, int origin_count, MPI_Datatype
new_ptr->origin_datatype = origin_datatype;
new_ptr->target_rank = target_rank;
MPIR_T_PVAR_TIMER_END(RMA, rma_rmaqueue_set);
mpi_errno = MPIDI_CH3I_Win_enqueue_op(win_ptr, new_ptr);
if (mpi_errno)
MPIU_ERR_POP(mpi_errno);
MPIR_T_PVAR_TIMER_START(RMA, rma_rmaqueue_set);
/* if source or target datatypes are derived, increment their
* reference counts */
if (!MPIR_DATATYPE_IS_PREDEFINED(origin_datatype)) {
......@@ -301,6 +317,8 @@ int MPIDI_Get(void *origin_addr, int origin_count, MPI_Datatype
new_ptr->piggyback_lock_candidate = 1;
}
MPIR_T_PVAR_TIMER_END(RMA, rma_rmaqueue_set);
mpi_errno = MPIDI_CH3I_RMA_Make_progress_target(win_ptr, target_rank, &made_progress);
if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
......@@ -405,6 +423,8 @@ int MPIDI_Accumulate(const void *origin_addr, int origin_count, MPI_Datatype
mpi_errno = MPIDI_CH3I_Win_get_op(win_ptr, &new_ptr);
if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
MPIR_T_PVAR_TIMER_START(RMA, rma_rmaqueue_set);
accum_pkt = &(new_ptr->pkt.accum);
MPIDI_Pkt_init(accum_pkt, MPIDI_CH3_PKT_ACCUMULATE);
......@@ -424,10 +444,14 @@ int MPIDI_Accumulate(const void *origin_addr, int origin_count, MPI_Datatype
new_ptr->origin_datatype = origin_datatype;
new_ptr->target_rank = target_rank;
MPIR_T_PVAR_TIMER_END(RMA, rma_rmaqueue_set);
mpi_errno = MPIDI_CH3I_Win_enqueue_op(win_ptr, new_ptr);
if (mpi_errno)
MPIU_ERR_POP(mpi_errno);
MPIR_T_PVAR_TIMER_START(RMA, rma_rmaqueue_set);
/* if source or target datatypes are derived, increment their
* reference counts */
if (!MPIR_DATATYPE_IS_PREDEFINED(origin_datatype)) {
......@@ -468,6 +492,8 @@ int MPIDI_Accumulate(const void *origin_addr, int origin_count, MPI_Datatype
}
}
MPIR_T_PVAR_TIMER_END(RMA, rma_rmaqueue_set);
mpi_errno = MPIDI_CH3I_RMA_Make_progress_target(win_ptr, target_rank, &made_progress);
if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
......@@ -576,6 +602,8 @@ int MPIDI_Get_accumulate(const void *origin_addr, int origin_count,
/* TODO: Can we use the MPIDI_RMA_ACC_CONTIG optimization? */
MPIR_T_PVAR_TIMER_START(RMA, rma_rmaqueue_set);
if (op == MPI_NO_OP) {
/* Convert GAcc to a Get */
MPIDI_CH3_Pkt_get_t *get_pkt = &(new_ptr->pkt.get);
......@@ -678,6 +706,8 @@ int MPIDI_Get_accumulate(const void *origin_addr, int origin_count,
}
}
MPIR_T_PVAR_TIMER_END(RMA, rma_rmaqueue_set);
mpi_errno = MPIDI_CH3I_Win_enqueue_op(win_ptr, new_ptr);
if (mpi_errno)
MPIU_ERR_POP(mpi_errno);
......@@ -781,6 +811,8 @@ int MPIDI_Compare_and_swap(const void *origin_addr, const void *compare_addr,
mpi_errno = MPIDI_CH3I_Win_get_op(win_ptr, &new_ptr);
if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
MPIR_T_PVAR_TIMER_START(RMA, rma_rmaqueue_set);
cas_pkt = &(new_ptr->pkt.cas);
MPIDI_Pkt_init(cas_pkt, MPIDI_CH3_PKT_CAS);
cas_pkt->addr = (char *) win_ptr->base_addrs[target_rank] +
......@@ -800,6 +832,8 @@ int MPIDI_Compare_and_swap(const void *origin_addr, const void *compare_addr,
new_ptr->target_rank = target_rank;
new_ptr->piggyback_lock_candidate = 1; /* CAS is always able to piggyback LOCK */
MPIR_T_PVAR_TIMER_END(RMA, rma_rmaqueue_set);
mpi_errno = MPIDI_CH3I_Win_enqueue_op(win_ptr, new_ptr);
if (mpi_errno)
MPIU_ERR_POP(mpi_errno);
......@@ -900,6 +934,8 @@ int MPIDI_Fetch_and_op(const void *origin_addr, void *result_addr,
mpi_errno = MPIDI_CH3I_Win_get_op(win_ptr, &new_ptr);
if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
MPIR_T_PVAR_TIMER_START(RMA, rma_rmaqueue_set);
if (op == MPI_NO_OP) {
/* Convert FOP to a Get */
MPIDI_CH3_Pkt_get_t *get_pkt = &(new_ptr->pkt.get);
......@@ -958,6 +994,8 @@ int MPIDI_Fetch_and_op(const void *origin_addr, void *result_addr,
}
}
MPIR_T_PVAR_TIMER_END(RMA, rma_rmaqueue_set);
mpi_errno = MPIDI_CH3I_Win_enqueue_op(win_ptr, new_ptr);
if (mpi_errno)
MPIU_ERR_POP(mpi_errno);
......
......@@ -7,6 +7,169 @@
#include "mpidimpl.h"
#include "mpidrma.h"
MPIR_T_PVAR_DOUBLE_TIMER_DECL(RMA, rma_rmapkt_put);
MPIR_T_PVAR_DOUBLE_TIMER_DECL(RMA, rma_rmapkt_get);
MPIR_T_PVAR_DOUBLE_TIMER_DECL(RMA, rma_rmapkt_acc);
MPIR_T_PVAR_DOUBLE_TIMER_DECL(RMA, rma_rmapkt_get_accum);
MPIR_T_PVAR_DOUBLE_TIMER_DECL(RMA, rma_rmapkt_cas);
MPIR_T_PVAR_DOUBLE_TIMER_DECL(RMA, rma_rmapkt_fop);
MPIR_T_PVAR_DOUBLE_TIMER_DECL(RMA, rma_rmapkt_get_resp);
MPIR_T_PVAR_DOUBLE_TIMER_DECL(RMA, rma_rmapkt_get_accum_resp);
MPIR_T_PVAR_DOUBLE_TIMER_DECL(RMA, rma_rmapkt_cas_resp);
MPIR_T_PVAR_DOUBLE_TIMER_DECL(RMA, rma_rmapkt_fop_resp);
MPIR_T_PVAR_DOUBLE_TIMER_DECL(RMA, rma_rmapkt_lock);
MPIR_T_PVAR_DOUBLE_TIMER_DECL(RMA, rma_rmapkt_lock_granted);
MPIR_T_PVAR_DOUBLE_TIMER_DECL(RMA, rma_rmapkt_unlock);
MPIR_T_PVAR_DOUBLE_TIMER_DECL(RMA, rma_rmapkt_flush);
MPIR_T_PVAR_DOUBLE_TIMER_DECL(RMA, rma_rmapkt_flush_ack);
MPIR_T_PVAR_DOUBLE_TIMER_DECL(RMA, rma_rmapkt_decr_at_cnt);
void MPIDI_CH3_RMA_Init_pkthandler_pvars(void)
{
/* rma_rmapkt_put */
MPIR_T_PVAR_TIMER_REGISTER_STATIC(RMA,
MPI_DOUBLE,
rma_rmapkt_put,
MPI_T_VERBOSITY_MPIDEV_DETAIL,
MPI_T_BIND_NO_OBJECT,
MPIR_T_PVAR_FLAG_READONLY,
"RMA", "RMA:PKTHANDLER for Put (in seconds)");
/* rma_rmapkt_get */
MPIR_T_PVAR_TIMER_REGISTER_STATIC(RMA,
MPI_DOUBLE,
rma_rmapkt_get,
MPI_T_VERBOSITY_MPIDEV_DETAIL,
MPI_T_BIND_NO_OBJECT,
MPIR_T_PVAR_FLAG_READONLY,
"RMA", "RMA:PKTHANDLER for Get (in seconds)");
/* rma_rmapkt_acc */
MPIR_T_PVAR_TIMER_REGISTER_STATIC(RMA,
MPI_DOUBLE,
rma_rmapkt_acc,
MPI_T_VERBOSITY_MPIDEV_DETAIL,
MPI_T_BIND_NO_OBJECT,
MPIR_T_PVAR_FLAG_READONLY,
"RMA", "RMA:PKTHANDLER for Accumulate (in seconds)");
/* rma_rmapkt_get_accum */
MPIR_T_PVAR_TIMER_REGISTER_STATIC(RMA,
MPI_DOUBLE,
rma_rmapkt_get_accum,
MPI_T_VERBOSITY_MPIDEV_DETAIL,
MPI_T_BIND_NO_OBJECT,
MPIR_T_PVAR_FLAG_READONLY,
"RMA", "RMA:PKTHANDLER for Get-Accumulate (in seconds)");
/* rma_rmapkt_cas */
MPIR_T_PVAR_TIMER_REGISTER_STATIC(RMA,
MPI_DOUBLE,
rma_rmapkt_cas,
MPI_T_VERBOSITY_MPIDEV_DETAIL,
MPI_T_BIND_NO_OBJECT,
MPIR_T_PVAR_FLAG_READONLY,
"RMA", "RMA:PKTHANDLER for Compare-and-swap (in seconds)");
/* rma_rmapkt_fop */
MPIR_T_PVAR_TIMER_REGISTER_STATIC(RMA,
MPI_DOUBLE,
rma_rmapkt_fop,
MPI_T_VERBOSITY_MPIDEV_DETAIL,
MPI_T_BIND_NO_OBJECT,
MPIR_T_PVAR_FLAG_READONLY,
"RMA", "RMA:PKTHANDLER for Fetch-and-op (in seconds)");
/* rma_rmapkt_get_resp */
MPIR_T_PVAR_TIMER_REGISTER_STATIC(RMA,
MPI_DOUBLE,
rma_rmapkt_get_resp,
MPI_T_VERBOSITY_MPIDEV_DETAIL,
MPI_T_BIND_NO_OBJECT,
MPIR_T_PVAR_FLAG_READONLY,
"RMA", "RMA:PKTHANDLER for Get response (in seconds)");
/* rma_rmapkt_get_accum_resp */
MPIR_T_PVAR_TIMER_REGISTER_STATIC(RMA,
MPI_DOUBLE,
rma_rmapkt_get_accum_resp,
MPI_T_VERBOSITY_MPIDEV_DETAIL,
MPI_T_BIND_NO_OBJECT,
MPIR_T_PVAR_FLAG_READONLY,
"RMA", "RMA:PKTHANDLER for Get-Accumulate response (in seconds)");
/* rma_rmapkt_cas_resp */
MPIR_T_PVAR_TIMER_REGISTER_STATIC(RMA,
MPI_DOUBLE,
rma_rmapkt_cas_resp,
MPI_T_VERBOSITY_MPIDEV_DETAIL,
MPI_T_BIND_NO_OBJECT,
MPIR_T_PVAR_FLAG_READONLY,
"RMA", "RMA:PKTHANDLER for Compare-and-Swap response (in seconds)");
/* rma_rmapkt_fop_resp */
MPIR_T_PVAR_TIMER_REGISTER_STATIC(RMA,
MPI_DOUBLE,
rma_rmapkt_fop_resp,
MPI_T_VERBOSITY_MPIDEV_DETAIL,
MPI_T_BIND_NO_OBJECT,
MPIR_T_PVAR_FLAG_READONLY,
"RMA", "RMA:PKTHANDLER for Fetch-and-op response (in seconds)");
/* rma_rmapkt_lock */
MPIR_T_PVAR_TIMER_REGISTER_STATIC(RMA,
MPI_DOUBLE,
rma_rmapkt_lock,
MPI_T_VERBOSITY_MPIDEV_DETAIL,
MPI_T_BIND_NO_OBJECT,
MPIR_T_PVAR_FLAG_READONLY,
"RMA", "RMA:PKTHANDLER for Lock (in seconds)");
/* rma_rmapkt_lock_granted */
MPIR_T_PVAR_TIMER_REGISTER_STATIC(RMA,
MPI_DOUBLE,
rma_rmapkt_lock_granted,
MPI_T_VERBOSITY_MPIDEV_DETAIL,
MPI_T_BIND_NO_OBJECT,
MPIR_T_PVAR_FLAG_READONLY,
"RMA", "RMA:PKTHANDLER for Lock-Granted (in seconds)");
/* rma_rmapkt_unlock */
MPIR_T_PVAR_TIMER_REGISTER_STATIC(RMA,
MPI_DOUBLE,
rma_rmapkt_unlock,
MPI_T_VERBOSITY_MPIDEV_DETAIL,
MPI_T_BIND_NO_OBJECT,
MPIR_T_PVAR_FLAG_READONLY,
"RMA", "RMA:PKTHANDLER for Unlock (in seconds)");
/* rma_rmapkt_flush */
MPIR_T_PVAR_TIMER_REGISTER_STATIC(RMA,
MPI_DOUBLE,
rma_rmapkt_flush,
MPI_T_VERBOSITY_MPIDEV_DETAIL,
MPI_T_BIND_NO_OBJECT,
MPIR_T_PVAR_FLAG_READONLY,
"RMA", "RMA:PKTHANDLER for Flush (in seconds)");
/* rma_rmapkt_flush_ack */
MPIR_T_PVAR_TIMER_REGISTER_STATIC(RMA,
MPI_DOUBLE,
rma_rmapkt_flush_ack,
MPI_T_VERBOSITY_MPIDEV_DETAIL,
MPI_T_BIND_NO_OBJECT,
MPIR_T_PVAR_FLAG_READONLY,
"RMA", "RMA:PKTHANDLER for Flush-Ack (in seconds)");
/* rma_rmapkt_decr_at_cnt */
MPIR_T_PVAR_TIMER_REGISTER_STATIC(RMA,
MPI_DOUBLE,
rma_rmapkt_decr_at_cnt,
MPI_T_VERBOSITY_MPIDEV_DETAIL,
MPI_T_BIND_NO_OBJECT,
MPIR_T_PVAR_FLAG_READONLY,
"RMA", "RMA:PKTHANDLER for Decr-At-Cnt (in seconds)");
}
/* ------------------------------------------------------------------------ */
/*
......@@ -37,6 +200,8 @@ int MPIDI_CH3_PktHandler_Put(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
MPIU_DBG_MSG(CH3_OTHER, VERBOSE, "received put pkt");
MPIR_T_PVAR_TIMER_START(RMA, rma_rmapkt_put);
MPIU_Assert(put_pkt->target_win_handle != MPI_WIN_NULL);
MPID_Win_get_ptr(put_pkt->target_win_handle, win_ptr);
......@@ -174,6 +339,7 @@ int MPIDI_CH3_PktHandler_Put(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
fn_exit:
MPIR_T_PVAR_TIMER_END(RMA, rma_rmapkt_put);
MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_PKTHANDLER_PUT);
return mpi_errno;
fn_fail:
......@@ -203,6 +369,8 @@ int MPIDI_CH3_PktHandler_Get(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
MPIU_DBG_MSG(CH3_OTHER, VERBOSE, "received get pkt");
MPIR_T_PVAR_TIMER_START(RMA, rma_rmapkt_get);
MPIU_Assert(get_pkt->target_win_handle != MPI_WIN_NULL);
MPID_Win_get_ptr(get_pkt->target_win_handle, win_ptr);
......@@ -326,6 +494,7 @@ int MPIDI_CH3_PktHandler_Get(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
}
fn_exit:
MPIR_T_PVAR_TIMER_END(RMA, rma_rmapkt_get);
MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_PKTHANDLER_GET);
return mpi_errno;
fn_fail:
......@@ -356,6 +525,8 @@ int MPIDI_CH3_PktHandler_Accumulate(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
MPIU_DBG_MSG(CH3_OTHER, VERBOSE, "received accumulate pkt");
MPIR_T_PVAR_TIMER_START(RMA, rma_rmapkt_acc);
MPIU_Assert(accum_pkt->target_win_handle != MPI_WIN_NULL);
MPID_Win_get_ptr(accum_pkt->target_win_handle, win_ptr);
......@@ -500,6 +671,7 @@ int MPIDI_CH3_PktHandler_Accumulate(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
}
fn_exit:
MPIR_T_PVAR_TIMER_END(RMA, rma_rmapkt_acc);
MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_PKTHANDLER_ACCUMULATE);
return mpi_errno;
fn_fail:
......@@ -531,6 +703,8 @@ int MPIDI_CH3_PktHandler_GetAccumulate(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
MPIU_DBG_MSG(CH3_OTHER, VERBOSE, "received accumulate pkt");
MPIR_T_PVAR_TIMER_START(RMA, rma_rmapkt_get_accum);
MPIU_Assert(get_accum_pkt->target_win_handle != MPI_WIN_NULL);
MPID_Win_get_ptr(get_accum_pkt->target_win_handle, win_ptr);
......@@ -677,6 +851,7 @@ int MPIDI_CH3_PktHandler_GetAccumulate(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
}
fn_exit:
MPIR_T_PVAR_TIMER_END(RMA, rma_rmapkt_get_accum);
MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_PKTHANDLER_GETACCUMULATE);
return mpi_errno;
fn_fail:
......@@ -706,6 +881,8 @@ int MPIDI_CH3_PktHandler_CAS(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
MPIU_DBG_MSG(CH3_OTHER, VERBOSE, "received CAS pkt");
MPIR_T_PVAR_TIMER_START(RMA, rma_rmapkt_cas);
MPIU_Assert(cas_pkt->target_win_handle != MPI_WIN_NULL);
MPID_Win_get_ptr(cas_pkt->target_win_handle, win_ptr);
......@@ -789,6 +966,7 @@ int MPIDI_CH3_PktHandler_CAS(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
}
fn_exit:
MPIR_T_PVAR_TIMER_END(RMA, rma_rmapkt_cas);
MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_PKTHANDLER_CAS);
return mpi_errno;
fn_fail:
......@@ -817,6 +995,8 @@ int MPIDI_CH3_PktHandler_CASResp(MPIDI_VC_t * vc ATTRIBUTE((unused)),
MPIU_DBG_MSG(CH3_OTHER, VERBOSE, "received CAS response pkt");
MPIR_T_PVAR_TIMER_START(RMA, rma_rmapkt_cas_resp);
MPID_Win_get_ptr(cas_resp_pkt->source_win_handle, win_ptr);
/* decrement ack_counter on this target */
......@@ -843,6 +1023,7 @@ int MPIDI_CH3_PktHandler_CASResp(MPIDI_VC_t * vc ATTRIBUTE((unused)),
*rreqp = NULL;
fn_exit:
MPIR_T_PVAR_TIMER_END(RMA, rma_rmapkt_cas_resp);
MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_PKTHANDLER_CASRESP);
return mpi_errno;
fn_fail:
......@@ -870,6 +1051,8 @@ int MPIDI_CH3_PktHandler_FOP(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
MPIU_DBG_MSG(CH3_OTHER, VERBOSE, "received FOP pkt");
MPIR_T_PVAR_TIMER_START(RMA, rma_rmapkt_fop);
MPID_Win_get_ptr(fop_pkt->target_win_handle, win_ptr);
(*buflen) = sizeof(MPIDI_CH3_Pkt_t);
......@@ -951,6 +1134,7 @@ int MPIDI_CH3_PktHandler_FOP(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
}
fn_exit:
MPIR_T_PVAR_TIMER_END(RMA, rma_rmapkt_fop);
MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_PKTHANDLER_FOP);
return mpi_errno;
/* --BEGIN ERROR HANDLING-- */
......@@ -979,6 +1163,8 @@ int MPIDI_CH3_PktHandler_FOPResp(MPIDI_VC_t * vc ATTRIBUTE((unused)),
MPIU_DBG_MSG(CH3_OTHER, VERBOSE, "received FOP response pkt");
MPIR_T_PVAR_TIMER_START(RMA, rma_rmapkt_fop_resp);
MPID_Win_get_ptr(fop_resp_pkt->source_win_handle, win_ptr);
/* Copy data to result buffer on orgin */
......@@ -1004,6 +1190,7 @@ int MPIDI_CH3_PktHandler_FOPResp(MPIDI_VC_t * vc ATTRIBUTE((unused)),
*rreqp = NULL;
fn_exit:
MPIR_T_PVAR_TIMER_END(RMA, rma_rmapkt_fop_resp);
MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_PKTHANDLER_FOPRESP);
return mpi_errno;
/* --BEGIN ERROR HANDLING-- */
......@@ -1035,6 +1222,8 @@ int MPIDI_CH3_PktHandler_Get_AccumResp(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
MPIU_DBG_MSG(CH3_OTHER, VERBOSE, "received Get-Accumulate response pkt");
MPIR_T_PVAR_TIMER_START(RMA, rma_rmapkt_get_accum_resp);
MPID_Win_get_ptr(get_accum_resp_pkt->source_win_handle, win_ptr);
/* decrement ack_counter on target */
......@@ -1071,6 +1260,7 @@ int MPIDI_CH3_PktHandler_Get_AccumResp(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
*buflen = data_len + sizeof(MPIDI_CH3_Pkt_t);
fn_exit:
MPIR_T_PVAR_TIMER_END(RMA, rma_rmapkt_get_accum_resp);
MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_PKTHANDLER_GET_ACCUM_RESP);
return mpi_errno;
fn_fail:
......@@ -1094,6 +1284,8 @@ int MPIDI_CH3_PktHandler_Lock(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
MPIU_DBG_MSG(CH3_OTHER, VERBOSE, "received lock pkt");
MPIR_T_PVAR_TIMER_START(RMA, rma_rmapkt_lock);
*buflen = sizeof(MPIDI_CH3_Pkt_t);
MPID_Win_get_ptr(lock_pkt->target_win_handle, win_ptr);
......@@ -1111,6 +1303,7 @@ int MPIDI_CH3_PktHandler_Lock(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
*rreqp = NULL;
fn_fail:
MPIR_T_PVAR_TIMER_END(RMA, rma_rmapkt_lock);
MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_PKTHANDLER_LOCK);
return mpi_errno;
}
......@@ -1139,6 +1332,8 @@ int MPIDI_CH3_PktHandler_GetResp(MPIDI_VC_t * vc ATTRIBUTE((unused)),
MPIU_DBG_MSG(CH3_OTHER, VERBOSE, "received get response pkt");
MPIR_T_PVAR_TIMER_START(RMA, rma_rmapkt_get_resp);
MPID_Win_get_ptr(get_resp_pkt->source_win_handle, win_ptr);
/* decrement ack_counter on target */
......@@ -1175,6 +1370,7 @@ int MPIDI_CH3_PktHandler_GetResp(MPIDI_VC_t * vc ATTRIBUTE((unused)),
*buflen = data_len + sizeof(MPIDI_CH3_Pkt_t);
fn_exit:
MPIR_T_PVAR_TIMER_END(RMA, rma_rmapkt_get_resp);
MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_PKTHANDLER_GETRESP);
return mpi_errno;
fn_fail:
......@@ -1198,6 +1394,8 @@ int MPIDI_CH3_PktHandler_LockGranted(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
MPIU_DBG_MSG(CH3_OTHER, VERBOSE, "received lock granted pkt");
MPIR_T_PVAR_TIMER_START(RMA, rma_rmapkt_lock_granted);
*buflen = sizeof(MPIDI_CH3_Pkt_t);
MPID_Win_get_ptr(lock_granted_pkt->source_win_handle, win_ptr);
......@@ -1208,6 +1406,7 @@ int MPIDI_CH3_PktHandler_LockGranted(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
*rreqp = NULL;
MPIDI_CH3_Progress_signal_completion();
MPIR_T_PVAR_TIMER_END(RMA, rma_rmapkt_lock_granted);
MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_PKTHANDLER_LOCKGRANTED);
fn_exit:
return MPI_SUCCESS;
......@@ -1232,6 +1431,8 @@ int MPIDI_CH3_PktHandler_FlushAck(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
MPIU_DBG_MSG(CH3_OTHER, VERBOSE, "received shared lock ops done pkt");