Commit 097c9628 authored by Xin Zhao's avatar Xin Zhao
Browse files

Code-refactor: move piggyback work from op routines to issue rountines

No reviewer.
parent 3a05784f
......@@ -342,11 +342,23 @@ static int issue_put_op(MPIDI_RMA_Op_t * rma_op, MPID_Win *win_ptr,
if (flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK)
put_pkt->lock_type = target_ptr->lock_type;
MPIDI_Comm_get_vc_set_active(comm_ptr, rma_op->target_rank, &vc);
MPID_Datatype_get_size_macro(rma_op->origin_datatype, origin_type_size);
MPIU_Assign_trunc(len, rma_op->origin_count * origin_type_size, size_t);
if (!rma_op->is_dt) {
/* Fill origin data into packet header IMMED area as much as possible */
MPIU_Assign_trunc(put_pkt->immed_len,
MPIR_MIN(len, (MPIDI_RMA_IMMED_BYTES/origin_type_size)*origin_type_size),
int);
if (put_pkt->immed_len > 0) {
void *src = rma_op->origin_addr, *dest = put_pkt->data;
mpi_errno = immed_copy(src, dest, (size_t)put_pkt->immed_len);
if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
}
}
MPIDI_Comm_get_vc_set_active(comm_ptr, rma_op->target_rank, &vc);
if (len == (size_t)put_pkt->immed_len) {
/* All origin data is in packet header, issue the header. */
MPIU_THREAD_CS_ENTER(CH3COMM, vc);
......@@ -400,14 +412,27 @@ static int issue_acc_op(MPIDI_RMA_Op_t *rma_op, MPID_Win *win_ptr,
rma_op->request = NULL;
accum_pkt->flags |= flags;
if (flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK)
accum_pkt->lock_type = target_ptr->lock_type;
MPIDI_Comm_get_vc_set_active(comm_ptr, rma_op->target_rank, &vc);
MPID_Datatype_get_size_macro(rma_op->origin_datatype, origin_type_size);
MPIU_Assign_trunc(len, rma_op->origin_count * origin_type_size, size_t);
if (!rma_op->is_dt) {
/* Fill origin data into packet header IMMED area as much as possible */
MPIU_Assign_trunc(accum_pkt->immed_len,
MPIR_MIN(len, (MPIDI_RMA_IMMED_BYTES/origin_type_size)*origin_type_size),
int);
if (accum_pkt->immed_len > 0) {
void *src = rma_op->origin_addr, *dest = accum_pkt->data;
mpi_errno = immed_copy(src, dest, (size_t)accum_pkt->immed_len);
if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
}
}
MPIDI_Comm_get_vc_set_active(comm_ptr, rma_op->target_rank, &vc);
if (len == (size_t)accum_pkt->immed_len) {
/* All origin data is in packet header, issue the header. */
MPIU_THREAD_CS_ENTER(CH3COMM, vc);
......@@ -485,14 +510,32 @@ static int issue_get_acc_op(MPIDI_RMA_Op_t *rma_op, MPID_Win *win_ptr,
get_accum_pkt->request_handle = resp_req->handle;
get_accum_pkt->flags |= flags;
if (!rma_op->is_dt) {
/* Only fill IMMED data in response packet when both origin and target
buffers are basic datatype. */
get_accum_pkt->flags |= MPIDI_CH3_PKT_FLAG_RMA_IMMED_RESP;
}
if (flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK)
get_accum_pkt->lock_type = target_ptr->lock_type;
MPIDI_Comm_get_vc_set_active(comm_ptr, rma_op->target_rank, &vc);
MPID_Datatype_get_size_macro(rma_op->origin_datatype, origin_type_size);
MPIU_Assign_trunc(len, rma_op->origin_count * origin_type_size, size_t);
if (!rma_op->is_dt) {
/* Fill origin data into packet header IMMED area as much as possible */
MPIU_Assign_trunc(get_accum_pkt->immed_len,
MPIR_MIN(len, (MPIDI_RMA_IMMED_BYTES/origin_type_size)*origin_type_size),
int);
if (get_accum_pkt->immed_len > 0) {
void *src = rma_op->origin_addr, *dest = get_accum_pkt->data;
mpi_errno = immed_copy(src, dest, (size_t)get_accum_pkt->immed_len);
if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
}
}
MPIDI_Comm_get_vc_set_active(comm_ptr, rma_op->target_rank, &vc);
if (len == (size_t)get_accum_pkt->immed_len) {
/* All origin data is in packet header, issue the header. */
MPIU_THREAD_CS_ENTER(CH3COMM, vc);
......@@ -599,7 +642,14 @@ static int issue_get_op(MPIDI_RMA_Op_t * rma_op, MPID_Win * win_ptr,
}
get_pkt->request_handle = rma_op->request->handle;
get_pkt->flags |= flags;
if (!rma_op->is_dt) {
/* Only fill IMMED data in response packet when both origin and target
buffers are basic datatype. */
get_pkt->flags |= MPIDI_CH3_PKT_FLAG_RMA_IMMED_RESP;
}
if (flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK)
get_pkt->lock_type = target_ptr->lock_type;
......@@ -745,6 +795,8 @@ static int issue_fop_op(MPIDI_RMA_Op_t * rma_op,
MPID_Comm *comm_ptr = win_ptr->comm_ptr;
MPIDI_CH3_Pkt_fop_t *fop_pkt = &rma_op->pkt.fop;
MPID_Request *resp_req = NULL;
size_t len;
MPI_Aint origin_type_size;
int mpi_errno = MPI_SUCCESS;
MPIDI_STATE_DECL(MPID_STATE_ISSUE_FOP_OP);
......@@ -772,6 +824,19 @@ static int issue_fop_op(MPIDI_RMA_Op_t * rma_op,
if (flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK)
fop_pkt->lock_type = target_ptr->lock_type;
MPID_Datatype_get_size_macro(rma_op->origin_datatype, origin_type_size);
MPIU_Assign_trunc(len, rma_op->origin_count * origin_type_size, size_t);
/* Fill origin data into packet header IMMED area as much as possible */
MPIU_Assign_trunc(fop_pkt->immed_len,
MPIR_MIN(len, (MPIDI_RMA_IMMED_BYTES/origin_type_size)*origin_type_size),
int);
if (fop_pkt->immed_len > 0) {
void *src = rma_op->origin_addr, *dest = fop_pkt->data;
mpi_errno = immed_copy(src, dest, (size_t)fop_pkt->immed_len);
if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
}
MPIDI_Comm_get_vc_set_active(comm_ptr, rma_op->target_rank, &vc);
/* All origin data is in packet header, issue the header. */
......
......@@ -174,27 +174,12 @@ int MPIDI_CH3I_Put(const void *origin_addr, int origin_count, MPI_Datatype
new_ptr->is_dt = 1;
}
/* If both origin and target are basic datatype, try to
copy origin data to packet header as much as possible. */
/* Judge if this operation is an piggyback candidate */
if (!new_ptr->is_dt) {
size_t len;
MPI_Aint origin_type_size;
MPID_Datatype_get_size_macro(new_ptr->origin_datatype, origin_type_size);
/* length of origin data */
MPIU_Assign_trunc(len, new_ptr->origin_count * origin_type_size, size_t);
/* length of origin data that can fit into immed area in pkt header */
MPIU_Assign_trunc(put_pkt->immed_len,
MPIR_MIN(len, (MPIDI_RMA_IMMED_BYTES / origin_type_size) * origin_type_size),
int);
if (put_pkt->immed_len > 0) {
void *src = new_ptr->origin_addr, *dest = put_pkt->data;
/* copy data from origin buffer to immed area in packet header */
mpi_errno = immed_copy(src, dest, (size_t)put_pkt->immed_len);
if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
}
if (len <= MPIR_MAX(MPIDI_RMA_IMMED_BYTES,
MPIR_CVAR_CH3_RMA_OP_PIGGYBACK_LOCK_DATA_SIZE))
new_ptr->piggyback_lock_candidate = 1;
......@@ -356,12 +341,9 @@ int MPIDI_CH3I_Get(void *origin_addr, int origin_count, MPI_Datatype
new_ptr->is_dt = 1;
}
/* Judge if this operation is an piggyback candidate. */
if (!new_ptr->is_dt) {
new_ptr->piggyback_lock_candidate = 1;
/* Only fill IMMED data in response packet when both origin and target
buffers are basic datatype. */
get_pkt->flags |= MPIDI_CH3_PKT_FLAG_RMA_IMMED_RESP;
}
MPIR_T_PVAR_TIMER_END(RMA, rma_rmaqueue_set);
......@@ -524,27 +506,12 @@ int MPIDI_CH3I_Accumulate(const void *origin_addr, int origin_count, MPI_Datatyp
new_ptr->is_dt = 1;
}
/* If both origin and target are basic datatype, try to
copy origin data to packet header as much as possible. */
/* Judge if this operation is an piggyback candidate. */
if (!new_ptr->is_dt) {
size_t len;
MPI_Aint origin_type_size;
MPID_Datatype_get_size_macro(new_ptr->origin_datatype, origin_type_size);
/* length of origin data */
MPIU_Assign_trunc(len, new_ptr->origin_count * origin_type_size, size_t);
/* length of origin data that can fit into immed areas in packet header */
MPIU_Assign_trunc(accum_pkt->immed_len,
MPIR_MIN(len, (MPIDI_RMA_IMMED_BYTES / origin_type_size) * origin_type_size),
int);
if (accum_pkt->immed_len > 0) {
void *src = new_ptr->origin_addr, *dest = accum_pkt->data;
/* copy data from origin buffer to immed area in packet header */
mpi_errno = immed_copy(src, dest, (size_t)accum_pkt->immed_len);
if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
}
if (len <= MPIR_MAX(MPIDI_RMA_IMMED_BYTES,
MPIR_CVAR_CH3_RMA_OP_PIGGYBACK_LOCK_DATA_SIZE))
new_ptr->piggyback_lock_candidate = 1;
......@@ -697,12 +664,9 @@ int MPIDI_CH3I_Get_accumulate(const void *origin_addr, int origin_count,
new_ptr->is_dt = 1;
}
/* Judge if this operation is a piggyback candidate */
if (!new_ptr->is_dt) {
new_ptr->piggyback_lock_candidate = 1;
/* Only fill IMMED data in response packet when both origin and target
buffers are basic datatype. */
get_pkt->flags |= MPIDI_CH3_PKT_FLAG_RMA_IMMED_RESP;
}
}
......@@ -747,34 +711,15 @@ int MPIDI_CH3I_Get_accumulate(const void *origin_addr, int origin_count,
new_ptr->is_dt = 1;
}
/* If all buffers are basic datatype, try to copy origin data to
packet header as much as possible. */
/* Judge if this operation is a piggyback candidate */
if (!new_ptr->is_dt) {
size_t len;
MPI_Aint origin_type_size;
MPID_Datatype_get_size_macro(new_ptr->origin_datatype, origin_type_size);
/* length of origin data */
MPIU_Assign_trunc(len, new_ptr->origin_count * origin_type_size, size_t);
/* length of origin data that can fit into immed area in packet header */
MPIU_Assign_trunc(get_accum_pkt->immed_len,
MPIR_MIN(len, (MPIDI_RMA_IMMED_BYTES / origin_type_size) * origin_type_size),
int);
if (get_accum_pkt->immed_len > 0) {
void *src = new_ptr->origin_addr, *dest = get_accum_pkt->data;
/* copy data from origin buffer to immed area in packet header */
mpi_errno = immed_copy(src, dest, (size_t)get_accum_pkt->immed_len);
if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
}
if (len <= MPIR_MAX(MPIDI_RMA_IMMED_BYTES,
MPIR_CVAR_CH3_RMA_OP_PIGGYBACK_LOCK_DATA_SIZE))
new_ptr->piggyback_lock_candidate = 1;
/* Only fill IMMED data in response packet when both origin and target
buffers are basic datatype. */
get_accum_pkt->flags |= MPIDI_CH3_PKT_FLAG_RMA_IMMED_RESP;
}
}
......@@ -1140,7 +1085,6 @@ int MPIDI_Fetch_and_op(const void *origin_addr, void *result_addr,
get_pkt->source_win_handle = win_ptr->handle;
get_pkt->origin_rank = rank;
get_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE;
get_pkt->flags |= MPIDI_CH3_PKT_FLAG_RMA_IMMED_RESP;
new_ptr->origin_addr = result_addr;
new_ptr->origin_count = 1;
......@@ -1150,8 +1094,6 @@ int MPIDI_Fetch_and_op(const void *origin_addr, void *result_addr,
}
else {
MPIDI_CH3_Pkt_fop_t *fop_pkt = &(new_ptr->pkt.fop);
size_t len;
MPI_Aint origin_type_size;
MPIDI_Pkt_init(fop_pkt, MPIDI_CH3_PKT_FOP);
fop_pkt->addr = (char *) win_ptr->base_addrs[target_rank] +
......@@ -1171,21 +1113,6 @@ int MPIDI_Fetch_and_op(const void *origin_addr, void *result_addr,
new_ptr->result_datatype = datatype;
new_ptr->target_rank = target_rank;
new_ptr->piggyback_lock_candidate = 1;
MPID_Datatype_get_size_macro(new_ptr->origin_datatype, origin_type_size);
/* length of origin data */
MPIU_Assign_trunc(len, new_ptr->origin_count * origin_type_size, size_t);
/* length of origin data that can fit into immed area in pkt header */
MPIU_Assign_trunc(fop_pkt->immed_len,
MPIR_MIN(len, (MPIDI_RMA_IMMED_BYTES / origin_type_size) * origin_type_size),
int);
if (fop_pkt->immed_len > 0) {
void *src = new_ptr->origin_addr, *dest = fop_pkt->data;
/* copy data from origin buffer to immed area in packet header */
mpi_errno = immed_copy(src, dest, (size_t)fop_pkt->immed_len);
if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
}
}
new_ptr->ureq = NULL; /* reset user request */
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment