Commit ce8bc310 authored by Xin Zhao's avatar Xin Zhao Committed by Pavan Balaji
Browse files

Bug-fix: make RMA work correctly with pair basic type.



The original implementation of RMA does not consider pair basic
types (e.g. MPI_FLOAT_INT, MPI_DOUBLE_INT). It only
works correctly with builtin datatypes (e.g. MPI_INT, MPI_FLOAT).
This patch makes the RMA work correctly with pair basic types.

The bug is that: (1) when performing the ACC computation, the original
implementation uses 'eltype' in the datatype structure, which is set
when all basic elements in this datatype have the same builtin
datatype. When basic elements have different builtin datatypes, like
pair datatypes, the 'eltype' is set to MPI_DATATYPE_NULL. This makes
the ACC computation be unable to work with pair types; (2) for all
basic type of data, the original implementation assumes that
they are all contiguous and issues them in an unpacked manner
with length of data size (count*type_size). This is incorrect for
pair datatypes, because most pair datatypes are non-contiguous
(type_extent != type_size).

In the previous patch, we already made 'eltype' to store basic
type instead of builtin type. In this patch, we fixed this
bug by (1) modify ACC computation to treat 'eltype' as basic
type; (2) For non-contiguous basic type data, we use the noncontig
API so that it will be issued in a packed manner.
Signed-off-by: Pavan Balaji's avatarPavan Balaji <balaji@anl.gov>
parent 67b69b2a
......@@ -188,6 +188,7 @@ static int issue_from_origin_buffer(MPIDI_RMA_Op_t * rma_op, MPIDI_VC_t * vc)
MPI_Aint origin_type_size;
MPI_Datatype target_datatype;
MPID_Datatype *target_dtp = NULL, *origin_dtp = NULL;
int is_origin_contig;
MPID_IOV iov[MPID_IOV_LIMIT];
int mpi_errno = MPI_SUCCESS;
MPIDI_STATE_DECL(MPID_STATE_ISSUE_FROM_ORIGIN_BUFFER);
......@@ -214,13 +215,14 @@ static int issue_from_origin_buffer(MPIDI_RMA_Op_t * rma_op, MPIDI_VC_t * vc)
}
MPID_Datatype_get_size_macro(rma_op->origin_datatype, origin_type_size);
MPID_Datatype_is_contig(rma_op->origin_datatype, &is_origin_contig);
iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) & (rma_op->pkt);
iov[0].MPID_IOV_LEN = sizeof(rma_op->pkt);
if (target_dtp == NULL) {
/* basic datatype on target */
if (origin_dtp == NULL) {
if (is_origin_contig) {
/* basic datatype on origin */
int iovcnt = 2;
......@@ -231,6 +233,17 @@ static int issue_from_origin_buffer(MPIDI_RMA_Op_t * rma_op, MPIDI_VC_t * vc)
mpi_errno = MPIDI_CH3_iStartMsgv(vc, iov, iovcnt, &rma_op->request);
MPIU_THREAD_CS_EXIT(CH3COMM, vc);
MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**ch3|rmamsg");
if (origin_dtp != NULL) {
if (rma_op->request == NULL) {
MPID_Datatype_release(origin_dtp);
}
else {
/* this will cause the datatype to be freed when the request
* is freed. */
rma_op->request->dev.datatype_ptr = origin_dtp;
}
}
}
else {
/* derived datatype on origin */
......@@ -244,9 +257,11 @@ static int issue_from_origin_buffer(MPIDI_RMA_Op_t * rma_op, MPIDI_VC_t * vc)
MPIU_ERR_CHKANDJUMP1(rma_op->request->dev.segment_ptr == NULL, mpi_errno,
MPI_ERR_OTHER, "**nomem", "**nomem %s", "MPID_Segment_alloc");
rma_op->request->dev.datatype_ptr = origin_dtp;
/* this will cause the datatype to be freed when the request
* is freed. */
if (origin_dtp != NULL) {
rma_op->request->dev.datatype_ptr = origin_dtp;
/* this will cause the datatype to be freed when the request
* is freed. */
}
MPID_Segment_init(rma_op->origin_addr, rma_op->origin_count,
rma_op->origin_datatype, rma_op->request->dev.segment_ptr, 0);
rma_op->request->dev.segment_first = 0;
......
......@@ -355,10 +355,13 @@ static inline int MPIDI_CH3I_Shm_acc_op(const void *origin_addr, int origin_coun
DLOOP_VECTOR *dloop_vec;
MPI_Aint first, last;
int vec_len, i, type_size, count;
MPI_Aint type_extent;
MPI_Datatype type;
MPI_Aint true_lb, true_extent, extent;
void *tmp_buf = NULL, *target_buf;
const void *source_buf;
MPI_Aint curr_len;
void *curr_loc;
if (origin_datatype != target_datatype) {
/* first copy the data into a temporary buffer with
......@@ -411,16 +414,44 @@ static inline int MPIDI_CH3I_Shm_acc_op(const void *origin_addr, int origin_coun
source_buf = (tmp_buf != NULL) ? (const void *) tmp_buf : origin_addr;
target_buf = (char *) base + disp_unit * target_disp;
type = dtp->eltype;
type_size = MPID_Datatype_get_basic_size(type);
MPIU_Assert(type != MPI_DATATYPE_NULL);
MPID_Datatype_get_size_macro(type, type_size);
MPID_Datatype_get_extent_macro(type, type_extent);
if (shm_op)
MPIDI_CH3I_SHM_MUTEX_LOCK(win_ptr);
for (i = 0; i < vec_len; i++) {
MPIU_Assign_trunc(count, (dloop_vec[i].DLOOP_VECTOR_LEN) / type_size, int);
(*uop) ((char *) source_buf + MPIU_PtrToAint(dloop_vec[i].DLOOP_VECTOR_BUF),
(char *) target_buf + MPIU_PtrToAint(dloop_vec[i].DLOOP_VECTOR_BUF),
&count, &type);
i = 0;
curr_loc = dloop_vec[0].DLOOP_VECTOR_BUF;
curr_len = dloop_vec[0].DLOOP_VECTOR_LEN;
while (i != vec_len) {
if (curr_len < type_size) {
MPIU_Assert(i != vec_len);
i++;
curr_len += dloop_vec[i].DLOOP_VECTOR_LEN;
continue;
}
MPIU_Assign_trunc(count, curr_len/type_size, int);
(*uop)((char *)source_buf + MPIU_PtrToAint(curr_loc),
(char *)target_buf + MPIU_PtrToAint(curr_loc),
&count, &type);
if (curr_len % type_size == 0) {
i++;
if (i != vec_len) {
curr_loc = dloop_vec[i].DLOOP_VECTOR_BUF;
curr_len = dloop_vec[i].DLOOP_VECTOR_LEN;
}
}
else {
curr_loc = (void *)((char *)curr_loc + type_extent * count);
curr_len -= type_size * count;
}
}
if (shm_op)
MPIDI_CH3I_SHM_MUTEX_UNLOCK(win_ptr);
......@@ -533,6 +564,9 @@ static inline int MPIDI_CH3I_Shm_get_acc_op(const void *origin_addr, int origin_
MPI_Aint true_lb, true_extent, extent;
void *tmp_buf = NULL, *target_buf;
const void *source_buf;
MPI_Aint type_extent;
MPI_Aint curr_len;
void *curr_loc;
if (origin_datatype != target_datatype) {
/* first copy the data into a temporary buffer with
......@@ -581,13 +615,39 @@ static inline int MPIDI_CH3I_Shm_get_acc_op(const void *origin_addr, int origin_
source_buf = (tmp_buf != NULL) ? (const void *) tmp_buf : origin_addr;
target_buf = (char *) base + disp_unit * target_disp;
type = dtp->eltype;
type_size = MPID_Datatype_get_basic_size(type);
for (i = 0; i < vec_len; i++) {
MPIU_Assign_trunc(count, (dloop_vec[i].DLOOP_VECTOR_LEN) / type_size, int);
(*uop) ((char *) source_buf + MPIU_PtrToAint(dloop_vec[i].DLOOP_VECTOR_BUF),
(char *) target_buf + MPIU_PtrToAint(dloop_vec[i].DLOOP_VECTOR_BUF),
&count, &type);
MPIU_Assert(type != MPI_DATATYPE_NULL);
MPID_Datatype_get_size_macro(type, type_size);
MPID_Datatype_get_extent_macro(type, type_extent);
i = 0;
curr_loc = dloop_vec[0].DLOOP_VECTOR_BUF;
curr_len = dloop_vec[0].DLOOP_VECTOR_LEN;
while (i != vec_len) {
if (curr_len < type_size) {
MPIU_Assert(i != vec_len);
i++;
curr_len += dloop_vec[i].DLOOP_VECTOR_LEN;
continue;
}
MPIU_Assign_trunc(count, curr_len/type_size, int);
(*uop)((char *)source_buf + MPIU_PtrToAint(curr_loc),
(char *)target_buf + MPIU_PtrToAint(curr_loc),
&count, &type);
if (curr_len % type_size == 0) {
i++;
if (i != vec_len) {
curr_loc = dloop_vec[i].DLOOP_VECTOR_BUF;
curr_len = dloop_vec[i].DLOOP_VECTOR_LEN;
}
}
else {
curr_loc = (void *)((char *)curr_loc + type_extent * count);
curr_len -= type_size * count;
}
}
MPID_Segment_free(segp);
......
......@@ -355,7 +355,9 @@ static inline int enqueue_lock_origin(MPID_Win * win_ptr, MPIDI_VC_t * vc,
}
else {
MPI_Aint type_size = 0;
MPI_Aint type_extent;
MPIDI_msg_sz_t recv_data_sz = 0;
MPIDI_msg_sz_t buf_size;
MPID_Request *req = NULL;
MPI_Datatype target_dtp;
int target_count;
......@@ -368,18 +370,20 @@ static inline int enqueue_lock_origin(MPID_Win * win_ptr, MPIDI_VC_t * vc,
MPIDI_CH3_PKT_RMA_GET_TARGET_DATATYPE((*pkt), target_dtp, mpi_errno);
MPIDI_CH3_PKT_RMA_GET_TARGET_COUNT((*pkt), target_count, mpi_errno);
MPID_Datatype_get_extent_macro(target_dtp, type_extent);
MPID_Datatype_get_size_macro(target_dtp, type_size);
recv_data_sz = type_size * target_count;
buf_size = type_extent * target_count;
if (new_ptr != NULL) {
if (win_ptr->current_lock_data_bytes + recv_data_sz < MPIR_CVAR_CH3_RMA_LOCK_DATA_BYTES) {
new_ptr->data = MPIU_Malloc(recv_data_sz);
if (win_ptr->current_lock_data_bytes + buf_size < MPIR_CVAR_CH3_RMA_LOCK_DATA_BYTES) {
new_ptr->data = MPIU_Malloc(buf_size);
}
if (new_ptr->data == NULL) {
/* Note that there are two possible reasons to make new_ptr->data to be NULL:
* (1) win_ptr->current_lock_data_bytes + recv_data_sz >= MPIR_CVAR_CH3_RMA_LOCK_DATA_BYTES;
* (2) MPIU_Malloc(recv_data_sz) failed.
* (1) win_ptr->current_lock_data_bytes + buf_size >= MPIR_CVAR_CH3_RMA_LOCK_DATA_BYTES;
* (2) MPIU_Malloc(buf_size) failed.
* In such cases, we cannot allocate memory for lock data, so we give up
* buffering lock data, however, we still buffer lock request.
*/
......@@ -413,8 +417,8 @@ static inline int enqueue_lock_origin(MPID_Win * win_ptr, MPIDI_VC_t * vc,
data_discarded = 1;
}
else {
win_ptr->current_lock_data_bytes += recv_data_sz;
new_ptr->data_size = recv_data_sz;
win_ptr->current_lock_data_bytes += buf_size;
new_ptr->data_size = buf_size;
}
}
......@@ -795,9 +799,11 @@ static inline int do_accumulate_op(void *source_buf, void *target_buf,
DLOOP_VECTOR *dloop_vec;
MPI_Aint first, last;
int vec_len, i, count;
MPI_Aint type_size;
MPI_Aint type_extent, type_size;
MPI_Datatype type;
MPID_Datatype *dtp;
MPI_Aint curr_len;
void *curr_loc;
segp = MPID_Segment_alloc();
/* --BEGIN ERROR HANDLING-- */
......@@ -831,12 +837,37 @@ static inline int do_accumulate_op(void *source_buf, void *target_buf,
MPID_Segment_pack_vector(segp, first, &last, dloop_vec, &vec_len);
type = dtp->eltype;
MPIU_Assert(type != MPI_DATATYPE_NULL);
MPID_Datatype_get_size_macro(type, type_size);
for (i = 0; i < vec_len; i++) {
MPIU_Assign_trunc(count, (dloop_vec[i].DLOOP_VECTOR_LEN) / type_size, int);
(*uop) ((char *) source_buf + MPIU_PtrToAint(dloop_vec[i].DLOOP_VECTOR_BUF),
(char *) target_buf + MPIU_PtrToAint(dloop_vec[i].DLOOP_VECTOR_BUF),
&count, &type);
MPID_Datatype_get_extent_macro(type, type_extent);
i = 0;
curr_loc = dloop_vec[0].DLOOP_VECTOR_BUF;
curr_len = dloop_vec[0].DLOOP_VECTOR_LEN;
while (i != vec_len) {
if (curr_len < type_size) {
MPIU_Assert(i != vec_len);
i++;
curr_len += dloop_vec[i].DLOOP_VECTOR_LEN;
continue;
}
MPIU_Assign_trunc(count, curr_len / type_size, int);
(*uop) ((char *) source_buf + MPIU_PtrToAint(curr_loc),
(char *) target_buf + MPIU_PtrToAint(curr_loc), &count, &type);
if (curr_len % type_size == 0) {
i++;
if (i != vec_len) {
curr_loc = dloop_vec[i].DLOOP_VECTOR_BUF;
curr_len = dloop_vec[i].DLOOP_VECTOR_LEN;
}
}
else {
curr_loc = (void *) ((char *) curr_loc + type_extent * count);
curr_len -= type_size * count;
}
}
MPID_Segment_free(segp);
......
......@@ -219,6 +219,7 @@ int MPIDI_CH3_ReqHandler_GaccumRecvComplete(MPIDI_VC_t * vc, MPID_Request * rreq
MPID_IOV iov[MPID_IOV_LIMIT];
MPI_Aint true_lb, true_extent;
int iovcnt;
int is_contig;
MPIU_CHKPMEM_DECL(1);
MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_REQHANDLER_GACCUMRECVCOMPLETE);
......@@ -239,6 +240,8 @@ int MPIDI_CH3_ReqHandler_GaccumRecvComplete(MPIDI_VC_t * vc, MPID_Request * rreq
MPID_Datatype_get_size_macro(rreq->dev.datatype, type_size);
MPID_Datatype_is_contig(rreq->dev.datatype, &is_contig);
/* Copy data into a temporary buffer */
resp_req = MPID_Request_create();
MPIU_ERR_CHKANDJUMP(resp_req == NULL, mpi_errno, MPI_ERR_OTHER, "**nomemreq");
......@@ -250,7 +253,7 @@ int MPIDI_CH3_ReqHandler_GaccumRecvComplete(MPIDI_VC_t * vc, MPID_Request * rreq
if (win_ptr->shm_allocated == TRUE)
MPIDI_CH3I_SHM_MUTEX_LOCK(win_ptr);
if (MPIR_DATATYPE_IS_PREDEFINED(rreq->dev.datatype)) {
if (is_contig) {
MPIU_Memcpy(resp_req->dev.user_buf, rreq->dev.real_user_buf,
rreq->dev.user_count * type_size);
}
......@@ -340,6 +343,7 @@ int MPIDI_CH3_ReqHandler_FOPRecvComplete(MPIDI_VC_t * vc, MPID_Request * rreq, i
int iovcnt;
MPIDI_CH3_Pkt_t upkt;
MPIDI_CH3_Pkt_fop_resp_t *fop_resp_pkt = &upkt.fop_resp;
int is_contig;
MPIU_CHKPMEM_DECL(1);
MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_REQHANDLER_FOPRECVCOMPLETE);
......@@ -349,6 +353,8 @@ int MPIDI_CH3_ReqHandler_FOPRecvComplete(MPIDI_VC_t * vc, MPID_Request * rreq, i
MPID_Datatype_get_size_macro(rreq->dev.datatype, type_size);
MPID_Datatype_is_contig(rreq->dev.datatype, &is_contig);
/* Create response request */
resp_req = MPID_Request_create();
MPIU_ERR_CHKANDJUMP(resp_req == NULL, mpi_errno, MPI_ERR_OTHER, "**nomemreq");
......@@ -368,7 +374,23 @@ int MPIDI_CH3_ReqHandler_FOPRecvComplete(MPIDI_VC_t * vc, MPID_Request * rreq, i
MPIDI_CH3I_SHM_MUTEX_LOCK(win_ptr);
/* Copy data into a temporary buffer in response request */
MPIU_Memcpy(resp_req->dev.user_buf, rreq->dev.real_user_buf, type_size);
if (is_contig) {
MPIU_Memcpy(resp_req->dev.user_buf, rreq->dev.real_user_buf, type_size);
}
else {
MPID_Segment *seg = MPID_Segment_alloc();
MPI_Aint last = type_size;
if (seg == NULL) {
if (win_ptr->shm_allocated == TRUE)
MPIDI_CH3I_SHM_MUTEX_UNLOCK(win_ptr);
}
MPIU_ERR_CHKANDJUMP1(seg == NULL, mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %s",
"MPID_Segment");
MPID_Segment_init(rreq->dev.real_user_buf, 1, rreq->dev.datatype, seg, 0);
MPID_Segment_pack(seg, 0, &last, resp_req->dev.user_buf);
MPID_Segment_free(seg);
}
/* Perform accumulate computation */
if (rreq->dev.op != MPI_NO_OP) {
......@@ -917,6 +939,7 @@ static inline int perform_get_in_lock_queue(MPID_Win * win_ptr, MPIDI_RMA_Lock_e
size_t len;
int iovcnt;
MPID_IOV iov[MPID_IOV_LIMIT];
int is_contig;
int mpi_errno = MPI_SUCCESS;
/* Piggyback candidate should have basic datatype for target datatype. */
......@@ -963,31 +986,54 @@ static inline int perform_get_in_lock_queue(MPID_Win * win_ptr, MPIDI_RMA_Lock_e
MPID_Datatype_get_size_macro(get_pkt->datatype, type_size);
MPIU_Assign_trunc(len, get_pkt->count * type_size, size_t);
MPID_Datatype_is_contig(get_pkt->datatype, &is_contig);
if (get_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_IMMED_RESP) {
void *src = (void *) (get_pkt->addr), *dest = (void *) (get_resp_pkt->info.data);
mpi_errno = immed_copy(src, dest, len);
if (mpi_errno != MPI_SUCCESS)
MPIU_ERR_POP(mpi_errno);
}
if (get_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_IMMED_RESP) {
/* All origin data is in packet header, issue the header. */
iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) get_resp_pkt;
iov[0].MPID_IOV_LEN = sizeof(*get_resp_pkt);
iovcnt = 1;
mpi_errno = MPIDI_CH3_iSendv(lock_entry->vc, sreq, iov, iovcnt);
if (mpi_errno != MPI_SUCCESS) {
MPID_Request_release(sreq);
MPIU_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**ch3|rmamsg");
}
}
else {
else if (is_contig) {
iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) get_resp_pkt;
iov[0].MPID_IOV_LEN = sizeof(*get_resp_pkt);
iov[1].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) (get_pkt->addr);
iov[1].MPID_IOV_LEN = get_pkt->count * type_size;
iovcnt = 2;
mpi_errno = MPIDI_CH3_iSendv(lock_entry->vc, sreq, iov, iovcnt);
if (mpi_errno != MPI_SUCCESS) {
MPID_Request_release(sreq);
MPIU_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**ch3|rmamsg");
}
}
else {
iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) get_resp_pkt;
iov[0].MPID_IOV_LEN = sizeof(*get_resp_pkt);
mpi_errno = MPIDI_CH3_iSendv(lock_entry->vc, sreq, iov, iovcnt);
if (mpi_errno != MPI_SUCCESS) {
MPID_Request_release(sreq);
MPIU_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**ch3|rmamsg");
sreq->dev.segment_ptr = MPID_Segment_alloc();
MPIU_ERR_CHKANDJUMP1(sreq->dev.segment_ptr == NULL, mpi_errno,
MPI_ERR_OTHER, "**nomem", "**nomem %s", "MPID_Segment_alloc");
MPID_Segment_init(get_pkt->addr, get_pkt->count,
get_pkt->datatype, sreq->dev.segment_ptr, 0);
sreq->dev.segment_first = 0;
sreq->dev.segment_size = get_pkt->count * type_size;
mpi_errno = lock_entry->vc->sendNoncontig_fn(lock_entry->vc, sreq,
iov[0].MPID_IOV_BUF, iov[0].MPID_IOV_LEN);
MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**ch3|rmamsg");
}
fn_exit:
......@@ -1051,6 +1097,7 @@ static inline int perform_get_acc_in_lock_queue(MPID_Win * win_ptr,
size_t len;
int iovcnt;
MPID_IOV iov[MPID_IOV_LIMIT];
int is_contig;
int mpi_errno = MPI_SUCCESS;
/* Piggyback candidate should have basic datatype for target datatype. */
......@@ -1081,6 +1128,8 @@ static inline int perform_get_acc_in_lock_queue(MPID_Win * win_ptr,
MPIDI_Pkt_init(get_accum_resp_pkt, MPIDI_CH3_PKT_GET_ACCUM_RESP_IMMED);
}
MPID_Datatype_is_contig(get_accum_pkt->datatype, &is_contig);
/* length of target data */
MPIU_Assign_trunc(len, get_accum_pkt->count * type_size, size_t);
......@@ -1098,9 +1147,24 @@ static inline int perform_get_acc_in_lock_queue(MPID_Win * win_ptr,
MPIU_ERR_POP(mpi_errno);
}
}
else {
else if (is_contig) {
MPIU_Memcpy(sreq->dev.user_buf, get_accum_pkt->addr, get_accum_pkt->count * type_size);
}
else {
MPID_Segment *seg = MPID_Segment_alloc();
MPI_Aint last = type_size * get_accum_pkt->count;
if (seg == NULL) {
if (win_ptr->shm_allocated == TRUE)
MPIDI_CH3I_SHM_MUTEX_UNLOCK(win_ptr);
}
MPIU_ERR_CHKANDJUMP1(seg == NULL, mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %s",
"MPID_Segment");
MPID_Segment_init(get_accum_pkt->addr, get_accum_pkt->count, get_accum_pkt->datatype, seg,
0);
MPID_Segment_pack(seg, 0, &last, sreq->dev.user_buf);
MPID_Segment_free(seg);
}
if (get_accum_pkt->type == MPIDI_CH3_PKT_GET_ACCUM_IMMED) {
/* All data fits in packet header */
......@@ -1175,6 +1239,7 @@ static inline int perform_fop_in_lock_queue(MPID_Win * win_ptr, MPIDI_RMA_Lock_e
MPI_Aint type_size;
MPID_IOV iov[MPID_IOV_LIMIT];
int iovcnt;
int is_contig;
int mpi_errno = MPI_SUCCESS;
/* Piggyback candidate should have basic datatype for target datatype. */
......@@ -1188,6 +1253,8 @@ static inline int perform_fop_in_lock_queue(MPID_Win * win_ptr, MPIDI_RMA_Lock_e
MPID_Datatype_get_size_macro(fop_pkt->datatype, type_size);
MPID_Datatype_is_contig(fop_pkt->datatype, &is_contig);
if (fop_pkt->flags & MPIDI_CH3_PKT_FOP_IMMED) {
MPIDI_Pkt_init(fop_resp_pkt, MPIDI_CH3_PKT_FOP_RESP_IMMED);
}
......@@ -1238,9 +1305,23 @@ static inline int perform_fop_in_lock_queue(MPID_Win * win_ptr, MPIDI_RMA_Lock_e
MPIU_ERR_POP(mpi_errno);
}
}
else {
else if (is_contig) {
MPIU_Memcpy(resp_req->dev.user_buf, fop_pkt->addr, type_size);
}
else {
MPID_Segment *seg = MPID_Segment_alloc();
MPI_Aint last = type_size;
if (seg == NULL) {
if (win_ptr->shm_allocated == TRUE)
MPIDI_CH3I_SHM_MUTEX_UNLOCK(win_ptr);
}
MPIU_ERR_CHKANDJUMP1(seg == NULL, mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %s",
"MPID_Segment");
MPID_Segment_init(fop_pkt->addr, 1, fop_pkt->datatype, seg, 0);
MPID_Segment_pack(seg, 0, &last, resp_req->dev.user_buf);
MPID_Segment_free(seg);
}
/* Apply the op */
if (fop_pkt->op != MPI_NO_OP) {
......
......@@ -124,6 +124,7 @@ int MPIDI_CH3I_Put(const void *origin_addr, int origin_count, MPI_Datatype
MPI_Aint origin_type_size;
size_t immed_len, len;
int use_immed_pkt = FALSE;
int is_origin_contig, is_target_contig;
/* queue it up */
mpi_errno = MPIDI_CH3I_Win_get_op(win_ptr, &new_ptr);
......@@ -159,11 +160,14 @@ int MPIDI_CH3I_Put(const void *origin_addr, int origin_count, MPI_Datatype
new_ptr->is_dt = 1;
}
MPID_Datatype_is_contig(origin_datatype, &is_origin_contig);
MPID_Datatype_is_contig(target_datatype, &is_target_contig);
MPID_Datatype_get_size_macro(origin_datatype, origin_type_size);
MPIU_Assign_trunc(len, origin_count * origin_type_size, size_t);
/* Judge if we can use IMMED data packet */
if (!new_ptr->is_dt) {
if (!new_ptr->is_dt && is_origin_contig && is_target_contig) {
MPIU_Assign_trunc(immed_len,
(MPIDI_RMA_IMMED_BYTES / origin_type_size) * origin_type_size,
size_t);
......@@ -318,6 +322,7 @@ int MPIDI_CH3I_Get(void *origin_addr, int origin_count, MPI_Datatype
MPI_Aint target_type_size;
size_t immed_len, len;
int use_immed_resp_pkt = FALSE;
int is_origin_contig, is_target_contig;
/* queue it up */
mpi_errno = MPIDI_CH3I_Win_get_op(win_ptr, &new_ptr);
......@@ -353,11 +358,14 @@ int MPIDI_CH3I_Get(void *origin_addr, int origin_count, MPI_Datatype
new_ptr->is_dt = 1;
}
MPID_Datatype_is_contig(origin_datatype, &is_origin_contig);
MPID_Datatype_is_contig(target_datatype, &is_target_contig);
MPID_Datatype_get_size_macro(target_datatype, target_type_size);
MPIU_Assign_trunc(len, target_count * target_type_size, size_t);
/* Judge if we can use IMMED data response packet */
if (!new_ptr->is_dt) {
if (!new_ptr->is_dt && is_origin_contig && is_target_contig) {
MPIU_Assign_trunc(immed_len,
(MPIDI_RMA_IMMED_BYTES / target_type_size) * target_type_size,
size_t);
......@@ -501,6 +509,7 @@ int MPIDI_CH3I_Accumulate(const void *origin_addr, int origin_count, MPI_Datatyp
MPI_Aint origin_type_size;
size_t immed_len, len;
int use_immed_pkt = FALSE;
int is_origin_contig, is_target_contig;
/* queue it up */
mpi_errno = MPIDI_CH3I_Win_get_op(win_ptr, &new_ptr);
......@@ -538,8 +547,11 @@ int MPIDI_CH3I_Accumulate(const void *origin_addr, int origin_count, MPI_Datatyp
MPID_Datatype_get_size_macro(origin_datatype, origin_type_size);
MPIU_Assign_trunc(len, origin_count * origin_type_size, size_t);
MPID_Datatype_is_contig(origin_datatype, &is_origin_contig);
MPID_Datatype_is_contig(target_datatype, &is_target_contig);
/* Judge if we can use IMMED data packet */
if (!new_ptr->is_dt) {
if (!new_ptr->is_dt && is_origin_contig && is_target_contig) {
MPIU_Assign_trunc(immed_len,
(MPIDI_RMA_IMMED_BYTES / origin_type_size) * origin_type_size,
size_t);
......@@ -712,6 +724,7 @@ int MPIDI_CH3I_Get_accumulate(const void *origin_addr, int origin_count,
MPI_Aint target_type_size;
size_t len, immed_len;
int use_immed_resp_pkt = FALSE;
int is_result_contig, is_target_contig;
/******************** Setting operation struct areas ***********************/
......@@ -740,8 +753,11 @@ int MPIDI_CH3I_Get_accumulate(const void *origin_addr, int origin_count,
MPID_Datatype_get_size_macro(target_datatype, target_type_size);
MPIU_Assign_trunc(len, target_count * target_type_size, size_t);
MPID_Datatype_is_contig(result_datatype, &is_result_contig);
MPID_Datatype_is_contig(target_datatype, &is_target_contig);
/* Judge if we can use IMMED data response packet */
if (!new_ptr->is_dt) {
if (!new_ptr->is_dt && is_result_contig && is_target_contig) {
MPIU_Assign_trunc(immed_len,
(MPIDI_RMA_IMMED_BYTES / target_type_size) * target_type_size,
size_t);
......@@ -777,6 +793,7 @@ int MPIDI_CH3I_Get_accumulate(const void *origin_addr, int origin_count,
MPI_Aint origin_type_size;
size_t immed_len, orig_len;
int use_immed_pkt = FALSE;
int is_origin_contig, is_target_contig, is_result_contig;
/******************** Setting operation struct areas ***********************/
......@@ -815,8 +832,12 @@ int MPIDI_CH3I_Get_accumulate(const void *origin_addr, int origin_count,
MPID_Datatype_get_size_macro(origin_datatype, origin_type_size);
MPIU_Assign_trunc(orig_len, origin_count * origin_type_size, size_t);
MPID_Datatype_is_contig(origin_datatype, &is_origin_contig);
MPID_Datatype_is_contig(target_datatype, &is_target_contig);
MPID_Datatype_is_contig(result_datatype, &is_result_contig);
/* Judge if we can use IMMED data packet */
if (!new_ptr->is_dt) {
if (!new_ptr->is_dt && is_origin_contig && is_target_contig && is_result_contig) {
MPIU_Assign_trunc(immed_len,
(MPIDI_RMA_IMMED_BYTES / origin_type_size) * origin_type_size,
size_t);
......@@ -1231,6 +1252,7 @@ int MPIDI_Fetch_and_op(const void *origin_addr, void *result_addr,
MPI_Aint target_type_size;
size_t immed_len;
int use_immed_resp_pkt = FALSE;