Commit 211e8986 authored by Norio Yamaguchi's avatar Norio Yamaguchi Committed by Huiwei Lu
Browse files

Code cleanup in netmod-IB


Signed-off-by: default avatarHuiwei Lu <huiweilu@mcs.anl.gov>
parent 0bdc7e74
......@@ -1900,8 +1900,8 @@ int MPID_nem_ib_com_put_scratch_pad(int condesc, uint64_t wr_id, uint64_t offset
/* rkey is defined in MPID_nem_ib_com_reg_mr_connect */
dprintf("MPID_nem_ib_com_put_scratch_pad,wr.rdma.remote_addr=%llx\n",
(unsigned long long) conp->icom_sr[MPID_NEM_IB_COM_SCRATCH_PAD_INITIATOR].wr.
rdma.remote_addr);
(unsigned long long) conp->icom_sr[MPID_NEM_IB_COM_SCRATCH_PAD_INITIATOR].wr.rdma.
remote_addr);
#ifdef HAVE_LIBDCFA
ib_errno = ibv_post_send(conp->icom_qp, &conp->icom_sr[MPID_NEM_IB_COM_SCRATCH_PAD_INITIATOR]);
......@@ -1965,8 +1965,8 @@ int MPID_nem_ib_com_get_scratch_pad(int condesc,
/* rkey is defined in MPID_nem_ib_com_reg_mr_connect */
dprintf("MPID_nem_ib_com_get_scratch_pad,wr.rdma.remote_addr=%llx\n",
(unsigned long long) conp->icom_sr[MPID_NEM_IB_COM_SCRATCH_PAD_GET].wr.
rdma.remote_addr);
(unsigned long long) conp->icom_sr[MPID_NEM_IB_COM_SCRATCH_PAD_GET].wr.rdma.
remote_addr);
#ifdef HAVE_LIBDCFA
ib_errno = ibv_post_send(conp->icom_qp, &conp->icom_sr[MPID_NEM_IB_COM_SCRATCH_PAD_GET]);
......@@ -2032,8 +2032,8 @@ int MPID_nem_ib_com_cas_scratch_pad(int condesc,
conp->icom_sr[MPID_NEM_IB_COM_SCRATCH_PAD_CAS].wr.atomic.swap = swap;
dprintf("MPID_nem_ib_com_cas_scratch_pad,wr.rdma.remote_addr=%llx\n",
(unsigned long long) conp->icom_sr[MPID_NEM_IB_COM_SCRATCH_PAD_CAS].wr.
rdma.remote_addr);
(unsigned long long) conp->icom_sr[MPID_NEM_IB_COM_SCRATCH_PAD_CAS].wr.rdma.
remote_addr);
#ifdef HAVE_LIBDCFA
ib_errno = ibv_post_send(conp->icom_qp, &conp->icom_sr[MPID_NEM_IB_COM_SCRATCH_PAD_CAS]);
......@@ -2456,7 +2456,7 @@ char *MPID_nem_ib_com_strerror(int err)
goto fn_exit;
}
else {
r = (char *)strerror_tbl[-err];
r = (char *) strerror_tbl[-err];
}
fn_exit:
return r;
......
......@@ -484,7 +484,7 @@ static inline int MPID_nem_ib_cbf_hash1(uint64_t addr)
(((addr >> (MPID_nem_ib_cbf_lognslot * 0)) & (MPID_nem_ib_cbf_nslot - 1)) ^
((addr >> (MPID_nem_ib_cbf_lognslot * 3)) & (MPID_nem_ib_cbf_nslot - 1)) ^
(((addr >> (MPID_nem_ib_cbf_lognslot * 6)) & (MPID_nem_ib_cbf_nslot - 1))
+ 1)) & (MPID_nem_ib_cbf_nslot - 1);
+ 1)) & (MPID_nem_ib_cbf_nslot - 1);
}
static inline int MPID_nem_ib_cbf_hash2(uint64_t addr)
......@@ -494,7 +494,7 @@ static inline int MPID_nem_ib_cbf_hash2(uint64_t addr)
(((addr >> (MPID_nem_ib_cbf_lognslot * 1)) & (MPID_nem_ib_cbf_nslot - 1)) ^
((addr >> (MPID_nem_ib_cbf_lognslot * 4)) & (MPID_nem_ib_cbf_nslot - 1)) ^
(((addr >> (MPID_nem_ib_cbf_lognslot * 7)) & (MPID_nem_ib_cbf_nslot - 1))
+ 1)) & (MPID_nem_ib_cbf_nslot - 1);
+ 1)) & (MPID_nem_ib_cbf_nslot - 1);
}
static inline int MPID_nem_ib_cbf_hash3(uint64_t addr)
......@@ -504,7 +504,7 @@ static inline int MPID_nem_ib_cbf_hash3(uint64_t addr)
(((addr >> (MPID_nem_ib_cbf_lognslot * 2)) & (MPID_nem_ib_cbf_nslot - 1)) ^
((addr >> (MPID_nem_ib_cbf_lognslot * 5)) & (MPID_nem_ib_cbf_nslot - 1)) ^
(((addr >> (MPID_nem_ib_cbf_lognslot * 8)) & (MPID_nem_ib_cbf_nslot - 1))
+ 2)) & (MPID_nem_ib_cbf_nslot - 1);
+ 2)) & (MPID_nem_ib_cbf_nslot - 1);
}
......
......@@ -494,7 +494,8 @@ int MPID_nem_ib_init(MPIDI_PG_t * pg_p, int pg_rank, char **bc_val_p, int *val_m
for (i = 0; i < MPID_nem_ib_nranks; i++) {
if (i != MPID_nem_ib_myrank) {
for (j = 0; j < MPID_NEM_IB_COM_MAX_RQ_CAPACITY; j++) {
MPID_nem_ib_com_scratch_pad_recv(MPID_nem_ib_scratch_pad_fds[i], sizeof(MPID_nem_ib_cm_notify_send_t));
MPID_nem_ib_com_scratch_pad_recv(MPID_nem_ib_scratch_pad_fds[i],
sizeof(MPID_nem_ib_cm_notify_send_t));
}
}
}
......@@ -855,11 +856,9 @@ int MPID_nem_ib_vc_init(MPIDI_VC_t * vc)
MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_obtain_pointer");
int ntrial = 0;
volatile MPID_nem_ib_com_qp_state_t *rstate =
(MPID_nem_ib_com_qp_state_t *) ((uint8_t *)
MPID_nem_ib_com_scratch_pad->icom_mem
[MPID_NEM_IB_COM_SCRATCH_PAD_TO] +
vc->pg_rank * sizeof(MPID_nem_ib_com_qp_state_t));
volatile MPID_nem_ib_com_qp_state_t *rstate = (MPID_nem_ib_com_qp_state_t *)
((uint8_t *) MPID_nem_ib_com_scratch_pad->icom_mem[MPID_NEM_IB_COM_SCRATCH_PAD_TO]
+ vc->pg_rank * sizeof(MPID_nem_ib_com_qp_state_t));
dprintf("ib_init,rstate=%p,*rstate=%08x\n", rstate, *((uint32_t *) rstate));
while (rstate->state != MPID_NEM_IB_COM_QP_STATE_RTR) {
__asm__ __volatile__("pause;":::"memory");
......@@ -1005,8 +1004,8 @@ int MPID_nem_ib_vc_terminate(MPIDI_VC_t * vc)
dprintf
("vc_terminate,before,%d->%d,diff-rsr=%d,l diff-lsr=%d,sendq_empty=%d,ncqe=%d,pending_sends=%d\n",
MPID_nem_ib_myrank, vc->pg_rank, MPID_nem_ib_diff16(vc_ib->ibcom->rsr_seq_num_tail,
vc_ib->ibcom->
rsr_seq_num_tail_last_sent),
vc_ib->
ibcom->rsr_seq_num_tail_last_sent),
MPID_nem_ib_diff16(vc_ib->ibcom->sseq_num, vc_ib->ibcom->lsr_seq_num_tail),
MPID_nem_ib_sendq_empty(vc_ib->sendq), MPID_nem_ib_ncqe, VC_FIELD(vc, pending_sends));
......
......@@ -323,8 +323,8 @@ int MPID_nem_ib_lmt_start_recv(struct MPIDI_VC *vc, struct MPID_Request *req, MP
length = s_cookie_buf->max_msg_sz;
}
REQ_FIELD(req, max_msg_sz) = s_cookie_buf->max_msg_sz; /* store initiator's max_msg_sz */
REQ_FIELD(req, seg_num) = s_cookie_buf->seg_num; /* store number of segments */
REQ_FIELD(req, max_msg_sz) = s_cookie_buf->max_msg_sz; /* store initiator's max_msg_sz */
REQ_FIELD(req, seg_num) = s_cookie_buf->seg_num; /* store number of segments */
/* try to issue RDMA-read command */
int slack = 1; /* slack for control packet bringing sequence number */
......
......@@ -1270,20 +1270,14 @@ int MPID_nem_ib_recv_buf_released(struct MPIDI_VC *vc, void *user_data)
MPID_nem_ib_ringbuf_headtail_t *headtail =
(MPID_nem_ib_ringbuf_headtail_t *) ((uint8_t *) MPID_nem_ib_scratch_pad +
MPID_NEM_IB_RINGBUF_OFF_HEAD);
uint16_t index_tail =
vc_ib->ibcom->remote_ringbuf->type ==
MPID_NEM_IB_RINGBUF_EXCLUSIVE ? ((uint16_t) (vc_ib->ibcom->rsr_seq_num_tail + 1) %
vc_ib->ibcom->
remote_ringbuf->nslot) : ((uint16_t) (headtail->tail +
1) %
vc_ib->ibcom->remote_ringbuf->
nslot);
uint16_t index_tail = vc_ib->ibcom->remote_ringbuf->type == MPID_NEM_IB_RINGBUF_EXCLUSIVE ?
((uint16_t) (vc_ib->ibcom->rsr_seq_num_tail + 1) % vc_ib->ibcom->remote_ringbuf-> nslot) :
((uint16_t) (headtail->tail + 1) % vc_ib->ibcom->remote_ringbuf->nslot);
dprintf("released,index_tail=%d\n", index_tail);
dprintf("released,%016lx\n", vc_ib->ibcom->remote_ringbuf->remote_released[index_tail / 64]);
if (1 || (index_tail & 7) || MPID_nem_ib_diff16(index_slot, index_tail) >= vc_ib->ibcom->remote_ringbuf->nslot - 8) { /* avoid wrap-around */
while (1) {
if (((vc_ib->ibcom->remote_ringbuf->
remote_released[index_tail / 64] >> (index_tail & 63)) & 1) == 1) {
if (((vc_ib->ibcom-> remote_ringbuf->remote_released[index_tail / 64] >> (index_tail & 63)) & 1) == 1) {
if (vc_ib->ibcom->remote_ringbuf->type == MPID_NEM_IB_RINGBUF_EXCLUSIVE) {
vc_ib->ibcom->rsr_seq_num_tail += 1;
dprintf("exclusive ringbuf,remote_tail,incremented to %d\n",
......@@ -1414,7 +1408,7 @@ int MPID_nem_ib_PktHandler_EagerSend(MPIDI_VC_t * vc,
* see MPID_nem_ib_iSendContig
*/
//ch3_pkt->type = MPIDI_CH3_PKT_EAGER_SEND;
dprintf("ib_poll.c,before PktHandler_EagerSend,buflen=%ld\n", *buflen);
dprintf("ib_poll.c,before PktHandler_EagerSend,buflen=%ld\n", *buflen);
MPIDI_msg_sz_t ch3_buflen = *buflen - sizeof(MPID_nem_ib_pkt_prefix_t);
mpi_errno = MPIDI_CH3_PktHandler_EagerSend(vc, (MPIDI_CH3_Pkt_t *) ch3_pkt, &ch3_buflen, rreqp);
dprintf("ib_poll.c,after PktHandler_EagerSend,buflen=%ld\n", ch3_buflen);
......@@ -1660,8 +1654,7 @@ int MPID_nem_ib_pkt_RTS_handler(MPIDI_VC_t * vc,
MPID_nem_ib_ncqe < MPID_NEM_IB_COM_MAX_CQ_CAPACITY - slack) {
mpi_errno =
MPID_nem_ib_lmt_start_recv_core(req, rts_pkt->addr, rts_pkt->rkey, length,
write_to_buf, REQ_FIELD(req, max_msg_sz),
last);
write_to_buf, REQ_FIELD(req, max_msg_sz), last);
if (mpi_errno) {
MPIU_ERR_POP(mpi_errno);
}
......@@ -1955,8 +1948,8 @@ int MPID_nem_ib_cm_drain_scq()
MPID_nem_ib_cm_ringbuf_tail) < MPID_NEM_IB_CM_NSEG) {
MPID_nem_ib_cm_cmd_syn_t *cmd =
(MPID_nem_ib_cm_cmd_syn_t *) shadow_cm->req->ibcom->
icom_mem[MPID_NEM_IB_COM_SCRATCH_PAD_FROM];
(MPID_nem_ib_cm_cmd_syn_t *) shadow_cm->req->
ibcom->icom_mem[MPID_NEM_IB_COM_SCRATCH_PAD_FROM];
MPID_NEM_IB_CM_COMPOSE_SYN(cmd, shadow_cm->req);
cmd->responder_ringbuf_index =
shadow_cm->req->responder_ringbuf_index =
......@@ -2481,8 +2474,8 @@ int MPID_nem_ib_cm_poll_syn()
MPID_nem_ib_cm_ringbuf_tail) < MPID_NEM_IB_CM_NSEG) {
MPID_nem_ib_cm_cmd_synack_t *cmd =
(MPID_nem_ib_cm_cmd_synack_t *) req->ibcom->
icom_mem[MPID_NEM_IB_COM_SCRATCH_PAD_FROM];
(MPID_nem_ib_cm_cmd_synack_t *) req->
ibcom->icom_mem[MPID_NEM_IB_COM_SCRATCH_PAD_FROM];
if (is_synack) {
dprintf("cm_poll_syn,sending synack,%d->%d[%d],connection_tx=%d\n",
MPID_nem_ib_myrank, syn->initiator_rank, req->ringbuf_index,
......@@ -2722,8 +2715,8 @@ int MPID_nem_ib_cm_poll()
dprintf("cm_poll,sending ack1,req=%p,ringbuf_index=%d\n", req,
req->ringbuf_index);
MPID_nem_ib_cm_cmd_ack1_t *cmd =
(MPID_nem_ib_cm_cmd_ack1_t *) req->ibcom->
icom_mem[MPID_NEM_IB_COM_SCRATCH_PAD_FROM];
(MPID_nem_ib_cm_cmd_ack1_t *) req->
ibcom->icom_mem[MPID_NEM_IB_COM_SCRATCH_PAD_FROM];
MPID_NEM_IB_CM_COMPOSE_ACK1(cmd, req, synack->responder_req);
dprintf
("cm_poll,composing ack1,cmd->responder_req=%p,cmd->rmem=%p,rkey=%08x,ringbuf_nslot=%d,remote_vc=%p\n",
......@@ -2863,8 +2856,8 @@ int MPID_nem_ib_cm_poll()
req, req->ringbuf_index, req->initiator_rank,
req->ibcom->outstanding_connection_tx);
MPID_nem_ib_cm_cmd_ack2_t *cmd =
(MPID_nem_ib_cm_cmd_ack2_t *) req->ibcom->
icom_mem[MPID_NEM_IB_COM_SCRATCH_PAD_FROM];
(MPID_nem_ib_cm_cmd_ack2_t *) req->
ibcom->icom_mem[MPID_NEM_IB_COM_SCRATCH_PAD_FROM];
MPID_NEM_IB_CM_COMPOSE_ACK2(cmd, ack1->initiator_req);
MPID_nem_ib_cm_cmd_shadow_t *shadow = (MPID_nem_ib_cm_cmd_shadow_t *)
MPIU_Malloc(sizeof(MPID_nem_ib_cm_cmd_shadow_t));
......
......@@ -132,8 +132,7 @@ static inline void __lru_queue_display()
int i = 0;
for (i = 0; i < MPID_NEM_IB_COM_REG_MR_NLINE; i++) {
dprintf("---- hash %d\n", i);
for (p =
(struct MPID_nem_ib_com_reg_mr_cache_entry_t *)
for (p = (struct MPID_nem_ib_com_reg_mr_cache_entry_t *)
MPID_nem_ib_com_reg_mr_cache[i].lru_next;
p != (struct MPID_nem_ib_com_reg_mr_cache_entry_t *) &MPID_nem_ib_com_reg_mr_cache[i];
p = (struct MPID_nem_ib_com_reg_mr_cache_entry_t *) p->lru_next) {
......@@ -357,8 +356,7 @@ int MPID_nem_ib_com_register_cache_release()
}
for (i = 0; i < MPID_NEM_IB_COM_REG_MR_NLINE; i++) {
for (p =
(struct MPID_nem_ib_com_reg_mr_cache_entry_t *)
for (p = (struct MPID_nem_ib_com_reg_mr_cache_entry_t *)
MPID_nem_ib_com_reg_mr_cache[i].lru_next;
p !=
(struct MPID_nem_ib_com_reg_mr_cache_entry_t *) &MPID_nem_ib_com_reg_mr_cache[i];) {
......
......@@ -62,53 +62,54 @@ static int MPID_nem_ib_iSendContig_core(MPIDI_VC_t * vc, MPID_Request * sreq, vo
s_data = data;
s_data_sz = data_sz;
/* If request length is too long, create LMT packet */
if (MPID_NEM_IB_NETMOD_HDR_SIZEOF(vc_ib->ibcom->local_ringbuf_type)
+ sizeof(MPIDI_CH3_Pkt_t) + data_sz
> MPID_NEM_IB_COM_RDMABUF_SZSEG - sizeof(MPID_nem_ib_netmod_trailer_t)) {
pkt_netmod.type = MPIDI_NEM_PKT_NETMOD;
pkt_netmod.subtype = MPIDI_NEM_IB_PKT_RMA_LMT_RTS;
void *write_from_buf = data;
uint32_t max_msg_sz;
MPID_nem_ib_com_get_info_conn(vc_ib->sc->fd, MPID_NEM_IB_COM_INFOKEY_PATTR_MAX_MSG_SZ,
&max_msg_sz, sizeof(uint32_t));
MPID_nem_ib_rma_lmt_cookie_t *s_cookie_buf = (MPID_nem_ib_rma_lmt_cookie_t *) MPIU_Malloc(sizeof(MPID_nem_ib_rma_lmt_cookie_t));
sreq->ch.s_cookie = s_cookie_buf;
s_cookie_buf->tail = *((uint8_t *) ((uint8_t *) write_from_buf + data_sz - sizeof(uint8_t)));
/* put IB rkey */
struct MPID_nem_ib_com_reg_mr_cache_entry_t *mr_cache =
MPID_nem_ib_com_reg_mr_fetch(write_from_buf, data_sz, 0,
MPID_NEM_IB_COM_REG_MR_GLOBAL);
MPIU_ERR_CHKANDJUMP(!mr_cache, mpi_errno, MPI_ERR_OTHER,
"**MPID_nem_ib_com_reg_mr_fetch");
struct ibv_mr *mr = mr_cache->mr;
REQ_FIELD(sreq, lmt_mr_cache) = (void *) mr_cache;
/* If request length is too long, create LMT packet */
if (MPID_NEM_IB_NETMOD_HDR_SIZEOF(vc_ib->ibcom->local_ringbuf_type)
+ sizeof(MPIDI_CH3_Pkt_t) + data_sz
> MPID_NEM_IB_COM_RDMABUF_SZSEG - sizeof(MPID_nem_ib_netmod_trailer_t)) {
pkt_netmod.type = MPIDI_NEM_PKT_NETMOD;
pkt_netmod.subtype = MPIDI_NEM_IB_PKT_RMA_LMT_RTS;
void *write_from_buf = data;
uint32_t max_msg_sz;
MPID_nem_ib_com_get_info_conn(vc_ib->sc->fd, MPID_NEM_IB_COM_INFOKEY_PATTR_MAX_MSG_SZ,
&max_msg_sz, sizeof(uint32_t));
MPID_nem_ib_rma_lmt_cookie_t *s_cookie_buf =
(MPID_nem_ib_rma_lmt_cookie_t *) MPIU_Malloc(sizeof(MPID_nem_ib_rma_lmt_cookie_t));
sreq->ch.s_cookie = s_cookie_buf;
s_cookie_buf->tail =
*((uint8_t *) ((uint8_t *) write_from_buf + data_sz - sizeof(uint8_t)));
/* put IB rkey */
struct MPID_nem_ib_com_reg_mr_cache_entry_t *mr_cache =
MPID_nem_ib_com_reg_mr_fetch(write_from_buf, data_sz, 0,
MPID_NEM_IB_COM_REG_MR_GLOBAL);
MPIU_ERR_CHKANDJUMP(!mr_cache, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_reg_mr_fetch");
struct ibv_mr *mr = mr_cache->mr;
REQ_FIELD(sreq, lmt_mr_cache) = (void *) mr_cache;
#ifdef HAVE_LIBDCFA
s_cookie_buf->addr = (void *) mr->host_addr;
s_cookie_buf->addr = (void *) mr->host_addr;
#else
s_cookie_buf->addr = write_from_buf;
s_cookie_buf->addr = write_from_buf;
#endif
s_cookie_buf->rkey = mr->rkey;
s_cookie_buf->len = data_sz;
s_cookie_buf->sender_req_id = sreq->handle;
s_cookie_buf->max_msg_sz = max_msg_sz;
/* set for ib_com_isend */
prefix = (void *)&pkt_netmod;
sz_prefix = sizeof(MPIDI_CH3_Pkt_t);
s_data = (void *)s_cookie_buf;
s_data_sz = sizeof(MPID_nem_ib_rma_lmt_cookie_t);
/* Release Request, when sender receives DONE packet. */
int incomplete;
MPIDI_CH3U_Request_increment_cc(sreq, &incomplete); // decrement in drain_scq and pkt_rma_lmt_getdone
}
s_cookie_buf->rkey = mr->rkey;
s_cookie_buf->len = data_sz;
s_cookie_buf->sender_req_id = sreq->handle;
s_cookie_buf->max_msg_sz = max_msg_sz;
/* set for ib_com_isend */
prefix = (void *) &pkt_netmod;
sz_prefix = sizeof(MPIDI_CH3_Pkt_t);
s_data = (void *) s_cookie_buf;
s_data_sz = sizeof(MPID_nem_ib_rma_lmt_cookie_t);
/* Release Request, when sender receives DONE packet. */
int incomplete;
MPIDI_CH3U_Request_increment_cc(sreq, &incomplete); // decrement in drain_scq and pkt_rma_lmt_getdone
}
/* packet handlers including MPIDI_CH3_PktHandler_EagerSend and MPID_nem_handle_pkt assume this */
hdr_sz = sizeof(MPIDI_CH3_Pkt_t);
......@@ -473,7 +474,8 @@ static int MPID_nem_ib_SendNoncontig_core(MPIDI_VC_t * vc, MPID_Request * sreq,
last = sreq->dev.segment_size; /* segment_size is byte offset */
if (last > 0) {
REQ_FIELD(sreq, lmt_pack_buf) = MPIU_Malloc((size_t) (sreq->dev.segment_size - sreq->dev.segment_first));
REQ_FIELD(sreq, lmt_pack_buf) =
MPIU_Malloc((size_t) (sreq->dev.segment_size - sreq->dev.segment_first));
MPIU_ERR_CHKANDJUMP(!REQ_FIELD(sreq, lmt_pack_buf), mpi_errno, MPI_ERR_OTHER,
"**outofmemory");
MPID_Segment_pack(sreq->dev.segment_ptr, sreq->dev.segment_first, &last,
......@@ -481,56 +483,56 @@ static int MPID_nem_ib_SendNoncontig_core(MPIDI_VC_t * vc, MPID_Request * sreq,
MPIU_Assert(last == sreq->dev.segment_size);
}
data = (void *)REQ_FIELD(sreq, lmt_pack_buf);
data = (void *) REQ_FIELD(sreq, lmt_pack_buf);
data_sz = last;
/* If request length is too long, create LMT packet */
if ( MPID_NEM_IB_NETMOD_HDR_SIZEOF(vc_ib->ibcom->local_ringbuf_type)
+ sizeof(MPIDI_CH3_Pkt_t) + sreq->dev.segment_size - sreq->dev.segment_first
> MPID_NEM_IB_COM_RDMABUF_SZSEG - sizeof(MPID_nem_ib_netmod_trailer_t)) {
pkt_netmod.type = MPIDI_NEM_PKT_NETMOD;
/* If request length is too long, create LMT packet */
if (MPID_NEM_IB_NETMOD_HDR_SIZEOF(vc_ib->ibcom->local_ringbuf_type)
+ sizeof(MPIDI_CH3_Pkt_t) + sreq->dev.segment_size - sreq->dev.segment_first
> MPID_NEM_IB_COM_RDMABUF_SZSEG - sizeof(MPID_nem_ib_netmod_trailer_t)) {
pkt_netmod.type = MPIDI_NEM_PKT_NETMOD;
pkt_netmod.subtype = MPIDI_NEM_IB_PKT_RMA_LMT_RTS;
pkt_netmod.subtype = MPIDI_NEM_IB_PKT_RMA_LMT_RTS;
void *write_from_buf = REQ_FIELD(sreq, lmt_pack_buf);
void *write_from_buf = REQ_FIELD(sreq, lmt_pack_buf);
uint32_t max_msg_sz;
MPID_nem_ib_com_get_info_conn(vc_ib->sc->fd, MPID_NEM_IB_COM_INFOKEY_PATTR_MAX_MSG_SZ,
&max_msg_sz, sizeof(uint32_t));
uint32_t max_msg_sz;
MPID_nem_ib_com_get_info_conn(vc_ib->sc->fd, MPID_NEM_IB_COM_INFOKEY_PATTR_MAX_MSG_SZ,
&max_msg_sz, sizeof(uint32_t));
MPID_nem_ib_rma_lmt_cookie_t *s_cookie_buf = (MPID_nem_ib_rma_lmt_cookie_t *) MPIU_Malloc(sizeof(MPID_nem_ib_rma_lmt_cookie_t));
MPID_nem_ib_rma_lmt_cookie_t *s_cookie_buf =
(MPID_nem_ib_rma_lmt_cookie_t *) MPIU_Malloc(sizeof(MPID_nem_ib_rma_lmt_cookie_t));
sreq->ch.s_cookie = s_cookie_buf;
sreq->ch.s_cookie = s_cookie_buf;
s_cookie_buf->tail = *((uint8_t *) ((uint8_t *) write_from_buf + last - sizeof(uint8_t)));
/* put IB rkey */
struct MPID_nem_ib_com_reg_mr_cache_entry_t *mr_cache =
MPID_nem_ib_com_reg_mr_fetch(write_from_buf, last, 0,
MPID_NEM_IB_COM_REG_MR_GLOBAL);
MPIU_ERR_CHKANDJUMP(!mr_cache, mpi_errno, MPI_ERR_OTHER,
"**MPID_nem_ib_com_reg_mr_fetch");
struct ibv_mr *mr = mr_cache->mr;
REQ_FIELD(sreq, lmt_mr_cache) = (void *) mr_cache;
s_cookie_buf->tail = *((uint8_t *) ((uint8_t *) write_from_buf + last - sizeof(uint8_t)));
/* put IB rkey */
struct MPID_nem_ib_com_reg_mr_cache_entry_t *mr_cache =
MPID_nem_ib_com_reg_mr_fetch(write_from_buf, last, 0,
MPID_NEM_IB_COM_REG_MR_GLOBAL);
MPIU_ERR_CHKANDJUMP(!mr_cache, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_reg_mr_fetch");
struct ibv_mr *mr = mr_cache->mr;
REQ_FIELD(sreq, lmt_mr_cache) = (void *) mr_cache;
#ifdef HAVE_LIBDCFA
s_cookie_buf->addr = (void *) mr->host_addr;
s_cookie_buf->addr = (void *) mr->host_addr;
#else
s_cookie_buf->addr = write_from_buf;
s_cookie_buf->addr = write_from_buf;
#endif
s_cookie_buf->rkey = mr->rkey;
s_cookie_buf->len = last;
s_cookie_buf->sender_req_id = sreq->handle;
s_cookie_buf->max_msg_sz = max_msg_sz;
/* set for ib_com_isend */
prefix = (void *)&pkt_netmod;
prefix_sz = sizeof(MPIDI_CH3_Pkt_t);
data = (void *)s_cookie_buf;
data_sz = sizeof(MPID_nem_ib_rma_lmt_cookie_t);
/* Release Request, when sender receives DONE packet. */
int incomplete;
MPIDI_CH3U_Request_increment_cc(sreq, &incomplete); // decrement in drain_scq and pkt_rma_lmt_getdone
}
s_cookie_buf->rkey = mr->rkey;
s_cookie_buf->len = last;
s_cookie_buf->sender_req_id = sreq->handle;
s_cookie_buf->max_msg_sz = max_msg_sz;
/* set for ib_com_isend */
prefix = (void *) &pkt_netmod;
prefix_sz = sizeof(MPIDI_CH3_Pkt_t);
data = (void *) s_cookie_buf;
data_sz = sizeof(MPID_nem_ib_rma_lmt_cookie_t);
/* Release Request, when sender receives DONE packet. */
int incomplete;
MPIDI_CH3U_Request_increment_cc(sreq, &incomplete); // decrement in drain_scq and pkt_rma_lmt_getdone
}
/* packet handlers assume this */
hdr_sz = sizeof(MPIDI_CH3_Pkt_t);
......@@ -873,8 +875,8 @@ int MPID_nem_ib_send_progress(MPIDI_VC_t * vc)
}
case MPIDI_NEM_IB_PKT_REPLY_SEQ_NUM:{
MPID_nem_ib_pkt_reply_seq_num_t *_pkt =
(MPID_nem_ib_pkt_reply_seq_num_t *) sreq->dev.iov[0].
MPID_IOV_BUF;
(MPID_nem_ib_pkt_reply_seq_num_t *) sreq->dev.
iov[0].MPID_IOV_BUF;
dprintf
("send_progress,MPIDI_NEM_IB_PKT_REPLY_SEQ_NUM,rsr_seq_num_tail=%d\n",
vc_ib->ibcom->rsr_seq_num_tail);
......@@ -1016,8 +1018,7 @@ int MPID_nem_ib_cm_progress()
("cm_progress,retry CAS,responder_rank=%d,req=%p,decided=%ld,vt=%ld,backoff=%ld\n",
sreq->responder_rank, sreq, sreq->retry_decided,
MPID_nem_ib_progress_engine_vt, sreq->retry_backoff);
shadow =
(MPID_nem_ib_cm_cmd_shadow_t *)
shadow = (MPID_nem_ib_cm_cmd_shadow_t *)
MPIU_Malloc(sizeof(MPID_nem_ib_cm_cmd_shadow_t));
shadow->type = sreq->state;
shadow->req = sreq;
......@@ -1030,13 +1031,13 @@ int MPID_nem_ib_cm_progress()
("cm_progress,retry CAS_RELEASE,responder_rank=%d,req=%p,decided=%ld,vt=%ld,backoff=%ld\n",
sreq->responder_rank, sreq, sreq->retry_decided,
MPID_nem_ib_progress_engine_vt, sreq->retry_backoff);
shadow =
(MPID_nem_ib_cm_cmd_shadow_t *)
shadow = (MPID_nem_ib_cm_cmd_shadow_t *)
MPIU_Malloc(sizeof(MPID_nem_ib_cm_cmd_shadow_t));
shadow->type = sreq->state;
shadow->req = sreq;
mpi_errno = MPID_nem_ib_cm_cas_release_core(sreq->responder_rank, shadow);
MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_cm_cas_release_core");
MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER,
"**MPID_nem_ib_cm_cas_release_core");
break;
case MPID_NEM_IB_CM_SYN:
if (is_conn_established(sreq->responder_rank)) {
......@@ -1061,8 +1062,7 @@ int MPID_nem_ib_cm_progress()
((MPID_nem_ib_cm_cmd_syn_t *) & sreq->cmd)->initiator_rank = MPID_nem_ib_myrank;
MPID_nem_ib_cm_ringbuf_head++;
shadow =
(MPID_nem_ib_cm_cmd_shadow_t *)
shadow = (MPID_nem_ib_cm_cmd_shadow_t *)
MPIU_Malloc(sizeof(MPID_nem_ib_cm_cmd_shadow_t));
shadow->type = sreq->state;
shadow->req = sreq;
......@@ -1080,8 +1080,7 @@ int MPID_nem_ib_cm_progress()
((MPID_nem_ib_cm_cmd_syn_t *) & sreq->cmd)->initiator_rank = MPID_nem_ib_myrank;
shadow =
(MPID_nem_ib_cm_cmd_shadow_t *)
shadow = (MPID_nem_ib_cm_cmd_shadow_t *)
MPIU_Malloc(sizeof(MPID_nem_ib_cm_cmd_shadow_t));
shadow->type = sreq->state;
shadow->req = sreq;
......@@ -1103,8 +1102,7 @@ int MPID_nem_ib_cm_progress()
MPID_nem_ib_cm_ringbuf_head;
sreq->initiator_ringbuf_index = MPID_nem_ib_cm_ringbuf_head;
MPID_nem_ib_cm_ringbuf_head++;
shadow =
(MPID_nem_ib_cm_cmd_shadow_t *)
shadow = (MPID_nem_ib_cm_cmd_shadow_t *)
MPIU_Malloc(sizeof(MPID_nem_ib_cm_cmd_shadow_t));
shadow->type = sreq->state;
shadow->req = sreq;
......@@ -1118,8 +1116,7 @@ int MPID_nem_ib_cm_progress()
"**MPID_nem_ib_cm_send_core");
break;
case MPID_NEM_IB_CM_ACK1:
shadow =
(MPID_nem_ib_cm_cmd_shadow_t *)
shadow = (MPID_nem_ib_cm_cmd_shadow_t *)
MPIU_Malloc(sizeof(MPID_nem_ib_cm_cmd_shadow_t));
shadow->type = sreq->state;
shadow->req = sreq;
......@@ -1133,8 +1130,7 @@ int MPID_nem_ib_cm_progress()
"**MPID_nem_ib_cm_send_core");
break;
case MPID_NEM_IB_CM_ACK2:
shadow =
(MPID_nem_ib_cm_cmd_shadow_t *)
shadow = (MPID_nem_ib_cm_cmd_shadow_t *)
MPIU_Malloc(sizeof(MPID_nem_ib_cm_cmd_shadow_t));
shadow->type = sreq->state;
shadow->req = sreq;
......@@ -1149,8 +1145,7 @@ int MPID_nem_ib_cm_progress()
break;
case MPID_NEM_IB_CM_ALREADY_ESTABLISHED:
case MPID_NEM_IB_CM_RESPONDER_IS_CONNECTING:
shadow =
(MPID_nem_ib_cm_cmd_shadow_t *)
shadow = (MPID_nem_ib_cm_cmd_shadow_t *)
MPIU_Malloc(sizeof(MPID_nem_ib_cm_cmd_shadow_t));
shadow->type = sreq->state;
shadow->req = sreq;
......@@ -1236,13 +1231,8 @@ int MPID_nem_ib_cm_cas_core(int rank, MPID_nem_ib_cm_cmd_shadow_t * shadow)
(unsigned long) conp->icom_rmem[MPID_NEM_IB_COM_SCRATCH_PAD_TO] + 0);
/* Compare-and-swap rank to acquire communication manager port */
ibcom_errno =
MPID_nem_ib_com_cas_scratch_pad(MPID_nem_ib_scratch_pad_fds[rank],
(uint64_t) shadow,
0,
MPID_NEM_IB_CM_RELEASED,
MPID_nem_ib_myrank/*rank*/, /*debug*/
&shadow->buf_from, &shadow->buf_from_sz);
ibcom_errno = MPID_nem_ib_com_cas_scratch_pad(MPID_nem_ib_scratch_pad_fds[rank], (uint64_t) shadow, 0, MPID_NEM_IB_CM_RELEASED, MPID_nem_ib_myrank /*rank */ , /*debug */
&shadow->buf_from, &shadow->buf_from_sz);
MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_cas_scratch_pad");
MPID_nem_ib_ncqe_scratch_pad += 1;
......@@ -1338,13 +1328,8 @@ int MPID_nem_ib_cm_cas_release_core(int rank, MPID_nem_ib_cm_cmd_shadow_t * shad
(unsigned long) conp->icom_rmem[MPID_NEM_IB_COM_SCRATCH_PAD_TO] + 0);
/* Compare-and-swap rank to acquire communication manager port */
ibcom_errno =
MPID_nem_ib_com_cas_scratch_pad(MPID_nem_ib_scratch_pad_fds[rank],
(uint64_t) shadow,
0,
MPID_nem_ib_myrank,
MPID_NEM_IB_CM_RELEASED/*rank*/, /*debug*/
&shadow->buf_from, &shadow->buf_from_sz);
ibcom_errno = MPID_nem_ib_com_cas_scratch_pad(MPID_nem_ib_scratch_pad_fds[rank], (uint64_t) shadow, 0, MPID_nem_ib_myrank, MPID_NEM_IB_CM_RELEASED /*rank */ , /*debug */
&shadow->buf_from, &shadow->buf_from_sz);
MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_cas_scratch_pad");
MPID_nem_ib_ncqe_scratch_pad += 1;
......@@ -1531,11 +1516,13 @@ int MPID_nem_ib_cm_notify_progress(void)
}
ibcom_errno = MPID_nem_ib_cm_notify_send(sreq->pg_rank, sreq->my_rank);
MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_cm_notify_send");
MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER,
"**MPID_nem_ib_cm_notify_send");
/* unlink sreq */
if (prev_sreq != NULL) {
MPID_nem_ib_cm_notify_sendq_next(prev_sreq) = MPID_nem_ib_cm_notify_sendq_next(sreq);
MPID_nem_ib_cm_notify_sendq_next(prev_sreq) =
MPID_nem_ib_cm_notify_sendq_next(sreq);
}
else {
MPID_nem_ib_cm_notify_sendq_head(MPID_nem_ib_cm_notify_sendq) =
......@@ -1641,8 +1628,7 @@ int MPID_nem_ib_ringbuf_ask_fetch(MPIDI_VC_t * vc)
* from the same process */
VC_FIELD(vc, ibcom->ask_guard) = 1;
MPID_nem_ib_ringbuf_cmd_shadow_t *shadow =
(MPID_nem_ib_ringbuf_cmd_shadow_t *)
MPID_nem_ib_ringbuf_cmd_shadow_t *shadow = (MPID_nem_ib_ringbuf_cmd_shadow_t *)
MPIU_Malloc(sizeof(MPID_nem_ib_ringbuf_cmd_shadow_t));
shadow->type = req->state;
shadow->req = req;
......@@ -1732,8 +1718,7 @@ int MPID_nem_ib_ringbuf_ask_cas(MPIDI_VC_t * vc, MPID_nem_ib_ringbuf_req_t * req
dprintf("ask_cas,core\n");
req->state = MPID_NEM_IB_RINGBUF_ASK_CAS;
MPID_nem_ib_ringbuf_cmd_shadow_t *shadow =