Commit 854bc551 authored by Darius Buntinas's avatar Darius Buntinas
Browse files

[svn-r10372] portals netmod: added debugging messages and added id initialization in recv path

parent 18a8d645
......@@ -10,9 +10,9 @@
#include <mpid_nem_impl.h>
#include <portals4.h>
#define PTL_MAX_EAGER (64*1024) /* 64K */
#define PTL_MAX_EAGER (64*1024UL) /* 64K */
#define PTL_LARGE_THRESHOLD (64*1024) /* 64K */
#define PTL_LARGE_THRESHOLD (64*1024UL) /* 64K */
extern ptl_handle_ni_t MPIDI_nem_ptl_ni;
extern ptl_pt_index_t MPIDI_nem_ptl_pt;
......@@ -131,14 +131,16 @@ typedef struct {
#define NPTL_MATCH_TAG_OFFSET 32
#define NPTL_MATCH_CTX_OFFSET 16
#define NPTL_MATCH_RANK_MASK (((ptl_match_bits_t)(1) << 16) - 1)
#define NPTL_MATCH_TAG_MASK ((((ptl_match_bits_t)(1) << 16) - 1) << NPTL_MATCH_TAG_OFFSET)
#define NPTL_MATCH(tag_, ctx_) (((ptl_match_bits_t)(tag_) << NPTL_MATCH_TAG_OFFSET) | \
((ptl_match_bits_t)(ctx_) << NPTL_MATCH_CTX_OFFSET) | \
((ptl_match_bits_t)(MPIDI_Process.my_pg_rank)))
#define NPTL_MATCH_CTX_MASK ((((ptl_match_bits_t)(1) << 16) - 1) << NPTL_MATCH_CTX_OFFSET)
#define NPTL_MATCH_TAG_MASK ((((ptl_match_bits_t)(1) << 32) - 1) << NPTL_MATCH_TAG_OFFSET)
#define NPTL_MATCH(tag_, ctx_, rank_) (((ptl_match_bits_t)(tag_) << NPTL_MATCH_TAG_OFFSET) | \
((ptl_match_bits_t)(ctx_) << NPTL_MATCH_CTX_OFFSET) | \
((ptl_match_bits_t)(rank_)))
#define NPTL_MATCH_IGNORE NPTL_MATCH_RANK_MASK
#define NPTL_MATCH_IGNORE_ANY_TAG (NPTL_MATCH_IGNORE | NPTL_MATCH_TAG_MASK)
#define NPTL_MATCH_GET_RANK(match_bits_) ((match_bits_) & NPTL_MATCH_RANK_MASK)
#define NPTL_MATCH_GET_CTX(match_bits_) (((match_bits_) & NPTL_MATCH_CTX_MASK) >> NPTL_MATCH_CTX_OFFSET)
#define NPTL_MATCH_GET_TAG(match_bits_) ((match_bits_) >> NPTL_MATCH_TAG_OFFSET)
int MPID_nem_ptl_send_init(void);
......@@ -159,7 +161,6 @@ void MPI_nem_ptl_pack_byte(MPID_Segment *segment, MPI_Aint first, MPI_Aint last,
MPID_nem_ptl_pack_overflow_t *overflow);
int MPID_nem_ptl_unpack_byte(MPID_Segment *segment, MPI_Aint first, MPI_Aint last, void *buf,
MPID_nem_ptl_pack_overflow_t *overflow);
const char *MPID_nem_ptl_strerror(int ret);
/* comm override functions */
int MPID_nem_ptl_recv_posted(struct MPIDI_VC *vc, struct MPID_Request *req);
......@@ -181,5 +182,42 @@ int MPID_nem_ptl_anysource_improbe(int tag, MPID_Comm * comm, int context_offset
MPI_Status * status);
void MPID_nem_ptl_anysource_posted(MPID_Request *rreq);
int MPID_nem_ptl_anysource_matched(MPID_Request *rreq);
int MPID_nem_ptl_init_id(MPIDI_VC_t *vc);
/* debugging */
const char *MPID_nem_ptl_strerror(int ret);
const char *MPID_nem_ptl_strevent(const ptl_event_t *ev);
const char *MPID_nem_ptl_strnifail(ptl_ni_fail_t ni_fail);
const char *MPID_nem_ptl_strlist(ptl_list_t list);
#define DBG_MSG_PUT(md_, data_sz_, pg_rank_, match_, header_) do { \
MPIU_DBG_MSG_FMT(CH3_CHANNEL, VERBOSE, (MPIU_DBG_FDEST, "PtlPut: md=%s data_sz=%lu pg_rank=%d", md_, data_sz_, pg_rank_)); \
MPIU_DBG_MSG_FMT(CH3_CHANNEL, VERBOSE, (MPIU_DBG_FDEST, " tag=%#lx ctx=%#lx rank=%ld match=%#lx", \
NPTL_MATCH_GET_TAG(match_), NPTL_MATCH_GET_CTX(match_), NPTL_MATCH_GET_RANK(match_), match_)); \
MPIU_DBG_MSG_FMT(CH3_CHANNEL, VERBOSE, (MPIU_DBG_FDEST, " flags=%c%c%c seqnum=%ld data_sz=%ld header=%#lx", \
header_ & NPTL_SSEND ? 'S':' ', \
header_ & NPTL_LARGE ? 'L':' ', \
header_ & NPTL_MULTIPLE ? 'M':' ', \
NPTL_HEADER_GET_SEQNUM(header_), NPTL_HEADER_GET_LENGTH(header_), header_)); \
\
} while(0)
#define DBG_MSG_MEAPPEND(pt_, pg_rank_, me_, usr_ptr_) do { \
MPIU_DBG_MSG_FMT(CH3_CHANNEL, VERBOSE, (MPIU_DBG_FDEST, "PtlMEAppend: pt=%s pg_rank=%d me.length=%lu IOV=%d usr_ptr=%p", \
pt_, pg_rank_, me_.length, me_.options & PTL_IOVEC, usr_ptr_)); \
MPIU_DBG_MSG_FMT(CH3_CHANNEL, VERBOSE, (MPIU_DBG_FDEST, " tag=%#lx ctx=%#lx rank=%ld match=%#lx ignore=%#lx", \
NPTL_MATCH_GET_TAG(me_.match_bits), NPTL_MATCH_GET_CTX(me_.match_bits), \
NPTL_MATCH_GET_RANK(me_.match_bits), me_.match_bits, me_.ignore_bits)); \
} while(0)
#define DBG_MSG_MESearch(pt_, pg_rank_, me_, usr_ptr_) do { \
MPIU_DBG_MSG_FMT(CH3_CHANNEL, VERBOSE, (MPIU_DBG_FDEST, "PtlMESearch: pt=%s pg_rank=%d usr_ptr=%p", pt_, pg_rank_, usr_ptr_)); \
MPIU_DBG_MSG_FMT(CH3_CHANNEL, VERBOSE, (MPIU_DBG_FDEST, " tag=%#lx ctx=%#lx rank=%ld match=%#lx ignore=%#lx", \
NPTL_MATCH_GET_TAG(me_.match_bits), NPTL_MATCH_GET_CTX(me_.match_bits), \
NPTL_MATCH_GET_RANK(me_.match_bits), me_.match_bits, me_.ignore_bits)); \
} while(0)
#endif /* PTL_IMPL_H */
......@@ -5,6 +5,7 @@
*/
#include "ptl_impl.h"
#include <pmi.h>
#ifdef ENABLE_CHECKPOINTING
#error Checkpointing not implemented
......@@ -387,6 +388,43 @@ int MPID_nem_ptl_vc_terminated(MPIDI_VC_t *vc)
goto fn_exit;
}
#undef FUNCNAME
#define FUNCNAME MPID_nem_ptl_init_id
#undef FCNAME
#define FCNAME MPIU_QUOTE(FUNCNAME)
int MPID_nem_ptl_init_id(MPIDI_VC_t *vc)
{
int mpi_errno = MPI_SUCCESS;
MPID_nem_ptl_vc_area *const vc_ptl = VC_PTL(vc);
char *bc;
int pmi_errno;
int val_max_sz;
MPIU_CHKLMEM_DECL(1);
MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_PTL_INIT_ID);
MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_PTL_INIT_ID);
pmi_errno = PMI_KVS_Get_value_length_max(&val_max_sz);
MPIU_ERR_CHKANDJUMP1(pmi_errno, mpi_errno, MPI_ERR_OTHER, "**fail", "**fail %d", pmi_errno);
MPIU_CHKLMEM_MALLOC(bc, char *, val_max_sz, mpi_errno, "bc");
mpi_errno = vc->pg->getConnInfo(vc->pg_rank, bc, val_max_sz, vc->pg);
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
mpi_errno = MPID_nem_ptl_get_id_from_bc(bc, &vc_ptl->id, &vc_ptl->pt, &vc_ptl->ptc);
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
vc_ptl->id_initialized = TRUE;
fn_exit:
MPIU_CHKLMEM_FREEALL();
MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_PTL_INIT_ID);
return mpi_errno;
fn_fail:
goto fn_exit;
}
#define CASE_STR(x) case x: return #x
const char *MPID_nem_ptl_strerror(int ret)
......@@ -411,3 +449,50 @@ const char *MPID_nem_ptl_strerror(int ret)
default: return "UNKNOWN";
}
}
const char *MPID_nem_ptl_strnifail(ptl_ni_fail_t ni_fail)
{
switch (ni_fail) {
CASE_STR(PTL_NI_OK);
CASE_STR(PTL_NI_UNDELIVERABLE);
CASE_STR(PTL_NI_DROPPED);
CASE_STR(PTL_NI_PT_DISABLED);
CASE_STR(PTL_NI_PERM_VIOLATION);
CASE_STR(PTL_NI_OP_VIOLATION);
CASE_STR(PTL_NI_NO_MATCH);
CASE_STR(PTL_NI_SEGV);
default: return "UNKNOWN";
}
}
const char *MPID_nem_ptl_strevent(const ptl_event_t *ev)
{
switch (ev->type) {
CASE_STR(PTL_EVENT_GET);
CASE_STR(PTL_EVENT_GET_OVERFLOW);
CASE_STR(PTL_EVENT_PUT);
CASE_STR(PTL_EVENT_PUT_OVERFLOW);
CASE_STR(PTL_EVENT_ATOMIC);
CASE_STR(PTL_EVENT_ATOMIC_OVERFLOW);
CASE_STR(PTL_EVENT_FETCH_ATOMIC);
CASE_STR(PTL_EVENT_FETCH_ATOMIC_OVERFLOW);
CASE_STR(PTL_EVENT_REPLY);
CASE_STR(PTL_EVENT_SEND);
CASE_STR(PTL_EVENT_ACK);
CASE_STR(PTL_EVENT_PT_DISABLED);
CASE_STR(PTL_EVENT_LINK);
CASE_STR(PTL_EVENT_AUTO_UNLINK);
CASE_STR(PTL_EVENT_AUTO_FREE);
CASE_STR(PTL_EVENT_SEARCH);
default: return "UNKNOWN";
}
}
const char *MPID_nem_ptl_strlist(ptl_list_t pt_list)
{
switch (pt_list) {
CASE_STR(PTL_OVERFLOW_LIST);
CASE_STR(PTL_PRIORITY_LIST);
default: return "UNKNOWN";
}
}
......@@ -12,9 +12,11 @@
#define OVERFLOW_LENGTH (1024*1024)
#define NUM_OVERFLOW_ME 5
#if 0
static char recvbuf[BUFLEN][NUMBUFS];
static ptl_le_t recvbuf_le[NUMBUFS];
static ptl_handle_le_t recvbuf_le_handle[NUMBUFS];
#endif
static ptl_handle_me_t overflow_me_handle[NUM_OVERFLOW_ME];
static void *overflow_buf[NUM_OVERFLOW_ME];
......@@ -29,8 +31,10 @@ int MPID_nem_ptl_poll_init(void)
{
int mpi_errno = MPI_SUCCESS;
int i;
#if 0
int ret;
ptl_process_t id_any;
#endif
MPIU_CHKPMEM_DECL(NUM_OVERFLOW_ME);
MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_PTL_POLL_INIT);
......@@ -85,17 +89,21 @@ int MPID_nem_ptl_poll_finalize(void)
MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_PTL_POLL_FINALIZE);
for (i = 0; i < NUM_OVERFLOW_ME; ++i)
for (i = 0; i < NUM_OVERFLOW_ME; ++i) {
if (overflow_me_handle[i] != PTL_INVALID_HANDLE) {
ret = PtlMEUnlink(overflow_me_handle[i]);
MPIU_ERR_CHKANDJUMP1(ret, mpi_errno, MPI_ERR_OTHER, "**ptlmeunlink", "**ptlmeunlink %s", MPID_nem_ptl_strerror(ret));
}
MPIU_Free(overflow_buf[i]);
}
#if 0
for (i = 0; i < NUMBUFS; ++i) {
ret = PtlLEUnlink(recvbuf_le_handle[i]);
MPIU_ERR_CHKANDJUMP1(ret, mpi_errno, MPI_ERR_OTHER, "**ptlleunlink", "**ptlleunlink %s", MPID_nem_ptl_strerror(ret));
}
#endif
fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_PTL_POLL_FINALIZE);
return mpi_errno;
......@@ -164,9 +172,14 @@ int MPID_nem_ptl_poll(int is_blocking_poll)
break;
MPIU_ERR_CHKANDJUMP(ret == PTL_EQ_DROPPED, mpi_errno, MPI_ERR_OTHER, "**eqdropped");
MPIU_ERR_CHKANDJUMP1(ret, mpi_errno, MPI_ERR_OTHER, "**ptleqget", "**ptleqget %s", MPID_nem_ptl_strerror(ret));
MPIU_DBG_MSG_FMT(CH3_CHANNEL, VERBOSE, (MPIU_DBG_FDEST, "Received event %s ni_fail=%s list=%s user_ptr=%p hdr_data=%#lx\n",
MPID_nem_ptl_strevent(&event), MPID_nem_ptl_strnifail(event.ni_fail_type),
MPID_nem_ptl_strlist(event.ptl_list), event.user_ptr, event.hdr_data));
switch (event.type) {
case PTL_EVENT_PUT:
if (event.ptl_list == PTL_OVERFLOW_LIST)
break;
case PTL_EVENT_PUT_OVERFLOW:
case PTL_EVENT_GET:
case PTL_EVENT_ACK:
......
......@@ -80,7 +80,7 @@ int MPID_nem_ptl_iprobe(MPIDI_VC_t *vc, int source, int tag, MPID_Comm *comm, in
me.uid = PTL_UID_ANY;
me.options = ( PTL_ME_OP_PUT | PTL_ME_USE_ONCE );
me.min_free = 0;
me.match_bits = NPTL_MATCH(tag, comm->context_id + context_offset);
me.match_bits = NPTL_MATCH(tag, comm->context_id + context_offset, source);
if (source == MPI_ANY_SOURCE)
me.match_id = id_any;
......@@ -95,7 +95,8 @@ int MPID_nem_ptl_iprobe(MPIDI_VC_t *vc, int source, int tag, MPID_Comm *comm, in
/* submit a search request */
ret = PtlMESearch(MPIDI_nem_ptl_ni, MPIDI_nem_ptl_pt, &me, PTL_SEARCH_ONLY, req);
MPIU_ERR_CHKANDJUMP1(ret, mpi_errno, MPI_ERR_OTHER, "**ptlmesearch", "**ptlmesearch %s", MPID_nem_ptl_strerror(ret));
DBG_MSG_MESearch("REG", vc->pg_rank, me, req);
/* wait for search request to complete */
do {
mpi_errno = MPID_nem_ptl_poll(FALSE);
......
......@@ -367,12 +367,18 @@ int MPID_nem_ptl_recv_posted(MPIDI_VC_t *vc, MPID_Request *rreq)
me.uid = PTL_UID_ANY;
me.options = ( PTL_ME_OP_PUT | PTL_ME_IS_ACCESSIBLE | PTL_ME_EVENT_LINK_DISABLE |
PTL_ME_EVENT_UNLINK_DISABLE | PTL_ME_USE_ONCE );
if (vc == NULL)
if (vc == NULL) {
/* MPI_ANY_SOURCE receive */
me.match_id = id_any;
else
} else {
if (!vc_ptl->id_initialized) {
mpi_errno = MPID_nem_ptl_init_id(vc);
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
}
me.match_id = vc_ptl->id;
me.match_bits = NPTL_MATCH(rreq->dev.match.parts.tag, rreq->dev.match.parts.context_id);
}
me.match_bits = NPTL_MATCH(rreq->dev.match.parts.tag, rreq->dev.match.parts.context_id, rreq->dev.match.parts.rank);
me.ignore_bits = NPTL_MATCH_IGNORE;
me.min_free = 0;
......@@ -451,7 +457,7 @@ int MPID_nem_ptl_recv_posted(MPIDI_VC_t *vc, MPID_Request *rreq)
ret = PtlMEAppend(MPIDI_nem_ptl_ni, MPIDI_nem_ptl_pt, &me, PTL_PRIORITY_LIST, rreq, &REQ_PTL(rreq)->me);
MPIU_ERR_CHKANDJUMP1(ret, mpi_errno, MPI_ERR_OTHER, "**ptlmeappend", "**ptlmeappend %s", MPID_nem_ptl_strerror(ret));
DBG_MSG_MEAPPEND("REG", vc->pg_rank, me, rreq);
fn_exit:
MPIU_CHKPMEM_COMMIT();
......
......@@ -5,7 +5,6 @@
*/
#include "ptl_impl.h"
#include <pmi.h>
#include <mpl_utlist.h>
#define NUMBUFS 20
......@@ -104,42 +103,6 @@ int MPID_nem_ptl_send_finalize(void)
goto fn_exit;
}
#undef FUNCNAME
#define FUNCNAME init_id
#undef FCNAME
#define FCNAME MPIU_QUOTE(FUNCNAME)
static int init_id(MPIDI_VC_t *vc)
{
int mpi_errno = MPI_SUCCESS;
MPID_nem_ptl_vc_area *const vc_ptl = VC_PTL(vc);
char *bc;
int pmi_errno;
int val_max_sz;
MPIU_CHKLMEM_DECL(1);
MPIDI_STATE_DECL(MPID_STATE_INIT_ID);
MPIDI_FUNC_ENTER(MPID_STATE_INIT_ID);
pmi_errno = PMI_KVS_Get_value_length_max(&val_max_sz);
MPIU_ERR_CHKANDJUMP1(pmi_errno, mpi_errno, MPI_ERR_OTHER, "**fail", "**fail %d", pmi_errno);
MPIU_CHKLMEM_MALLOC(bc, char *, val_max_sz, mpi_errno, "bc");
mpi_errno = vc->pg->getConnInfo(vc->pg_rank, bc, val_max_sz, vc->pg);
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
mpi_errno = MPID_nem_ptl_get_id_from_bc(bc, &vc_ptl->id, &vc_ptl->pt, &vc_ptl->ptc);
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
vc_ptl->id_initialized = TRUE;
fn_exit:
MPIU_CHKLMEM_FREEALL();
MPIDI_FUNC_EXIT(MPID_STATE_INIT_ID);
return mpi_errno;
fn_fail:
goto fn_exit;
}
#undef FUNCNAME
#define FUNCNAME MPID_nem_ptl_sendq_complete_with_error
#undef FCNAME
......@@ -205,7 +168,7 @@ static inline int send_pkt(MPIDI_VC_t *vc, void **vhdr_p, void **vdata_p, MPIDI_
MPIDI_FUNC_ENTER(MPID_STATE_SEND_PKT);
if (!vc_ptl->id_initialized) {
mpi_errno = init_id(vc);
mpi_errno = MPID_nem_ptl_init_id(vc);
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
}
......@@ -271,7 +234,7 @@ static int send_noncontig_pkt(MPIDI_VC_t *vc, MPID_Request *sreq, void **vhdr_p,
MPID_nem_ptl_init_req(sreq);
if (!vc_ptl->id_initialized) {
mpi_errno = init_id(vc);
mpi_errno = MPID_nem_ptl_init_id(vc);
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
}
......@@ -839,7 +802,7 @@ static int send_msg(ptl_hdr_data_t ssend_flag, struct MPIDI_VC *vc, const void *
MPID_nem_ptl_request_create_sreq(sreq, mpi_errno, comm);
if (!vc_ptl->id_initialized) {
mpi_errno = init_id(vc);
mpi_errno = MPID_nem_ptl_init_id(vc);
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
}
......@@ -850,9 +813,10 @@ static int send_msg(ptl_hdr_data_t ssend_flag, struct MPIDI_VC *vc, const void *
if (dt_contig) {
REQ_PTL(sreq)->event_handler = handler_send_complete;
ret = PtlPut(MPIDI_nem_ptl_global_md, (ptl_size_t)buf, data_sz, PTL_ACK_REQ, vc_ptl->id, vc_ptl->pt,
NPTL_MATCH(tag, comm->context_id + context_offset), 0, sreq,
NPTL_MATCH(tag, comm->context_id + context_offset, comm->rank), 0, sreq,
NPTL_HEADER(ssend_flag, 0, data_sz));
MPIU_ERR_CHKANDJUMP1(ret, mpi_errno, MPI_ERR_OTHER, "**ptlput", "**ptlput %s", MPID_nem_ptl_strerror(ret));
DBG_MSG_PUT("global", data_sz, vc->pg_rank, NPTL_MATCH(tag, comm->context_id + context_offset, comm->rank), NPTL_HEADER(ssend_flag, 0, data_sz));
goto fn_exit;
}
......@@ -879,9 +843,10 @@ static int send_msg(ptl_hdr_data_t ssend_flag, struct MPIDI_VC *vc, const void *
REQ_PTL(sreq)->event_handler = handler_send_complete;
ret = PtlPut(REQ_PTL(sreq)->md, (ptl_size_t)buf, data_sz, PTL_ACK_REQ, vc_ptl->id, vc_ptl->pt,
NPTL_MATCH(tag, comm->context_id + context_offset), 0, sreq,
NPTL_MATCH(tag, comm->context_id + context_offset, comm->rank), 0, sreq,
NPTL_HEADER(ssend_flag, 0, data_sz));
MPIU_ERR_CHKANDJUMP1(ret, mpi_errno, MPI_ERR_OTHER, "**ptlput", "**ptlput %s", MPID_nem_ptl_strerror(ret));
DBG_MSG_PUT("sreq", data_sz, vc->pg_rank, NPTL_MATCH(tag, comm->context_id + context_offset, comm->rank), NPTL_HEADER(ssend_flag, 0, data_sz));
goto fn_exit;
}
......@@ -893,9 +858,10 @@ static int send_msg(ptl_hdr_data_t ssend_flag, struct MPIDI_VC *vc, const void *
MPIU_Assert(last == sreq->dev.segment_size);
REQ_PTL(sreq)->event_handler = handler_send_complete;
ret = PtlPut(MPIDI_nem_ptl_global_md, (ptl_size_t)REQ_PTL(sreq)->chunk_buffer[0], data_sz, PTL_ACK_REQ,
vc_ptl->id, vc_ptl->pt, NPTL_MATCH(tag, comm->context_id + context_offset), 0, sreq,
vc_ptl->id, vc_ptl->pt, NPTL_MATCH(tag, comm->context_id + context_offset, comm->rank), 0, sreq,
NPTL_HEADER(ssend_flag, 0, data_sz));
MPIU_ERR_CHKANDJUMP1(ret, mpi_errno, MPI_ERR_OTHER, "**ptlput", "**ptlput %s", MPID_nem_ptl_strerror(ret));
DBG_MSG_PUT("global", data_sz, vc->pg_rank, NPTL_MATCH(tag, comm->context_id + context_offset, comm->rank), NPTL_HEADER(ssend_flag, 0, data_sz));
goto fn_exit;
}
......@@ -915,14 +881,16 @@ static int send_msg(ptl_hdr_data_t ssend_flag, struct MPIDI_VC *vc, const void *
ret = PtlMEAppend(MPIDI_nem_ptl_ni, MPIDI_nem_ptl_control_pt, &me, PTL_PRIORITY_LIST, sreq, &REQ_PTL(sreq)->me);
MPIU_ERR_CHKANDJUMP1(ret, mpi_errno, MPI_ERR_OTHER, "**ptlmeappend", "**ptlmeappend %s", MPID_nem_ptl_strerror(ret));
DBG_MSG_MEAPPEND("CTL", vc->pg_rank, me, sreq);
REQ_PTL(sreq)->large = TRUE;
REQ_PTL(sreq)->event_handler = handler_large;
ret = PtlPut(MPIDI_nem_ptl_global_md, (ptl_size_t)buf, PTL_LARGE_THRESHOLD, PTL_ACK_REQ, vc_ptl->id, vc_ptl->pt,
NPTL_MATCH(tag, comm->context_id + context_offset), 0, sreq,
NPTL_MATCH(tag, comm->context_id + context_offset, comm->rank), 0, sreq,
NPTL_HEADER(ssend_flag | NPTL_LARGE, me.match_bits, data_sz));
MPIU_ERR_CHKANDJUMP1(ret, mpi_errno, MPI_ERR_OTHER, "**ptlput", "**ptlput %s", MPID_nem_ptl_strerror(ret));
DBG_MSG_PUT("global", PTL_LARGE_THRESHOLD, vc->pg_rank, NPTL_MATCH(tag, comm->context_id + context_offset, comm->rank), NPTL_HEADER(ssend_flag | NPTL_LARGE, me.match_bits, data_sz));
goto fn_exit;
}
......@@ -971,6 +939,7 @@ static int send_msg(ptl_hdr_data_t ssend_flag, struct MPIDI_VC *vc, const void *
ret = PtlMEAppend(MPIDI_nem_ptl_ni, MPIDI_nem_ptl_control_pt, &me, PTL_PRIORITY_LIST, sreq,
&REQ_PTL(sreq)->me);
MPIU_ERR_CHKANDJUMP1(ret, mpi_errno, MPI_ERR_OTHER, "**ptlmeappend", "**ptlmeappend %s", MPID_nem_ptl_strerror(ret));
DBG_MSG_MEAPPEND("CTL", vc->pg_rank, me, sreq);
/* Create MD for first chunk */
md.start = sreq->dev.iov;
......@@ -985,9 +954,10 @@ static int send_msg(ptl_hdr_data_t ssend_flag, struct MPIDI_VC *vc, const void *
REQ_PTL(sreq)->event_handler = handler_large;
ret = PtlPut(REQ_PTL(sreq)->md, 0, PTL_LARGE_THRESHOLD, PTL_ACK_REQ, vc_ptl->id, vc_ptl->pt,
NPTL_MATCH(tag, comm->context_id + context_offset), 0, sreq,
NPTL_MATCH(tag, comm->context_id + context_offset, comm->rank), 0, sreq,
NPTL_HEADER(ssend_flag | NPTL_LARGE, me.match_bits, data_sz));
MPIU_ERR_CHKANDJUMP1(ret, mpi_errno, MPI_ERR_OTHER, "**ptlput", "**ptlput %s", MPID_nem_ptl_strerror(ret));
DBG_MSG_PUT("req", PTL_LARGE_THRESHOLD, vc->pg_rank, NPTL_MATCH(tag, comm->context_id + context_offset, comm->rank), NPTL_HEADER(ssend_flag | NPTL_LARGE, me.match_bits, data_sz));
goto fn_exit;
}
}
......@@ -1015,14 +985,16 @@ static int send_msg(ptl_hdr_data_t ssend_flag, struct MPIDI_VC *vc, const void *
ret = PtlMEAppend(MPIDI_nem_ptl_ni, MPIDI_nem_ptl_control_pt, &me, PTL_PRIORITY_LIST, sreq, &REQ_PTL(sreq)->me);
MPIU_ERR_CHKANDJUMP1(ret, mpi_errno, MPI_ERR_OTHER, "**ptlmeappend", "**ptlmeappend %s", MPID_nem_ptl_strerror(ret));
DBG_MSG_MEAPPEND("CTL", vc->pg_rank, me, sreq);
REQ_PTL(sreq)->large = TRUE;
REQ_PTL(sreq)->event_handler = handler_large;
ret = PtlPut(MPIDI_nem_ptl_global_md, (ptl_size_t)REQ_PTL(sreq)->chunk_buffer[0], PTL_LARGE_THRESHOLD, PTL_ACK_REQ,
vc_ptl->id, vc_ptl->pt, NPTL_MATCH(tag, comm->context_id + context_offset), 0, sreq,
vc_ptl->id, vc_ptl->pt, NPTL_MATCH(tag, comm->context_id + context_offset, comm->rank), 0, sreq,
NPTL_HEADER(ssend_flag | NPTL_LARGE, me.match_bits, data_sz));
MPIU_ERR_CHKANDJUMP1(ret, mpi_errno, MPI_ERR_OTHER, "**ptlput", "**ptlput %s", MPID_nem_ptl_strerror(ret));
DBG_MSG_PUT("global", PTL_LARGE_THRESHOLD, vc->pg_rank, NPTL_MATCH(tag, comm->context_id + context_offset, comm->rank), NPTL_HEADER(ssend_flag | NPTL_LARGE, me.match_bits, data_sz));
goto fn_exit;
#if 0
......@@ -1060,7 +1032,7 @@ static int send_msg(ptl_hdr_data_t ssend_flag, struct MPIDI_VC *vc, const void *
REQ_PTL(sreq)->event_handler = handler_large_multi;
ret = PtlPut(MPIDI_nem_ptl_global_md, (ptl_size_t)REQ_PTL(sreq_)->chunk_buffer[0], PTL_LARGE_THRESHOLD, PTL_ACK_REQ, vc_ptl->id,
vc_ptl->pt, NPTL_MATCH(tag, comm->context_id + context_offset), 0, sreq,
vc_ptl->pt, NPTL_MATCH(tag, comm->context_id + context_offset, comm->rank), 0, sreq,
NPTL_HEADER(ssend_flag | NPTL_LARGE | NPTL_MULTIPLE, me.match_bits, data_sz));
MPIU_ERR_CHKANDJUMP1(ret, mpi_errno, MPI_ERR_OTHER, "**ptlput", "**ptlput %s", MPID_nem_ptl_strerror(ret));
#endif
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment