Commit d39a7363 authored by Charles J Archer's avatar Charles J Archer Committed by Pavan Balaji
Browse files

Open Fabrics Working Group (OFIWG) Netmod Support

 * Implements a tag matching interface netmod over the OFIWG Scalable Fabric Interfaces (SFI)
parent e38618b0
......@@ -13,3 +13,4 @@ include $(top_srcdir)/src/mpid/ch3/channels/nemesis/netmod/scif/Makefile.mk
include $(top_srcdir)/src/mpid/ch3/channels/nemesis/netmod/portals4/Makefile.mk
include $(top_srcdir)/src/mpid/ch3/channels/nemesis/netmod/ib/Makefile.mk
include $(top_srcdir)/src/mpid/ch3/channels/nemesis/netmod/mxm/Makefile.mk
include $(top_srcdir)/src/mpid/ch3/channels/nemesis/netmod/sfi/Makefile.mk
## -*- Mode: Makefile; -*-
## vim: set ft=automake :
##
## (C) 2011 by Argonne National Laboratory.
## See COPYRIGHT in top-level directory.
##
if BUILD_NEMESIS_NETMOD_SFI
mpi_core_sources += \
src/mpid/ch3/channels/nemesis/netmod/sfi/sfi_init.c \
src/mpid/ch3/channels/nemesis/netmod/sfi/sfi_cm.c \
src/mpid/ch3/channels/nemesis/netmod/sfi/sfi_tagged.c \
src/mpid/ch3/channels/nemesis/netmod/sfi/sfi_msg.c \
src/mpid/ch3/channels/nemesis/netmod/sfi/sfi_data.c \
src/mpid/ch3/channels/nemesis/netmod/sfi/sfi_progress.c
errnames_txt_files += src/mpid/ch3/channels/nemesis/netmod/sfi/errnames.txt
endif
**sfi_avmap:SFI get address vector map failed
**sfi_avmap %s %d %s %s:SFI address vector map failed (%s:%d:%s:%s)
**sfi_tsendto:SFI tagged sendto failed
**sfi_tsendto %s %d %s %s:SFI tagged sendto failed (%s:%d:%s:%s)
**sfi_trecvfrom:SFI tagged recvfrom failed
**sfi_trecvfrom %s %d %s %s:SFI tagged recvfrom failed (%s:%d:%s:%s)
**sfi_getinfo:SFI getinfo() failed
**sfi_getinfo %s %d %s %s:SFI getinfo() failed (%s:%d:%s:%s)
**sfi_openep:SFI endpoint open failed
**sfi_openep %s %d %s %s:SFI endpoint open failed (%s:%d:%s:%s)
**sfi_openfabric:SFI fabric open failure
**sfi_openfabric %s %d %s %s:SFI fabric open failed (%s:%d:%s:%s)
**sfi_opendomain:SFI domain open failure
**sfi_opendomain %s %d %s %s:SFI domain open failed (%s:%d:%s:%s)
**sfi_opencq:SFI event queue create failure
**sfi_opencq %s %d %s %s:SFI event queue create failed (%s:%d:%s:%s)
**sfi_avopen:SFI address vector open failed
**sfi_avopen %s %d %s %s:SFI address vector open failed (%s:%d:%s:%s)
**sfi_bind:SFI resource bind failure
**sfi_bind %s %d %s %s:SFI resource bind failed (%s:%d:%s:%s)
**sfi_ep_enable:SFI endpoint enable failed
**sfi_ep_enable %s %d %s %s:SFI endpoint enable failed (%s:%d:%s:%s)
**sfi_getname:SFI get endpoint name failed
**sfi_getname %s %d %s %s:SFI get endpoint name failed (%s:%d:%s:%s)
**sfi_avclose:SFI av close failed
**sfi_avclose %s %d %s %s:SFI av close failed (%s:%d:%s:%s)
**sfi_epclose:SFI endpoint close failed
**sfi_epclose %s %d %s %s:SFI endpoint close failed (%s:%d:%s:%s)
**sfi_cqclose:SFI cq close failed
**sfi_cqclose %s %d %s %s:SFI cq close failed (%s:%d:%s:%s)
**sfi_mrclose:SFI mr close failed
**sfi_mrclose %s %d %s %s:SFI mr close failed (%s:%d:%s:%s)
**sfi_fabricclose:SFI fabric close failed
**sfi_fabricclose %s %d %s %s:SFI fabric close failed (%s:%d:%s:%s)
**sfi_domainclose:SFI domain close failed
**sfi_domainclose %s %d %s %s:SFI domain close failed (%s:%d:%s:%s)
**sfi_tsearch:SFI tsearch failed
**sfi_tsearch %s %d %s %s:SFI tsearch failed (%s:%d:%s:%s)
**sfi_poll:SFI poll failed
**sfi_poll %s %d %s %s:SFI poll failed (%s:%d:%s:%s)
**sfi_cancel:SFI cancel failed
**sfi_cancel %s %d %s %s:SFI cancel failed (%s:%d:%s:%s)
This diff is collapsed.
/*
* (C) 2006 by Argonne National Laboratory.
* See COPYRIGHT in top-level directory.
*
* Portions of this code were written by Intel Corporation.
* Copyright (C) 2011-2012 Intel Corporation. Intel provides this material
* to Argonne National Laboratory subject to Software Grant and Corporate
* Contributor License Agreement dated February 8, 2012.
*/
#include "sfi_impl.h"
MPID_nem_sfi_global_t gl_data;
/* ************************************************************************** */
/* Netmod Function Table */
/* ************************************************************************** */
MPIDI_Comm_ops_t _g_comm_ops = {
MPID_nem_sfi_recv_posted, /* recv_posted */
MPID_nem_sfi_send, /* send */
MPID_nem_sfi_send, /* rsend */
MPID_nem_sfi_ssend, /* ssend */
MPID_nem_sfi_isend, /* isend */
MPID_nem_sfi_isend, /* irsend */
MPID_nem_sfi_issend, /* issend */
NULL, /* send_init */
NULL, /* bsend_init */
NULL, /* rsend_init */
NULL, /* ssend_init */
NULL, /* startall */
MPID_nem_sfi_cancel_send, /* cancel_send */
MPID_nem_sfi_cancel_recv, /* cancel_recv */
NULL, /* probe */
MPID_nem_sfi_iprobe, /* iprobe */
MPID_nem_sfi_improbe /* improbe */
};
MPID_nem_netmod_funcs_t MPIDI_nem_sfi_funcs = {
MPID_nem_sfi_init,
MPID_nem_sfi_finalize,
#ifdef ENABLE_CHECKPOINTING
NULL,
NULL,
NULL,
#endif
MPID_nem_sfi_poll,
MPID_nem_sfi_get_business_card,
MPID_nem_sfi_connect_to_root,
MPID_nem_sfi_vc_init,
MPID_nem_sfi_vc_destroy,
MPID_nem_sfi_vc_terminate,
MPID_nem_sfi_anysource_iprobe,
MPID_nem_sfi_anysource_improbe,
};
/*
* (C) 2006 by Argonne National Laboratory.
* See COPYRIGHT in top-level directory.
*
* Portions of this code were written by Intel Corporation.
* Copyright (C) 2011-2012 Intel Corporation. Intel provides this material
* to Argonne National Laboratory subject to Software Grant and Corporate
* Contributor License Agreement dated February 8, 2012.
*/
#ifndef SFI_IMPL_H
#define SFI_IMPL_H
#include "mpid_nem_impl.h"
#include "mpihandlemem.h"
#include "pmi.h"
#include <rdma/fabric.h>
#include <rdma/fi_errno.h>
#include <rdma/fi_endpoint.h>
#include <rdma/fi_domain.h>
#include <rdma/fi_tagged.h>
#include <rdma/fi_cm.h>
#include <netdb.h>
/* ************************************************************************** */
/* Type Definitions */
/* ************************************************************************** */
typedef struct iovec iovec_t;
typedef struct fi_info info_t;
typedef struct fi_cq_attr cq_attr_t;
typedef struct fi_av_attr av_attr_t;
typedef struct fi_domain_attr domain_attr_t;
typedef struct fi_tx_ctx_attr tx_ctx_attr_t;
typedef struct fi_cq_tagged_entry cq_tagged_entry_t;
typedef struct fi_cq_err_entry cq_err_entry_t;
typedef struct fi_context context_t;
typedef int (*event_callback_fn) (cq_tagged_entry_t * wc, MPID_Request *);
typedef int (*req_fn) (MPIDI_VC_t *, MPID_Request *, int *);
/* ******************************** */
/* Global Object for state tracking */
/* ******************************** */
typedef struct {
fi_addr_t bound_addr; /* This ranks bound address */
fi_addr_t any_addr; /* Specifies any source */
size_t bound_addrlen; /* length of the bound address */
struct fid_fabric *fabric; /* fabric object */
struct fid_domain *domain; /* domain object */
struct fid_ep *endpoint; /* endpoint object */
struct fid_cq *cq; /* completion queue */
struct fid_av *av; /* address vector */
struct fid_mr *mr; /* memory region */
MPIDI_PG_t *pg_p; /* MPI Process group */
MPIDI_VC_t *cm_vcs; /* temporary VC's */
MPID_Request *persistent_req; /* Unexpected request queue */
MPID_Request *conn_req; /* Connection request */
MPIDI_Comm_ops_t comm_ops;
} MPID_nem_sfi_global_t;
/* ******************************** */
/* Device channel specific data */
/* This is per destination */
/* ******************************** */
typedef struct {
fi_addr_t direct_addr; /* Remote SFI address */
int ready; /* VC ready state */
int is_cmvc; /* Cleanup VC */
MPIDI_VC_t *next; /* VC queue */
} MPID_nem_sfi_vc_t;
#define VC_SFI(vc) ((MPID_nem_sfi_vc_t *)vc->ch.netmod_area.padding)
/* ******************************** */
/* Per request object data */
/* SFI/Netmod specific */
/* ******************************** */
typedef struct {
context_t sfi_context; /* Context Object */
void *addr; /* SFI Address */
event_callback_fn event_callback; /* Callback Event */
char *pack_buffer; /* MPI Pack Buffer */
int pack_buffer_size; /* Pack buffer size */
int match_state; /* State of the match */
int req_started; /* Request state */
MPIDI_VC_t *vc; /* VC paired with this request */
uint64_t tag; /* 64 bit tag request */
MPID_Request *parent; /* Parent request */
} MPID_nem_sfi_req_t;
#define REQ_SFI(req) ((MPID_nem_sfi_req_t *)((req)->ch.netmod_area.padding))
/* ******************************** */
/* Logging and function macros */
/* ******************************** */
#undef FUNCNAME
#define FUNCNAME nothing
#define BEGIN_FUNC(FUNCNAME) \
MPIDI_STATE_DECL(FUNCNAME); \
MPIDI_FUNC_ENTER(FUNCNAME);
#define END_FUNC(FUNCNAME) \
MPIDI_FUNC_EXIT(FUNCNAME);
#define END_FUNC_RC(FUNCNAME) \
fn_exit: \
MPIDI_FUNC_EXIT(FUNCNAME); \
return mpi_errno; \
fn_fail: \
goto fn_exit;
#define __SHORT_FILE__ \
(strrchr(__FILE__,'/') \
? strrchr(__FILE__,'/')+1 \
: __FILE__ \
)
#define DECL_FUNC(FUNCNAME) MPIU_QUOTE(FUNCNAME)
#define SFI_COMPILE_TIME_ASSERT(expr_) \
do { switch(0) { case 0: case (expr_): default: break; } } while (0)
#define FI_RC(FUNC,STR) \
do \
{ \
ssize_t _ret = FUNC; \
MPIU_ERR_##CHKANDJUMP4(_ret<0, \
mpi_errno, \
MPI_ERR_OTHER, \
"**sfi_"#STR, \
"**sfi_"#STR" %s %d %s %s", \
__SHORT_FILE__, \
__LINE__, \
FCNAME, \
fi_strerror(-_ret)); \
} while (0)
#define PMI_RC(FUNC,STR) \
do \
{ \
pmi_errno = FUNC; \
MPIU_ERR_##CHKANDJUMP4(pmi_errno!=PMI_SUCCESS, \
mpi_errno, \
MPI_ERR_OTHER, \
"**sfi_"#STR, \
"**sfi_"#STR" %s %d %s %s", \
__SHORT_FILE__, \
__LINE__, \
FCNAME, \
#STR); \
} while (0)
#define MPI_RC(FUNC) \
do \
{ \
mpi_errno = FUNC; \
if (mpi_errno) MPIU_ERR_POP(mpi_errno); \
} while (0);
#define VC_READY_CHECK(vc) \
({ \
if (1 != VC_SFI(vc)->ready) { \
MPI_RC(MPID_nem_sfi_vc_connect(vc)); \
} \
})
#define SFI_ADDR_INIT(src, vc, remote_proc) \
({ \
if (MPI_ANY_SOURCE != src) { \
MPIU_Assert(vc != NULL); \
VC_READY_CHECK(vc); \
remote_proc = VC_SFI(vc)->direct_addr; \
} else { \
MPIU_Assert(vc == NULL); \
remote_proc = gl_data.any_addr; \
} \
})
#define NO_PGID 0
/* **************************************************************************
* match/ignore bit manipulation
* **************************************************************************
* 0123 4567 01234567 0123 4567 01234567 0123 4567 01234567 01234567 01234567
* | | |
* ^ | context id | source | message tag
* | | | |
* +---- protocol
* ************************************************************************** */
#define MPID_PROTOCOL_MASK (0xF000000000000000ULL)
#define MPID_CONTEXT_MASK (0x0FFFF00000000000ULL)
#define MPID_SOURCE_MASK (0x00000FFFF0000000ULL)
#define MPID_TAG_MASK (0x000000000FFFFFFFULL)
#define MPID_PGID_MASK (0x00000000FFFFFFFFULL)
#define MPID_PSOURCE_MASK (0x0000FFFF00000000ULL)
#define MPID_PORT_NAME_MASK (0x0FFF000000000000ULL)
#define MPID_SYNC_SEND (0x1000000000000000ULL)
#define MPID_SYNC_SEND_ACK (0x2000000000000000ULL)
#define MPID_MSG_RTS (0x3000000000000000ULL)
#define MPID_MSG_CTS (0x4000000000000000ULL)
#define MPID_MSG_DATA (0x5000000000000000ULL)
#define MPID_CONN_REQ (0x6000000000000000ULL)
#define MPID_SOURCE_SHIFT (16)
#define MPID_TAG_SHIFT (28)
#define MPID_PSOURCE_SHIFT (16)
#define MPID_PORT_SHIFT (32)
#define SFI_KVSAPPSTRLEN 1024
/* ******************************** */
/* Request manipulation inlines */
/* ******************************** */
static inline void MPID_nem_sfi_init_req(MPID_Request * req)
{
memset(REQ_SFI(req), 0, sizeof(MPID_nem_sfi_req_t));
}
static inline int MPID_nem_sfi_create_req(MPID_Request ** request, int refcnt)
{
int mpi_errno = MPI_SUCCESS;
MPID_Request *req;
req = MPID_Request_create();
MPIU_Assert(req);
MPIU_Object_set_ref(req, refcnt);
MPID_nem_sfi_init_req(req);
*request = req;
return mpi_errno;
}
/* ******************************** */
/* Tag Manipulation inlines */
/* ******************************** */
static inline uint64_t init_sendtag(MPIR_Context_id_t contextid, int source, int tag, uint64_t type)
{
uint64_t match_bits;
match_bits = contextid;
match_bits = (match_bits << MPID_SOURCE_SHIFT);
match_bits |= source;
match_bits = (match_bits << MPID_TAG_SHIFT);
match_bits |= (MPID_TAG_MASK & tag) | type;
return match_bits;
}
/* receive posting */
static inline uint64_t init_recvtag(uint64_t * mask_bits,
MPIR_Context_id_t contextid, int source, int tag)
{
uint64_t match_bits = 0;
*mask_bits = MPID_SYNC_SEND;
match_bits = contextid;
match_bits = (match_bits << MPID_SOURCE_SHIFT);
if (MPI_ANY_SOURCE == source) {
match_bits = (match_bits << MPID_TAG_SHIFT);
*mask_bits |= MPID_SOURCE_MASK;
}
else {
match_bits |= source;
match_bits = (match_bits << MPID_TAG_SHIFT);
}
if (MPI_ANY_TAG == tag)
*mask_bits |= MPID_TAG_MASK;
else
match_bits |= (MPID_TAG_MASK & tag);
return match_bits;
}
static inline int get_tag(uint64_t match_bits)
{
return ((int) (match_bits & MPID_TAG_MASK));
}
static inline int get_source(uint64_t match_bits)
{
return ((int) ((match_bits & MPID_SOURCE_MASK) >> (MPID_TAG_SHIFT)));
}
static inline int get_psource(uint64_t match_bits)
{
return ((int) ((match_bits & MPID_PSOURCE_MASK) >> (MPID_PORT_SHIFT)));
}
static inline int get_pgid(uint64_t match_bits)
{
return ((int) (match_bits & MPID_PGID_MASK));
}
static inline int get_port(uint64_t match_bits)
{
return ((int) ((match_bits & MPID_PORT_NAME_MASK) >> MPID_TAG_SHIFT));
}
/* ************************************************************************** */
/* MPICH Comm Override and Netmod functions */
/* ************************************************************************** */
int MPID_nem_sfi_recv_posted(struct MPIDI_VC *vc, struct MPID_Request *req);
int MPID_nem_sfi_send(struct MPIDI_VC *vc, const void *buf, int count,
MPI_Datatype datatype, int dest, int tag, MPID_Comm * comm,
int context_offset, struct MPID_Request **request);
int MPID_nem_sfi_isend(struct MPIDI_VC *vc, const void *buf, int count,
MPI_Datatype datatype, int dest, int tag, MPID_Comm * comm,
int context_offset, struct MPID_Request **request);
int MPID_nem_sfi_ssend(struct MPIDI_VC *vc, const void *buf, int count,
MPI_Datatype datatype, int dest, int tag, MPID_Comm * comm,
int context_offset, struct MPID_Request **request);
int MPID_nem_sfi_issend(struct MPIDI_VC *vc, const void *buf, int count,
MPI_Datatype datatype, int dest, int tag, MPID_Comm * comm,
int context_offset, struct MPID_Request **request);
int MPID_nem_sfi_cancel_send(struct MPIDI_VC *vc, struct MPID_Request *sreq);
int MPID_nem_sfi_cancel_recv(struct MPIDI_VC *vc, struct MPID_Request *rreq);
int MPID_nem_sfi_iprobe(struct MPIDI_VC *vc, int source, int tag, MPID_Comm * comm,
int context_offset, int *flag, MPI_Status * status);
int MPID_nem_sfi_improbe(struct MPIDI_VC *vc, int source, int tag, MPID_Comm * comm,
int context_offset, int *flag, MPID_Request ** message,
MPI_Status * status);
int MPID_nem_sfi_anysource_iprobe(int tag, MPID_Comm * comm, int context_offset,
int *flag, MPI_Status * status);
int MPID_nem_sfi_anysource_improbe(int tag, MPID_Comm * comm, int context_offset,
int *flag, MPID_Request ** message, MPI_Status * status);
void MPID_nem_sfi_anysource_posted(MPID_Request * rreq);
int MPID_nem_sfi_anysource_matched(MPID_Request * rreq);
int MPID_nem_sfi_send_data(cq_tagged_entry_t * wc, MPID_Request * sreq);
int MPID_nem_sfi_SendNoncontig(MPIDI_VC_t * vc, MPID_Request * sreq,
void *hdr, MPIDI_msg_sz_t hdr_sz);
int MPID_nem_sfi_iStartContigMsg(MPIDI_VC_t * vc, void *hdr, MPIDI_msg_sz_t hdr_sz,
void *data, MPIDI_msg_sz_t data_sz, MPID_Request ** sreq_ptr);
int MPID_nem_sfi_iSendContig(MPIDI_VC_t * vc, MPID_Request * sreq, void *hdr,
MPIDI_msg_sz_t hdr_sz, void *data, MPIDI_msg_sz_t data_sz);
/* ************************************************************************** */
/* SFI utility functions : not exposed as a netmod public API */
/* ************************************************************************** */
#define MPID_NONBLOCKING_POLL 0
#define MPID_BLOCKING_POLL 1
int MPID_nem_sfi_init(MPIDI_PG_t * pg_p, int pg_rank, char **bc_val_p, int *val_max_sz_p);
int MPID_nem_sfi_finalize(void);
int MPID_nem_sfi_vc_init(MPIDI_VC_t * vc);
int MPID_nem_sfi_get_business_card(int my_rank, char **bc_val_p, int *val_max_sz_p);
int MPID_nem_sfi_poll(int in_blocking_poll);
int MPID_nem_sfi_vc_terminate(MPIDI_VC_t * vc);
int MPID_nem_sfi_vc_connect(MPIDI_VC_t * vc);
int MPID_nem_sfi_connect_to_root(const char *business_card, MPIDI_VC_t * new_vc);
int MPID_nem_sfi_vc_destroy(MPIDI_VC_t * vc);
int MPID_nem_sfi_cm_init(MPIDI_PG_t * pg_p, int pg_rank);
int MPID_nem_sfi_cm_finalize();
extern MPID_nem_sfi_global_t gl_data;
extern MPIDI_Comm_ops_t _g_comm_ops;
#endif
This diff is collapsed.
/*
* (C) 2006 by Argonne National Laboratory.
* See COPYRIGHT in top-level directory.
*
* Portions of this code were written by Intel Corporation.
* Copyright (C) 2011-2012 Intel Corporation. Intel provides this material
* to Argonne National Laboratory subject to Software Grant and Corporate
* Contributor License Agreement dated February 8, 2012.
*/
#include "sfi_impl.h"
/* ------------------------------------------------------------------------ */
/* GET_PGID_AND_SET_MATCH macro looks up the process group to find the */
/* correct rank in multiple process groups. The "contigmsg" family of apis */
/* work on a global scope, not on a communicator scope(like tagged MPI.) */
/* The pgid matching is used for uniquely scoping the tag, usually in */
/* intercomms and dynamic process management where there are multiple */
/* global world spaces with similar ranks in the global space */
/* ------------------------------------------------------------------------ */
#define GET_PGID_AND_SET_MATCH() \
({ \
if (vc->pg) { \
MPIDI_PG_IdToNum(gl_data.pg_p, &pgid); \
} else { \
pgid = NO_PGID; \
} \
match_bits = (uint64_t)MPIR_Process.comm_world->rank << \
(MPID_PORT_SHIFT); \
if (0 == pgid) { \
match_bits |= (uint64_t)vc->port_name_tag<< \
(MPID_PORT_SHIFT+MPID_PSOURCE_SHIFT); \
} \
match_bits |= pgid; \
match_bits |= MPID_MSG_RTS; \
})
/* ------------------------------------------------------------------------ */
/* START_COMM is common code used by the nemesis netmod functions: */
/* iSendContig */
/* SendNoncontig */
/* iStartContigMsg */
/* These routines differ slightly in their behaviors, but can share common */
/* code to perform the send. START_COMM provides that common code, which */
/* is based on a tagged rendezvous message. */
/* The rendezvous is implemented with an RTS-CTS-Data send protocol: */
/* CTS_POST() | | */
/* RTS_SEND() | -------------------------------> | ue_callback()(sfi_cm.c)*/
/* | | pack_buffer() */
/* | | DATA_POST() */
/* | | RTS_POST() */
/* | | CTS_SEND() */
/* CTS_MATCH() | <------------------------------- | */
/* DATA_SEND() | ===============================> | handle_packet() */
/* | | notify_ch3_pkt() */
/* v v */
/* ------------------------------------------------------------------------ */
#define START_COMM() \
({ \
GET_PGID_AND_SET_MATCH(); \
VC_READY_CHECK(vc); \
c = 1; \
MPID_cc_incr(sreq->cc_ptr, &c); \
MPID_cc_incr(sreq->cc_ptr, &c); \
REQ_SFI(sreq)->event_callback = MPID_nem_sfi_data_callback; \
REQ_SFI(sreq)->pack_buffer = pack_buffer; \
REQ_SFI(sreq)->pack_buffer_size = pkt_len; \
REQ_SFI(sreq)->vc = vc; \
REQ_SFI(sreq)->tag = match_bits; \
\
MPID_nem_sfi_create_req(&cts_req, 1); \
cts_req->dev.OnDataAvail = NULL; \
cts_req->dev.next = NULL; \
REQ_SFI(cts_req)->event_callback = MPID_nem_sfi_cts_recv_callback; \
REQ_SFI(cts_req)->parent = sreq; \
\
FI_RC(fi_trecvfrom(gl_data.endpoint, \
NULL, \
0, \
gl_data.mr, \
VC_SFI(vc)->direct_addr, \
match_bits | MPID_MSG_CTS, \
0, /* Exact tag match, no ignore bits */ \
&(REQ_SFI(cts_req)->sfi_context)),trecvfrom); \
FI_RC(fi_tsendto(gl_data.endpoint, \
&REQ_SFI(sreq)->pack_buffer_size, \
sizeof(REQ_SFI(sreq)->pack_buffer_size), \
gl_data.mr, \
VC_SFI(vc)->direct_addr, \
match_bits, \
&(REQ_SFI(sreq)->sfi_context)),tsendto); \
})
/* ------------------------------------------------------------------------ */
/* General handler for RTS-CTS-Data protocol. Waits for the cc counter */
/* to hit two (send RTS and receive CTS decrementers) before kicking off the*/
/* bulk data transfer. On data send completion, the request can be freed */
/* ------------------------------------------------------------------------ */
#undef FCNAME
#define FCNAME DECL_FUNC(MPID_nem_sfi_data_callback)
static int MPID_nem_sfi_data_callback(cq_tagged_entry_t * wc, MPID_Request * sreq)
{
int complete = 0, mpi_errno = MPI_SUCCESS;
MPIDI_VC_t *vc;
req_fn reqFn;
uint64_t tag = 0;
BEGIN_FUNC(FCNAME);
if (sreq->cc == 2) {
vc = REQ_SFI(sreq)->vc;
REQ_SFI(sreq)->tag = tag | MPID_MSG_DATA;
FI_RC(fi_tsendto(gl_data.endpoint,
REQ_SFI(sreq)->pack_buffer,
REQ_SFI(sreq)->pack_buffer_size,
gl_data.mr,
VC_SFI(vc)->direct_addr,
wc->tag | MPID_MSG_DATA, (void *) &(REQ_SFI(sreq)->sfi_context)), tsendto);
}
if (sreq->cc == 1) {
if (REQ_SFI(sreq)->pack_buffer)
MPIU_Free(REQ_SFI(sreq)->pack_buffer);
reqFn = sreq->dev.OnDataAvail;
if (!reqFn) {
MPIDI_CH3U_Request_complete(sreq);
}
else {