ptl_impl.h 15.6 KB
Newer Older
1
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
2 3 4 5 6 7 8 9 10 11 12
/*
 *  (C) 2012 by Argonne National Laboratory.
 *      See COPYRIGHT in top-level directory.
 */

#ifndef PTL_IMPL_H
#define PTL_IMPL_H

#include <mpid_nem_impl.h>
#include <portals4.h>

13
#define PTL_MAX_EAGER (64*1024UL) /* 64K */
14

15
#define PTL_LARGE_THRESHOLD (64*1024UL) /* 64K */
16

17 18
extern ptl_handle_ni_t MPIDI_nem_ptl_ni;
extern ptl_pt_index_t  MPIDI_nem_ptl_pt;
19
extern ptl_pt_index_t  MPIDI_nem_ptl_get_pt; /* portal for gets by receiver */
20
extern ptl_pt_index_t  MPIDI_nem_ptl_control_pt; /* portal for MPICH control messages */
21
extern ptl_pt_index_t  MPIDI_nem_ptl_rpt_pt; /* portal for MPICH control messages */
22
extern ptl_handle_eq_t MPIDI_nem_ptl_origin_eq;
23 24 25 26 27
extern ptl_handle_eq_t MPIDI_nem_ptl_eq;
extern ptl_handle_eq_t MPIDI_nem_ptl_get_eq;
extern ptl_handle_eq_t MPIDI_nem_ptl_control_eq;
extern ptl_handle_eq_t MPIDI_nem_ptl_origin_eq;
extern ptl_handle_eq_t MPIDI_nem_ptl_rpt_eq;
28 29

extern ptl_handle_md_t MPIDI_nem_ptl_global_md;
30
extern ptl_ni_limits_t MPIDI_nem_ptl_ni_limits;
31 32 33 34 35 36 37 38 39

#define MPID_NEM_PTL_MAX_OVERFLOW_DATA 32 /* that's way more than we need */
typedef struct MPID_nem_ptl_pack_overflow
{
    MPI_Aint len;
    MPI_Aint offset;
    char buf[MPID_NEM_PTL_MAX_OVERFLOW_DATA];
} MPID_nem_ptl_pack_overflow_t;

40 41
typedef int (* event_handler_fn)(const ptl_event_t *e);

42 43
#define MPID_NEM_PTL_NUM_CHUNK_BUFFERS 2

44
typedef struct {
45
    struct MPID_nem_ptl_pack_overflow overflow[MPID_NEM_PTL_NUM_CHUNK_BUFFERS];
46
    int noncontig;
47 48
    int large;
    ptl_handle_md_t md;
49 50
    ptl_handle_me_t put_me;
    ptl_handle_me_t *get_me_p;
51
    int num_gets;
52
    int put_done;
53
    ptl_size_t chunk_offset;
54
    void *chunk_buffer[MPID_NEM_PTL_NUM_CHUNK_BUFFERS];
55
    MPIDI_msg_sz_t bytes_put;
56 57
    int found; /* used in probes with PtlMESearch() */
    event_handler_fn event_handler;
58 59 60
} MPID_nem_ptl_req_area;

/* macro for ptl private in req */
61 62 63
static inline MPID_nem_ptl_req_area * REQ_PTL(MPID_Request *req) {
    return (MPID_nem_ptl_req_area *)req->ch.netmod_area.padding;
}
64

65
#define MPID_nem_ptl_init_req(req_) do {                        \
66 67
        int i;                                                  \
        for (i = 0; i < MPID_NEM_PTL_NUM_CHUNK_BUFFERS; ++i) {  \
68
            REQ_PTL(req_)->overflow[i].len  = 0;                \
69
            REQ_PTL(req_)->chunk_buffer[i] = NULL;              \
70
        }                                                       \
71 72 73
        REQ_PTL(req_)->noncontig     = FALSE;                   \
        REQ_PTL(req_)->large         = FALSE;                   \
        REQ_PTL(req_)->md            = PTL_INVALID_HANDLE;      \
74 75
        REQ_PTL(req_)->put_me        = PTL_INVALID_HANDLE;      \
        REQ_PTL(req_)->get_me_p      = NULL;                    \
76
        REQ_PTL(req_)->num_gets      = 0;                       \
77
        REQ_PTL(req_)->put_done     = 0;                       \
78
        REQ_PTL(req_)->event_handler = NULL;                    \
79
        REQ_PTL(req_)->chunk_offset  = 0;                       \
80 81
    } while (0)

82 83 84 85 86 87 88 89 90 91 92
#define MPID_nem_ptl_request_create_sreq(sreq_, errno_, comm_) do {                                             \
        (sreq_) = MPIU_Handle_obj_alloc(&MPID_Request_mem);                                                     \
        MPIU_ERR_CHKANDJUMP1((sreq_) == NULL, mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %s", "request");    \
        MPIU_Object_set_ref((sreq_), 2);                                                                        \
        (sreq_)->kind               = MPID_REQUEST_SEND;                                                        \
        MPIR_Comm_add_ref(comm_);                                                                               \
        (sreq_)->comm               = comm_;                                                                    \
        (sreq_)->greq_fns           = NULL;                                                                     \
        MPID_cc_set(&(sreq_)->cc, 1);                                                                           \
        (sreq_)->cc_ptr             = &(sreq_)->cc;                                                             \
        (sreq_)->status.MPI_ERROR   = MPI_SUCCESS;                                                              \
Pavan Balaji's avatar
Pavan Balaji committed
93
        MPIR_STATUS_SET_CANCEL_BIT((sreq_)->status, FALSE);                                                           \
94 95 96 97
        (sreq_)->dev.cancel_pending = FALSE;                                                                    \
        (sreq_)->dev.state          = 0;                                                                        \
        (sreq_)->dev.datatype_ptr   = NULL;                                                                     \
        (sreq_)->dev.segment_ptr    = NULL;                                                                     \
98
        (sreq_)->dev.tmpbuf         = NULL;                                                                     \
99 100
                                                                                                                \
        MPID_nem_ptl_init_req(sreq_);                                                                           \
101
    } while (0)
102 103 104 105

typedef struct {
    ptl_process_t id;
    ptl_pt_index_t pt;
106
    ptl_pt_index_t ptg;
107
    ptl_pt_index_t ptc;
108
    ptl_pt_index_t ptr;
109
    ptl_pt_index_t ptrg;
Antonio J. Pena's avatar
Antonio J. Pena committed
110
    ptl_pt_index_t ptrc;
111 112 113 114 115
    int id_initialized; /* TRUE iff id and pt have been initialized */
    MPIDI_msg_sz_t num_queued_sends; /* number of reqs for this vc in sendq */
} MPID_nem_ptl_vc_area;

/* macro for ptl private in VC */
116
#define VC_PTL(vc) ((MPID_nem_ptl_vc_area *)vc->ch.netmod_area.padding)
117

118 119 120
/* Header bit fields
   bit   field
   ----  -------------------
121 122 123
   63    single/multiple
   62    large/small
   61    ssend
124
   0-60  length
125 126 127

   Note: This means we support no more than 2^24 processes.
*/
128 129 130 131
#define NPTL_SSEND             ((ptl_hdr_data_t)1<<61)
#define NPTL_LARGE             ((ptl_hdr_data_t)1<<62)
#define NPTL_MULTIPLE          ((ptl_hdr_data_t)1<<63)

132
#define NPTL_LENGTH_MASK     (((ptl_hdr_data_t)1<<61)-1)
133

134 135
#define NPTL_HEADER(flags_, length_) ((flags_) | (ptl_hdr_data_t)(length_))
#define NPTL_HEADER_GET_LENGTH(hdr_) ((hdr_) & NPTL_LENGTH_MASK)
136

137
#define NPTL_MAX_LENGTH NPTL_LENGTH_MASK
138 139 140 141 142 143 144 145 146 147 148 149

/* The comm_world rank of the sender is stored in the match_bits, but they are
   ignored when matching.
   bit   field
   ----  -------------------
   32-63  Tag
   16-31  Context id
    0-15  Rank
*/

#define NPTL_MATCH_TAG_OFFSET 32
#define NPTL_MATCH_CTX_OFFSET 16
150
#define NPTL_MATCH_RANK_MASK (((ptl_match_bits_t)(1) << 16) - 1)
151 152
#define NPTL_MATCH_CTX_MASK ((((ptl_match_bits_t)(1) << 16) - 1) << NPTL_MATCH_CTX_OFFSET)
#define NPTL_MATCH_TAG_MASK ((((ptl_match_bits_t)(1) << 32) - 1) << NPTL_MATCH_TAG_OFFSET)
153 154 155
#define NPTL_MATCH(tag_, ctx_, rank_) ((((ptl_match_bits_t)(tag_) << NPTL_MATCH_TAG_OFFSET) & NPTL_MATCH_TAG_MASK) | \
                                       (((ptl_match_bits_t)(ctx_) << NPTL_MATCH_CTX_OFFSET) & NPTL_MATCH_CTX_MASK) | \
                                       ((ptl_match_bits_t)(rank_) & NPTL_MATCH_RANK_MASK))
156
#define NPTL_MATCH_IGNORE (NPTL_MATCH_RANK_MASK | (ptl_match_bits_t)(MPIR_TAG_ERROR_BIT | MPIR_TAG_PROC_FAILURE_BIT) << 32)
157 158
#define NPTL_MATCH_IGNORE_ANY_TAG (NPTL_MATCH_IGNORE | NPTL_MATCH_TAG_MASK)

159
#define NPTL_MATCH_GET_RANK(match_bits_) ((match_bits_) & NPTL_MATCH_RANK_MASK)
160
#define NPTL_MATCH_GET_CTX(match_bits_) (((match_bits_) & NPTL_MATCH_CTX_MASK) >> NPTL_MATCH_CTX_OFFSET)
161
#define NPTL_MATCH_GET_TAG(match_bits_) ((match_bits_) >> NPTL_MATCH_TAG_OFFSET)
162

163 164
int MPID_nem_ptl_nm_init(void);
int MPID_nem_ptl_nm_finalize(void);
Antonio J. Pena's avatar
Antonio J. Pena committed
165
int MPID_nem_ptl_nm_ctl_event_handler(const ptl_event_t *e);
166 167 168 169 170 171 172 173 174 175
int MPID_nem_ptl_sendq_complete_with_error(MPIDI_VC_t *vc, int req_errno);
int MPID_nem_ptl_SendNoncontig(MPIDI_VC_t *vc, MPID_Request *sreq, void *hdr, MPIDI_msg_sz_t hdr_sz);
int MPID_nem_ptl_iStartContigMsg(MPIDI_VC_t *vc, void *hdr, MPIDI_msg_sz_t hdr_sz, void *data, MPIDI_msg_sz_t data_sz,
                                   MPID_Request **sreq_ptr);
int MPID_nem_ptl_iSendContig(MPIDI_VC_t *vc, MPID_Request *sreq, void *hdr, MPIDI_msg_sz_t hdr_sz,
                               void *data, MPIDI_msg_sz_t data_sz);
int MPID_nem_ptl_poll_init(void);
int MPID_nem_ptl_poll_finalize(void);
int MPID_nem_ptl_poll(int is_blocking_poll);
int MPID_nem_ptl_vc_terminated(MPIDI_VC_t *vc);
176
int MPID_nem_ptl_get_id_from_bc(const char *business_card, ptl_process_t *id, ptl_pt_index_t *pt, ptl_pt_index_t *ptg,
177
                                ptl_pt_index_t *ptc, ptl_pt_index_t *ptr, ptl_pt_index_t *ptrg, ptl_pt_index_t *ptrc);
178 179 180 181
void MPI_nem_ptl_pack_byte(MPID_Segment *segment, MPI_Aint first, MPI_Aint last, void *buf,
                           MPID_nem_ptl_pack_overflow_t *overflow);
int MPID_nem_ptl_unpack_byte(MPID_Segment *segment, MPI_Aint first, MPI_Aint last, void *buf,
                             MPID_nem_ptl_pack_overflow_t *overflow);
182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197

/* comm override functions */
int MPID_nem_ptl_recv_posted(struct MPIDI_VC *vc, struct MPID_Request *req);
/* isend is also used to implement send, rsend and irsend */
int MPID_nem_ptl_isend(struct MPIDI_VC *vc, const void *buf, int count, MPI_Datatype datatype, int dest, int tag,
                       MPID_Comm *comm, int context_offset, struct MPID_Request **request);
/* issend is also used to implement ssend */
int MPID_nem_ptl_issend(struct MPIDI_VC *vc, const void *buf, int count, MPI_Datatype datatype, int dest, int tag,
                        MPID_Comm *comm, int context_offset, struct MPID_Request **request);
int MPID_nem_ptl_cancel_send(struct MPIDI_VC *vc,  struct MPID_Request *sreq);
int MPID_nem_ptl_cancel_recv(struct MPIDI_VC *vc,  struct MPID_Request *rreq);
int MPID_nem_ptl_probe(struct MPIDI_VC *vc,  int source, int tag, MPID_Comm *comm, int context_offset, MPI_Status *status);
int MPID_nem_ptl_iprobe(struct MPIDI_VC *vc,  int source, int tag, MPID_Comm *comm, int context_offset, int *flag,
                        MPI_Status *status);
int MPID_nem_ptl_improbe(struct MPIDI_VC *vc,  int source, int tag, MPID_Comm *comm, int context_offset, int *flag,
                         MPID_Request **message, MPI_Status *status);
198 199 200 201 202
int MPID_nem_ptl_anysource_iprobe(int tag, MPID_Comm * comm, int context_offset, int *flag, MPI_Status * status);
int MPID_nem_ptl_anysource_improbe(int tag, MPID_Comm * comm, int context_offset, int *flag, MPID_Request **message,
                                   MPI_Status * status);
void MPID_nem_ptl_anysource_posted(MPID_Request *rreq);
int MPID_nem_ptl_anysource_matched(MPID_Request *rreq);
203 204
int MPID_nem_ptl_init_id(MPIDI_VC_t *vc);

205 206 207 208 209 210 211
int MPID_nem_ptl_lmt_initiate_lmt(MPIDI_VC_t *vc, MPIDI_CH3_Pkt_t *rts_pkt, MPID_Request *req);
int MPID_nem_ptl_lmt_start_recv(MPIDI_VC_t *vc,  MPID_Request *rreq, MPID_IOV s_cookie);
int MPID_nem_ptl_lmt_start_send(MPIDI_VC_t *vc, MPID_Request *sreq, MPID_IOV r_cookie);
int MPID_nem_ptl_lmt_handle_cookie(MPIDI_VC_t *vc, MPID_Request *req, MPID_IOV s_cookie);
int MPID_nem_ptl_lmt_done_send(MPIDI_VC_t *vc, MPID_Request *req);
int MPID_nem_ptl_lmt_done_recv(MPIDI_VC_t *vc, MPID_Request *req);

212 213 214 215 216 217 218 219 220 221 222 223
/* packet handlers */

int MPID_nem_ptl_pkt_cancel_send_req_handler(MPIDI_VC_t *vc, MPIDI_CH3_Pkt_t *pkt,
                                             MPIDI_msg_sz_t *buflen, MPID_Request **rreqp);
int MPID_nem_ptl_pkt_cancel_send_resp_handler(MPIDI_VC_t *vc, MPIDI_CH3_Pkt_t *pkt,
                                              MPIDI_msg_sz_t *buflen, MPID_Request **rreqp);

/* local packet types */

typedef enum MPIDI_nem_ptl_pkt_type {
    MPIDI_NEM_PTL_PKT_CANCEL_SEND_REQ,
    MPIDI_NEM_PTL_PKT_CANCEL_SEND_RESP,
224
    MPIDI_NEM_PTL_PKT_INVALID = -1 /* force signed, to avoid warnings */
225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241
} MPIDI_nem_ptl_pkt_type_t;

typedef struct MPIDI_nem_ptl_pkt_cancel_send_req
{
    MPIDI_CH3_Pkt_type_t type;
    unsigned subtype;
    MPIDI_Message_match match;
    MPI_Request sender_req_id;
} MPIDI_nem_ptl_pkt_cancel_send_req_t;

typedef struct MPIDI_nem_ptl_pkt_cancel_send_resp
{
    MPIDI_CH3_Pkt_type_t type;
    unsigned subtype;
    MPI_Request sender_req_id;
    int ack;
} MPIDI_nem_ptl_pkt_cancel_send_resp_t;
242 243 244 245 246 247 248 249

/* debugging */
const char *MPID_nem_ptl_strerror(int ret);
const char *MPID_nem_ptl_strevent(const ptl_event_t *ev);
const char *MPID_nem_ptl_strnifail(ptl_ni_fail_t ni_fail);
const char *MPID_nem_ptl_strlist(ptl_list_t list);

#define DBG_MSG_PUT(md_, data_sz_, pg_rank_, match_, header_) do {                                                                          \
250
        MPIU_DBG_MSG_FMT(CH3_CHANNEL, VERBOSE, (MPIU_DBG_FDEST, "MPID_nem_ptl_rptl_put: md=%s data_sz=%lu pg_rank=%d", md_, data_sz_, pg_rank_));          \
251 252
        MPIU_DBG_MSG_FMT(CH3_CHANNEL, VERBOSE, (MPIU_DBG_FDEST, "        tag=%#lx ctx=%#lx rank=%ld match=%#lx",                            \
                                                NPTL_MATCH_GET_TAG(match_), NPTL_MATCH_GET_CTX(match_), NPTL_MATCH_GET_RANK(match_), match_)); \
253
        MPIU_DBG_MSG_FMT(CH3_CHANNEL, VERBOSE, (MPIU_DBG_FDEST, "        flags=%c%c%c data_sz=%ld header=%#lx",                             \
254 255 256
                                                header_ & NPTL_SSEND ? 'S':' ',                                                             \
                                                header_ & NPTL_LARGE ? 'L':' ',                                                             \
                                                header_ & NPTL_MULTIPLE ? 'M':' ',                                                          \
257
                                                NPTL_HEADER_GET_LENGTH(header_), header_));                                                 \
258 259 260 261 262 263
    } while(0)

#define DBG_MSG_GET(md_, data_sz_, pg_rank_, match_) do {                                                                                   \
        MPIU_DBG_MSG_FMT(CH3_CHANNEL, VERBOSE, (MPIU_DBG_FDEST, "PtlGet: md=%s data_sz=%lu pg_rank=%d", md_, data_sz_, pg_rank_));          \
        MPIU_DBG_MSG_FMT(CH3_CHANNEL, VERBOSE, (MPIU_DBG_FDEST, "        tag=%#lx ctx=%#lx rank=%ld match=%#lx",                            \
                                                NPTL_MATCH_GET_TAG(match_), NPTL_MATCH_GET_CTX(match_), NPTL_MATCH_GET_RANK(match_), match_)); \
264 265 266
    } while(0)

#define DBG_MSG_MEAPPEND(pt_, pg_rank_, me_, usr_ptr_) do {                                                                                 \
267 268
        MPIU_DBG_MSG_FMT(CH3_CHANNEL, VERBOSE, (MPIU_DBG_FDEST, "PtlMEAppend: pt=%s pg_rank=%d me.start=%p me.length=%lu is_IOV=%d usr_ptr=%p", \
                                                pt_, pg_rank_, me_.start, me_.length, me_.options & PTL_IOVEC, usr_ptr_));                  \
269 270 271 272 273 274 275 276 277 278 279 280 281
        MPIU_DBG_MSG_FMT(CH3_CHANNEL, VERBOSE, (MPIU_DBG_FDEST, "             tag=%#lx ctx=%#lx rank=%ld match=%#lx ignore=%#lx",           \
                                                NPTL_MATCH_GET_TAG(me_.match_bits), NPTL_MATCH_GET_CTX(me_.match_bits),                     \
                                                NPTL_MATCH_GET_RANK(me_.match_bits), me_.match_bits, me_.ignore_bits));                     \
    } while(0)
    
#define DBG_MSG_MESearch(pt_, pg_rank_, me_, usr_ptr_) do {                                                                             \
        MPIU_DBG_MSG_FMT(CH3_CHANNEL, VERBOSE, (MPIU_DBG_FDEST, "PtlMESearch: pt=%s pg_rank=%d usr_ptr=%p", pt_, pg_rank_, usr_ptr_));  \
        MPIU_DBG_MSG_FMT(CH3_CHANNEL, VERBOSE, (MPIU_DBG_FDEST, "             tag=%#lx ctx=%#lx rank=%ld match=%#lx ignore=%#lx",       \
                                                NPTL_MATCH_GET_TAG(me_.match_bits), NPTL_MATCH_GET_CTX(me_.match_bits),                 \
                                                NPTL_MATCH_GET_RANK(me_.match_bits), me_.match_bits, me_.ignore_bits));                 \
    } while(0)
    

282 283

#endif /* PTL_IMPL_H */