ptl_nm.c 17.2 KB
Newer Older
1
2
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
/*
Antonio J. Pena's avatar
Antonio J. Pena committed
3
 *  (C) 2014 by Argonne National Laboratory.
4
5
6
7
 *      See COPYRIGHT in top-level directory.
 */

#include "ptl_impl.h"
Antonio J. Pena's avatar
Antonio J. Pena committed
8
#include "stddef.h"  /* C99; for offsetof */
9
#include <mpl_utlist.h>
10
#include "rptl.h"
11

Antonio J. Pena's avatar
Antonio J. Pena committed
12
13
#define NUM_RECV_BUFS 50
#define CTL_TAG 0
14
#define GET_TAG 1
15
16
#define PAYLOAD_SIZE  (PTL_MAX_EAGER - sizeof(MPIDI_CH3_Pkt_t))
#define SENDBUF_SIZE(sent_sz_) (sizeof(MPIDI_CH3_Pkt_t) + (sent_sz_))
Antonio J. Pena's avatar
Antonio J. Pena committed
17
18
19
#define SENDBUF(req_) REQ_PTL(req_)->chunk_buffer[0]
#define TMPBUF(req_) REQ_PTL(req_)->chunk_buffer[1]

20
static char recvbufs[NUM_RECV_BUFS * PTL_MAX_EAGER];
Antonio J. Pena's avatar
Antonio J. Pena committed
21
22
23
static ptl_me_t mes[NUM_RECV_BUFS];
static ptl_handle_me_t me_handles[NUM_RECV_BUFS];
static unsigned long long put_cnt = 0;  /* required to not finalizing too early */
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43

#undef FUNCNAME
#define FUNCNAME MPID_nem_ptl_nm_init
#undef FCNAME
#define FCNAME MPIU_QUOTE(FUNCNAME)
int MPID_nem_ptl_nm_init(void)
{
    int mpi_errno = MPI_SUCCESS;
    int i;
    int ret;
    ptl_process_t id_any;
    MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_PTL_NM_INIT);

    MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_PTL_NM_INIT);

    /* init recv */
    id_any.phys.pid = PTL_PID_ANY;
    id_any.phys.nid = PTL_NID_ANY;
    
    for (i = 0; i < NUM_RECV_BUFS; ++i) {
44
45
        mes[i].start = &recvbufs[i * PTL_MAX_EAGER];
        mes[i].length = PTL_MAX_EAGER;
Antonio J. Pena's avatar
Antonio J. Pena committed
46
47
48
49
50
        mes[i].ct_handle = PTL_CT_NONE;
        mes[i].uid = PTL_UID_ANY;
        mes[i].options = (PTL_ME_OP_PUT | PTL_ME_USE_ONCE | PTL_ME_EVENT_UNLINK_DISABLE |
                         PTL_ME_EVENT_LINK_DISABLE | PTL_ME_IS_ACCESSIBLE);
        mes[i].match_id = id_any;
51
52
        mes[i].match_bits = NPTL_MATCH(CTL_TAG, 0, MPI_ANY_SOURCE);
        mes[i].ignore_bits = NPTL_MATCH_IGNORE;
Antonio J. Pena's avatar
Antonio J. Pena committed
53

54
55
        ret = MPID_nem_ptl_me_append(MPIDI_nem_ptl_ni, MPIDI_nem_ptl_control_pt, &mes[i],
                                     PTL_PRIORITY_LIST, (void *)(uint64_t)i, &me_handles[i]);
Antonio J. Pena's avatar
Antonio J. Pena committed
56
57
        MPIU_ERR_CHKANDJUMP1(ret, mpi_errno, MPI_ERR_OTHER, "**ptlmeappend", "**ptlmeappend %s",
                             MPID_nem_ptl_strerror(ret));
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
    }

 fn_exit:
    MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_PTL_NM_INIT);
    return mpi_errno;
 fn_fail:
    goto fn_exit;
}

#undef FUNCNAME
#define FUNCNAME MPID_nem_ptl_nm_finalize
#undef FCNAME
#define FCNAME MPIU_QUOTE(FUNCNAME)
int MPID_nem_ptl_nm_finalize(void)
{
    int mpi_errno = MPI_SUCCESS;
    int ret;
    int i;
    MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_PTL_NM_FINALIZE);

    MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_PTL_NM_FINALIZE);

Antonio J. Pena's avatar
Antonio J. Pena committed
80
81
    while (put_cnt) MPID_nem_ptl_poll(1);  /* Wait for puts to finish */

82
    for (i = 0; i < NUM_RECV_BUFS; ++i) {
Antonio J. Pena's avatar
Antonio J. Pena committed
83
84
85
        ret = PtlMEUnlink(me_handles[i]);
        MPIU_ERR_CHKANDJUMP1(ret, mpi_errno, MPI_ERR_OTHER, "**ptlmeunlink", "**ptlmeunlink %s",
                             MPID_nem_ptl_strerror(ret));
86
87
88
89
90
91
92
93
94
95
    }

 fn_exit:
    MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_PTL_NM_FINALIZE);
    return mpi_errno;
 fn_fail:
    goto fn_exit;
}

#undef FUNCNAME
Antonio J. Pena's avatar
Antonio J. Pena committed
96
#define FUNCNAME meappend_large
97
98
#undef FCNAME
#define FCNAME MPIU_QUOTE(FUNCNAME)
99
static inline int meappend_large(ptl_process_t id, MPID_Request *req, ptl_match_bits_t match_bits, void *buf, size_t remaining)
100
{
Antonio J. Pena's avatar
Antonio J. Pena committed
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
    int mpi_errno = MPI_SUCCESS;
    int ret;
    ptl_me_t me;
    MPIDI_STATE_DECL(MPID_STATE_MEAPPEND_LARGE);

    MPIDI_FUNC_ENTER(MPID_STATE_MEAPPEND_LARGE);

    me.start = buf;
    me.length = remaining < MPIDI_nem_ptl_ni_limits.max_msg_size ?
                    remaining : MPIDI_nem_ptl_ni_limits.max_msg_size;
    me.ct_handle = PTL_CT_NONE;
    me.uid = PTL_UID_ANY;
    me.options = ( PTL_ME_OP_GET | PTL_ME_USE_ONCE | PTL_ME_IS_ACCESSIBLE |
                   PTL_ME_EVENT_LINK_DISABLE | PTL_ME_EVENT_UNLINK_DISABLE );
    me.match_id = id;
116
    me.match_bits = match_bits;
117
    me.ignore_bits = NPTL_MATCH_IGNORE;
Antonio J. Pena's avatar
Antonio J. Pena committed
118
119
120
121
122
    me.min_free = 0;

    while (remaining) {
        ptl_handle_me_t foo_me_handle;

123
        ++REQ_PTL(req)->num_gets;
Antonio J. Pena's avatar
Antonio J. Pena committed
124

125
126
        ret = MPID_nem_ptl_me_append(MPIDI_nem_ptl_ni, MPIDI_nem_ptl_control_pt, &me, PTL_PRIORITY_LIST, req,
                                     &foo_me_handle);
Antonio J. Pena's avatar
Antonio J. Pena committed
127
128
        MPIU_ERR_CHKANDJUMP1(ret, mpi_errno, MPI_ERR_OTHER, "**ptlmeappend", "**ptlmeappend %s",
                             MPID_nem_ptl_strerror(ret));
129
        MPIU_DBG_MSG_FMT(CH3_CHANNEL, VERBOSE, (MPIU_DBG_FDEST, "PtlMEAppend(req=%p tag=%#lx)", req, NPTL_MATCH_GET_TAG(match_bits)));
Antonio J. Pena's avatar
Antonio J. Pena committed
130
131
132
133
134

        me.start = (char *)me.start + me.length;
        remaining -= me.length;
        if (remaining < MPIDI_nem_ptl_ni_limits.max_msg_size)
            me.length = remaining;
135
    }
Pavan Balaji's avatar
Pavan Balaji committed
136

Antonio J. Pena's avatar
Antonio J. Pena committed
137
138
139
140
141
 fn_exit:
    MPIDI_FUNC_EXIT(MPID_STATE_MEAPPEND_LARGE);
    return mpi_errno;
 fn_fail:
    goto fn_exit;
142
143
144
145
146
147
}

#undef FUNCNAME
#define FUNCNAME send_pkt
#undef FCNAME
#define FCNAME MPIU_QUOTE(FUNCNAME)
Antonio J. Pena's avatar
Antonio J. Pena committed
148
149
static inline int send_pkt(MPIDI_VC_t *vc, void *hdr_p, void *data_p, MPIDI_msg_sz_t data_sz,
                           MPID_Request *sreq)
150
151
152
153
{
    int mpi_errno = MPI_SUCCESS;
    MPID_nem_ptl_vc_area *const vc_ptl = VC_PTL(vc);
    int ret;
154
    char *sendbuf;
Antonio J. Pena's avatar
Antonio J. Pena committed
155
156
    const size_t sent_sz = data_sz < PAYLOAD_SIZE ? data_sz : PAYLOAD_SIZE;
    const size_t sendbuf_sz = SENDBUF_SIZE(sent_sz);
157
    const size_t remaining = data_sz - sent_sz;
158
    ptl_match_bits_t match_bits = NPTL_MATCH(CTL_TAG, 0, MPIDI_Process.my_pg_rank);
159
160
161
162
    MPIDI_STATE_DECL(MPID_STATE_SEND_PKT);

    MPIDI_FUNC_ENTER(MPID_STATE_SEND_PKT);
    
Antonio J. Pena's avatar
Antonio J. Pena committed
163
164
    sendbuf = MPIU_Malloc(sendbuf_sz);
    MPIU_Assert(sendbuf != NULL);
165
    MPIU_Memcpy(sendbuf, hdr_p, sizeof(MPIDI_CH3_Pkt_t));
Antonio J. Pena's avatar
Antonio J. Pena committed
166
    TMPBUF(sreq) = NULL;
167
    REQ_PTL(sreq)->num_gets = 0;
168
    REQ_PTL(sreq)->put_done = 0;
169

Antonio J. Pena's avatar
Antonio J. Pena committed
170
    if (data_sz) {
171
172
        MPIU_Memcpy(sendbuf + sizeof(MPIDI_CH3_Pkt_t), data_p, sent_sz);
        if (remaining)  /* Post MEs for the remote gets */
173
            mpi_errno = meappend_large(vc_ptl->id, sreq, NPTL_MATCH(GET_TAG, 0, MPIDI_Process.my_pg_rank),
174
                                       (char *)data_p + sent_sz, remaining);
Antonio J. Pena's avatar
Antonio J. Pena committed
175
176
            if (mpi_errno)
                goto fn_fail;
177
178
    }

Antonio J. Pena's avatar
Antonio J. Pena committed
179
180
181
182
    SENDBUF(sreq) = sendbuf;
    REQ_PTL(sreq)->event_handler = MPID_nem_ptl_nm_ctl_event_handler;

    ret = MPID_nem_ptl_rptl_put(MPIDI_nem_ptl_global_md, (ptl_size_t)sendbuf, sendbuf_sz, PTL_NO_ACK_REQ,
183
                                vc_ptl->id, vc_ptl->ptc, match_bits, 0, sreq, NPTL_HEADER(0, remaining));
Antonio J. Pena's avatar
Antonio J. Pena committed
184
185
186
187
188
    MPIU_ERR_CHKANDJUMP1(ret, mpi_errno, MPI_ERR_OTHER, "**ptlput", "**ptlput %s",
                         MPID_nem_ptl_strerror(ret));
    MPIU_DBG_MSG_FMT(CH3_CHANNEL, VERBOSE, (MPIU_DBG_FDEST, "PtlPut(size=%lu id=(%#x,%#x) pt=%#x)",
                                            sendbuf_sz, vc_ptl->id.phys.nid,
                                            vc_ptl->id.phys.pid, vc_ptl->ptc));
189
190
    ++put_cnt;

191
192
193
194
195
196
197
198
199
200
201
 fn_exit:
    MPIDI_FUNC_EXIT(MPID_STATE_SEND_PKT);
    return mpi_errno;
 fn_fail:
    goto fn_exit;
}

#undef FUNCNAME
#define FUNCNAME send_noncontig_pkt
#undef FCNAME
#define FCNAME MPIU_QUOTE(FUNCNAME)
Antonio J. Pena's avatar
Antonio J. Pena committed
202
static int send_noncontig_pkt(MPIDI_VC_t *vc, MPID_Request *sreq, void *hdr_p)
203
204
205
206
{
    int mpi_errno = MPI_SUCCESS;
    MPID_nem_ptl_vc_area *const vc_ptl = VC_PTL(vc);
    int ret;
207
    char *sendbuf;
208
209
    const size_t data_sz = sreq->dev.segment_size - sreq->dev.segment_first;
    const size_t sent_sz = data_sz < PAYLOAD_SIZE ? data_sz : PAYLOAD_SIZE;
210
    const size_t sendbuf_sz = SENDBUF_SIZE(sent_sz);
211
    const size_t remaining = data_sz - sent_sz;
212
    ptl_match_bits_t match_bits = NPTL_MATCH(CTL_TAG, 0, MPIDI_Process.my_pg_rank);
213
214
215
    MPIDI_STATE_DECL(MPID_STATE_SEND_NONCONTIG_PKT);
    MPIDI_FUNC_ENTER(MPID_STATE_SEND_NONCONTIG_PKT);

Antonio J. Pena's avatar
Antonio J. Pena committed
216
217
    sendbuf = MPIU_Malloc(sendbuf_sz);
    MPIU_Assert(sendbuf != NULL);
218
    MPIU_Memcpy(sendbuf, hdr_p, sizeof(MPIDI_CH3_Pkt_t));
Antonio J. Pena's avatar
Antonio J. Pena committed
219
    TMPBUF(sreq) = NULL;
220
    REQ_PTL(sreq)->num_gets = 0;
221
    REQ_PTL(sreq)->put_done = 0;
222

223
224
225
226
    if (data_sz) {
        MPIDI_msg_sz_t first = sreq->dev.segment_first;
        MPIDI_msg_sz_t last = sreq->dev.segment_first + sent_sz;
        MPID_Segment_pack(sreq->dev.segment_ptr, first, &last, sendbuf + sizeof(MPIDI_CH3_Pkt_t));
227

228
229
        if (remaining) {  /* Post MEs for the remote gets */
            TMPBUF(sreq) = MPIU_Malloc(remaining);
230
            first = last;
231
            last = sreq->dev.segment_size;
232
            MPID_Segment_pack(sreq->dev.segment_ptr, first, &last, TMPBUF(sreq));
Antonio J. Pena's avatar
Antonio J. Pena committed
233
            MPIU_Assert(last == sreq->dev.segment_size);
234

235
            mpi_errno = meappend_large(vc_ptl->id, sreq, NPTL_MATCH(GET_TAG, 0, MPIDI_Process.my_pg_rank), TMPBUF(sreq), remaining);
Antonio J. Pena's avatar
Antonio J. Pena committed
236
237
            if (mpi_errno)
                goto fn_fail;
238
239
240
        }
    }

Antonio J. Pena's avatar
Antonio J. Pena committed
241
242
    SENDBUF(sreq) = sendbuf;
    REQ_PTL(sreq)->event_handler = MPID_nem_ptl_nm_ctl_event_handler;
243

Antonio J. Pena's avatar
Antonio J. Pena committed
244
    ret = MPID_nem_ptl_rptl_put(MPIDI_nem_ptl_global_md, (ptl_size_t)sendbuf, sendbuf_sz, PTL_NO_ACK_REQ,
245
                                vc_ptl->id, vc_ptl->ptc, match_bits, 0, sreq, NPTL_HEADER(0, remaining));
Antonio J. Pena's avatar
Antonio J. Pena committed
246
247
248
249
250
    MPIU_ERR_CHKANDJUMP1(ret, mpi_errno, MPI_ERR_OTHER, "**ptlput", "**ptlput %s",
                         MPID_nem_ptl_strerror(ret));
    MPIU_DBG_MSG_FMT(CH3_CHANNEL, VERBOSE, (MPIU_DBG_FDEST, "PtlPut(size=%lu id=(%#x,%#x) pt=%#x)",
                                            sendbuf_sz, vc_ptl->id.phys.nid,
                                            vc_ptl->id.phys.pid, vc_ptl->ptc));
251
    ++put_cnt;
252
253

 fn_exit:
Antonio J. Pena's avatar
Antonio J. Pena committed
254
    MPIDI_FUNC_EXIT(MPID_STATE_SEND_NONCONTIG_PKT);
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
    return mpi_errno;
 fn_fail:
    goto fn_exit;
}


#undef FUNCNAME
#define FUNCNAME MPID_nem_ptl_SendNoncontig
#undef FCNAME
#define FCNAME MPIU_QUOTE(FUNCNAME)
int MPID_nem_ptl_SendNoncontig(MPIDI_VC_t *vc, MPID_Request *sreq, void *hdr, MPIDI_msg_sz_t hdr_sz)
{
    int mpi_errno = MPI_SUCCESS;
    MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_PTL_SENDNONCONTIG);

    MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_PTL_SENDNONCONTIG);
    
    MPIU_Assert(hdr_sz <= sizeof(MPIDI_CH3_Pkt_t));
Antonio J. Pena's avatar
Antonio J. Pena committed
273
    mpi_errno = send_noncontig_pkt(vc, sreq, hdr);
274
275
276
277
278
279
280
281
282
283
284
285
286
    if (mpi_errno) MPIU_ERR_POP(mpi_errno);
    
 fn_exit:
    MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_PTL_SENDNONCONTIG);
    return mpi_errno;
 fn_fail:
    goto fn_exit;
}

#undef FUNCNAME
#define FUNCNAME MPID_nem_ptl_iStartContigMsg
#undef FCNAME
#define FCNAME MPIU_QUOTE(FUNCNAME)
Antonio J. Pena's avatar
Antonio J. Pena committed
287
288
int MPID_nem_ptl_iStartContigMsg(MPIDI_VC_t *vc, void *hdr, MPIDI_msg_sz_t hdr_sz, void *data,
                                 MPIDI_msg_sz_t data_sz, MPID_Request **sreq_ptr)
289
290
291
292
293
294
295
296
{
    int mpi_errno = MPI_SUCCESS;
    MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_PTL_ISTARTCONTIGMSG);

    MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_PTL_ISTARTCONTIGMSG);
    MPIU_Assert(hdr_sz <= sizeof(MPIDI_CH3_Pkt_t));

    /* create a request */
Antonio J. Pena's avatar
Antonio J. Pena committed
297
298
299
300
301
302
303
304
    *sreq_ptr = MPID_Request_create();
    MPIU_Assert(*sreq_ptr != NULL);
    MPIU_Object_set_ref(*sreq_ptr, 2);
    (*sreq_ptr)->kind = MPID_REQUEST_SEND;
    (*sreq_ptr)->dev.OnDataAvail = NULL;
    (*sreq_ptr)->dev.user_buf = NULL;

    mpi_errno = send_pkt(vc, hdr, data, data_sz, *sreq_ptr);
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
    if (mpi_errno) MPIU_ERR_POP(mpi_errno);

 fn_exit:
    MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_PTL_ISTARTCONTIGMSG);
    return mpi_errno;
 fn_fail:
    goto fn_exit;
}

#undef FUNCNAME
#define FUNCNAME MPID_nem_ptl_iSendContig
#undef FCNAME
#define FCNAME MPIU_QUOTE(FUNCNAME)
int MPID_nem_ptl_iSendContig(MPIDI_VC_t *vc, MPID_Request *sreq, void *hdr, MPIDI_msg_sz_t hdr_sz,
                               void *data, MPIDI_msg_sz_t data_sz)
{
    int mpi_errno = MPI_SUCCESS;
    MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_PTL_ISENDCONTIG);

    MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_PTL_ISENDCONTIG);
    MPIU_Assert(hdr_sz <= sizeof(MPIDI_CH3_Pkt_t));
    
Antonio J. Pena's avatar
Antonio J. Pena committed
327
    mpi_errno = send_pkt(vc, hdr, data, data_sz, sreq);
328
329
330
331
332
333
334
335
336
337
    if (mpi_errno) MPIU_ERR_POP(mpi_errno);
    
 fn_exit:
    MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_PTL_ISENDCONTIG);
    return mpi_errno;
 fn_fail:
    goto fn_exit;
}

#undef FUNCNAME
Antonio J. Pena's avatar
Antonio J. Pena committed
338
#define FUNCNAME on_data_avail
339
340
#undef FCNAME
#define FCNAME MPIU_QUOTE(FUNCNAME)
Antonio J. Pena's avatar
Antonio J. Pena committed
341
static inline void on_data_avail(MPID_Request * req)
342
{
343
    int (*reqFn) (MPIDI_VC_t *, MPID_Request *, int *);
Antonio J. Pena's avatar
Antonio J. Pena committed
344
    MPIDI_STATE_DECL(MPID_STATE_ON_DATA_AVAIL);
345

Antonio J. Pena's avatar
Antonio J. Pena committed
346
    MPIDI_FUNC_ENTER(MPID_STATE_ON_DATA_AVAIL);
347

Antonio J. Pena's avatar
Antonio J. Pena committed
348
349
350
    reqFn = req->dev.OnDataAvail;
    if (!reqFn) {
        MPIDI_CH3U_Request_complete(req);
351
352
        MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, ".... complete");
    }
Antonio J. Pena's avatar
Antonio J. Pena committed
353
354
355
356
357
    else {
        int complete;
        MPIDI_VC_t *vc = req->ch.vc;
        reqFn(vc, req, &complete);
        MPIU_Assert(complete == TRUE);
358
    }
359
360
361

    --put_cnt;

Antonio J. Pena's avatar
Antonio J. Pena committed
362
    MPIDI_FUNC_EXIT(MPID_STATE_ON_DATA_AVAIL);
363
364
365
}

#undef FUNCNAME
Antonio J. Pena's avatar
Antonio J. Pena committed
366
#define FUNCNAME MPID_nem_ptl_nm_ctl_event_handler
367
368
#undef FCNAME
#define FCNAME MPIU_QUOTE(FUNCNAME)
Antonio J. Pena's avatar
Antonio J. Pena committed
369
int MPID_nem_ptl_nm_ctl_event_handler(const ptl_event_t *e)
370
371
{
    int mpi_errno = MPI_SUCCESS;
Antonio J. Pena's avatar
Antonio J. Pena committed
372
    MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_PTL_NM_CTL_EVENT_HANDLER);
373

Antonio J. Pena's avatar
Antonio J. Pena committed
374
375
376
    MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_PTL_NM_CTL_EVENT_HANDLER);

    switch(e->type) {
377
378

    case PTL_EVENT_PUT:
379
        {
Antonio J. Pena's avatar
Antonio J. Pena committed
380
381
            int ret;
            const uint64_t buf_idx = (uint64_t) e->user_ptr;
382
            const size_t packet_sz = e->mlength;
Antonio J. Pena's avatar
Antonio J. Pena committed
383
384
            MPIDI_VC_t *vc;
            MPID_nem_ptl_vc_area * vc_ptl;
385
            ptl_size_t remaining = NPTL_HEADER_GET_LENGTH(e->hdr_data);
Antonio J. Pena's avatar
Antonio J. Pena committed
386

387
            MPIU_Assert(e->start == &recvbufs[buf_idx * PTL_MAX_EAGER]);
Antonio J. Pena's avatar
Antonio J. Pena committed
388

389
            MPIDI_PG_Get_vc(MPIDI_Process.my_pg, NPTL_MATCH_GET_RANK(e->match_bits), &vc);
Antonio J. Pena's avatar
Antonio J. Pena committed
390
391
            vc_ptl = VC_PTL(vc);

392
393
            if (remaining == 0) {
                mpi_errno = MPID_nem_handle_pkt(vc, &recvbufs[buf_idx * PTL_MAX_EAGER], packet_sz);
Antonio J. Pena's avatar
Antonio J. Pena committed
394
395
396
397
398
399
400
401
402
403
404
405
                if (mpi_errno)
                    MPIU_ERR_POP(mpi_errno);
            }
            else {
                int incomplete;
                size_t size;
                char *buf_ptr;

                MPID_Request *req = MPID_Request_create();
                MPIU_Assert(req != NULL);
                MPIDI_CH3U_Request_decrement_cc(req, &incomplete);  /* We'll increment it below */
                REQ_PTL(req)->event_handler = MPID_nem_ptl_nm_ctl_event_handler;
406
                REQ_PTL(req)->bytes_put = packet_sz + remaining;
Antonio J. Pena's avatar
Antonio J. Pena committed
407
408
                TMPBUF(req) = MPIU_Malloc(REQ_PTL(req)->bytes_put);
                MPIU_Assert(TMPBUF(req) != NULL);
409
                MPIU_Memcpy(TMPBUF(req), &recvbufs[buf_idx * PTL_MAX_EAGER], packet_sz);
Antonio J. Pena's avatar
Antonio J. Pena committed
410
411
412

                req->ch.vc = vc;

413
                size = remaining < MPIDI_nem_ptl_ni_limits.max_msg_size ? remaining : MPIDI_nem_ptl_ni_limits.max_msg_size;
Antonio J. Pena's avatar
Antonio J. Pena committed
414
                buf_ptr = (char *)TMPBUF(req) + packet_sz;
415
                while (remaining) {
Antonio J. Pena's avatar
Antonio J. Pena committed
416
417
                    MPIDI_CH3U_Request_increment_cc(req, &incomplete);  /* Will be decremented - and eventually freed in REPLY */
                    ret = MPID_nem_ptl_rptl_get(MPIDI_nem_ptl_global_md, (ptl_size_t)buf_ptr,
418
                                                size, vc_ptl->id, vc_ptl->ptc, NPTL_MATCH(GET_TAG, 0, MPIDI_Process.my_pg_rank), 0, req);
Antonio J. Pena's avatar
Antonio J. Pena committed
419
420
421
                    MPIU_ERR_CHKANDJUMP1(ret, mpi_errno, MPI_ERR_OTHER, "**ptlget", "**ptlget %s",
                                         MPID_nem_ptl_strerror(ret));
                    MPIU_DBG_MSG_FMT(CH3_CHANNEL, VERBOSE, (MPIU_DBG_FDEST,
422
                                                            "PtlGet(size=%lu id=(%#x,%#x) pt=%#x tag=%d)", size,
Antonio J. Pena's avatar
Antonio J. Pena committed
423
                                                            vc_ptl->id.phys.nid,
424
                                                            vc_ptl->id.phys.pid, vc_ptl->ptc, GET_TAG));
Antonio J. Pena's avatar
Antonio J. Pena committed
425
                    buf_ptr += size;
426
427
428
                    remaining -= size;
                    if (remaining < MPIDI_nem_ptl_ni_limits.max_msg_size)
                        size = remaining;
Antonio J. Pena's avatar
Antonio J. Pena committed
429
430
431
432
                }
            }

            /* Repost the recv buffer */
433
434
            ret = MPID_nem_ptl_me_append(MPIDI_nem_ptl_ni, MPIDI_nem_ptl_control_pt, &mes[buf_idx],
                                         PTL_PRIORITY_LIST, e->user_ptr /* buf_idx */, &me_handles[buf_idx]);
Antonio J. Pena's avatar
Antonio J. Pena committed
435
436
437
            MPIU_ERR_CHKANDJUMP1(ret, mpi_errno, MPI_ERR_OTHER, "**ptlmeappend",
                                 "**ptlmeappend %s", MPID_nem_ptl_strerror(ret));
        }
438
        break;
Antonio J. Pena's avatar
Antonio J. Pena committed
439

440
441
442
443
444
445
    case PTL_EVENT_GET:
        {
            MPID_Request *const req = e->user_ptr;

            if (--REQ_PTL(req)->num_gets == 0) {
                MPIU_Free(TMPBUF(req));
446
                if (REQ_PTL(req)->put_done)
447
448
449
450
451
452
453
454
455
456
                    on_data_avail(req);  /* Otherwise we'll do it on the SEND */
            }
        }
        break;

    case PTL_EVENT_SEND:
        {
            MPID_Request *const req = e->user_ptr;

            MPIU_Free(SENDBUF(req));
457
            REQ_PTL(req)->put_done = 1;
458
459
460
461
462
            if (REQ_PTL(req)->num_gets == 0)  /* Otherwise GET will do it */
                on_data_avail(req);
        }
        break;

Antonio J. Pena's avatar
Antonio J. Pena committed
463
464
465
    case PTL_EVENT_REPLY:
        {
            int incomplete;
466
            MPID_Request *const req = e->user_ptr;
Antonio J. Pena's avatar
Antonio J. Pena committed
467

468
            MPIDI_CH3U_Request_decrement_cc(req, &incomplete);
Antonio J. Pena's avatar
Antonio J. Pena committed
469
            if (!incomplete) {
470
                mpi_errno = MPID_nem_handle_pkt(req->ch.vc, TMPBUF(req), REQ_PTL(req)->bytes_put);
Antonio J. Pena's avatar
Antonio J. Pena committed
471
472
473
474
                if (mpi_errno)
                    MPIU_ERR_POP(mpi_errno);

                /* Free resources */
475
476
                MPIU_Free(TMPBUF(req));
                MPID_Request_release(req);
Antonio J. Pena's avatar
Antonio J. Pena committed
477
478
            }
        }
479
        break;
Antonio J. Pena's avatar
Antonio J. Pena committed
480

481
482
483
484
485
486
487
    default:
        MPIU_Error_printf("Received unexpected event type: %d %s", e->type, MPID_nem_ptl_strevent(e));
        MPIU_ERR_INTERNALANDJUMP(mpi_errno, "Unexpected event type");
        break;
    }

 fn_exit:
Antonio J. Pena's avatar
Antonio J. Pena committed
488
    MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_PTL_NM_CTL_EVENT_HANDLER);
489
490
491
492
    return mpi_errno;
 fn_fail:
    goto fn_exit;
}