helper_fns.c 24.5 KB
Newer Older
1
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
2
3
4
5
6
7
8
9
10
/*
 *
 *  (C) 2001 by Argonne National Laboratory.
 *      See COPYRIGHT in top-level directory.
 */

#include "mpiimpl.h"
#include "datatype.h"

11
#define COPY_BUFFER_SZ 16384
12
13
14
#if !defined(MPIC_REQUEST_PTR_ARRAY_SIZE)
#define MPIC_REQUEST_PTR_ARRAY_SIZE 64
#endif
15

16
17
18
19
20
/* These functions are used in the implementation of collective
   operations. They are wrappers around MPID send/recv functions. They do
   sends/receives by setting the context offset to
   MPID_CONTEXT_INTRA_COLL or MPID_CONTEXT_INTER_COLL. */

21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
#undef FUNCNAME
#define FUNCNAME MPIC_Probe
#undef FCNAME
#define FCNAME MPIU_QUOTE(FUNCNAME)
int MPIC_Probe(int source, int tag, MPI_Comm comm, MPI_Status *status)
{
    int mpi_errno = MPI_SUCCESS;
    int context_id;
    MPID_Comm *comm_ptr;

    MPID_Comm_get_ptr( comm, comm_ptr );

    context_id = (comm_ptr->comm_kind == MPID_INTRACOMM) ?
        MPID_CONTEXT_INTRA_COLL : MPID_CONTEXT_INTER_COLL;
    
    mpi_errno = MPID_Probe(source, tag, comm_ptr, context_id, status);
    if (mpi_errno != MPI_SUCCESS) goto fn_fail;

 fn_exit:
    return mpi_errno;
 fn_fail:
    goto fn_exit;
}


46
47
48
49
#undef FUNCNAME
#define FUNCNAME MPIR_Localcopy
#undef FCNAME
#define FCNAME "MPIR_Localcopy"
50
int MPIR_Localcopy(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
51
52
                   void *recvbuf, int recvcount, MPI_Datatype recvtype)
{
53
54
55
    int mpi_errno = MPI_SUCCESS;
    int sendtype_iscontig, recvtype_iscontig;
    MPI_Aint sendsize, recvsize, sdata_sz, rdata_sz, copy_sz;
56
    MPI_Aint true_extent, sendtype_true_lb, recvtype_true_lb;
57
    MPIU_CHKLMEM_DECL(1);
58
    MPID_MPI_STATE_DECL(MPID_STATE_MPIR_LOCALCOPY);
59

60
61
    MPID_MPI_FUNC_ENTER(MPID_STATE_MPIR_LOCALCOPY);

62
63
    MPID_Datatype_get_size_macro(sendtype, sendsize);
    MPID_Datatype_get_size_macro(recvtype, recvsize);
64

65
66
    sdata_sz = sendsize * sendcount;
    rdata_sz = recvsize * recvcount;
67

68
    /* if there is no data to copy, bail out */
69
70
    if (!sdata_sz || !rdata_sz)
        goto fn_exit;
71
72
73

#if defined(HAVE_ERROR_CHECKING)
    if (sdata_sz > rdata_sz) {
74
75
76
77
        MPIU_ERR_SET2(mpi_errno, MPI_ERR_TRUNCATE, "**truncate", "**truncate %d %d", sdata_sz, rdata_sz);
        copy_sz = rdata_sz;
    }
    else
78
#endif /* HAVE_ERROR_CHECKING */
79
        copy_sz = sdata_sz;
80

81
82
83
    /* Builtin types is the common case; optimize for it */
    if ((HANDLE_GET_KIND(sendtype) == HANDLE_KIND_BUILTIN) &&
        HANDLE_GET_KIND(recvtype) == HANDLE_KIND_BUILTIN) {
84
85
        MPIU_Memcpy(recvbuf, sendbuf, copy_sz);
        goto fn_exit;
86
    }
87

88
89
90
    MPIR_Datatype_iscontig(sendtype, &sendtype_iscontig);
    MPIR_Datatype_iscontig(recvtype, &recvtype_iscontig);

91
92
    MPIR_Type_get_true_extent_impl(sendtype, &sendtype_true_lb, &true_extent);
    MPIR_Type_get_true_extent_impl(recvtype, &recvtype_true_lb, &true_extent);
93
94

    if (sendtype_iscontig && recvtype_iscontig)
95
96
97
98
99
100
101
102
103
    {
#if defined(HAVE_ERROR_CHECKING)
        MPIU_ERR_CHKMEMCPYANDJUMP(mpi_errno,
                                  ((char *)recvbuf + recvtype_true_lb),
                                  ((char *)sendbuf + sendtype_true_lb),
                                  copy_sz);
#endif
        MPIU_Memcpy(((char *) recvbuf + recvtype_true_lb),
               ((char *) sendbuf + sendtype_true_lb),
104
               copy_sz);
105
    }
106
107
108
    else if (sendtype_iscontig)
    {
        MPID_Segment seg;
109
	MPI_Aint last;
110
111
112
113

	MPID_Segment_init(recvbuf, recvcount, recvtype, &seg, 0);
	last = copy_sz;
	MPID_Segment_unpack(&seg, 0, &last, (char*)sendbuf + sendtype_true_lb);
114
        MPIU_ERR_CHKANDJUMP(last != copy_sz, mpi_errno, MPI_ERR_TYPE, "**dtypemismatch");
115
116
117
118
    }
    else if (recvtype_iscontig)
    {
        MPID_Segment seg;
119
	MPI_Aint last;
120
121
122
123
124
125
126
127
128
129
130
131
132
133

	MPID_Segment_init(sendbuf, sendcount, sendtype, &seg, 0);
	last = copy_sz;
	MPID_Segment_pack(&seg, 0, &last, (char*)recvbuf + recvtype_true_lb);
        MPIU_ERR_CHKANDJUMP(last != copy_sz, mpi_errno, MPI_ERR_TYPE, "**dtypemismatch");
    }
    else
    {
	char * buf;
	MPIDI_msg_sz_t buf_off;
	MPID_Segment sseg;
	MPIDI_msg_sz_t sfirst;
	MPID_Segment rseg;
	MPIDI_msg_sz_t rfirst;
134
135

        MPIU_CHKLMEM_MALLOC(buf, char *, COPY_BUFFER_SZ, mpi_errno, "buf");
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184

	MPID_Segment_init(sendbuf, sendcount, sendtype, &sseg, 0);
	MPID_Segment_init(recvbuf, recvcount, recvtype, &rseg, 0);

	sfirst = 0;
	rfirst = 0;
	buf_off = 0;
	
	while (1)
	{
	    MPI_Aint last;
	    char * buf_end;

	    if (copy_sz - sfirst > COPY_BUFFER_SZ - buf_off)
	    {
		last = sfirst + (COPY_BUFFER_SZ - buf_off);
	    }
	    else
	    {
		last = copy_sz;
	    }
	    
	    MPID_Segment_pack(&sseg, sfirst, &last, buf + buf_off);
	    MPIU_Assert(last > sfirst);
	    
	    buf_end = buf + buf_off + (last - sfirst);
	    sfirst = last;
	    
	    MPID_Segment_unpack(&rseg, rfirst, &last, buf);
	    MPIU_Assert(last > rfirst);

	    rfirst = last;

	    if (rfirst == copy_sz)
	    {
		/* successful completion */
		break;
	    }

            /* if the send side finished, but the recv side couldn't unpack it, there's a datatype mismatch */
            MPIU_ERR_CHKANDJUMP(sfirst == copy_sz, mpi_errno, MPI_ERR_TYPE, "**dtypemismatch");        

            /* if not all data was unpacked, copy it to the front of the buffer for next time */
	    buf_off = sfirst - rfirst;
	    if (buf_off > 0)
	    {
		memmove(buf, buf_end - buf_off, buf_off);
	    }
	}
185
186
    }
    
187
    
188
  fn_exit:
189
    MPIU_CHKLMEM_FREEALL();
190
    MPID_MPI_FUNC_EXIT(MPID_STATE_MPIR_LOCALCOPY);
191
192
193
194
195
196
197
    return mpi_errno;

  fn_fail:
    goto fn_exit;
}


198
199
200
201
/* FIXME: For the brief-global and finer-grain control, we must ensure that
   the global lock is *not* held when this routine is called. (unless we change
   progress_start/end to grab the lock, in which case we must *still* make
   sure that the lock is not held when this routine is called). */
202
203
204
205
#undef FUNCNAME
#define FUNCNAME MPIC_Wait
#undef FCNAME
#define FCNAME "MPIC_Wait"
206
int MPIC_Wait(MPID_Request * request_ptr, mpir_errflag_t *errflag)
207
{
Rajeev Thakur's avatar
Rajeev Thakur committed
208
    int mpi_errno = MPI_SUCCESS;
209
210
211
    MPIDI_STATE_DECL(MPID_STATE_MPIC_WAIT);

    MPIDI_PT2PT_FUNC_ENTER(MPID_STATE_MPIC_WAIT);
212
213
214
215
216
217

    MPIU_DBG_MSG_S(PT2PT, TYPICAL, "IN: errflag = %s", *errflag?"TRUE":"FALSE");

    if (request_ptr->kind == MPID_REQUEST_SEND)
        request_ptr->status.MPI_TAG = 0;

218
    if (!MPID_Request_is_complete(request_ptr))
219
220
221
222
    {
	MPID_Progress_state progress_state;
	
	MPID_Progress_start(&progress_state);
223
        while (!MPID_Request_is_complete(request_ptr))
224
225
226
227
228
229
230
	{
	    mpi_errno = MPID_Progress_wait(&progress_state);
	    if (mpi_errno) { MPIU_ERR_POP(mpi_errno); }
	}
	MPID_Progress_end(&progress_state);
    }

Wesley Bland's avatar
Wesley Bland committed
231
232
    if (request_ptr->kind == MPID_REQUEST_RECV)
        MPIR_Process_status(&request_ptr->status, errflag);
233
234
235

 fn_exit:
    MPIU_DBG_MSG_D(PT2PT, TYPICAL, "OUT: errflag = %d", *errflag);
236
    MPIDI_PT2PT_FUNC_EXIT(MPID_STATE_MPIC_WAIT);
237
    return mpi_errno;
238
239
240
 fn_fail:
    /* --BEGIN ERROR HANDLING-- */
    goto fn_exit;
241
    /* --END ERROR HANDLING-- */
242
}
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265


/* Fault-tolerance versions.  When a process fails, collectives will
   still complete, however the result may be invalid.  Processes
   directly communicating with the failed process can detect the
   failure, however another mechanism is needed to commuinicate the
   failure to other processes receiving the invalid data.  To do this
   we introduce the _ft versions of the MPIC_ helper functions.  These
   functions take a pointer to an error flag.  When this is set to
   TRUE, the send functions will communicate the failure to the
   receiver.  If a function detects a failure, either by getting a
   failure in the communication operation, or by receiving an error
   indicator from a remote process, it sets the error flag to TRUE.

   In this implementation, we indicate an error to a remote process by
   sending an empty message instead of the requested buffer.  When a
   process receives an empty message, it knows to set the error flag.
   We count on the fact that collectives that exchange data (as
   opposed to barrier) will never send an empty message.  The barrier
   collective will not communicate failure information this way, but
   this is OK since there is no data that can be received corrupted. */

#undef FUNCNAME
266
#define FUNCNAME MPIC_Send
267
268
#undef FCNAME
#define FCNAME MPIU_QUOTE(FUNCNAME)
269
int MPIC_Send(const void *buf, int count, MPI_Datatype datatype, int dest, int tag,
270
                 MPID_Comm *comm_ptr, mpir_errflag_t *errflag)
271
272
{
    int mpi_errno = MPI_SUCCESS;
273
274
    int context_id;
    MPID_Request *request_ptr = NULL;
Wesley Bland's avatar
Wesley Bland committed
275
    MPIDI_STATE_DECL(MPID_STATE_MPIC_SEND);
276

Wesley Bland's avatar
Wesley Bland committed
277
    MPIDI_FUNC_ENTER(MPID_STATE_MPIC_SEND);
278

Wesley Bland's avatar
Wesley Bland committed
279
    MPIU_DBG_MSG_D(PT2PT, TYPICAL, "IN: errflag = %d", *errflag);
280

281
282
283
    MPIU_ERR_CHKANDJUMP1((count < 0), mpi_errno, MPI_ERR_COUNT,
                         "**countneg", "**countneg %d", count);

Wesley Bland's avatar
Wesley Bland committed
284
285
286
287
288
    switch(*errflag) {
        case MPIR_ERR_NONE:
            break;
        case MPIR_ERR_PROC_FAILED:
            MPIR_TAG_SET_PROC_FAILURE_BIT(tag);
289
        default:
Wesley Bland's avatar
Wesley Bland committed
290
291
            MPIR_TAG_SET_ERROR_BIT(tag);
    }
292

293
294
295
296
297
298
299
    context_id = (comm_ptr->comm_kind == MPID_INTRACOMM) ?
        MPID_CONTEXT_INTRA_COLL : MPID_CONTEXT_INTER_COLL;

    mpi_errno = MPID_Send(buf, count, datatype, dest, tag, comm_ptr,
                          context_id, &request_ptr);
    if (mpi_errno) MPIU_ERR_POP(mpi_errno);
    if (request_ptr) {
300
        mpi_errno = MPIC_Wait(request_ptr, errflag);
301
302
303
        if (mpi_errno) MPIU_ERR_POP(mpi_errno);
        MPID_Request_release(request_ptr);
    }
304

305
 fn_exit:
306
    MPIU_DBG_MSG_D(PT2PT, TYPICAL, "OUT: errflag = %d", *errflag);
Wesley Bland's avatar
Wesley Bland committed
307
    MPIDI_FUNC_EXIT(MPID_STATE_MPIC_SEND);
308
    return mpi_errno;
309
 fn_fail:
310
311
    /* --BEGIN ERROR HANDLING-- */
    if (request_ptr) MPID_Request_release(request_ptr);
312
313
314
315
316
317
318
    if (mpi_errno && !*errflag) {
        if (MPIX_ERR_PROC_FAILED == MPIR_ERR_GET_CLASS(mpi_errno)) {
            *errflag = MPIR_ERR_PROC_FAILED;
        } else {
            *errflag = MPIR_ERR_OTHER;
        }
    }
319
    goto fn_exit;
320
    /* --END ERROR HANDLING-- */
321
322
323
}

#undef FUNCNAME
324
#define FUNCNAME MPIC_Recv
325
326
#undef FCNAME
#define FCNAME MPIU_QUOTE(FUNCNAME)
327
int MPIC_Recv(void *buf, int count, MPI_Datatype datatype, int source, int tag,
328
                 MPID_Comm *comm_ptr, MPI_Status *status, mpir_errflag_t *errflag)
329
330
{
    int mpi_errno = MPI_SUCCESS;
331
    int context_id;
332
    MPI_Status mystatus;
333
    MPID_Request *request_ptr = NULL;
Wesley Bland's avatar
Wesley Bland committed
334
    MPIDI_STATE_DECL(MPID_STATE_MPIC_RECV);
335

Wesley Bland's avatar
Wesley Bland committed
336
    MPIDI_FUNC_ENTER(MPID_STATE_MPIC_RECV);
337

Wesley Bland's avatar
Wesley Bland committed
338
    MPIU_DBG_MSG_D(PT2PT, TYPICAL, "IN: errflag = %d", *errflag);
339

340
341
    MPIU_ERR_CHKANDJUMP1((count < 0), mpi_errno, MPI_ERR_COUNT,
                         "**countneg", "**countneg %d", count);
342

343
344
    context_id = (comm_ptr->comm_kind == MPID_INTRACOMM) ?
        MPID_CONTEXT_INTRA_COLL : MPID_CONTEXT_INTER_COLL;
345

346
347
    if (status == MPI_STATUS_IGNORE)
        status = &mystatus;
348

349
350
    mpi_errno = MPID_Recv(buf, count, datatype, source, tag, comm_ptr,
                          context_id, status, &request_ptr);
351
    if (mpi_errno) MPIU_ERR_POP(mpi_errno);
352
    if (request_ptr) {
353
        mpi_errno = MPIC_Wait(request_ptr, errflag);
354
        if (mpi_errno != MPI_SUCCESS)
355
            MPIU_ERR_POP(mpi_errno);
356
357
358

        *status = request_ptr->status;
        mpi_errno = status->MPI_ERROR;
359
        MPID_Request_release(request_ptr);
360
361
    } else {
        MPIR_Process_status(status, errflag);
362
    }
363

364
365
    if (MPI_SUCCESS == MPIR_ERR_GET_CLASS(status->MPI_ERROR)) {
        MPIU_Assert(status->MPI_TAG == tag);
366
367
368
    }

 fn_exit:
Wesley Bland's avatar
Wesley Bland committed
369
    MPIU_DBG_MSG_D(PT2PT, TYPICAL, "OUT: errflag = %d", *errflag);
Wesley Bland's avatar
Wesley Bland committed
370
    MPIDI_FUNC_EXIT(MPID_STATE_MPIC_RECV);
371
372
    return mpi_errno;
 fn_fail:
373
374
    /* --BEGIN ERROR HANDLING-- */
    if (request_ptr) MPID_Request_release(request_ptr);
375
    goto fn_exit;
376
    /* --END ERROR HANDLING-- */
377
378
379
}

#undef FUNCNAME
380
#define FUNCNAME MPIC_Ssend
381
382
#undef FCNAME
#define FCNAME MPIU_QUOTE(FUNCNAME)
383
int MPIC_Ssend(const void *buf, int count, MPI_Datatype datatype, int dest, int tag,
384
                  MPID_Comm *comm_ptr, mpir_errflag_t *errflag)
385
386
{
    int mpi_errno = MPI_SUCCESS;
387
388
    int context_id;
    MPID_Request *request_ptr = NULL;
Wesley Bland's avatar
Wesley Bland committed
389
    MPIDI_STATE_DECL(MPID_STATE_MPIC_SSEND);
390

Wesley Bland's avatar
Wesley Bland committed
391
    MPIDI_FUNC_ENTER(MPID_STATE_MPIC_SSEND);
392

Wesley Bland's avatar
Wesley Bland committed
393
    MPIU_DBG_MSG_D(PT2PT, TYPICAL, "IN: errflag = %d", *errflag);
394
395
396
397
398
399
400

    MPIU_ERR_CHKANDJUMP1((count < 0), mpi_errno, MPI_ERR_COUNT,
            "**countneg", "**countneg %d", count);

    context_id = (comm_ptr->comm_kind == MPID_INTRACOMM) ?
        MPID_CONTEXT_INTRA_COLL : MPID_CONTEXT_INTER_COLL;

401
402
403
404
405
406
407
408
    switch(*errflag) {
        case MPIR_ERR_NONE:
            break;
        case MPIR_ERR_PROC_FAILED:
            MPIR_TAG_SET_PROC_FAILURE_BIT(tag);
        default:
            MPIR_TAG_SET_ERROR_BIT(tag);
    }
409

410
411
412
413
    mpi_errno = MPID_Ssend(buf, count, datatype, dest, tag, comm_ptr,
                           context_id, &request_ptr);
    if (mpi_errno) MPIU_ERR_POP(mpi_errno);
    if (request_ptr) {
414
        mpi_errno = MPIC_Wait(request_ptr, errflag);
415
416
417
        if (mpi_errno) MPIU_ERR_POP(mpi_errno);
        MPID_Request_release(request_ptr);
    }
418

419
 fn_exit:
420
    MPIU_DBG_MSG_D(PT2PT, TYPICAL, "OUT: errflag = %d", *errflag);
Wesley Bland's avatar
Wesley Bland committed
421
    MPIDI_FUNC_EXIT(MPID_STATE_MPIC_SSEND);
422
    return mpi_errno;
423
 fn_fail:
424
425
    /* --BEGIN ERROR HANDLING-- */
    if (request_ptr) MPID_Request_release(request_ptr);
426
427
428
429
430
431
432
    if (mpi_errno && !*errflag) {
        if (MPIX_ERR_PROC_FAILED == MPIR_ERR_GET_CLASS(mpi_errno)) {
            *errflag = MPIR_ERR_PROC_FAILED;
        } else {
            *errflag = MPIR_ERR_OTHER;
        }
    }
433
    goto fn_exit;
434
    /* --END ERROR HANDLING-- */
435
436
437
}

#undef FUNCNAME
438
#define FUNCNAME MPIC_Sendrecv
439
440
#undef FCNAME
#define FCNAME MPIU_QUOTE(FUNCNAME)
441
int MPIC_Sendrecv(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
442
443
                     int dest, int sendtag, void *recvbuf, int recvcount,
                     MPI_Datatype recvtype, int source, int recvtag,
444
                     MPID_Comm *comm_ptr, MPI_Status *status, mpir_errflag_t *errflag)
445
446
{
    int mpi_errno = MPI_SUCCESS;
447
    int context_id;
448
    MPI_Status mystatus;
449
    MPID_Request *recv_req_ptr = NULL, *send_req_ptr = NULL;
Wesley Bland's avatar
Wesley Bland committed
450
    MPIDI_STATE_DECL(MPID_STATE_MPIC_SENDRECV);
451

Wesley Bland's avatar
Wesley Bland committed
452
    MPIDI_FUNC_ENTER(MPID_STATE_MPIC_SENDRECV);
453
454
455

    MPIU_DBG_MSG_S(PT2PT, TYPICAL, "IN: errflag = %s", *errflag?"TRUE":"FALSE");

456
457
458
459
460
461
462
463
    MPIU_ERR_CHKANDJUMP1((sendcount < 0), mpi_errno, MPI_ERR_COUNT,
                         "**countneg", "**countneg %d", sendcount);
    MPIU_ERR_CHKANDJUMP1((recvcount < 0), mpi_errno, MPI_ERR_COUNT,
                         "**countneg", "**countneg %d", recvcount);

    context_id = (comm_ptr->comm_kind == MPID_INTRACOMM) ?
        MPID_CONTEXT_INTRA_COLL : MPID_CONTEXT_INTER_COLL;

464
    if (status == MPI_STATUS_IGNORE) status = &mystatus;
Wesley Bland's avatar
Wesley Bland committed
465
466
467
468
469
    switch(*errflag) {
        case MPIR_ERR_NONE:
            break;
        case MPIR_ERR_PROC_FAILED:
            MPIR_TAG_SET_PROC_FAILURE_BIT(sendtag);
470
        default:
Wesley Bland's avatar
Wesley Bland committed
471
472
            MPIR_TAG_SET_ERROR_BIT(sendtag);
    }
473

474
475
    mpi_errno = MPID_Irecv(recvbuf, recvcount, recvtype, source, recvtag,
                           comm_ptr, context_id, &recv_req_ptr);
476
    if (mpi_errno) MPIU_ERR_POP(mpi_errno);
477
    mpi_errno = MPID_Isend(sendbuf, sendcount, sendtype, dest, sendtag,
478
479
480
                           comm_ptr, context_id, &send_req_ptr);
    if (mpi_errno) MPIU_ERR_POP(mpi_errno);

481
    mpi_errno = MPIC_Wait(send_req_ptr, errflag);
482
    if (mpi_errno) MPIU_ERR_POP(mpi_errno);
483
    mpi_errno = MPIC_Wait(recv_req_ptr, errflag);
484
485
486
487
    if (mpi_errno) MPIU_ERR_POPFATAL(mpi_errno);

    *status = recv_req_ptr->status;

488
489
    if (mpi_errno == MPI_SUCCESS) {
        mpi_errno = recv_req_ptr->status.MPI_ERROR;
490

491
492
        if (mpi_errno == MPI_SUCCESS) {
            mpi_errno = send_req_ptr->status.MPI_ERROR;
493
494
        }
    }
495

496
497
    MPID_Request_release(send_req_ptr);
    MPID_Request_release(recv_req_ptr);
Wesley Bland's avatar
Wesley Bland committed
498

499
 fn_exit:
Wesley Bland's avatar
Wesley Bland committed
500
    MPIU_DBG_MSG_D(PT2PT, TYPICAL, "OUT: errflag = %d", *errflag);
501

Wesley Bland's avatar
Wesley Bland committed
502
    MPIDI_FUNC_EXIT(MPID_STATE_MPIC_SENDRECV);
503
504
    return mpi_errno;
 fn_fail:
505
506
507
508
    if (send_req_ptr)
        MPID_Request_release(send_req_ptr);
    if (recv_req_ptr)
        MPID_Request_release(recv_req_ptr);
509
510
511
    goto fn_exit;
}

512
513
514
/* NOTE: for regular collectives (as opposed to irregular collectives) calling
 * this function repeatedly will almost always be slower than performing the
 * equivalent inline because of the overhead of the repeated malloc/free */
515
#undef FUNCNAME
516
#define FUNCNAME MPIC_Sendrecv_replace
517
518
#undef FCNAME
#define FCNAME MPIU_QUOTE(FUNCNAME)
519
int MPIC_Sendrecv_replace(void *buf, int count, MPI_Datatype datatype,
520
521
                             int dest, int sendtag,
                             int source, int recvtag,
522
                             MPID_Comm *comm_ptr, MPI_Status *status, mpir_errflag_t *errflag)
523
524
525
{
    int mpi_errno = MPI_SUCCESS;
    MPI_Status mystatus;
526
    MPIR_Context_id_t context_id_offset;
527
528
    MPID_Request *sreq = NULL;
    MPID_Request *rreq = NULL;
529
    void *tmpbuf = NULL;
530
531
    MPI_Aint tmpbuf_size = 0;
    MPI_Aint tmpbuf_count = 0;
532
    MPIU_CHKLMEM_DECL(1);
Wesley Bland's avatar
Wesley Bland committed
533
    MPIDI_STATE_DECL(MPID_STATE_MPIC_SENDRECV_REPLACE);
534
535
536
537
#ifdef MPID_LOG_ARROWS
    /* The logging macros log sendcount and recvcount */
    int sendcount = count, recvcount = count;
#endif
538

Wesley Bland's avatar
Wesley Bland committed
539
    MPIDI_FUNC_ENTER(MPID_STATE_MPIC_SENDRECV_REPLACE);
540

Wesley Bland's avatar
Wesley Bland committed
541
    MPIU_DBG_MSG_D(PT2PT, TYPICAL, "IN: errflag = %d", *errflag);
542

543
544
545
    MPIU_ERR_CHKANDJUMP1((count < 0), mpi_errno, MPI_ERR_COUNT,
                         "**countneg", "**countneg %d", count);

546
    if (status == MPI_STATUS_IGNORE) status = &mystatus;
Wesley Bland's avatar
Wesley Bland committed
547
548
549
550
551
    switch(*errflag) {
        case MPIR_ERR_NONE:
            break;
        case MPIR_ERR_PROC_FAILED:
            MPIR_TAG_SET_PROC_FAILURE_BIT(sendtag);
552
        default:
Wesley Bland's avatar
Wesley Bland committed
553
554
            MPIR_TAG_SET_ERROR_BIT(sendtag);
    }
555

556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
    context_id_offset = (comm_ptr->comm_kind == MPID_INTRACOMM) ?
        MPID_CONTEXT_INTRA_COLL : MPID_CONTEXT_INTER_COLL;

    if (count > 0 && dest != MPI_PROC_NULL) {
        MPIR_Pack_size_impl(count, datatype, &tmpbuf_size);
        MPIU_CHKLMEM_MALLOC(tmpbuf, void *, tmpbuf_size, mpi_errno, "temporary send buffer");

        mpi_errno = MPIR_Pack_impl(buf, count, datatype, tmpbuf, tmpbuf_size, &tmpbuf_count);
        if (mpi_errno) MPIU_ERR_POP(mpi_errno);
    }

    mpi_errno = MPID_Irecv(buf, count, datatype, source, recvtag,
                           comm_ptr, context_id_offset, &rreq);
    if (mpi_errno) MPIU_ERR_POP(mpi_errno);

    mpi_errno = MPID_Isend(tmpbuf, tmpbuf_count, MPI_PACKED, dest,
                           sendtag, comm_ptr, context_id_offset, &sreq);
    if (mpi_errno != MPI_SUCCESS) {
        /* --BEGIN ERROR HANDLING-- */
        /* FIXME: should we cancel the pending (possibly completed) receive
         * request or wait for it to complete? */
        MPID_Request_release(rreq);
        MPIU_ERR_POP(mpi_errno);
        /* --END ERROR HANDLING-- */
    }

582
583
584
585
    mpi_errno = MPIC_Wait(sreq, errflag);
    if (mpi_errno) MPIU_ERR_POP(mpi_errno);
    mpi_errno = MPIC_Wait(rreq, errflag);
    if (mpi_errno) MPIU_ERR_POP(mpi_errno);
586
587
588
589
590
591
592
593
594
595
596
597
598
599

    *status = rreq->status;

    if (mpi_errno == MPI_SUCCESS) {
        mpi_errno = rreq->status.MPI_ERROR;

        if (mpi_errno == MPI_SUCCESS) {
            mpi_errno = sreq->status.MPI_ERROR;
        }
    }

    MPID_Request_release(sreq);
    MPID_Request_release(rreq);

600
 fn_exit:
601
    MPIU_CHKLMEM_FREEALL();
Wesley Bland's avatar
Wesley Bland committed
602
    MPIU_DBG_MSG_D(PT2PT, TYPICAL, "OUT: errflag = %d", *errflag);
Wesley Bland's avatar
Wesley Bland committed
603
    MPIDI_FUNC_EXIT(MPID_STATE_MPIC_SENDRECV_REPLACE);
604
605
    return mpi_errno;
 fn_fail:
606
607
608
609
     if (sreq)
         MPID_Request_release(sreq);
     if (rreq)
         MPID_Request_release(rreq);
610
611
612
613
    goto fn_exit;
}

#undef FUNCNAME
614
#define FUNCNAME MPIC_Isend
615
616
#undef FCNAME
#define FCNAME MPIU_QUOTE(FUNCNAME)
617
int MPIC_Isend(const void *buf, int count, MPI_Datatype datatype, int dest, int tag,
618
                  MPID_Comm *comm_ptr, MPID_Request **request_ptr, mpir_errflag_t *errflag)
619
620
{
    int mpi_errno = MPI_SUCCESS;
621
    int context_id;
Wesley Bland's avatar
Wesley Bland committed
622
    MPIDI_STATE_DECL(MPID_STATE_MPIC_ISEND);
623

Wesley Bland's avatar
Wesley Bland committed
624
    MPIDI_FUNC_ENTER(MPID_STATE_MPIC_ISEND);
625

Wesley Bland's avatar
Wesley Bland committed
626
    MPIU_DBG_MSG_D(PT2PT, TYPICAL, "IN: errflag = %d", *errflag);
627

628
629
630
    MPIU_ERR_CHKANDJUMP1((count < 0), mpi_errno, MPI_ERR_COUNT,
                         "**countneg", "**countneg %d", count);

Wesley Bland's avatar
Wesley Bland committed
631
632
633
634
635
    switch(*errflag) {
        case MPIR_ERR_NONE:
            break;
        case MPIR_ERR_PROC_FAILED:
            MPIR_TAG_SET_PROC_FAILURE_BIT(tag);
636
        default:
Wesley Bland's avatar
Wesley Bland committed
637
638
            MPIR_TAG_SET_ERROR_BIT(tag);
    }
639

640
641
642
643
    context_id = (comm_ptr->comm_kind == MPID_INTRACOMM) ?
        MPID_CONTEXT_INTRA_COLL : MPID_CONTEXT_INTER_COLL;

    mpi_errno = MPID_Isend(buf, count, datatype, dest, tag, comm_ptr,
644
            context_id, request_ptr);
645
646
    if (mpi_errno) MPIU_ERR_POP(mpi_errno);

647
 fn_exit:
Wesley Bland's avatar
Wesley Bland committed
648
    MPIDI_FUNC_EXIT(MPID_STATE_MPIC_ISEND);
649
    return mpi_errno;
650
651
 fn_fail:
    goto fn_exit;
652
653
}

Wesley Bland's avatar
Wesley Bland committed
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
#undef FUNCNAME
#define FUNCNAME MPIC_Issend
#undef FCNAME
#define FCNAME MPIU_QUOTE(FUNCNAME)
int MPIC_Issend(const void *buf, int count, MPI_Datatype datatype, int dest, int tag,
                  MPID_Comm *comm_ptr, MPID_Request **request_ptr, mpir_errflag_t *errflag)
{
    int mpi_errno = MPI_SUCCESS;
    int context_id;
    MPIDI_STATE_DECL(MPID_STATE_MPIC_ISSEND);

    MPIDI_FUNC_ENTER(MPID_STATE_MPIC_ISSEND);

    MPIU_DBG_MSG_D(PT2PT, TYPICAL, "IN: errflag = %d", *errflag);

    MPIU_ERR_CHKANDJUMP1((count < 0), mpi_errno, MPI_ERR_COUNT,
                         "**countneg", "**countneg %d", count);

    switch(*errflag) {
        case MPIR_ERR_NONE:
            break;
        case MPIR_ERR_PROC_FAILED:
            MPIR_TAG_SET_PROC_FAILURE_BIT(tag);
        default:
            MPIR_TAG_SET_ERROR_BIT(tag);
    }

    context_id = (comm_ptr->comm_kind == MPID_INTRACOMM) ?
        MPID_CONTEXT_INTRA_COLL : MPID_CONTEXT_INTER_COLL;

    mpi_errno = MPID_Issend(buf, count, datatype, dest, tag, comm_ptr,
            context_id, request_ptr);
    if (mpi_errno) MPIU_ERR_POP(mpi_errno);

 fn_exit:
    MPIDI_FUNC_EXIT(MPID_STATE_MPIC_ISSEND);
    return mpi_errno;
 fn_fail:
    goto fn_exit;
}

695
#undef FUNCNAME
696
#define FUNCNAME MPIC_Irecv
697
698
#undef FCNAME
#define FCNAME MPIU_QUOTE(FUNCNAME)
699
int MPIC_Irecv(void *buf, int count, MPI_Datatype datatype, int source,
700
                  int tag, MPID_Comm *comm_ptr, MPID_Request **request_ptr)
701
702
{
    int mpi_errno = MPI_SUCCESS;
703
    int context_id;
Wesley Bland's avatar
Wesley Bland committed
704
    MPIDI_STATE_DECL(MPID_STATE_MPIC_IRECV);
705

Wesley Bland's avatar
Wesley Bland committed
706
    MPIDI_FUNC_ENTER(MPID_STATE_MPIC_IRECV);
707

708
709
710
711
712
713
714
    MPIU_ERR_CHKANDJUMP1((count < 0), mpi_errno, MPI_ERR_COUNT,
                         "**countneg", "**countneg %d", count);

    context_id = (comm_ptr->comm_kind == MPID_INTRACOMM) ?
        MPID_CONTEXT_INTRA_COLL : MPID_CONTEXT_INTER_COLL;

    mpi_errno = MPID_Irecv(buf, count, datatype, source, tag, comm_ptr,
715
            context_id, request_ptr);
716
717
    if (mpi_errno) MPIU_ERR_POP(mpi_errno);

718
 fn_exit:
Wesley Bland's avatar
Wesley Bland committed
719
    MPIDI_FUNC_EXIT(MPID_STATE_MPIC_IRECV);
720
    return mpi_errno;
721
722
 fn_fail:
    goto fn_exit;
723
724
725
726
}


#undef FUNCNAME
727
#define FUNCNAME MPIC_Waitall
728
729
#undef FCNAME
#define FCNAME MPIU_QUOTE(FUNCNAME)
730
int MPIC_Waitall(int numreq, MPID_Request *requests[], MPI_Status statuses[], mpir_errflag_t *errflag)
731
732
733
{
    int mpi_errno = MPI_SUCCESS;
    int i;
734
735
736
737
    MPI_Request request_ptr_array[MPIC_REQUEST_PTR_ARRAY_SIZE];
    MPI_Request *request_ptrs = request_ptr_array;
    MPI_Status status_static_array[MPIC_REQUEST_PTR_ARRAY_SIZE];
    MPI_Status *status_array = statuses;
Wesley Bland's avatar
Wesley Bland committed
738
    MPIDI_STATE_DECL(MPID_STATE_MPIC_WAITALL);
739
    MPIU_CHKLMEM_DECL(2);
740

Wesley Bland's avatar
Wesley Bland committed
741
    MPIDI_FUNC_ENTER(MPID_STATE_MPIC_WAITALL);
742
743
744

    MPIU_DBG_MSG_S(PT2PT, TYPICAL, "IN: errflag = %s", *errflag?"TRUE":"FALSE");

745
746
747
    if (statuses == MPI_STATUSES_IGNORE) {
        status_array = status_static_array;
    }
748

749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
    if (numreq > MPIC_REQUEST_PTR_ARRAY_SIZE) {
        MPIU_CHKLMEM_MALLOC(request_ptrs, MPI_Request *, numreq * sizeof(MPI_Request), mpi_errno, "request pointers");
        MPIU_CHKLMEM_MALLOC(status_array, MPI_Status *, numreq * sizeof(MPI_Status), mpi_errno, "status objects");
    }

    for (i = 0; i < numreq; ++i) {
        /* The MPI_TAG field is not set for send operations, so if we want
        to check for the error bit in the tag below, we should initialize all
        tag fields here. */
        status_array[i].MPI_TAG = 0;

        /* Convert the MPID_Request objects to MPI_Request objects */
        request_ptrs[i] = requests[i]->handle;
    }

    mpi_errno = MPIR_Waitall_impl(numreq, request_ptrs, status_array);
765

766
767
768
    /* The errflag value here is for all requests, not just a single one.  If
     * in the future, this function is used for multiple collectives at a
     * single time, we may have to change that. */
769
    for (i = 0; i < numreq; ++i) {
770
        MPIR_Process_status(&status_array[i], errflag);
771
772
773
    }

 fn_exit:
774
775
776
    if (numreq > MPIC_REQUEST_PTR_ARRAY_SIZE)
        MPIU_CHKLMEM_FREEALL();

Wesley Bland's avatar
Wesley Bland committed
777
    MPIU_DBG_MSG_D(PT2PT, TYPICAL, "OUT: errflag = %d", *errflag);
Wesley Bland's avatar
Wesley Bland committed
778
    MPIDI_FUNC_EXIT(MPID_STATE_MPIC_WAITALL);
779
780
781
782
    return mpi_errno;
 fn_fail:
    goto fn_exit;
}