helper_fns.c 24.7 KB
Newer Older
1
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
2 3 4 5 6 7 8 9 10
/*
 *
 *  (C) 2001 by Argonne National Laboratory.
 *      See COPYRIGHT in top-level directory.
 */

#include "mpiimpl.h"
#include "datatype.h"

11
#define COPY_BUFFER_SZ 16384
12 13 14
#if !defined(MPIC_REQUEST_PTR_ARRAY_SIZE)
#define MPIC_REQUEST_PTR_ARRAY_SIZE 64
#endif
15

16 17 18 19 20
/* These functions are used in the implementation of collective
   operations. They are wrappers around MPID send/recv functions. They do
   sends/receives by setting the context offset to
   MPID_CONTEXT_INTRA_COLL or MPID_CONTEXT_INTER_COLL. */

21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45
#undef FUNCNAME
#define FUNCNAME MPIC_Probe
#undef FCNAME
#define FCNAME MPIU_QUOTE(FUNCNAME)
int MPIC_Probe(int source, int tag, MPI_Comm comm, MPI_Status *status)
{
    int mpi_errno = MPI_SUCCESS;
    int context_id;
    MPID_Comm *comm_ptr;

    MPID_Comm_get_ptr( comm, comm_ptr );

    context_id = (comm_ptr->comm_kind == MPID_INTRACOMM) ?
        MPID_CONTEXT_INTRA_COLL : MPID_CONTEXT_INTER_COLL;
    
    mpi_errno = MPID_Probe(source, tag, comm_ptr, context_id, status);
    if (mpi_errno != MPI_SUCCESS) goto fn_fail;

 fn_exit:
    return mpi_errno;
 fn_fail:
    goto fn_exit;
}


46 47 48 49
#undef FUNCNAME
#define FUNCNAME MPIR_Localcopy
#undef FCNAME
#define FCNAME "MPIR_Localcopy"
50 51
int MPIR_Localcopy(const void *sendbuf, MPI_Aint sendcount, MPI_Datatype sendtype,
                   void *recvbuf, MPI_Aint recvcount, MPI_Datatype recvtype)
52
{
53 54 55
    int mpi_errno = MPI_SUCCESS;
    int sendtype_iscontig, recvtype_iscontig;
    MPI_Aint sendsize, recvsize, sdata_sz, rdata_sz, copy_sz;
56
    MPI_Aint true_extent, sendtype_true_lb, recvtype_true_lb;
57
    MPIU_CHKLMEM_DECL(1);
58
    MPID_MPI_STATE_DECL(MPID_STATE_MPIR_LOCALCOPY);
59

60 61
    MPID_MPI_FUNC_ENTER(MPID_STATE_MPIR_LOCALCOPY);

62 63
    MPID_Datatype_get_size_macro(sendtype, sendsize);
    MPID_Datatype_get_size_macro(recvtype, recvsize);
64

65 66
    sdata_sz = sendsize * sendcount;
    rdata_sz = recvsize * recvcount;
67

68
    /* if there is no data to copy, bail out */
69 70
    if (!sdata_sz || !rdata_sz)
        goto fn_exit;
71 72 73

#if defined(HAVE_ERROR_CHECKING)
    if (sdata_sz > rdata_sz) {
74 75 76 77
        MPIU_ERR_SET2(mpi_errno, MPI_ERR_TRUNCATE, "**truncate", "**truncate %d %d", sdata_sz, rdata_sz);
        copy_sz = rdata_sz;
    }
    else
78
#endif /* HAVE_ERROR_CHECKING */
79
        copy_sz = sdata_sz;
80

81 82 83
    /* Builtin types is the common case; optimize for it */
    if ((HANDLE_GET_KIND(sendtype) == HANDLE_KIND_BUILTIN) &&
        HANDLE_GET_KIND(recvtype) == HANDLE_KIND_BUILTIN) {
84 85
        MPIU_Memcpy(recvbuf, sendbuf, copy_sz);
        goto fn_exit;
86
    }
87

88 89 90
    MPIR_Datatype_iscontig(sendtype, &sendtype_iscontig);
    MPIR_Datatype_iscontig(recvtype, &recvtype_iscontig);

91 92
    MPIR_Type_get_true_extent_impl(sendtype, &sendtype_true_lb, &true_extent);
    MPIR_Type_get_true_extent_impl(recvtype, &recvtype_true_lb, &true_extent);
93 94

    if (sendtype_iscontig && recvtype_iscontig)
95 96 97 98 99 100 101 102 103
    {
#if defined(HAVE_ERROR_CHECKING)
        MPIU_ERR_CHKMEMCPYANDJUMP(mpi_errno,
                                  ((char *)recvbuf + recvtype_true_lb),
                                  ((char *)sendbuf + sendtype_true_lb),
                                  copy_sz);
#endif
        MPIU_Memcpy(((char *) recvbuf + recvtype_true_lb),
               ((char *) sendbuf + sendtype_true_lb),
104
               copy_sz);
105
    }
106 107 108
    else if (sendtype_iscontig)
    {
        MPID_Segment seg;
109
	MPI_Aint last;
110 111 112 113

	MPID_Segment_init(recvbuf, recvcount, recvtype, &seg, 0);
	last = copy_sz;
	MPID_Segment_unpack(&seg, 0, &last, (char*)sendbuf + sendtype_true_lb);
114
        MPIU_ERR_CHKANDJUMP(last != copy_sz, mpi_errno, MPI_ERR_TYPE, "**dtypemismatch");
115 116 117 118
    }
    else if (recvtype_iscontig)
    {
        MPID_Segment seg;
119
	MPI_Aint last;
120 121 122 123 124 125 126 127 128 129 130 131 132 133

	MPID_Segment_init(sendbuf, sendcount, sendtype, &seg, 0);
	last = copy_sz;
	MPID_Segment_pack(&seg, 0, &last, (char*)recvbuf + recvtype_true_lb);
        MPIU_ERR_CHKANDJUMP(last != copy_sz, mpi_errno, MPI_ERR_TYPE, "**dtypemismatch");
    }
    else
    {
	char * buf;
	MPIDI_msg_sz_t buf_off;
	MPID_Segment sseg;
	MPIDI_msg_sz_t sfirst;
	MPID_Segment rseg;
	MPIDI_msg_sz_t rfirst;
134 135

        MPIU_CHKLMEM_MALLOC(buf, char *, COPY_BUFFER_SZ, mpi_errno, "buf");
136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184

	MPID_Segment_init(sendbuf, sendcount, sendtype, &sseg, 0);
	MPID_Segment_init(recvbuf, recvcount, recvtype, &rseg, 0);

	sfirst = 0;
	rfirst = 0;
	buf_off = 0;
	
	while (1)
	{
	    MPI_Aint last;
	    char * buf_end;

	    if (copy_sz - sfirst > COPY_BUFFER_SZ - buf_off)
	    {
		last = sfirst + (COPY_BUFFER_SZ - buf_off);
	    }
	    else
	    {
		last = copy_sz;
	    }
	    
	    MPID_Segment_pack(&sseg, sfirst, &last, buf + buf_off);
	    MPIU_Assert(last > sfirst);
	    
	    buf_end = buf + buf_off + (last - sfirst);
	    sfirst = last;
	    
	    MPID_Segment_unpack(&rseg, rfirst, &last, buf);
	    MPIU_Assert(last > rfirst);

	    rfirst = last;

	    if (rfirst == copy_sz)
	    {
		/* successful completion */
		break;
	    }

            /* if the send side finished, but the recv side couldn't unpack it, there's a datatype mismatch */
            MPIU_ERR_CHKANDJUMP(sfirst == copy_sz, mpi_errno, MPI_ERR_TYPE, "**dtypemismatch");        

            /* if not all data was unpacked, copy it to the front of the buffer for next time */
	    buf_off = sfirst - rfirst;
	    if (buf_off > 0)
	    {
		memmove(buf, buf_end - buf_off, buf_off);
	    }
	}
185 186
    }
    
187
    
188
  fn_exit:
189
    MPIU_CHKLMEM_FREEALL();
190
    MPID_MPI_FUNC_EXIT(MPID_STATE_MPIR_LOCALCOPY);
191 192 193 194 195 196 197
    return mpi_errno;

  fn_fail:
    goto fn_exit;
}


198 199 200 201
/* FIXME: For the brief-global and finer-grain control, we must ensure that
   the global lock is *not* held when this routine is called. (unless we change
   progress_start/end to grab the lock, in which case we must *still* make
   sure that the lock is not held when this routine is called). */
202 203 204 205
#undef FUNCNAME
#define FUNCNAME MPIC_Wait
#undef FCNAME
#define FCNAME "MPIC_Wait"
206
int MPIC_Wait(MPID_Request * request_ptr, mpir_errflag_t *errflag)
207
{
Rajeev Thakur's avatar
Rajeev Thakur committed
208
    int mpi_errno = MPI_SUCCESS;
209 210 211
    MPIDI_STATE_DECL(MPID_STATE_MPIC_WAIT);

    MPIDI_PT2PT_FUNC_ENTER(MPID_STATE_MPIC_WAIT);
212 213 214 215 216 217

    MPIU_DBG_MSG_S(PT2PT, TYPICAL, "IN: errflag = %s", *errflag?"TRUE":"FALSE");

    if (request_ptr->kind == MPID_REQUEST_SEND)
        request_ptr->status.MPI_TAG = 0;

218
    if (!MPID_Request_is_complete(request_ptr))
219 220 221 222
    {
	MPID_Progress_state progress_state;
	
	MPID_Progress_start(&progress_state);
223
        while (!MPID_Request_is_complete(request_ptr))
224 225 226 227 228 229 230
	{
	    mpi_errno = MPID_Progress_wait(&progress_state);
	    if (mpi_errno) { MPIU_ERR_POP(mpi_errno); }
	}
	MPID_Progress_end(&progress_state);
    }

Wesley Bland's avatar
Wesley Bland committed
231 232
    if (request_ptr->kind == MPID_REQUEST_RECV)
        MPIR_Process_status(&request_ptr->status, errflag);
233

234 235
    MPIR_TAG_CLEAR_ERROR_BITS(request_ptr->status.MPI_TAG);

236 237
 fn_exit:
    MPIU_DBG_MSG_D(PT2PT, TYPICAL, "OUT: errflag = %d", *errflag);
238
    MPIDI_PT2PT_FUNC_EXIT(MPID_STATE_MPIC_WAIT);
239
    return mpi_errno;
240 241 242
 fn_fail:
    /* --BEGIN ERROR HANDLING-- */
    goto fn_exit;
243
    /* --END ERROR HANDLING-- */
244
}
245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267


/* Fault-tolerance versions.  When a process fails, collectives will
   still complete, however the result may be invalid.  Processes
   directly communicating with the failed process can detect the
   failure, however another mechanism is needed to commuinicate the
   failure to other processes receiving the invalid data.  To do this
   we introduce the _ft versions of the MPIC_ helper functions.  These
   functions take a pointer to an error flag.  When this is set to
   TRUE, the send functions will communicate the failure to the
   receiver.  If a function detects a failure, either by getting a
   failure in the communication operation, or by receiving an error
   indicator from a remote process, it sets the error flag to TRUE.

   In this implementation, we indicate an error to a remote process by
   sending an empty message instead of the requested buffer.  When a
   process receives an empty message, it knows to set the error flag.
   We count on the fact that collectives that exchange data (as
   opposed to barrier) will never send an empty message.  The barrier
   collective will not communicate failure information this way, but
   this is OK since there is no data that can be received corrupted. */

#undef FUNCNAME
268
#define FUNCNAME MPIC_Send
269 270
#undef FCNAME
#define FCNAME MPIU_QUOTE(FUNCNAME)
271
int MPIC_Send(const void *buf, MPI_Aint count, MPI_Datatype datatype, int dest, int tag,
272
                 MPID_Comm *comm_ptr, mpir_errflag_t *errflag)
273 274
{
    int mpi_errno = MPI_SUCCESS;
275 276
    int context_id;
    MPID_Request *request_ptr = NULL;
Wesley Bland's avatar
Wesley Bland committed
277
    MPIDI_STATE_DECL(MPID_STATE_MPIC_SEND);
278

Wesley Bland's avatar
Wesley Bland committed
279
    MPIDI_FUNC_ENTER(MPID_STATE_MPIC_SEND);
280

Wesley Bland's avatar
Wesley Bland committed
281
    MPIU_DBG_MSG_D(PT2PT, TYPICAL, "IN: errflag = %d", *errflag);
282

283 284 285
    MPIU_ERR_CHKANDJUMP1((count < 0), mpi_errno, MPI_ERR_COUNT,
                         "**countneg", "**countneg %d", count);

Wesley Bland's avatar
Wesley Bland committed
286 287 288 289 290
    switch(*errflag) {
        case MPIR_ERR_NONE:
            break;
        case MPIR_ERR_PROC_FAILED:
            MPIR_TAG_SET_PROC_FAILURE_BIT(tag);
291
        default:
Wesley Bland's avatar
Wesley Bland committed
292 293
            MPIR_TAG_SET_ERROR_BIT(tag);
    }
294

295 296 297 298 299 300 301
    context_id = (comm_ptr->comm_kind == MPID_INTRACOMM) ?
        MPID_CONTEXT_INTRA_COLL : MPID_CONTEXT_INTER_COLL;

    mpi_errno = MPID_Send(buf, count, datatype, dest, tag, comm_ptr,
                          context_id, &request_ptr);
    if (mpi_errno) MPIU_ERR_POP(mpi_errno);
    if (request_ptr) {
302
        mpi_errno = MPIC_Wait(request_ptr, errflag);
303 304 305
        if (mpi_errno) MPIU_ERR_POP(mpi_errno);
        MPID_Request_release(request_ptr);
    }
306

307
 fn_exit:
308
    MPIU_DBG_MSG_D(PT2PT, TYPICAL, "OUT: errflag = %d", *errflag);
Wesley Bland's avatar
Wesley Bland committed
309
    MPIDI_FUNC_EXIT(MPID_STATE_MPIC_SEND);
310
    return mpi_errno;
311
 fn_fail:
312 313
    /* --BEGIN ERROR HANDLING-- */
    if (request_ptr) MPID_Request_release(request_ptr);
314 315 316 317 318 319 320
    if (mpi_errno && !*errflag) {
        if (MPIX_ERR_PROC_FAILED == MPIR_ERR_GET_CLASS(mpi_errno)) {
            *errflag = MPIR_ERR_PROC_FAILED;
        } else {
            *errflag = MPIR_ERR_OTHER;
        }
    }
321
    goto fn_exit;
322
    /* --END ERROR HANDLING-- */
323 324 325
}

#undef FUNCNAME
326
#define FUNCNAME MPIC_Recv
327 328
#undef FCNAME
#define FCNAME MPIU_QUOTE(FUNCNAME)
329
int MPIC_Recv(void *buf, MPI_Aint count, MPI_Datatype datatype, int source, int tag,
330
                 MPID_Comm *comm_ptr, MPI_Status *status, mpir_errflag_t *errflag)
331 332
{
    int mpi_errno = MPI_SUCCESS;
333
    int context_id;
334
    MPI_Status mystatus;
335
    MPID_Request *request_ptr = NULL;
Wesley Bland's avatar
Wesley Bland committed
336
    MPIDI_STATE_DECL(MPID_STATE_MPIC_RECV);
337

Wesley Bland's avatar
Wesley Bland committed
338
    MPIDI_FUNC_ENTER(MPID_STATE_MPIC_RECV);
339

Wesley Bland's avatar
Wesley Bland committed
340
    MPIU_DBG_MSG_D(PT2PT, TYPICAL, "IN: errflag = %d", *errflag);
341

342 343
    MPIU_ERR_CHKANDJUMP1((count < 0), mpi_errno, MPI_ERR_COUNT,
                         "**countneg", "**countneg %d", count);
344

345 346
    context_id = (comm_ptr->comm_kind == MPID_INTRACOMM) ?
        MPID_CONTEXT_INTRA_COLL : MPID_CONTEXT_INTER_COLL;
347

348 349
    if (status == MPI_STATUS_IGNORE)
        status = &mystatus;
350

351 352
    mpi_errno = MPID_Recv(buf, count, datatype, source, tag, comm_ptr,
                          context_id, status, &request_ptr);
353
    if (mpi_errno) MPIU_ERR_POP(mpi_errno);
354
    if (request_ptr) {
355
        mpi_errno = MPIC_Wait(request_ptr, errflag);
356
        if (mpi_errno != MPI_SUCCESS)
357
            MPIU_ERR_POP(mpi_errno);
358 359 360

        *status = request_ptr->status;
        mpi_errno = status->MPI_ERROR;
361
        MPID_Request_release(request_ptr);
362 363
    } else {
        MPIR_Process_status(status, errflag);
364 365

        MPIR_TAG_CLEAR_ERROR_BITS(status->MPI_TAG);
366
    }
367

368 369
    if (MPI_SUCCESS == MPIR_ERR_GET_CLASS(status->MPI_ERROR)) {
        MPIU_Assert(status->MPI_TAG == tag);
370 371 372
    }

 fn_exit:
Wesley Bland's avatar
Wesley Bland committed
373
    MPIU_DBG_MSG_D(PT2PT, TYPICAL, "OUT: errflag = %d", *errflag);
Wesley Bland's avatar
Wesley Bland committed
374
    MPIDI_FUNC_EXIT(MPID_STATE_MPIC_RECV);
375 376
    return mpi_errno;
 fn_fail:
377 378
    /* --BEGIN ERROR HANDLING-- */
    if (request_ptr) MPID_Request_release(request_ptr);
379
    goto fn_exit;
380
    /* --END ERROR HANDLING-- */
381 382 383
}

#undef FUNCNAME
384
#define FUNCNAME MPIC_Ssend
385 386
#undef FCNAME
#define FCNAME MPIU_QUOTE(FUNCNAME)
387
int MPIC_Ssend(const void *buf, MPI_Aint count, MPI_Datatype datatype, int dest, int tag,
388
                  MPID_Comm *comm_ptr, mpir_errflag_t *errflag)
389 390
{
    int mpi_errno = MPI_SUCCESS;
391 392
    int context_id;
    MPID_Request *request_ptr = NULL;
Wesley Bland's avatar
Wesley Bland committed
393
    MPIDI_STATE_DECL(MPID_STATE_MPIC_SSEND);
394

Wesley Bland's avatar
Wesley Bland committed
395
    MPIDI_FUNC_ENTER(MPID_STATE_MPIC_SSEND);
396

Wesley Bland's avatar
Wesley Bland committed
397
    MPIU_DBG_MSG_D(PT2PT, TYPICAL, "IN: errflag = %d", *errflag);
398 399 400 401 402 403 404

    MPIU_ERR_CHKANDJUMP1((count < 0), mpi_errno, MPI_ERR_COUNT,
            "**countneg", "**countneg %d", count);

    context_id = (comm_ptr->comm_kind == MPID_INTRACOMM) ?
        MPID_CONTEXT_INTRA_COLL : MPID_CONTEXT_INTER_COLL;

405 406 407 408 409 410 411 412
    switch(*errflag) {
        case MPIR_ERR_NONE:
            break;
        case MPIR_ERR_PROC_FAILED:
            MPIR_TAG_SET_PROC_FAILURE_BIT(tag);
        default:
            MPIR_TAG_SET_ERROR_BIT(tag);
    }
413

414 415 416 417
    mpi_errno = MPID_Ssend(buf, count, datatype, dest, tag, comm_ptr,
                           context_id, &request_ptr);
    if (mpi_errno) MPIU_ERR_POP(mpi_errno);
    if (request_ptr) {
418
        mpi_errno = MPIC_Wait(request_ptr, errflag);
419 420 421
        if (mpi_errno) MPIU_ERR_POP(mpi_errno);
        MPID_Request_release(request_ptr);
    }
422

423
 fn_exit:
424
    MPIU_DBG_MSG_D(PT2PT, TYPICAL, "OUT: errflag = %d", *errflag);
Wesley Bland's avatar
Wesley Bland committed
425
    MPIDI_FUNC_EXIT(MPID_STATE_MPIC_SSEND);
426
    return mpi_errno;
427
 fn_fail:
428 429
    /* --BEGIN ERROR HANDLING-- */
    if (request_ptr) MPID_Request_release(request_ptr);
430 431 432 433 434 435 436
    if (mpi_errno && !*errflag) {
        if (MPIX_ERR_PROC_FAILED == MPIR_ERR_GET_CLASS(mpi_errno)) {
            *errflag = MPIR_ERR_PROC_FAILED;
        } else {
            *errflag = MPIR_ERR_OTHER;
        }
    }
437
    goto fn_exit;
438
    /* --END ERROR HANDLING-- */
439 440 441
}

#undef FUNCNAME
442
#define FUNCNAME MPIC_Sendrecv
443 444
#undef FCNAME
#define FCNAME MPIU_QUOTE(FUNCNAME)
445 446
int MPIC_Sendrecv(const void *sendbuf, MPI_Aint sendcount, MPI_Datatype sendtype,
                     int dest, int sendtag, void *recvbuf, MPI_Aint recvcount,
447
                     MPI_Datatype recvtype, int source, int recvtag,
448
                     MPID_Comm *comm_ptr, MPI_Status *status, mpir_errflag_t *errflag)
449 450
{
    int mpi_errno = MPI_SUCCESS;
451
    int context_id;
452
    MPI_Status mystatus;
453
    MPID_Request *recv_req_ptr = NULL, *send_req_ptr = NULL;
Wesley Bland's avatar
Wesley Bland committed
454
    MPIDI_STATE_DECL(MPID_STATE_MPIC_SENDRECV);
455

Wesley Bland's avatar
Wesley Bland committed
456
    MPIDI_FUNC_ENTER(MPID_STATE_MPIC_SENDRECV);
457 458 459

    MPIU_DBG_MSG_S(PT2PT, TYPICAL, "IN: errflag = %s", *errflag?"TRUE":"FALSE");

460 461 462 463 464 465 466 467
    MPIU_ERR_CHKANDJUMP1((sendcount < 0), mpi_errno, MPI_ERR_COUNT,
                         "**countneg", "**countneg %d", sendcount);
    MPIU_ERR_CHKANDJUMP1((recvcount < 0), mpi_errno, MPI_ERR_COUNT,
                         "**countneg", "**countneg %d", recvcount);

    context_id = (comm_ptr->comm_kind == MPID_INTRACOMM) ?
        MPID_CONTEXT_INTRA_COLL : MPID_CONTEXT_INTER_COLL;

468
    if (status == MPI_STATUS_IGNORE) status = &mystatus;
Wesley Bland's avatar
Wesley Bland committed
469 470 471 472 473
    switch(*errflag) {
        case MPIR_ERR_NONE:
            break;
        case MPIR_ERR_PROC_FAILED:
            MPIR_TAG_SET_PROC_FAILURE_BIT(sendtag);
474
        default:
Wesley Bland's avatar
Wesley Bland committed
475 476
            MPIR_TAG_SET_ERROR_BIT(sendtag);
    }
477

478 479
    mpi_errno = MPID_Irecv(recvbuf, recvcount, recvtype, source, recvtag,
                           comm_ptr, context_id, &recv_req_ptr);
480
    if (mpi_errno) MPIU_ERR_POP(mpi_errno);
481
    mpi_errno = MPID_Isend(sendbuf, sendcount, sendtype, dest, sendtag,
482 483 484
                           comm_ptr, context_id, &send_req_ptr);
    if (mpi_errno) MPIU_ERR_POP(mpi_errno);

485
    mpi_errno = MPIC_Wait(send_req_ptr, errflag);
486
    if (mpi_errno) MPIU_ERR_POP(mpi_errno);
487
    mpi_errno = MPIC_Wait(recv_req_ptr, errflag);
488 489 490 491
    if (mpi_errno) MPIU_ERR_POPFATAL(mpi_errno);

    *status = recv_req_ptr->status;

492 493
    if (mpi_errno == MPI_SUCCESS) {
        mpi_errno = recv_req_ptr->status.MPI_ERROR;
494

495 496
        if (mpi_errno == MPI_SUCCESS) {
            mpi_errno = send_req_ptr->status.MPI_ERROR;
497 498
        }
    }
499

500 501
    MPID_Request_release(send_req_ptr);
    MPID_Request_release(recv_req_ptr);
Wesley Bland's avatar
Wesley Bland committed
502

503
 fn_exit:
Wesley Bland's avatar
Wesley Bland committed
504
    MPIU_DBG_MSG_D(PT2PT, TYPICAL, "OUT: errflag = %d", *errflag);
505

Wesley Bland's avatar
Wesley Bland committed
506
    MPIDI_FUNC_EXIT(MPID_STATE_MPIC_SENDRECV);
507 508
    return mpi_errno;
 fn_fail:
509 510 511 512
    if (send_req_ptr)
        MPID_Request_release(send_req_ptr);
    if (recv_req_ptr)
        MPID_Request_release(recv_req_ptr);
513 514 515
    goto fn_exit;
}

516 517 518
/* NOTE: for regular collectives (as opposed to irregular collectives) calling
 * this function repeatedly will almost always be slower than performing the
 * equivalent inline because of the overhead of the repeated malloc/free */
519
#undef FUNCNAME
520
#define FUNCNAME MPIC_Sendrecv_replace
521 522
#undef FCNAME
#define FCNAME MPIU_QUOTE(FUNCNAME)
523
int MPIC_Sendrecv_replace(void *buf, int count, MPI_Datatype datatype,
524 525
                             int dest, int sendtag,
                             int source, int recvtag,
526
                             MPID_Comm *comm_ptr, MPI_Status *status, mpir_errflag_t *errflag)
527 528 529
{
    int mpi_errno = MPI_SUCCESS;
    MPI_Status mystatus;
530
    MPIR_Context_id_t context_id_offset;
531 532
    MPID_Request *sreq = NULL;
    MPID_Request *rreq = NULL;
533
    void *tmpbuf = NULL;
534 535
    MPI_Aint tmpbuf_size = 0;
    MPI_Aint tmpbuf_count = 0;
536
    MPIU_CHKLMEM_DECL(1);
Wesley Bland's avatar
Wesley Bland committed
537
    MPIDI_STATE_DECL(MPID_STATE_MPIC_SENDRECV_REPLACE);
538 539 540 541
#ifdef MPID_LOG_ARROWS
    /* The logging macros log sendcount and recvcount */
    int sendcount = count, recvcount = count;
#endif
542

Wesley Bland's avatar
Wesley Bland committed
543
    MPIDI_FUNC_ENTER(MPID_STATE_MPIC_SENDRECV_REPLACE);
544

Wesley Bland's avatar
Wesley Bland committed
545
    MPIU_DBG_MSG_D(PT2PT, TYPICAL, "IN: errflag = %d", *errflag);
546

547 548 549
    MPIU_ERR_CHKANDJUMP1((count < 0), mpi_errno, MPI_ERR_COUNT,
                         "**countneg", "**countneg %d", count);

550
    if (status == MPI_STATUS_IGNORE) status = &mystatus;
Wesley Bland's avatar
Wesley Bland committed
551 552 553 554 555
    switch(*errflag) {
        case MPIR_ERR_NONE:
            break;
        case MPIR_ERR_PROC_FAILED:
            MPIR_TAG_SET_PROC_FAILURE_BIT(sendtag);
556
        default:
Wesley Bland's avatar
Wesley Bland committed
557 558
            MPIR_TAG_SET_ERROR_BIT(sendtag);
    }
559

560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585
    context_id_offset = (comm_ptr->comm_kind == MPID_INTRACOMM) ?
        MPID_CONTEXT_INTRA_COLL : MPID_CONTEXT_INTER_COLL;

    if (count > 0 && dest != MPI_PROC_NULL) {
        MPIR_Pack_size_impl(count, datatype, &tmpbuf_size);
        MPIU_CHKLMEM_MALLOC(tmpbuf, void *, tmpbuf_size, mpi_errno, "temporary send buffer");

        mpi_errno = MPIR_Pack_impl(buf, count, datatype, tmpbuf, tmpbuf_size, &tmpbuf_count);
        if (mpi_errno) MPIU_ERR_POP(mpi_errno);
    }

    mpi_errno = MPID_Irecv(buf, count, datatype, source, recvtag,
                           comm_ptr, context_id_offset, &rreq);
    if (mpi_errno) MPIU_ERR_POP(mpi_errno);

    mpi_errno = MPID_Isend(tmpbuf, tmpbuf_count, MPI_PACKED, dest,
                           sendtag, comm_ptr, context_id_offset, &sreq);
    if (mpi_errno != MPI_SUCCESS) {
        /* --BEGIN ERROR HANDLING-- */
        /* FIXME: should we cancel the pending (possibly completed) receive
         * request or wait for it to complete? */
        MPID_Request_release(rreq);
        MPIU_ERR_POP(mpi_errno);
        /* --END ERROR HANDLING-- */
    }

586 587 588 589
    mpi_errno = MPIC_Wait(sreq, errflag);
    if (mpi_errno) MPIU_ERR_POP(mpi_errno);
    mpi_errno = MPIC_Wait(rreq, errflag);
    if (mpi_errno) MPIU_ERR_POP(mpi_errno);
590 591 592 593 594 595 596 597 598 599 600 601 602 603

    *status = rreq->status;

    if (mpi_errno == MPI_SUCCESS) {
        mpi_errno = rreq->status.MPI_ERROR;

        if (mpi_errno == MPI_SUCCESS) {
            mpi_errno = sreq->status.MPI_ERROR;
        }
    }

    MPID_Request_release(sreq);
    MPID_Request_release(rreq);

604
 fn_exit:
605
    MPIU_CHKLMEM_FREEALL();
Wesley Bland's avatar
Wesley Bland committed
606
    MPIU_DBG_MSG_D(PT2PT, TYPICAL, "OUT: errflag = %d", *errflag);
Wesley Bland's avatar
Wesley Bland committed
607
    MPIDI_FUNC_EXIT(MPID_STATE_MPIC_SENDRECV_REPLACE);
608 609
    return mpi_errno;
 fn_fail:
610 611 612 613
     if (sreq)
         MPID_Request_release(sreq);
     if (rreq)
         MPID_Request_release(rreq);
614 615 616 617
    goto fn_exit;
}

#undef FUNCNAME
618
#define FUNCNAME MPIC_Isend
619 620
#undef FCNAME
#define FCNAME MPIU_QUOTE(FUNCNAME)
621
int MPIC_Isend(const void *buf, int count, MPI_Datatype datatype, int dest, int tag,
622
                  MPID_Comm *comm_ptr, MPID_Request **request_ptr, mpir_errflag_t *errflag)
623 624
{
    int mpi_errno = MPI_SUCCESS;
625
    int context_id;
Wesley Bland's avatar
Wesley Bland committed
626
    MPIDI_STATE_DECL(MPID_STATE_MPIC_ISEND);
627

Wesley Bland's avatar
Wesley Bland committed
628
    MPIDI_FUNC_ENTER(MPID_STATE_MPIC_ISEND);
629

Wesley Bland's avatar
Wesley Bland committed
630
    MPIU_DBG_MSG_D(PT2PT, TYPICAL, "IN: errflag = %d", *errflag);
631

632 633 634
    MPIU_ERR_CHKANDJUMP1((count < 0), mpi_errno, MPI_ERR_COUNT,
                         "**countneg", "**countneg %d", count);

Wesley Bland's avatar
Wesley Bland committed
635 636 637 638 639
    switch(*errflag) {
        case MPIR_ERR_NONE:
            break;
        case MPIR_ERR_PROC_FAILED:
            MPIR_TAG_SET_PROC_FAILURE_BIT(tag);
640
        default:
Wesley Bland's avatar
Wesley Bland committed
641 642
            MPIR_TAG_SET_ERROR_BIT(tag);
    }
643

644 645 646 647
    context_id = (comm_ptr->comm_kind == MPID_INTRACOMM) ?
        MPID_CONTEXT_INTRA_COLL : MPID_CONTEXT_INTER_COLL;

    mpi_errno = MPID_Isend(buf, count, datatype, dest, tag, comm_ptr,
648
            context_id, request_ptr);
649 650
    if (mpi_errno) MPIU_ERR_POP(mpi_errno);

651
 fn_exit:
Wesley Bland's avatar
Wesley Bland committed
652
    MPIDI_FUNC_EXIT(MPID_STATE_MPIC_ISEND);
653
    return mpi_errno;
654 655
 fn_fail:
    goto fn_exit;
656 657
}

Wesley Bland's avatar
Wesley Bland committed
658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698
#undef FUNCNAME
#define FUNCNAME MPIC_Issend
#undef FCNAME
#define FCNAME MPIU_QUOTE(FUNCNAME)
int MPIC_Issend(const void *buf, int count, MPI_Datatype datatype, int dest, int tag,
                  MPID_Comm *comm_ptr, MPID_Request **request_ptr, mpir_errflag_t *errflag)
{
    int mpi_errno = MPI_SUCCESS;
    int context_id;
    MPIDI_STATE_DECL(MPID_STATE_MPIC_ISSEND);

    MPIDI_FUNC_ENTER(MPID_STATE_MPIC_ISSEND);

    MPIU_DBG_MSG_D(PT2PT, TYPICAL, "IN: errflag = %d", *errflag);

    MPIU_ERR_CHKANDJUMP1((count < 0), mpi_errno, MPI_ERR_COUNT,
                         "**countneg", "**countneg %d", count);

    switch(*errflag) {
        case MPIR_ERR_NONE:
            break;
        case MPIR_ERR_PROC_FAILED:
            MPIR_TAG_SET_PROC_FAILURE_BIT(tag);
        default:
            MPIR_TAG_SET_ERROR_BIT(tag);
    }

    context_id = (comm_ptr->comm_kind == MPID_INTRACOMM) ?
        MPID_CONTEXT_INTRA_COLL : MPID_CONTEXT_INTER_COLL;

    mpi_errno = MPID_Issend(buf, count, datatype, dest, tag, comm_ptr,
            context_id, request_ptr);
    if (mpi_errno) MPIU_ERR_POP(mpi_errno);

 fn_exit:
    MPIDI_FUNC_EXIT(MPID_STATE_MPIC_ISSEND);
    return mpi_errno;
 fn_fail:
    goto fn_exit;
}

699
#undef FUNCNAME
700
#define FUNCNAME MPIC_Irecv
701 702
#undef FCNAME
#define FCNAME MPIU_QUOTE(FUNCNAME)
703
int MPIC_Irecv(void *buf, int count, MPI_Datatype datatype, int source,
704
                  int tag, MPID_Comm *comm_ptr, MPID_Request **request_ptr)
705 706
{
    int mpi_errno = MPI_SUCCESS;
707
    int context_id;
Wesley Bland's avatar
Wesley Bland committed
708
    MPIDI_STATE_DECL(MPID_STATE_MPIC_IRECV);
709

Wesley Bland's avatar
Wesley Bland committed
710
    MPIDI_FUNC_ENTER(MPID_STATE_MPIC_IRECV);
711

712 713 714 715 716 717 718
    MPIU_ERR_CHKANDJUMP1((count < 0), mpi_errno, MPI_ERR_COUNT,
                         "**countneg", "**countneg %d", count);

    context_id = (comm_ptr->comm_kind == MPID_INTRACOMM) ?
        MPID_CONTEXT_INTRA_COLL : MPID_CONTEXT_INTER_COLL;

    mpi_errno = MPID_Irecv(buf, count, datatype, source, tag, comm_ptr,
719
            context_id, request_ptr);
720 721
    if (mpi_errno) MPIU_ERR_POP(mpi_errno);

722
 fn_exit:
Wesley Bland's avatar
Wesley Bland committed
723
    MPIDI_FUNC_EXIT(MPID_STATE_MPIC_IRECV);
724
    return mpi_errno;
725 726
 fn_fail:
    goto fn_exit;
727 728 729 730
}


#undef FUNCNAME
731
#define FUNCNAME MPIC_Waitall
732 733
#undef FCNAME
#define FCNAME MPIU_QUOTE(FUNCNAME)
734
int MPIC_Waitall(int numreq, MPID_Request *requests[], MPI_Status statuses[], mpir_errflag_t *errflag)
735 736 737
{
    int mpi_errno = MPI_SUCCESS;
    int i;
738 739 740 741
    MPI_Request request_ptr_array[MPIC_REQUEST_PTR_ARRAY_SIZE];
    MPI_Request *request_ptrs = request_ptr_array;
    MPI_Status status_static_array[MPIC_REQUEST_PTR_ARRAY_SIZE];
    MPI_Status *status_array = statuses;
Wesley Bland's avatar
Wesley Bland committed
742
    MPIDI_STATE_DECL(MPID_STATE_MPIC_WAITALL);
743
    MPIU_CHKLMEM_DECL(2);
744

Wesley Bland's avatar
Wesley Bland committed
745
    MPIDI_FUNC_ENTER(MPID_STATE_MPIC_WAITALL);
746 747 748

    MPIU_DBG_MSG_S(PT2PT, TYPICAL, "IN: errflag = %s", *errflag?"TRUE":"FALSE");

749 750 751
    if (statuses == MPI_STATUSES_IGNORE) {
        status_array = status_static_array;
    }
752

753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768
    if (numreq > MPIC_REQUEST_PTR_ARRAY_SIZE) {
        MPIU_CHKLMEM_MALLOC(request_ptrs, MPI_Request *, numreq * sizeof(MPI_Request), mpi_errno, "request pointers");
        MPIU_CHKLMEM_MALLOC(status_array, MPI_Status *, numreq * sizeof(MPI_Status), mpi_errno, "status objects");
    }

    for (i = 0; i < numreq; ++i) {
        /* The MPI_TAG field is not set for send operations, so if we want
        to check for the error bit in the tag below, we should initialize all
        tag fields here. */
        status_array[i].MPI_TAG = 0;

        /* Convert the MPID_Request objects to MPI_Request objects */
        request_ptrs[i] = requests[i]->handle;
    }

    mpi_errno = MPIR_Waitall_impl(numreq, request_ptrs, status_array);
769

770 771 772
    /* The errflag value here is for all requests, not just a single one.  If
     * in the future, this function is used for multiple collectives at a
     * single time, we may have to change that. */
773
    for (i = 0; i < numreq; ++i) {
774
        MPIR_Process_status(&status_array[i], errflag);
775 776

        MPIR_TAG_CLEAR_ERROR_BITS(status_array[i].MPI_TAG);
777 778 779
    }

 fn_exit:
780 781 782
    if (numreq > MPIC_REQUEST_PTR_ARRAY_SIZE)
        MPIU_CHKLMEM_FREEALL();

Wesley Bland's avatar
Wesley Bland committed
783
    MPIU_DBG_MSG_D(PT2PT, TYPICAL, "OUT: errflag = %d", *errflag);
Wesley Bland's avatar
Wesley Bland committed
784
    MPIDI_FUNC_EXIT(MPID_STATE_MPIC_WAITALL);
785 786 787 788
    return mpi_errno;
 fn_fail:
    goto fn_exit;
}