Commit 5be10ce9 authored by Wesley Bland's avatar Wesley Bland Committed by Junchao Zhang
Browse files

Add MPIX_Comm_shrink functionality



This adds a new function MPIX_COMM_SHRINK. This is a communicator creation
function that creates a new communicator based on a previous communicator, but
excluding any failed processes.

As part of the operation, the shrink call needs to perform an agreement to
determine the group of failed processes. This is done using the algorithm
published by Hursey et al. in his EuroMPI '12 paper.

The list of failed processes is collected using a bit array. This happens via
a few new functions in the CH3 layer to create and send a bitarry to the
master process and receive an updated bitarray. Obviously, this is not a very
scalable implementation yet, but something better can easily be plugged in
here to replace the naïve implementation. This is also a use case for an
MPI_Recv_reduce for future reference.
Signed-off-by: default avatarJunchao Zhang <jczhang@mcs.anl.gov>
parent ee5173e3
......@@ -1538,6 +1538,7 @@ int MPI_T_category_changed(int *stamp);
int MPIX_Comm_failure_ack(MPI_Comm comm);
int MPIX_Comm_failure_get_acked(MPI_Comm comm, MPI_Group *failedgrp);
int MPIX_Comm_revoke(MPI_Comm comm);
int MPIX_Comm_shrink(MPI_Comm comm, MPI_Comm *newcomm);
/* End Prototypes */
......@@ -2176,6 +2177,7 @@ int PMPI_T_category_changed(int *stamp);
int PMPIX_Comm_failure_ack(MPI_Comm comm);
int PMPIX_Comm_failure_get_acked(MPI_Comm comm, MPI_Group *failedgrp);
int PMPIX_Comm_revoke(MPI_Comm comm);
int PMPIX_Comm_shrink(MPI_Comm comm, MPI_Comm *newcomm);
#endif /* MPI_BUILD_PROFILING */
......
......@@ -2784,6 +2784,36 @@ int MPID_Comm_failure_ack(MPID_Comm *comm);
@*/
int MPID_Comm_failure_get_acked(MPID_Comm *comm, MPID_Group **failed_group_ptr);
/*@
MPID_Comm_failed_bitarray - MPID function to get the bitarray including all of the failed processes
Input Parameters:
. comm - communicator
. acked - true if bitarray should contain only acked procs
Output Parameter:
. bitarray - Bit array containing all of the failed processes in comm
Return Value:
'MPI_SUCCESS' or a valid MPI error code.
@*/
int MPID_Comm_failed_bitarray(MPID_Comm *comm, uint32_t **bitarray, int acked);
/*@
MPID_Comm_get_all_failed_procs - Constructs a group of failed processes that it uniform over a communicator
Input Parameters:
. comm - communicator
. tag - Tag used to do communciation
Output Parameters:
. failed_grp - group of all failed processes
Return Value:
'MPI_SUCCESS' or a valid MPI error code.
@*/
int MPID_Comm_get_all_failed_procs(MPID_Comm *comm_ptr, MPID_Group **failed_group, int tag);
/*@
MPID_Comm_revoke - MPID entry point for MPI_Comm_revoke
......@@ -3781,7 +3811,8 @@ int MPID_VCR_Get_lpid(MPID_VCR vcr, int * lpid_ptr);
#define MPIR_TOPO_A_TAG 26
#define MPIR_TOPO_B_TAG 27
#define MPIR_REDUCE_SCATTER_BLOCK_TAG 28
#define MPIR_FIRST_NBC_TAG 29
#define MPIR_SHRINK_TAG 29
#define MPIR_FIRST_NBC_TAG 30
/* These macros must be used carefully. These macros will not work with
* negative tags. By definition, users are not to use negative tags and the
......@@ -4084,6 +4115,8 @@ void MPIR_Free_err_dyncodes( void );
int MPIR_Comm_idup_impl(MPID_Comm *comm_ptr, MPID_Comm **newcomm, MPID_Request **reqp);
int MPIR_Comm_shrink(MPID_Comm *comm_ptr, MPID_Comm **newcomm_ptr);
int MPIR_Allreduce_group(void *sendbuf, void *recvbuf, int count,
MPI_Datatype datatype, MPI_Op op, MPID_Comm *comm_ptr,
MPID_Group *group_ptr, int tag, int *errflag);
......
......@@ -29,7 +29,8 @@ mpi_sources += \
src/mpi/comm/comm_split_type.c \
src/mpi/comm/comm_failure_ack.c \
src/mpi/comm/comm_failure_get_acked.c \
src/mpi/comm/comm_revoke.c
src/mpi/comm/comm_revoke.c \
src/mpi/comm/comm_shrink.c
mpi_core_sources += \
src/mpi/comm/commutil.c
......
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
/*
* (C) 2001 by Argonne National Laboratory.
* See COPYRIGHT in top-level directory.
*/
#include "mpiimpl.h"
#include "mpicomm.h"
#include <stdint.h>
/* This function has multiple phases.
*
* In the first phase, all alive processes must collectively decide which
* processes are dead. This happens via a fault-tolerant all-reduce style
* algorithm. This is implemented via the recursive-doubling algorithm as a
* first pass for simplicity.
*
* In the second phase, the remaining processes must create a new communicator
* based on the group determined in the first phase. This phase simply uses
* the existing implementation of MPI_Comm_create_group. If the call to
* MPI_Comm_create_group fails, then the algorithm is restarted in phase one
* and a new group is determined.
*/
/* -- Begin Profiling Symbol Block for routine MPIX_Comm_shrink */
#if defined(HAVE_PRAGMA_WEAK)
#pragma weak MPIX_Comm_shrink = PMPIX_Comm_shrink
#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
#pragma _HP_SECONDARY_DEF PMPIX_Comm_shrink MPIX_Comm_shrink
#elif defined(HAVE_PRAGMA_CRI_DUP)
#pragma _CRI duplicate MPIX_Comm_shrink as PMPIX_Comm_shrink
#endif
/* -- End Profiling Symbol Block */
/* Define MPICH_MPI_FROM_PMPI if weak symbols are not supported to build
the MPI routines */
#ifndef MPICH_MPI_FROM_PMPI
#undef MPIX_Comm_shrink
#define MPIX_Comm_shrink PMPIX_Comm_shrink
#endif
#undef FUNCNAME
#define FUNCNAME MPIR_Comm_shrink
#undef FCNAME
#define FCNAME MPIU_QUOTE(FUNCNAME)
/* comm shrink impl; assumes that standard error checking has already taken
* place in the calling function */
int MPIR_Comm_shrink(MPID_Comm *comm_ptr, MPID_Comm **newcomm_ptr)
{
int mpi_errno = MPI_SUCCESS;
MPID_Group *global_failed, *comm_grp, *new_group_ptr;
int attempts = 0;
int errflag = 0, tmp_errflag = 0;
MPID_MPI_STATE_DECL(MPID_STATE_MPIR_COMM_SHRINK);
MPID_MPI_FUNC_ENTER(MPID_STATE_MPIR_COMM_SHRINK);
/* TODO - Implement this function for intercommunicators */
MPIR_Comm_group_impl(comm_ptr, &comm_grp);
do {
mpi_errno = MPID_Comm_get_all_failed_procs(comm_ptr, &global_failed, MPIR_SHRINK_TAG);
/* Ignore the mpi_errno value here as it will definitely communicate
* with failed procs */
mpi_errno = MPIR_Group_difference_impl(comm_grp, global_failed, &new_group_ptr);
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
if (MPID_Group_empty != global_failed) MPIR_Group_release(global_failed);
mpi_errno = MPIR_Comm_create_group(comm_ptr, new_group_ptr, MPIR_SHRINK_TAG, newcomm_ptr);
errflag = mpi_errno || *newcomm_ptr == NULL;
mpi_errno = MPIR_Allreduce_group(MPI_IN_PLACE, &errflag, 1, MPI_INT, MPI_MAX, comm_ptr,
new_group_ptr, MPIR_SHRINK_TAG, &tmp_errflag);
MPIR_Group_release(new_group_ptr);
if (errflag) MPIU_Object_set_ref(new_group_ptr, 0);
} while (errflag && ++attempts < 5);
if (errflag && attempts >= 5) goto fn_fail;
else mpi_errno = MPI_SUCCESS;
fn_exit:
MPIR_Group_release(comm_grp);
return mpi_errno;
fn_fail:
if (*newcomm_ptr) MPIU_Object_set_ref(*newcomm_ptr, 0);
MPIU_Object_set_ref(global_failed, 0);
MPIU_Object_set_ref(new_group_ptr, 0);
goto fn_exit;
}
#undef FUNCNAME
#define FUNCNAME MPIX_Comm_shrink
#undef FCNAME
#define FCNAME MPIU_QUOTE(FUNCNAME)
/*@
MPIX_Comm_shrink - Creates a new communitor from an existing communicator while
excluding failed processes
Input Parameters:
+ comm - communicator (handle)
Output Parameters:
. newcomm - new communicator (handle)
.N Threadsafe
.N Fortran
.N Errors
.N MPI_SUCCESS
.N MPI_ERR_COMM
@*/
int MPIX_Comm_shrink(MPI_Comm comm, MPI_Comm *newcomm)
{
int mpi_errno = MPI_SUCCESS;
MPID_Comm *comm_ptr = NULL, *newcomm_ptr;
MPID_MPI_STATE_DECL(MPID_STATE_MPIX_COMM_SHRINK);
MPIR_ERRTEST_INITIALIZED_ORDIE();
MPIU_THREAD_CS_ENTER(ALLFUNC,);
MPID_MPI_FUNC_ENTER(MPID_STATE_MPIX_COMM_SHRINK);
/* Validate parameters, and convert MPI object handles to object pointers */
# ifdef HAVE_ERROR_CHECKING
{
MPID_BEGIN_ERROR_CHECKS;
{
MPIR_ERRTEST_COMM(comm, mpi_errno);
}
MPID_END_ERROR_CHECKS;
MPID_Comm_get_ptr( comm, comm_ptr );
MPID_BEGIN_ERROR_CHECKS;
{
/* Validate comm_ptr */
MPID_Comm_valid_ptr( comm_ptr, mpi_errno );
if (MPIX_ERR_REVOKED != MPIR_ERR_GET_CLASS(mpi_errno) && mpi_errno)
goto fn_fail;
}
MPID_END_ERROR_CHECKS;
}
#else
{
MPID_Comm_get_ptr( comm, comm_ptr );
}
#endif
/* ... body of routine ... */
mpi_errno = MPIR_Comm_shrink(comm_ptr, &newcomm_ptr);
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
if (newcomm_ptr)
MPIU_OBJ_PUBLISH_HANDLE(*newcomm, newcomm_ptr->handle);
else
*newcomm = MPI_COMM_NULL;
/* ... end of body of routine ... */
fn_exit:
MPID_MPI_FUNC_EXIT(MPID_STATE_MPIX_COMM_SHRINK);
MPIU_THREAD_CS_EXIT(ALLFUNC,);
return mpi_errno;
fn_fail:
/* --BEGIN ERROR HANDLING-- */
#ifdef HAVE_ERROR_CHECKING
{
mpi_errno =
MPIR_Err_create_code(mpi_errno, MPIR_ERR_RECOVERABLE, FCNAME, __LINE__,
MPI_ERR_OTHER, "**mpix_comm_shrink",
"**mpix_comm_shrink %C %p", comm, newcomm);
}
#endif
mpi_errno = MPIR_Err_return_comm(comm_ptr, FCNAME, mpi_errno);
goto fn_exit;
/* --END ERROR HANDLING-- */
}
......@@ -1103,6 +1103,8 @@ is too big (> MPIU_SHMW_GHND_SZ)
**mpix_comm_failure_get_acked %C %p:MPIX_Comm_failure_get_acked(%C, group=%p) failed
**mpix_comm_revoke:MPIX_Comm_revoke failed
**mpix_comm_revoke %C:MPIX_Comm_revoke(%C) failed
**mpix_comm_shrink:MPIX_Comm_shrink failed
**mpix_comm_shrink %C %p:MPIX_Comm_shrink(%C, new_comm=%p) failed
**mpi_intercomm_create:MPI_Intercomm_create failed
**mpi_intercomm_create %C %d %C %d %d %p:MPI_Intercomm_create(%C, local_leader=%d, %C, remote_leader=%d, tag=%d, newintercomm=%p) failed
**mpi_intercomm_merge:MPI_Intercomm_merge failed
......
......@@ -31,6 +31,7 @@ mpi_core_sources += \
src/mpid/ch3/src/mpid_comm_disconnect.c \
src/mpid/ch3/src/mpid_comm_spawn_multiple.c \
src/mpid/ch3/src/mpid_comm_failure_ack.c \
src/mpid/ch3/src/mpid_comm_get_all_failed_procs.c \
src/mpid/ch3/src/mpid_comm_revoke.c \
src/mpid/ch3/src/mpid_finalize.c \
src/mpid/ch3/src/mpid_get_universe_size.c \
......
......@@ -941,11 +941,13 @@ int MPIDI_CH3U_Clean_recvq(MPID_Comm *comm_ptr)
match.parts.context_id = comm_ptr->recvcontext_id + MPID_CONTEXT_INTRA_COLL;
if (MATCH_WITH_LEFT_RIGHT_MASK(rreq->dev.match, match, mask)) {
MPIU_DBG_MSG_FMT(CH3_OTHER,VERBOSE,(MPIU_DBG_FDEST,
"cleaning up unexpected collective pkt rank=%d tag=%d contextid=%d",
rreq->dev.match.parts.rank, rreq->dev.match.parts.tag, rreq->dev.match.parts.context_id));
dequeue_and_set_error(&rreq, prev_rreq, &recvq_unexpected_head, &recvq_unexpected_tail, &error, MPI_PROC_NULL);
continue;
if (rreq->dev.match.parts.tag != MPIR_SHRINK_TAG) {
MPIU_DBG_MSG_FMT(CH3_OTHER,VERBOSE,(MPIU_DBG_FDEST,
"cleaning up unexpected collective pkt rank=%d tag=%d contextid=%d",
rreq->dev.match.parts.rank, rreq->dev.match.parts.tag, rreq->dev.match.parts.context_id));
dequeue_and_set_error(&rreq, prev_rreq, &recvq_unexpected_head, &recvq_unexpected_tail, &error, MPI_PROC_NULL);
continue;
}
}
prev_rreq = rreq;
......@@ -971,11 +973,13 @@ int MPIDI_CH3U_Clean_recvq(MPID_Comm *comm_ptr)
match.parts.context_id = comm_ptr->recvcontext_id + MPID_CONTEXT_INTRA_COLL;
if (MATCH_WITH_LEFT_RIGHT_MASK(rreq->dev.match, match, mask)) {
MPIU_DBG_MSG_FMT(CH3_OTHER,VERBOSE,(MPIU_DBG_FDEST,
"cleaning up unexpected collective pkt rank=%d tag=%d contextid=%d",
rreq->dev.match.parts.rank, rreq->dev.match.parts.tag, rreq->dev.match.parts.context_id));
dequeue_and_set_error(&rreq, prev_rreq, &recvq_posted_head, &recvq_posted_tail, &error, MPI_PROC_NULL);
continue;
if (rreq->dev.match.parts.tag != MPIR_SHRINK_TAG) {
MPIU_DBG_MSG_FMT(CH3_OTHER,VERBOSE,(MPIU_DBG_FDEST,
"cleaning up unexpected collective pkt rank=%d tag=%d contextid=%d",
rreq->dev.match.parts.rank, rreq->dev.match.parts.tag, rreq->dev.match.parts.context_id));
dequeue_and_set_error(&rreq, prev_rreq, &recvq_posted_head, &recvq_posted_tail, &error, MPI_PROC_NULL);
continue;
}
}
prev_rreq = rreq;
......
......@@ -69,3 +69,77 @@ fn_exit:
fn_fail:
goto fn_exit;
}
#undef FUNCNAME
#define FUNCNAME MPID_Comm_failed_bitarray
#undef FCNAME
#define FCNAME MPIU_QUOTE(FUNCNAME)
int MPID_Comm_failed_bitarray(MPID_Comm *comm_ptr, uint32_t **bitarray, int acked)
{
int mpi_errno = MPI_SUCCESS;
int size, i;
uint32_t bit;
int *failed_procs, *group_procs;
MPID_Group *failed_group, *comm_group;
MPIDI_STATE_DECL(MPID_STATE_COMM_FAILED_BITARRAY);
MPIU_CHKLMEM_DECL(2);
MPIDI_FUNC_ENTER(MPID_STATE_COMM_FAILED_BITARRAY);
/* TODO - Fix this for intercommunicators */
size = comm_ptr->local_size;
/* We can fit sizeof(uint32_t) * 8 ranks in one uint64_t so divide the
* size by that */
/* This buffer will be handed back to the calling function so we use a
* "real" malloc here and expect the caller to free the buffer later. The
* other buffers in this function are temporary and will be automatically
* cleaned up at the end of the function. */
*bitarray = (uint32_t *) MPIU_Malloc(sizeof(uint32_t) * (size / (sizeof(uint32_t) * 8)+1));
if (!(*bitarray)) {
fprintf(stderr, "Could not allocate space for bitarray\n");
PMPI_Abort(MPI_COMM_WORLD, 1);
}
for (i = 0; i <= size/(sizeof(uint32_t)*8); i++) *bitarray[i] = 0;
mpi_errno = MPIDI_CH3U_Check_for_failed_procs();
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
if (acked)
MPIDI_CH3U_Get_failed_group(comm_ptr->ch.last_ack_rank, &failed_group);
else
MPIDI_CH3U_Get_failed_group(-2, &failed_group);
if (failed_group == MPID_Group_empty) goto fn_exit;
MPIU_CHKLMEM_MALLOC(group_procs, int *, sizeof(int)*failed_group->size, mpi_errno, "group_procs");
for (i = 0; i < failed_group->size; i++) group_procs[i] = i;
MPIU_CHKLMEM_MALLOC(failed_procs, int *, sizeof(int)*failed_group->size, mpi_errno, "failed_procs");
MPIR_Comm_group_impl(comm_ptr, &comm_group);
MPIR_Group_translate_ranks_impl(failed_group, failed_group->size, group_procs, comm_group, failed_procs);
/* The bits will actually be ordered in decending order rather than
* ascending. This is purely for readability since it makes no practical
* difference. So if the bits look like this:
*
* 10001100 01001000 00000000 00000001
*
* Then processes 1, 5, 6, 9, 12, and 32 have failed. */
for (i = 0; i < failed_group->size; i++) {
bit = 0x80000000;
bit >>= failed_procs[i] % (sizeof(uint32_t) * 8);
*bitarray[failed_procs[i] / (sizeof(uint32_t) * 8)] |= bit;
}
MPIR_Group_free_impl(comm_group);
fn_exit:
MPIU_CHKLMEM_FREEALL();
return mpi_errno;
fn_fail:
goto fn_exit;
}
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
/*
* (C) 2011 by Argonne National Laboratory.
* See COPYRIGHT in top-level directory.
*/
#include "mpidimpl.h"
#ifdef USE_PMI2_API
#include "pmi2.h"
#else
#include "pmi.h"
#endif
/* Generates a bitarray based on orig_comm where all procs in group are marked with 1 */
static int *group_to_bitarray(MPID_Group *group, MPID_Comm *orig_comm) {
uint32_t *bitarray, mask;
int bitarray_size = orig_comm->local_size / 8 + orig_comm->local_size % 8 ? 1 : 0;
int *group_ranks, *comm_ranks, i, index;
bitarray = (int *) MPIU_Malloc(sizeof(int) * bitarray_size);
if (group == MPID_Group_empty) {
for (i = 0; i < bitarray_size; i++) bitarray[i] = 0;
return bitarray;
}
group_ranks = (int *) MPIU_Malloc(sizeof(int) * group->size);
comm_ranks = (int *) MPIU_Malloc(sizeof(int) * group->size);
for (i = 0; i < group->size; i++) group_ranks[i] = i;
for (i = 0; i < bitarray_size; i++) bitarray[i] = 0;
MPIR_Group_translate_ranks_impl(group, group->size, group_ranks,
orig_comm->local_group, comm_ranks);
for (i = 0; i < group->size && comm_ranks[i] != MPI_UNDEFINED; i++) {
index = comm_ranks[i] / 32;
mask = 0x80000000 >> comm_ranks[i] % 32;
bitarray[index] |= mask;
}
MPIU_Free(group_ranks);
MPIU_Free(comm_ranks);
return bitarray;
}
/* Generates an MPID_Group from a bitarray */
static MPID_Group *bitarray_to_group(MPID_Comm *comm_ptr, int *bitarray)
{
MPID_Group *ret_group;
MPID_Group *comm_group;
UT_array *ranks_array;
int i, found = 0;
utarray_new(ranks_array, &ut_int_icd);
MPIR_Comm_group_impl(comm_ptr, &comm_group);
/* Converts the bitarray into a utarray */
for (i = 0; i < comm_ptr->local_size; i++) {
if (bitarray[i/32] & (0x80000000 >> i % 32)) {
utarray_push_back(ranks_array, &i);
found++;
}
}
if (found)
/* Converts the utarray into a group */
MPIR_Group_incl_impl(comm_group, found, ut_int_array(ranks_array), &ret_group);
else
ret_group = MPID_Group_empty;
utarray_free(ranks_array);
MPIR_Group_release(comm_group);
return ret_group;
}
#undef FUNCNAME
#define FUNCNAME MPID_Comm_get_all_failed_procs
#undef FCNAME
#define FCNAME MPIU_QUOTE(FUNCNAME)
int MPID_Comm_get_all_failed_procs(MPID_Comm *comm_ptr, MPID_Group **failed_group, int tag)
{
int mpi_errno = MPI_SUCCESS;
int errflag = 0;
int i, j;
int *bitarray, *remote_bitarray, bitarray_size;
MPID_Group *local_fail;
MPIDI_STATE_DECL(MPID_STATE_MPID_COMM_GET_ALL_FAILED_PROCS);
MPID_MPI_FUNC_ENTER(MPID_STATE_MPID_COMM_GET_ALL_FAILED_PROCS);
/* Kick the progress engine in case it's been a while so we get all the
* latest updates about failures */
MPIDI_CH3I_Progress(NULL, false);
/* Generate the list of failed processes */
MPIDI_CH3U_Check_for_failed_procs();
mpi_errno = MPIDI_CH3U_Get_failed_group(-2, &local_fail);
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
/* Generate a bitarray based on the list of failed procs */
bitarray = group_to_bitarray(local_fail, comm_ptr);
bitarray_size = comm_ptr->local_size / 8 + comm_ptr->local_size % 8 ? 1 : 0;
remote_bitarray = MPIU_Malloc(sizeof(int) * bitarray_size);
/* For now, this will be implemented as a star with rank 0 serving as
* the source */
if (comm_ptr->rank == 0) {
for (i = 1; i < comm_ptr->local_size; i++) {
/* Get everyone's list of failed processes to aggregate */
mpi_errno = MPIC_Recv(remote_bitarray, bitarray_size, MPI_INT,
i, tag, comm_ptr->handle, MPI_STATUS_IGNORE, &errflag);
if (mpi_errno) continue;
/* Combine the received bitarray with my own */
for (j = 0; j < bitarray_size; j++)
bitarray[j] |= remote_bitarray[j];
}
for (i = 1; i < comm_ptr->local_size; i++) {
/* Send the list to each rank to be processed locally */
mpi_errno = MPIC_Ssend(bitarray, bitarray_size, MPI_INT, i,
tag, comm_ptr->handle, &errflag);
if (mpi_errno) errflag = 1;
}
/* Convert the bitarray into a group */
*failed_group = bitarray_to_group(comm_ptr, bitarray);
} else {
/* Send my bitarray to rank 0 */
mpi_errno = MPIC_Ssend(bitarray, bitarray_size, MPI_INT, 0,
tag, comm_ptr->handle, &errflag);
if (mpi_errno) errflag = 1;
/* Get the resulting bitarray back from rank 0 */
mpi_errno = MPIC_Recv(remote_bitarray, bitarray_size, MPI_INT, 0,
tag, comm_ptr->handle, MPI_STATUS_IGNORE, &errflag);
if (mpi_errno) errflag = 1;
/* Convert the bitarray into a group */
*failed_group = bitarray_to_group(comm_ptr, remote_bitarray);
}
MPIU_Free(bitarray);
MPIU_Free(remote_bitarray);
fn_exit:
MPID_MPI_FUNC_EXIT(MPID_STATE_MPID_COMM_GET_ALL_FAILED_PROCS);
return mpi_errno;
fn_fail:
goto fn_exit;
}
......@@ -33,7 +33,8 @@ int MPID_Iprobe(int source, int tag, MPID_Comm *comm, int context_offset,
}
/* Check to make sure the communicator hasn't already been revoked */
if (comm->revoked) {
if (comm->revoked &&
MPIR_SHRINK_TAG != MPIR_TAG_MASK_ERROR_BIT(tag & ~MPIR_Process.tagged_coll_mask)) {
MPIU_ERR_SETANDJUMP(mpi_errno,MPIX_ERR_REVOKED,"**revoked");
}
......
......@@ -32,7 +32,8 @@ int MPID_Irecv(void * buf, int count, MPI_Datatype datatype, int rank, int tag,
}
/* Check to make sure the communicator hasn't already been revoked */
if (comm->revoked) {
if (comm->revoked &&
MPIR_SHRINK_TAG != MPIR_TAG_MASK_ERROR_BIT(tag & ~MPIR_Process.tagged_coll_mask)) {
MPIU_DBG_MSG(CH3_OTHER,VERBOSE,"Comm has been revoked. Returning from MPID_IRECV.");
MPIU_ERR_SETANDJUMP(mpi_errno,MPIX_ERR_REVOKED,"**revoked");
}
......
......@@ -39,7 +39,8 @@ int MPID_Irsend(const void * buf, int count, MPI_Datatype datatype, int rank, in
rank, tag, comm->context_id + context_offset));
/* Check to make sure the communicator hasn't already been revoked */
if (comm->revoked) {
if (comm->revoked &&
MPIR_SHRINK_TAG != MPIR_TAG_MASK_ERROR_BIT(tag & ~MPIR_Process.tagged_coll_mask)) {
MPIU_ERR_SETANDJUMP(mpi_errno,MPIX_ERR_REVOKED,"**revoked");
}
......
......@@ -51,7 +51,8 @@ int MPID_Isend(const void * buf, int count, MPI_Datatype datatype, int rank,
rank, tag, comm->context_id + context_offset));
/* Check to make sure the communicator hasn't already been revoked */
if (comm->revoked) {
if (comm->revoked &&
MPIR_SHRINK_TAG != MPIR_TAG_MASK_ERROR_BIT(tag & ~MPIR_Process.tagged_coll_mask)) {
MPIU_DBG_MSG(CH3_OTHER,VERBOSE,"Communicator revoked. MPID_ISEND returning");
MPIU_ERR_SETANDJUMP(mpi_errno,MPIX_ERR_REVOKED,"**revoked");
}
......
......@@ -38,7 +38,8 @@ int MPID_Issend(const void * buf, int count, MPI_Datatype datatype, int rank, in
rank, tag, comm->context_id + context_offset));
/* Check to make sure the communicator hasn't already been revoked */