Commit c9aa03bb authored by Darius Buntinas's avatar Darius Buntinas
Browse files

[svn-r7186] Fixes to allow job to continue with a node failure. VCs...

[svn-r7186] Fixes to allow job to continue with a node failure.  VCs associated with the failed process enter a failed state, and cannot be used for communication, but communication is still allowed with other processes.  Reviewed by goodell@.
parent d69acce2
......@@ -880,6 +880,9 @@ is too big (> MPIU_SHMW_GHND_SZ)
**envvarparse:Unable to parse environment variable
**envvarparse %s:Unable to parse environment variable (%s)
**signal:signal() failed
**signal %s:signal() failed: %s
#
# mpi functions
#
......
......@@ -79,6 +79,10 @@ if test "$enable_fast" = "yes" ; then
AC_DEFINE(ENABLE_NO_YIELD,1,[Define to remove yield()s in polling loops])
fi
# make sure we support signal
AC_CHECK_HEADERS(signal.h)
AC_CHECK_FUNCS(signal)
# Check for gm, mx, elan, psm options
PAC_SET_HEADER_LIB_PATH(gm)
PAC_SET_HEADER_LIB_PATH(mx)
......
......@@ -170,7 +170,7 @@ typedef union MPIDI_CH3_nem_pkt
{ \
MPIU_Object_set_ref(_rts_req, 0); \
MPIDI_CH3_Request_destroy(_rts_req); \
MPIU_ERR_SETFATALANDJUMP(mpi_errno, MPI_ERR_OTHER, "**rtspkt"); \
MPIU_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**rtspkt"); \
} \
/* --END ERROR HANDLING-- */ \
if (_rts_req != NULL) \
......@@ -179,10 +179,9 @@ typedef union MPIDI_CH3_nem_pkt
{ \
MPIU_Object_set_ref(_rts_req, 0); \
MPIDI_CH3_Request_destroy(_rts_req); \
mpi_errno = MPIR_Err_create_code(_rts_req->status.MPI_ERROR, MPIR_ERR_FATAL, \
FCNAME, __LINE__, MPI_ERR_OTHER, "**rtspkt", 0); \
mpi_errno = _rts_req->status.MPI_ERROR; \
MPID_Request_release(_rts_req); \
goto fn_exit; \
MPIU_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**rtspkt"); \
} \
MPID_Request_release(_rts_req); \
} \
......
......@@ -797,9 +797,15 @@ int MPID_nem_tcp_connect(struct MPIDI_VC *const vc)
MPIU_Assert(vc != NULL);
/* Handle error case */
if (vc_tcp->state == MPID_NEM_TCP_VC_STATE_ERROR ||
vc->state == MPIDI_VC_STATE_MORIBUND) {
MPIU_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**vc_in_error_state");
}
/* We have an active connection, start polling more often */
MPID_nem_tcp_skip_polls = MAX_SKIP_POLLS_ACTIVE;
MPIDI_CHANGE_VC_STATE(vc, ACTIVE);
if (vc_tcp->state == MPID_NEM_TCP_VC_STATE_DISCONNECTED) {
......@@ -925,7 +931,8 @@ int MPID_nem_tcp_connect(struct MPIDI_VC *const vc)
resolution. */
}
else {
MPIU_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**vc_in_error_state");
/* We already handled the error case at the top of the routine. */
MPIU_Assertp(0);
}
fn_exit:
......@@ -1498,7 +1505,7 @@ static int MPID_nem_tcp_recv_handler (struct pollfd *pfd, sockconn_t *const sc)
the other side performs a tcp close() before we do and we
blow up here. */
MPIU_DBG_MSG(NEM_SOCK_DET, VERBOSE, "other side closed, but we're shutting down, closing sc");
mpi_errno = cleanup_and_free_sc_plfd(sc); /* QUIESCENT */
mpi_errno = MPID_nem_tcp_cleanup_on_error(sc_vc);
goto fn_exit;
}
else
......
......@@ -8,6 +8,9 @@
#include <sys/types.h>
#include <sys/socket.h>
#include <arpa/inet.h>
#ifdef HAVE_SIGNAL_H
#include <signal.h>
#endif
#define DBG_IFNAME 0
......@@ -99,7 +102,25 @@ int MPID_nem_tcp_init (MPIDI_PG_t *pg_p, int pg_rank, char **bc_val_p, int *val_
mpi_errno = MPID_nem_tcp_send_init();
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
#ifdef HAVE_SIGNAL
{
/* In order to be able to handle socket errors on our own, we need
to ignore SIGPIPE. This may cause problems for programs that
intend to handle SIGPIPE or count on being killed, but I expect
such programs are very rare, and I'm not sure what the best
solution would be anyway. */
void *ret;
ret = signal(SIGPIPE, SIG_IGN);
MPIU_ERR_CHKANDJUMP1(ret == SIG_ERR, mpi_errno, MPI_ERR_OTHER, "**signal", "**signal %s", MPIU_Strerror(errno));
if (ret != SIG_DFL && ret != SIG_IGN) {
/* The app has set its own signal handler. Replace the previous handler. */
ret = signal(SIGPIPE, ret);
MPIU_ERR_CHKANDJUMP1(ret == SIG_ERR, mpi_errno, MPI_ERR_OTHER, "**signal", "**signal %s", MPIU_Strerror(errno));
}
}
#endif
fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_TCP_INIT);
/* fprintf(stdout, FCNAME " Exit\n"); fflush(stdout); */
......
......@@ -17,7 +17,6 @@
int MPID_nem_finalize(void)
{
int mpi_errno = MPI_SUCCESS;
int pmi_errno;
MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_FINALIZE);
MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_FINALIZE);
......@@ -54,16 +53,6 @@ int MPID_nem_finalize(void)
#ifdef PAPI_MONITOR
my_papi_close();
#endif /*PAPI_MONITOR */
#ifdef USE_PMI2_API
/* FIXME: I removed this PMI_Barrier in PMI2. The barrier seems
not to be necessary, but I'm keeping this here until I'm
convinced that this isn't here to handle some timing-related
bug. DARIUS */
#else
pmi_errno = PMI_Barrier();
MPIU_ERR_CHKANDJUMP1 (pmi_errno != PMI_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**pmi_barrier", "**pmi_barrier %d", pmi_errno);
#endif
fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_FINALIZE);
......
......@@ -48,9 +48,7 @@ int MPIDI_CH3_SendNoncontig_iov( MPIDI_VC_t *vc, MPID_Request *sreq,
{
MPIU_Object_set_ref(sreq, 0);
MPIDI_CH3_Request_destroy(sreq);
mpi_errno = MPIR_Err_create_code(mpi_errno, MPIR_ERR_FATAL,
FCNAME, __LINE__, MPI_ERR_OTHER, "**ch3|eagermsg", 0);
goto fn_fail;
MPIU_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**ch3|eagermsg");
}
/* --END ERROR HANDLING-- */
......@@ -62,9 +60,7 @@ int MPIDI_CH3_SendNoncontig_iov( MPIDI_VC_t *vc, MPID_Request *sreq,
/* --BEGIN ERROR HANDLING-- */
MPIU_Object_set_ref(sreq, 0);
MPIDI_CH3_Request_destroy(sreq);
mpi_errno = MPIR_Err_create_code(mpi_errno, MPIR_ERR_RECOVERABLE,
FCNAME, __LINE__, MPI_ERR_OTHER, "**ch3|loadsendiov", 0);
goto fn_fail;
MPIU_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**ch3|loadsendiov");
/* --END ERROR HANDLING-- */
}
......@@ -187,7 +183,7 @@ int MPIDI_CH3_EagerContigSend( MPID_Request **sreq_p,
mpi_errno = MPIU_CALL(MPIDI_CH3,iStartMsgv(vc, iov, 2, sreq_p));
MPIU_THREAD_CS_EXIT(CH3COMM,vc);
if (mpi_errno != MPI_SUCCESS) {
MPIU_ERR_SETFATALANDJUMP(mpi_errno,MPI_ERR_OTHER,"**ch3|eagermsg");
MPIU_ERR_SETANDJUMP(mpi_errno,MPI_ERR_OTHER,"**ch3|eagermsg");
}
sreq = *sreq_p;
......@@ -263,7 +259,7 @@ int MPIDI_CH3_EagerContigShortSend( MPID_Request **sreq_p,
sizeof(*eagershort_pkt), sreq_p ));
MPIU_THREAD_CS_EXIT(CH3COMM,vc);
if (mpi_errno != MPI_SUCCESS) {
MPIU_ERR_SETFATALANDJUMP(mpi_errno,MPI_ERR_OTHER,"**ch3|eagermsg");
MPIU_ERR_SETANDJUMP(mpi_errno,MPI_ERR_OTHER,"**ch3|eagermsg");
}
sreq = *sreq_p;
if (sreq != NULL) {
......@@ -550,14 +546,14 @@ int MPIDI_CH3_EagerContigIsend( MPID_Request **sreq_p,
MPIU_Object_set_ref(sreq, 0);
MPIDI_CH3_Request_destroy(sreq);
*sreq_p = NULL;
mpi_errno = MPIR_Err_create_code(mpi_errno, MPIR_ERR_FATAL, FCNAME,
__LINE__, MPI_ERR_OTHER, "**ch3|eagermsg", 0);
goto fn_exit;
MPIU_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**ch3|eagermsg");
}
/* --END ERROR HANDLING-- */
fn_exit:
return mpi_errno;
fn_fail:
goto fn_exit;
}
/*
......
......@@ -82,8 +82,7 @@ int MPIDI_CH3_EagerSyncNoncontigSend( MPID_Request **sreq_p,
MPIU_Object_set_ref(sreq, 0);
MPIDI_CH3_Request_destroy(sreq);
*sreq_p = NULL;
mpi_errno = MPIR_Err_create_code(mpi_errno, MPIR_ERR_FATAL, FCNAME, __LINE__, MPI_ERR_OTHER, "**ch3|eagermsg", 0);
goto fn_fail;
MPIU_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**ch3|eagermsg");
}
/* --END ERROR HANDLING-- */
}
......@@ -155,13 +154,14 @@ int MPIDI_CH3_EagerSyncZero(MPID_Request **sreq_p, int rank, int tag,
MPIU_Object_set_ref(sreq, 0);
MPIDI_CH3_Request_destroy(sreq);
*sreq_p = NULL;
mpi_errno = MPIR_Err_create_code(mpi_errno, MPIR_ERR_FATAL, FCNAME, __LINE__, MPI_ERR_OTHER, "**ch3|eagermsg", 0);
goto fn_exit;
MPIU_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**ch3|eagermsg");
}
/* --END ERROR HANDLING-- */
fn_exit:
return mpi_errno;
fn_fail:
goto fn_exit;
}
/*
......
......@@ -220,10 +220,7 @@ int MPIDI_CH3U_VC_SendClose( MPIDI_VC_t *vc, int rank )
mpi_errno = MPIU_CALL(MPIDI_CH3,iStartMsg(vc, close_pkt,
sizeof(*close_pkt), &sreq));
if (mpi_errno != MPI_SUCCESS) {
MPIU_ERR_SET(mpi_errno,MPI_ERR_OTHER,
"**ch3|send_close_ack");
}
MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**ch3|send_close_ack");
if (sreq != NULL) {
/* There is still another reference being held by the channel. It
......@@ -231,14 +228,21 @@ int MPIDI_CH3U_VC_SendClose( MPIDI_VC_t *vc, int rank )
MPID_Request_release(sreq);
}
fn_exit:
MPIU_THREAD_CS_EXIT(CH3COMM,vc);
MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3U_VC_SENDCLOSE);
return mpi_errno;
fn_fail:
goto fn_exit;
}
/* Here is the matching code that processes a close packet when it is
received */
#undef FUNCNAME
#define FUNCNAME MPIDI_CH3_PktHandler_Close
#undef FCNAME
#define FCNAME MPIU_QUOTE(FUNCNAME)
int MPIDI_CH3_PktHandler_Close( MPIDI_VC_t *vc, MPIDI_CH3_Pkt_t *pkt,
MPIDI_msg_sz_t *buflen, MPID_Request **rreqp )
{
......@@ -258,10 +262,7 @@ int MPIDI_CH3_PktHandler_Close( MPIDI_VC_t *vc, MPIDI_CH3_Pkt_t *pkt,
vc->pg_rank);
mpi_errno = MPIU_CALL(MPIDI_CH3,iStartMsg(vc, resp_pkt,
sizeof(*resp_pkt), &resp_sreq));
if (mpi_errno != MPI_SUCCESS) {
MPIU_ERR_SETANDJUMP(mpi_errno,MPI_ERR_OTHER,
"**ch3|send_close_ack");
}
MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**ch3|send_close_ack");
if (resp_sreq != NULL)
{
......
......@@ -314,7 +314,7 @@ int MPIDI_CH3_ReqHandler_GetRespDerivedDTComplete( MPIDI_VC_t *vc,
MPIU_Object_set_ref(sreq, 0);
MPIDI_CH3_Request_destroy(sreq);
sreq = NULL;
MPIU_ERR_SETFATALANDJUMP(mpi_errno,MPI_ERR_OTHER,"**ch3|rmamsg");
MPIU_ERR_SETANDJUMP(mpi_errno,MPI_ERR_OTHER,"**ch3|rmamsg");
}
/* --END ERROR HANDLING-- */
......@@ -913,7 +913,7 @@ int MPIDI_CH3I_Send_pt_rma_done_pkt(MPIDI_VC_t *vc, MPI_Win source_win_handle)
sizeof(*pt_rma_done_pkt), &req));
/* MPIU_THREAD_CS_EXIT(CH3COMM,vc); */
if (mpi_errno != MPI_SUCCESS) {
MPIU_ERR_SETFATALANDJUMP(mpi_errno,MPI_ERR_OTHER,"**ch3|rmamsg");
MPIU_ERR_SETANDJUMP(mpi_errno,MPI_ERR_OTHER,"**ch3|rmamsg");
}
if (req != NULL)
......@@ -1023,7 +1023,7 @@ static int do_simple_get(MPID_Win *win_ptr, MPIDI_Win_lock_queue *lock_queue)
{
MPIU_Object_set_ref(req, 0);
MPIDI_CH3_Request_destroy(req);
MPIU_ERR_SETFATALANDJUMP(mpi_errno,MPI_ERR_OTHER,"**ch3|rmamsg");
MPIU_ERR_SETANDJUMP(mpi_errno,MPI_ERR_OTHER,"**ch3|rmamsg");
}
/* --END ERROR HANDLING-- */
......
......@@ -2374,7 +2374,7 @@ int MPIDI_CH3_PktHandler_Get( MPIDI_VC_t *vc, MPIDI_CH3_Pkt_t *pkt,
{
MPIU_Object_set_ref(req, 0);
MPIDI_CH3_Request_destroy(req);
MPIU_ERR_SETFATALANDJUMP(mpi_errno,MPI_ERR_OTHER,"**ch3|rmamsg");
MPIU_ERR_SETANDJUMP(mpi_errno,MPI_ERR_OTHER,"**ch3|rmamsg");
}
/* --END ERROR HANDLING-- */
......@@ -2853,7 +2853,7 @@ int MPIDI_CH3_PktHandler_LockGetUnlock( MPIDI_VC_t *vc, MPIDI_CH3_Pkt_t *pkt,
{
MPIU_Object_set_ref(req, 0);
MPIDI_CH3_Request_destroy(req);
MPIU_ERR_SETFATALANDJUMP(mpi_errno,MPI_ERR_OTHER,"**ch3|rmamsg");
MPIU_ERR_SETANDJUMP(mpi_errno,MPI_ERR_OTHER,"**ch3|rmamsg");
}
/* --END ERROR HANDLING-- */
}
......
......@@ -60,8 +60,7 @@ int MPIDI_CH3_RndvSend( MPID_Request **sreq_p, const void * buf, int count,
MPIU_Object_set_ref(sreq, 0);
MPIDI_CH3_Request_destroy(sreq);
*sreq_p = NULL;
mpi_errno = MPIR_Err_create_code(mpi_errno, MPIR_ERR_FATAL, FCNAME, __LINE__, MPI_ERR_OTHER, "**ch3|rtspkt", 0);
goto fn_exit;
MPIU_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**ch3|rtspkt");
}
/* --END ERROR HANDLING-- */
if (rts_sreq != NULL)
......@@ -71,9 +70,9 @@ int MPIDI_CH3_RndvSend( MPID_Request **sreq_p, const void * buf, int count,
MPIU_Object_set_ref(sreq, 0);
MPIDI_CH3_Request_destroy(sreq);
*sreq_p = NULL;
mpi_errno = MPIR_Err_create_code(rts_sreq->status.MPI_ERROR, MPIR_ERR_FATAL, FCNAME, __LINE__, MPI_ERR_OTHER, "**ch3|rtspkt", 0);
MPID_Request_release(rts_sreq);
goto fn_exit;
mpi_errno = rts_sreq->status.MPI_ERROR;
MPID_Request_release(rts_sreq);
MPIU_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**ch3|rtspkt");
}
MPID_Request_release(rts_sreq);
}
......@@ -85,8 +84,9 @@ int MPIDI_CH3_RndvSend( MPID_Request **sreq_p, const void * buf, int count,
engine, threads, etc.). */
fn_exit:
return mpi_errno;
fn_fail:
goto fn_exit;
}
/*
......
......@@ -93,7 +93,7 @@ int MPID_Irsend(const void * buf, int count, MPI_Datatype datatype, int rank, in
MPIU_Object_set_ref(sreq, 0);
MPIDI_CH3_Request_destroy(sreq);
sreq = NULL;
mpi_errno = MPIR_Err_create_code(mpi_errno, MPIR_ERR_FATAL, FCNAME, __LINE__, MPI_ERR_OTHER, "**ch3|eagermsg", 0);
MPIU_ERR_SET(mpi_errno, MPI_ERR_OTHER, "**ch3|eagermsg");
goto fn_exit;
}
/* --END ERROR HANDLING-- */
......
......@@ -112,8 +112,7 @@ int MPID_Isend(const void * buf, int count, MPI_Datatype datatype, int rank,
MPIU_Object_set_ref(sreq, 0);
MPIDI_CH3_Request_destroy(sreq);
sreq = NULL;
mpi_errno = MPIR_Err_create_code(mpi_errno, MPIR_ERR_FATAL, FCNAME,
__LINE__, MPI_ERR_OTHER, "**ch3|eagermsg", 0);
MPIU_ERR_SET(mpi_errno, MPI_ERR_OTHER, "**ch3|eagermsg");
goto fn_exit;
}
/* --END ERROR HANDLING-- */
......
......@@ -87,7 +87,7 @@ int MPID_Rsend(const void * buf, int count, MPI_Datatype datatype, int rank, int
/* --BEGIN ERROR HANDLING-- */
if (mpi_errno != MPI_SUCCESS)
{
mpi_errno = MPIR_Err_create_code(mpi_errno, MPIR_ERR_FATAL, FCNAME, __LINE__, MPI_ERR_OTHER, "**ch3|eagermsg", 0);
mpi_errno = MPIR_Err_create_code(mpi_errno, MPIR_ERR_RECOVERABLE, FCNAME, __LINE__, MPI_ERR_OTHER, "**ch3|eagermsg", 0);
goto fn_exit;
}
/* --END ERROR HANDLING-- */
......
......@@ -100,12 +100,7 @@ int MPID_Send(const void * buf, int count, MPI_Datatype datatype, int rank,
/* --BEGIN ERROR HANDLING-- */
if (mpi_errno != MPI_SUCCESS)
{
/* FIXME: this is a fatal error because a sequence number has
already been allocated. If sequence numbers are not
being used then this could be a recoverable error. A check
needs to be added that sets the error to fatal or
recoverable depending on the use of sequence numbers. */
MPIU_ERR_SETFATALANDJUMP(mpi_errno,MPI_ERR_OTHER,"**ch3|eagermsg");
MPIU_ERR_SETANDJUMP(mpi_errno,MPI_ERR_OTHER,"**ch3|eagermsg");
}
/* --END ERROR HANDLING-- */
if (sreq != NULL)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment