Commit ac07f982 authored by Lena Oden's avatar Lena Oden Committed by Kenneth Raffenetti
Browse files

Handling of discard connection to avoid reconnect



The loser of a head-to-head connection sometimes tries
to reconnect later, afer MPI_Finalize was called  This
can lead to several errors in the socket layer, depending
on the state of the disarded connection and the appereance
of the connection events. Refs #2180
This Patch has two ways to handle this:

1.)
Discarded connections are marked with CONN_STATE_DISCARD,
so they are hold from connection.  Furthermore, an error on
any discarded connection (because the remote side closed in
MPI_Finalize) is ignored and the connection is closed.

2.)
Add a finalize flag for process groups. If a process group is
closing and tries to close all VCs, a flag is set to mark this.
If the flag is set, a reconnection (in the socket state) is
refused and the connection is closed on both sided.

Both steps are necessary to catch all reconnection tries after
MPI_Finalize was called.
Signed-off-by: Kenneth Raffenetti's avatarKen Raffenetti <raffenet@mcs.anl.gov>
parent 394d46b7
......@@ -458,6 +458,14 @@ static int MPIDI_CH3I_Progress_handle_sock_event(MPIDU_Sock_event_t * event)
{
MPIDI_CH3I_Connection_t * conn =
(MPIDI_CH3I_Connection_t *) event->user_ptr;
/* If we have a READ event on a discarded connection, we probably have
an error on this connection, if the remote side is closed due to
MPI_Finalize. Since the connection is discareded (and therefore not needed)
it can be closed and the error can be ignored */
if(conn->state == CONN_STATE_DISCARD){
MPIDI_CH3_Sockconn_handle_close_event(conn);
break;
}
MPID_Request * rreq = conn->recv_active;
......
......@@ -92,6 +92,13 @@ typedef struct MPIDI_PG
find a particular process group. */
void * id;
/* Flag to mark a procress group which is finalizing. This means thay
the VCs for this process group are closing, (normally becuase
MPI_Finalize was called). This is required to avoid a reconnection
of the VCs when the PG is closed due to unused elements in the event
queue */
int finalize;
/* Replacement abstraction for connection information */
/* Connection information needed to access processes in this process
group and to share the data with other processes. The items are
......
......@@ -200,6 +200,7 @@ int MPIDI_PG_Create(int vct_sz, void * pg_id, MPIDI_PG_t ** pg_ptr)
MPIU_Object_set_ref(pg, 0);
pg->size = vct_sz;
pg->id = pg_id;
pg->finalize = 0;
/* Initialize the connection information to null. Use
the appropriate MPIDI_PG_InitConnXXX routine to set up these
fields */
......@@ -1216,6 +1217,7 @@ int MPIDI_PG_Close_VCs( void )
MPIDI_VC_GetStateString(vc->state)));
}
}
pg->finalize = 1;
pg = pg->next;
}
/* Note that we do not free the process groups within this routine, even
......
......@@ -598,11 +598,13 @@ int MPIDI_CH3_Sockconn_handle_connect_event( MPIDI_CH3I_Connection_t *conn,
}
/* --END ERROR HANDLING-- */
if (conn->state == CONN_STATE_CONNECTING) {
if (conn->state == CONN_STATE_CONNECTING || conn->state == CONN_STATE_DISCARD) {
MPIDI_CH3I_Pkt_sc_open_req_t *openpkt =
(MPIDI_CH3I_Pkt_sc_open_req_t *)&conn->pkt.type;
MPIU_DBG_CONNSTATECHANGE(conn->vc,conn,CONN_STATE_OPEN_CSEND);
conn->state = CONN_STATE_OPEN_CSEND;
if(conn->state == CONN_STATE_CONNECTING){
MPIU_DBG_CONNSTATECHANGE(conn->vc,conn,CONN_STATE_OPEN_CSEND);
conn->state = CONN_STATE_OPEN_CSEND;
}
MPIDI_Pkt_init(openpkt, MPIDI_CH3I_PKT_SC_OPEN_REQ);
openpkt->pg_id_len = (int) strlen(MPIDI_Process.my_pg->id) + 1;
openpkt->pg_rank = MPIR_Process.comm_world->rank;
......@@ -688,6 +690,16 @@ int MPIDI_CH3_Sockconn_handle_close_event( MPIDI_CH3I_Connection_t * conn )
not be referenced anymore in any case. */
conn->vc = NULL;
}
else if(conn->state == CONN_STATE_DISCARD) {
/* post close, so the socket is closed and memmory leaks are avoided */
MPIU_DBG_MSG(CH3_DISCONNECT,TYPICAL,"CLosing sock (Post_close)");
conn->state = CONN_STATE_CLOSING;
mpi_errno = MPIDU_Sock_post_close(conn->sock);
if (mpi_errno != MPI_SUCCESS) {
MPIU_ERR_POP(mpi_errno);
}
goto fn_exit;
}
else {
MPIU_Assert(conn->state == CONN_STATE_LISTENING);
MPIDI_CH3I_listener_conn = NULL;
......@@ -785,8 +797,9 @@ int MPIDI_CH3_Sockconn_handle_conn_event( MPIDI_CH3I_Connection_t * conn )
MPIDI_CH3I_Pkt_sc_open_resp_t *openpkt =
(MPIDI_CH3I_Pkt_sc_open_resp_t *)&conn->pkt.type;
/* FIXME: is this the correct assert? */
MPIU_Assert( conn->state == CONN_STATE_OPEN_CRECV );
if (openpkt->ack) {
if (openpkt->ack && conn->state != CONN_STATE_DISCARD) {
MPIU_Assert( conn->state == CONN_STATE_OPEN_CRECV );
MPIDI_CH3I_VC *vcch = &conn->vc->ch;
MPIU_DBG_CONNSTATECHANGE(conn->vc,conn,CONN_STATE_CONNECTED);
conn->state = CONN_STATE_CONNECTED;
......@@ -812,6 +825,11 @@ int MPIDI_CH3_Sockconn_handle_conn_event( MPIDI_CH3I_Connection_t * conn )
Why isn't it changed? Is there an assert here,
such as conn->vc->conn != conn (there is another connection
chosen for the vc)? */
/*Answer to FIXME */
/* Neither freed nor updated. This connection is the looser of
a head-to-head connection. The VC is still in use, but by
another sochekt connection. The refcount is not incremented
By chaning the assosiated connection. */
/* MPIU_Assert( conn->vc->ch.conn != conn ); */
/* Set the candidate vc for this connection to NULL (we
are discarding this connection because (I think) we
......@@ -824,6 +842,11 @@ int MPIDI_CH3_Sockconn_handle_conn_event( MPIDI_CH3I_Connection_t * conn )
conn->vc = NULL;
conn->state = CONN_STATE_CLOSING;
/* FIXME: What does post close do here? */
/* Answer to FIXME: */
/* Since the connection is discarded, the socket is
no longer needed and should be closed. This is initiated with the post
close command. This also caused that the socket is removed from the
socket set, so no more polling on this socket*/
MPIU_DBG_MSG(CH3_DISCONNECT,TYPICAL,"CLosing sock (Post_close)");
mpi_errno = MPIDU_Sock_post_close(conn->sock);
if (mpi_errno != MPI_SUCCESS) {
......@@ -881,6 +904,18 @@ int MPIDI_CH3_Sockconn_handle_connopen_event( MPIDI_CH3I_Connection_t * conn )
MPIDI_PG_Get_vc_set_active(pg, pg_rank, &vc);
MPIU_Assert(vc->pg_rank == pg_rank);
if(pg->finalize == 1) {
MPIDI_Pkt_init(openresp, MPIDI_CH3I_PKT_SC_OPEN_RESP);
openresp->ack = FALSE;
MPIU_DBG_CONNSTATECHANGE(conn->vc,conn,CONN_STATE_OPEN_LSEND);
conn->state = CONN_STATE_OPEN_LSEND;
mpi_errno = connection_post_send_pkt(conn);
if (mpi_errno != MPI_SUCCESS) {
MPIU_ERR_SETANDJUMP(mpi_errno,MPI_ERR_INTERN,
"**ch3|sock|open_lrecv_data");
}
goto fn_exit;
}
vcch = &vc->ch;
if (vcch->conn == NULL) {
/* no head-to-head connects, accept the connection */
......@@ -902,6 +937,11 @@ int MPIDI_CH3_Sockconn_handle_connopen_event( MPIDI_CH3I_Connection_t * conn )
MPIU_DBG_MSG_FMT(CH3_CONNECT,TYPICAL,(MPIU_DBG_FDEST,
"vc=%p,conn=%p:Accept head-to-head connection (my process group), discarding vcch->conn=%p",vc,conn, vcch->conn));
/* mark old connection */
MPIDI_CH3I_Connection_t *old_conn = vcch->conn;
MPIU_DBG_CONNSTATECHANGE(old_conn,old_conn,CONN_STATE_DISCARD);
old_conn->state = CONN_STATE_DISCARD;
/* accept connection */
MPIU_DBG_VCCHSTATECHANGE(vc,VC_STATE_CONNECTING);
vcch->state = MPIDI_CH3I_VC_STATE_CONNECTING;
......@@ -926,6 +966,10 @@ int MPIDI_CH3_Sockconn_handle_connopen_event( MPIDI_CH3I_Connection_t * conn )
if (strcmp(MPIDI_Process.my_pg->id, pg->id) < 0) {
MPIU_DBG_MSG_FMT(CH3_CONNECT,TYPICAL,(MPIU_DBG_FDEST,
"vc=%p,conn=%p:Accept head-to-head connection (two process groups), discarding vcch->conn=%p",vc,conn, vcch->conn));
/* mark old connection */
MPIDI_CH3I_Connection_t *old_conn = vcch->conn;
MPIU_DBG_CONNSTATECHANGE(old_conn,old_conn,CONN_STATE_DISCARD);
old_conn->state = CONN_STATE_DISCARD;
/* accept connection */
MPIU_DBG_VCCHSTATECHANGE(vc,VC_STATE_CONNECTING);
vcch->state = MPIDI_CH3I_VC_STATE_CONNECTING;
......@@ -973,11 +1017,13 @@ int MPIDI_CH3_Sockconn_handle_connwrite( MPIDI_CH3I_Connection_t * conn )
MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_SOCKCONN_HANDLE_CONNWRITE);
if (conn->state == CONN_STATE_OPEN_CSEND) {
if (conn->state == CONN_STATE_OPEN_CSEND || conn->state == CONN_STATE_DISCARD) {
/* finished sending open request packet */
/* post receive for open response packet */
MPIU_DBG_CONNSTATECHANGE(conn->vc,conn,CONN_STATE_OPEN_CRECV);
conn->state = CONN_STATE_OPEN_CRECV;
if(conn->state == CONN_STATE_OPEN_CSEND){
MPIU_DBG_CONNSTATECHANGE(conn->vc,conn,CONN_STATE_OPEN_CRECV);
conn->state = CONN_STATE_OPEN_CRECV;
}
mpi_errno = connection_post_recv_pkt(conn);
if (mpi_errno != MPI_SUCCESS) {
MPIU_ERR_POP(mpi_errno);
......
......@@ -119,6 +119,7 @@ const char * MPIDI_Conn_GetStateString(int state)
case CONN_STATE_CONNECTED: name = "CONN_STATE_CONNECTED"; break;
case CONN_STATE_CLOSING: name = "CONN_STATE_CLOSING"; break;
case CONN_STATE_CLOSED: name = "CONN_STATE_CLOSED"; break;
case CONN_STATE_DISCARD: name = "CONN_STATE_DISCARD"; break;
case CONN_STATE_FAILED: name = "CONN_STATE_FAILE"; break;
}
......
......@@ -25,6 +25,7 @@ typedef enum MPIDI_CH3I_Conn_state
CONN_STATE_CONNECTED,
CONN_STATE_CLOSING,
CONN_STATE_CLOSED,
CONN_STATE_DISCARD,
CONN_STATE_FAILED
} MPIDI_CH3I_Conn_state;
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment