Close remainig conns before sockset is destroyed

Loser of Head-to-Head connections are not necessarily
closed, if the sock set is destroyed. This patch
looks for all open connections, close the socket
and free the memory recourses. Fixes #2180
Signed-off-by: Ken Raffenetti <raffenet@mcs.anl.gov>

src/mpid/ch3/channels/sock/src/ch3_progress.c

@@ -375,6 +375,8 @@ int MPIDI_CH3I_Progress_init(void)
 int MPIDI_CH3I_Progress_finalize(void)
 {
     int mpi_errno;
+    MPIDI_CH3I_Connection_t *conn = NULL;
+
     MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3I_PROGRESS_FINALIZE);
 
     MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3I_PROGRESS_FINALIZE);
@@ -383,8 +385,16 @@ int MPIDI_CH3I_Progress_finalize(void)
     mpi_errno = MPIDU_CH3I_ShutdownListener();
     if (mpi_errno != MPI_SUCCESS) { MPIU_ERR_POP(mpi_errno); }
     
-    /* FIXME: Cleanly shutdown other socks and free connection structures. 
-       (close protocol?) */
+
+    /* Close open connections */
+    MPIDU_Sock_close_open_sockets(MPIDI_CH3I_sock_set,(void**) &conn);
+    while (conn != NULL) {
+        conn->state = CONN_STATE_CLOSING;
+        mpi_errno = MPIDI_CH3_Sockconn_handle_close_event(conn);
+	if (mpi_errno) { MPIU_ERR_POP(mpi_errno); }
+        MPIDU_Sock_close_open_sockets(MPIDI_CH3I_sock_set,(void**) &conn);
+    }
+
 
 
     /*

src/mpid/common/sock/mpidu_sock.h

@@ -228,6 +228,33 @@ Utility-Sock
 int MPIDU_Sock_create_set(MPIDU_Sock_set_t * set);
 
 
+/*@
+MPIDU_Sock_close_open_sockets - close the first open sockets of a sock_element
+
+Input Parameter:
+. set - set to be considered
+
+Output Parameter:
+. user_ptr - pointer to the user pointer pointer of a socket.
+
+Return value: a MPI error code with a Sock extended error class
++ MPI_SUCCESS - sock set successfully destroyed
+. MPIDU_SOCK_ERR_INIT - Sock module not initialized
+. MPIDU_SOCK_ERR_BAD_SET - invalid sock set
+. MPIDU_SOCK_ERR_NOMEM - unable to allocate required memory
+- MPIDU_SOCK_ERR_FAIL - unable to destroy the sock set (<BRT> because it still contained active sock objects?)
+
+
+Notes:
+This function only closes the first open socket of a sock_set and returns the
+user pointer of the sock-info structure. To close all sockets, the function must
+be called repeatedly, untiluser_ptr == NULL. The reason for this is
+that the overlying protocoll may need the user_ptr for further cleanup.
+
+@*/
+int MPIDU_Sock_close_open_sockets(struct MPIDU_Sock_set * sock_set, void** user_ptr );
+
+
 /*@
 MPIDU_Sock_destroy_set - destroy an existing sock set, releasing an internal resource associated with that set
 

src/mpid/common/sock/poll/sock_set.i

@@ -170,6 +170,38 @@ int MPIDU_Sock_create_set(struct MPIDU_Sock_set ** sock_setp)
     /* --END ERROR HANDLING-- */
 }
 
+#undef FUNCNAME
+#define FUNCNAME MPIDU_Sock_close_open_sockets
+#undef FCNAME
+#define FCNAME MPIU_QUOTE(FUNCNAME)
+int MPIDU_Sock_close_open_sockets(struct MPIDU_Sock_set * sock_set, void** user_ptr ){
+
+    int i;
+    int mpi_errno = MPI_SUCCESS;
+    struct pollinfo * pollinfos = NULL;
+    pollinfos = sock_set->pollinfos;
+    MPIDI_STATE_DECL(MPID_STATE_MPIDU_SOCK_CLOSE_OPEN_SOCKETS);
+
+    MPIDI_FUNC_ENTER(MPID_STATE_MPIDU_SOCK_CLOSE_OPEN_SOCKETS);
+
+    MPIDU_SOCKI_VERIFY_INIT(mpi_errno, fn_exit);
+    /* wakeup waiting socket if mullti-threades */
+    *user_ptr = NULL;
+    for (i = 0; i < sock_set->poll_array_elems; i++) {
+        if(pollinfos[i].sock != NULL &&  pollinfos[i].type != MPIDU_SOCKI_TYPE_INTERRUPTER){
+             close(pollinfos[i].fd);
+             MPIDU_Socki_sock_free(pollinfos[i].sock);
+            *user_ptr = pollinfos[i].user_ptr;
+             break;
+        }
+    }
+#ifdef USE_SOCK_VERIFY
+  fn_exit:
+#endif
+    MPIDI_FUNC_EXIT(MPID_STATE_MPIDU_SOCK_CLOSE_OPEN_SOCKETS);
+    return mpi_errno;
+}
+
 
 #undef FUNCNAME
 #define FUNCNAME MPIDU_Sock_destroy_set