[svn-r4411] Fixed nemesis to correctly set vc state (instead of setting all...

[svn-r4411] Fixed nemesis to correctly set vc state (instead of setting all vcs to active.  Renamed MPIDI_Comm_get_vc to MPIDI_Comm_get_vc_set_active to alert caller to side-effect.  Added MPIDI_CHANGE_VC_STATE macro to set the vc state and call debugging macro.  Changed all places where vc state is changed to use this macro.  Reviewed by goodell@.

src/mpid/ch3/channels/nemesis/include/mpidi_ch3_impl.h

@@ -59,7 +59,6 @@ extern struct MPID_Request *MPIDI_CH3I_active_send[CH3_NUM_QUEUES];
 
 #define MPIDI_CH3I_SendQ_empty(queue) (MPIDI_CH3I_sendq_head[queue] == NULL)
 
-
 int MPIDI_CH3I_Progress_init(void);
 int MPIDI_CH3I_Progress_finalize(void);
 

src/mpid/ch3/channels/nemesis/nemesis/netmod/mx/mx_init.c

@@ -278,6 +278,8 @@ MPID_nem_mx_vc_init (MPIDI_VC_t *vc)
 			MPID_NEM_MX_FILTER,MX_INFINITE,&(VC_FIELD(vc, remote_endpoint_addr)));
        MPIU_ERR_CHKANDJUMP1 (ret != MX_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**mx_connect", "**mx_connect %s", mx_strerror (ret));
        mx_set_endpoint_addr_context(VC_FIELD(vc, remote_endpoint_addr),(void *)vc);
+
+       MPIDI_CHANGE_VC_STATE(vc, ACTIVE);
    }
 #endif
    mx_get_info(MPID_nem_mx_local_endpoint, MX_COPY_SEND_MAX, NULL, 0, &threshold, sizeof(uint32_t));

src/mpid/ch3/channels/nemesis/nemesis/netmod/mx/mx_poll.c

@@ -581,7 +581,7 @@ MPID_nem_mx_handle_rreq(MPID_Request *req, mx_status_t status)
     mx_status_t  status;
     uint32_t     result;
     
-    MPIDI_Comm_get_vc(req->comm, req->status.MPI_SOURCE, &vc);   
+    MPIDI_Comm_get_vc_set_active(req->comm, req->status.MPI_SOURCE, &vc);   
     mpi_errno = vc->pg->getConnInfo(vc->pg_rank, business_card, MPID_NEM_MAX_NETMOD_STRING_LEN, vc->pg);
     if (mpi_errno) MPIU_ERR_POP(mpi_errno);
     mpi_errno = MPID_nem_mx_get_from_bc (business_card, &VC_FIELD(vc, remote_endpoint_id), &VC_FIELD(vc, remote_nic_id));

src/mpid/ch3/channels/nemesis/nemesis/netmod/newmad/newmad_poll.c

@@ -348,7 +348,7 @@ MPID_nem_newmad_handle_rreq(MPID_Request *req, nm_sr_request_t *nmad_request, nm
 	MPIU_Free(REQ_FIELD(req,iov));
     }
     
-    MPIDI_Comm_get_vc(req->comm, req->status.MPI_SOURCE, &vc);
+    MPIDI_Comm_get_vc_set_active(req->comm, req->status.MPI_SOURCE, &vc);
     MPIDI_CH3U_Handle_recv_req(vc, req, &complete);
     MPIU_Assert(complete == TRUE);
  fn_exit:

src/mpid/ch3/channels/nemesis/nemesis/netmod/tcp/socksm.c

@@ -750,6 +750,8 @@ int MPID_nem_tcp_connect(struct MPIDI_VC *const vc)
 
     MPIU_Assert(vc != NULL);
 
+    MPIDI_CHANGE_VC_STATE(vc, ACTIVE);
+
     if (((MPIDI_CH3I_VC *)vc->channel_private)->state == MPID_NEM_TCP_VC_STATE_DISCONNECTED) {
         struct sockaddr_in *sock_addr;
 	struct in_addr addr;

src/mpid/ch3/channels/nemesis/nemesis/netmod/tcp/tcp_send.c

@@ -212,6 +212,8 @@ int MPID_nem_tcp_conn_est (MPIDI_VC_t *vc)
 
     MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_TCP_CONN_EST);
 
+    MPIDI_CHANGE_VC_STATE(vc, ACTIVE);
+
     if (!SENDQ_EMPTY (VC_FIELD(vc, send_queue)))
     {
         SET_PLFD(vc);

src/mpid/ch3/channels/nemesis/nemesis/netmod/wintcp/socksm.c

@@ -840,6 +840,8 @@ int MPID_nem_newtcp_module_connect (struct MPIDI_VC *const vc)
 
     MPIU_Assert(vc != NULL);
 
+    MPIDI_CHANGE_VC_STATE(vc, ACTIVE);
+
     if (((MPIDI_CH3I_VC *)vc->channel_private)->state == MPID_NEM_NEWTCP_MODULE_VC_STATE_DISCONNECTED) {
         struct sockaddr_in *sock_addr;
         struct in_addr addr;

src/mpid/ch3/channels/nemesis/nemesis/netmod/wintcp/wintcp_send.c

@@ -211,6 +211,8 @@ int MPID_nem_newtcp_module_conn_est (MPIDI_VC_t *vc)
 
     MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_NEWTCP_MODULE_CONN_EST);
 
+    MPIDI_CHANGE_VC_STATE(vc, ACTIVE);
+
     if (!SENDQ_EMPTY (VC_FIELD(vc, send_queue)))
     {
         mpi_errno = MPID_nem_newtcp_module_conn_wr_enable(vc);

src/mpid/ch3/channels/nemesis/nemesis/src/mpid_nem_init.c

@@ -421,6 +421,8 @@ MPID_nem_vc_init (MPIDI_VC_t *vc)
 
     if (vc_ch->is_local)
     {
+        MPIDI_CHANGE_VC_STATE(vc, ACTIVE);
+        
 	vc_ch->fbox_out = &MPID_nem_mem_region.mailboxes.out[MPID_nem_mem_region.local_ranks[vc->lpid]]->mpich2;
 	vc_ch->fbox_in = &MPID_nem_mem_region.mailboxes.in[MPID_nem_mem_region.local_ranks[vc->lpid]]->mpich2;
 	vc_ch->recv_queue = MPID_nem_mem_region.RecvQ[vc->lpid];

src/mpid/ch3/channels/nemesis/nemesis/src/mpid_nem_lmt.c

@@ -73,7 +73,7 @@ int MPID_nem_lmt_RndvSend(MPID_Request **sreq_p, const void * buf, int count, MP
 
     MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_LMT_RNDVSEND);
 
-    MPIDI_Comm_get_vc(comm, rank, &vc);
+    MPIDI_Comm_get_vc_set_active(comm, rank, &vc);
 
     /* if the lmt functions are not set, fall back to the default rendezvous code */
     if (((MPIDI_CH3I_VC *)vc->channel_private)->lmt_initiate_lmt == NULL)

src/mpid/ch3/channels/nemesis/src/ch3_init.c

@@ -55,10 +55,8 @@ int MPIDI_CH3_Init(int has_parent, MPIDI_PG_t *pg_p, int pg_rank)
 
     for (i = 0; i < pg_p->size; i++)
     {
-	MPIDI_VC_t *vc;
-	MPIDI_PG_Get_vc_set_active (pg_p, i, &vc);
-	mpi_errno = MPIDI_CH3_VC_Init (vc);
-        if (mpi_errno) MPIU_ERR_POP (mpi_errno);
+	mpi_errno = MPIDI_CH3_VC_Init(&pg_p->vct[i]);
+        if (mpi_errno) MPIU_ERR_POP(mpi_errno);
     }
 
     mpi_errno = MPID_nem_coll_barrier_init();
@@ -134,8 +132,6 @@ int MPIDI_CH3_VC_Init( MPIDI_VC_t *vc )
         goto fn_exit;
 
     ((MPIDI_CH3I_VC *)vc->channel_private)->recv_active = NULL;
-    MPIU_DBG_VCSTATECHANGE(vc,VC_STATE_ACTIVE);
-    vc->state = MPIDI_VC_STATE_ACTIVE;
 
     mpi_errno = MPID_nem_vc_init (vc);
     if (mpi_errno) MPIU_ERR_POP (mpi_errno);
@@ -198,8 +194,7 @@ int MPIDI_CH3_Connect_to_root (const char *port_name, MPIDI_VC_t **new_vc)
     /* init channel portion of vc */
     MPIU_ERR_CHKANDJUMP (!nemesis_initialized, mpi_errno, MPI_ERR_OTHER, "**intern");
     ((MPIDI_CH3I_VC *)vc->channel_private)->recv_active = NULL;
-    MPIU_DBG_VCSTATECHANGE(vc,VC_STATE_ACTIVE);
-    vc->state = MPIDI_VC_STATE_ACTIVE;
+    MPIDI_CHANGE_VC_STATE(vc, ACTIVE);
 
     *new_vc = vc; /* we now have a valid, disconnected, temp VC */
 

src/mpid/ch3/channels/nemesis/src/ch3_progress.c

@@ -819,7 +819,7 @@ void MPIDI_CH3I_Posted_recv_enqueued(MPID_Request *rreq)
         int local_rank = -1;
 	MPIDI_VC_t *vc;
 
-	MPIDI_Comm_get_vc((rreq)->comm, (rreq)->dev.match.parts.rank, &vc);
+	MPIDI_Comm_get_vc_set_active((rreq)->comm, (rreq)->dev.match.parts.rank, &vc);
 #ifdef ENABLE_COMM_OVERRIDES
         /* call vc-specific handler */
 	if (vc->comm_ops && vc->comm_ops->recv_posted)
@@ -873,7 +873,7 @@ int MPIDI_CH3I_Posted_recv_dequeued(MPID_Request *rreq)
             goto fn_exit;
         
         /* don't use MPID_NEM_IS_LOCAL, it doesn't handle dynamic processes */
-        MPIDI_Comm_get_vc(rreq->comm, rreq->dev.match.parts.rank, &vc);
+        MPIDI_Comm_get_vc_set_active(rreq->comm, rreq->dev.match.parts.rank, &vc);
         MPIU_Assert(vc != NULL);
         if (!((MPIDI_CH3I_VC *)vc->channel_private)->is_local)
             goto fn_exit;

src/mpid/ch3/channels/sctp/src/sctp_util.c

@@ -271,7 +271,7 @@ void inline MPIDU_Sctp_stream_init(MPIDI_VC_t* vc, MPID_Request* req, int stream
 
   /* need to update state for upcalls to close protocol to fully work, and barriers */
   if(vc->state == MPIDI_VC_STATE_INACTIVE)
-      vc->state = MPIDI_VC_STATE_ACTIVE;
+      MPIDI_CHANGE_VC_STATE(vc, ACTIVE);
 
   if(req) {
     MPIDI_CH3I_SendQ_enqueue_x(vc, req, stream);

src/mpid/ch3/channels/shm/src/ch3_init.c

@@ -91,7 +91,7 @@ int MPIDI_CH3_Init(int has_parent, MPIDI_PG_t * pg, int pg_rank )
 	MPIDI_CH3I_VC *vcch = (MPIDI_CH3I_VC *)pg->vct[p].channel_private;
 	/* FIXME: the vc's must be set to active for the close protocol to 
 	   work in the shm channel */
-	pg->vct[p].state = MPIDI_VC_STATE_ACTIVE;
+        MPIDI_CHANGE_VC_STATE(&(pg->vct[p]), ACTIVE);
 	/* FIXME: Should the malloc be within the init? */
 	MPIDI_CH3_VC_Init( &pg->vct[p] );
 	/* FIXME: Need to free this request when the vc is removed */

src/mpid/ch3/include/mpidimpl.h

@@ -47,6 +47,11 @@
 int gethostname(char *name, size_t len);
 # endif
 
+#define MPIDI_CHANGE_VC_STATE(vc, new_state) do {               \
+        MPIU_DBG_VCSTATECHANGE(vc, VC_STATE_##new_state);       \
+        (vc)->state = MPIDI_VC_STATE_##new_state;               \
+    } while (0)
+
 /*S
   MPIDI_PG_t - Process group description
 
@@ -496,15 +501,15 @@ extern MPIDI_Process_t MPIDI_Process;
 /*------------------
   BEGIN COMM SECTION
   ------------------*/
-#define MPIDI_Comm_get_vc(comm_, rank_, vcp_)		\
-{							\
-    *(vcp_) = (comm_)->vcr[(rank_)];			\
-    if ((*(vcp_))->state == MPIDI_VC_STATE_INACTIVE)	\
-    {							\
-	MPIU_DBG_PrintVCState2(*(vcp_), MPIDI_VC_STATE_ACTIVE);  \
-	(*(vcp_))->state = MPIDI_VC_STATE_ACTIVE;	\
-    }							\
-}
+#define MPIDI_Comm_get_vc_set_active(comm_, rank_, vcp_) do {           \
+        *(vcp_) = (comm_)->vcr[(rank_)];                                \
+        if ((*(vcp_))->state == MPIDI_VC_STATE_INACTIVE)                \
+        {                                                               \
+            MPIU_DBG_PrintVCState2(*(vcp_), MPIDI_VC_STATE_ACTIVE);     \
+            MPIDI_CHANGE_VC_STATE((*(vcp_)), ACTIVE);                   \
+        }                                                               \
+    } while(0)
+
 /*----------------
   END COMM SECTION
   ----------------*/
@@ -584,16 +589,15 @@ int MPIDI_CH3_PG_Init( MPIDI_PG_t * );
     MPIU_DBG_MSG_FMT(REFCOUNT,TYPICAL,(MPIU_DBG_FDEST,\
          "Decr process group %p ref count to %d",pg_,pg_->ref_count));\
 }
-/* FIXME: What is the difference between get_vcr and get_vc? */
-#define MPIDI_PG_Get_vc_set_active(pg_, rank_, vcp_)    \
-{							\
-    *(vcp_) = &(pg_)->vct[rank_];			\
-    if ((*(vcp_))->state == MPIDI_VC_STATE_INACTIVE)	\
-    {							\
-	MPIU_DBG_PrintVCState2(*(vcp_), MPIDI_VC_STATE_ACTIVE);  \
-	(*(vcp_))->state = MPIDI_VC_STATE_ACTIVE;	\
-    }							\
-}
+
+#define MPIDI_PG_Get_vc_set_active(pg_, rank_, vcp_)  do {              \
+        *(vcp_) = &(pg_)->vct[rank_];                                   \
+        if ((*(vcp_))->state == MPIDI_VC_STATE_INACTIVE)                \
+        {                                                               \
+            MPIU_DBG_PrintVCState2(*(vcp_), MPIDI_VC_STATE_ACTIVE);     \
+            MPIDI_CHANGE_VC_STATE((*(vcp_)), ACTIVE);                   \
+        }                                                               \
+    } while(0)
 
 #define MPIDI_PG_Get_size(pg_) ((pg_)->size)
 

src/mpid/ch3/src/ch3u_eager.c

@@ -110,7 +110,7 @@ int MPIDI_CH3_EagerNoncontigSend( MPID_Request **sreq_p,
     eager_pkt->sender_req_id	= MPI_REQUEST_NULL;
     eager_pkt->data_sz		= data_sz;
     
-    MPIDI_Comm_get_vc(comm, rank, &vc);
+    MPIDI_Comm_get_vc_set_active(comm, rank, &vc);
 
     MPIDI_VC_FAI_send_seqnum(vc, seqnum);
     MPIDI_Pkt_set_seqnum(eager_pkt, seqnum);
@@ -178,7 +178,7 @@ int MPIDI_CH3_EagerContigSend( MPID_Request **sreq_p,
     iov[1].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) buf;
     iov[1].MPID_IOV_LEN = data_sz;
     
-    MPIDI_Comm_get_vc(comm, rank, &vc);
+    MPIDI_Comm_get_vc_set_active(comm, rank, &vc);
     MPIDI_VC_FAI_send_seqnum(vc, seqnum);
     MPIDI_Pkt_set_seqnum(eager_pkt, seqnum);
     
@@ -239,7 +239,7 @@ int MPIDI_CH3_EagerContigShortSend( MPID_Request **sreq_p,
        "sending contiguous short eager message, data_sz=" MPIDI_MSG_SZ_FMT,
 					data_sz));
 	    
-    MPIDI_Comm_get_vc(comm, rank, &vc);
+    MPIDI_Comm_get_vc_set_active(comm, rank, &vc);
     MPIDI_VC_FAI_send_seqnum(vc, seqnum);
     MPIDI_Pkt_set_seqnum(eagershort_pkt, seqnum);
 
@@ -515,7 +515,7 @@ int MPIDI_CH3_EagerContigIsend( MPID_Request **sreq_p,
     iov[1].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) buf;
     iov[1].MPID_IOV_LEN = data_sz;
     
-    MPIDI_Comm_get_vc(comm, rank, &vc);
+    MPIDI_Comm_get_vc_set_active(comm, rank, &vc);
     MPIDI_VC_FAI_send_seqnum(vc, seqnum);
     MPIDI_Pkt_set_seqnum(eager_pkt, seqnum);
     MPIDI_Request_set_seqnum(sreq, seqnum);

src/mpid/ch3/src/ch3u_eagersync.c

@@ -51,7 +51,7 @@ int MPIDI_CH3_EagerSyncNoncontigSend( MPID_Request **sreq_p,
     es_pkt->sender_req_id = sreq->handle;
     es_pkt->data_sz = data_sz;
 
-    MPIDI_Comm_get_vc(comm, rank, &vc);
+    MPIDI_Comm_get_vc_set_active(comm, rank, &vc);
     
     MPIDI_VC_FAI_send_seqnum(vc, seqnum);
     MPIDI_Pkt_set_seqnum(es_pkt, seqnum);
@@ -136,7 +136,7 @@ int MPIDI_CH3_EagerSyncZero(MPID_Request **sreq_p, int rank, int tag,
     es_pkt->sender_req_id = sreq->handle;
     es_pkt->data_sz = 0;
     
-    MPIDI_Comm_get_vc(comm, rank, &vc);
+    MPIDI_Comm_get_vc_set_active(comm, rank, &vc);
     MPIDI_VC_FAI_send_seqnum(vc, seqnum);
     MPIDI_Pkt_set_seqnum(es_pkt, seqnum);
     MPIDI_Request_set_seqnum(sreq, seqnum);

src/mpid/ch3/src/ch3u_handle_connection.c

@@ -49,8 +49,7 @@ int MPIDI_CH3U_Handle_connection(MPIDI_VC_t * vc, MPIDI_VC_Event_t event)
 	    {
 		case MPIDI_VC_STATE_CLOSE_ACKED:
 		{
-		    MPIU_DBG_VCSTATECHANGE(vc,VC_STATE_INACTIVE);
-		    vc->state = MPIDI_VC_STATE_INACTIVE;
+                    MPIDI_CHANGE_VC_STATE(vc, INACTIVE);
 		    /* FIXME: Decrement the reference count?  Who increments? */
 		    /* FIXME: The reference count is often already 0.  But
 		       not always */
@@ -179,13 +178,11 @@ int MPIDI_CH3U_VC_SendClose( MPIDI_VC_t *vc, int rank )
      * be changed before the close packet is sent.
      */
     if (vc->state == MPIDI_VC_STATE_ACTIVE) {
-	MPIU_DBG_VCSTATECHANGE(vc,VC_STATE_LOCAL_CLOSE);
-	vc->state = MPIDI_VC_STATE_LOCAL_CLOSE;
+        MPIDI_CHANGE_VC_STATE(vc, LOCAL_CLOSE);
     }
     else {
 	MPIU_Assert( vc->state == MPIDI_VC_STATE_REMOTE_CLOSE );
-	MPIU_DBG_VCSTATECHANGE(vc,VC_STATE_CLOSE_ACKED);
-	vc->state = MPIDI_VC_STATE_CLOSE_ACKED;
+        MPIDI_CHANGE_VC_STATE(vc, CLOSE_ACKED);
     }
 		
     mpi_errno = MPIU_CALL(MPIDI_CH3,iStartMsg(vc, close_pkt, 
@@ -246,8 +243,7 @@ int MPIDI_CH3_PktHandler_Close( MPIDI_VC_t *vc, MPIDI_CH3_Pkt_t *pkt,
 	    MPIU_DBG_MSG_D(CH3_DISCONNECT,TYPICAL,
 		   "received close(FALSE) from %d, moving to CLOSE_ACKED.",
 		   vc->pg_rank);
-	    MPIU_DBG_VCSTATECHANGE(vc,VC_STATE_CLOSE_ACKED);
-	    vc->state = MPIDI_VC_STATE_CLOSE_ACKED;
+            MPIDI_CHANGE_VC_STATE(vc, CLOSE_ACKED);
 	}
 #if 0
 	else if (vc->state == MPIDI_VC_STATE_CLOSE_ACKED) {
@@ -260,8 +256,8 @@ int MPIDI_CH3_PktHandler_Close( MPIDI_VC_t *vc, MPIDI_CH3_Pkt_t *pkt,
 	       the Assert in the next case) */
 	    MPIU_DBG_MSG(CH3_DISCONNECT,TYPICAL,
 			 "Saw CLOSE_ACKED while already in that state");
-	    vc->state = MPIDI_VC_STATE_REMOTE_CLOSE;
-	    /* We need this terminate to decrement the outstanding closes */
+            MPIDI_CHANGE_VC_STATE(vc, REMOTE_CLOSE);
+ 	    /* We need this terminate to decrement the outstanding closes */
 	    /* For example, with sockets, Connection_terminate will close
 	       the socket */
 	    mpi_errno = MPIU_CALL(MPIDI_CH3,Connection_terminate(vc));
@@ -278,8 +274,7 @@ int MPIDI_CH3_PktHandler_Close( MPIDI_VC_t *vc, MPIDI_CH3_Pkt_t *pkt,
                      "received close(FALSE) from %d, moving to REMOTE_CLOSE.",
 				   vc->pg_rank);
 	    MPIU_Assert(vc->state == MPIDI_VC_STATE_ACTIVE);
-	    MPIU_DBG_VCSTATECHANGE(vc,VC_STATE_REMOTE_CLOSE);
-	    vc->state = MPIDI_VC_STATE_REMOTE_CLOSE;
+            MPIDI_CHANGE_VC_STATE(vc, REMOTE_CLOSE);
 	}
     }
     else /* (close_pkt->ack == TRUE) */
@@ -289,9 +284,7 @@ int MPIDI_CH3_PktHandler_Close( MPIDI_VC_t *vc, MPIDI_CH3_Pkt_t *pkt,
 			       vc->pg_rank);
 	MPIU_Assert (vc->state == MPIDI_VC_STATE_LOCAL_CLOSE || 
 		     vc->state == MPIDI_VC_STATE_CLOSE_ACKED);
-	MPIU_DBG_VCSTATECHANGE(vc,VC_STATE_CLOSE_ACKED);
-	
-	vc->state = MPIDI_VC_STATE_CLOSE_ACKED;
+        MPIDI_CHANGE_VC_STATE(vc, CLOSE_ACKED);
 	/* For example, with sockets, Connection_terminate will close
 	   the socket */
 	mpi_errno = MPIU_CALL(MPIDI_CH3,Connection_terminate(vc));

src/mpid/ch3/src/ch3u_rma_sync.c

@@ -479,7 +479,7 @@ static int MPIDI_CH3I_Send_rma_msg(MPIDI_RMA_ops *rma_op, MPID_Win *win_ptr,
     */
 
     MPID_Comm_get_ptr(win_ptr->comm, comm_ptr);
-    MPIDI_Comm_get_vc(comm_ptr, rma_op->target_rank, &vc);
+    MPIDI_Comm_get_vc_set_active(comm_ptr, rma_op->target_rank, &vc);
 
     MPIDI_CH3I_DATATYPE_IS_PREDEFINED(rma_op->origin_datatype, predefined);
     if (!predefined)
@@ -714,7 +714,7 @@ static int MPIDI_CH3I_Recv_rma_msg(MPIDI_RMA_ops *rma_op, MPID_Win *win_ptr,
 */
 	    
     MPID_Comm_get_ptr(win_ptr->comm, comm_ptr);
-    MPIDI_Comm_get_vc(comm_ptr, rma_op->target_rank, &vc);
+    MPIDI_Comm_get_vc_set_active(comm_ptr, rma_op->target_rank, &vc);
 
     MPIDI_CH3I_DATATYPE_IS_PREDEFINED(rma_op->target_datatype, predefined);
     if (predefined)
@@ -1156,7 +1156,7 @@ int MPIDI_Win_complete(MPID_Win *win_ptr)
 	    put_pkt->target_win_handle = win_ptr->all_win_handles[dst];
 	    put_pkt->source_win_handle = win_ptr->handle;
 	    
-	    MPIDI_Comm_get_vc(comm_ptr, dst, &vc);
+	    MPIDI_Comm_get_vc_set_active(comm_ptr, dst, &vc);
 	    
 	    MPIU_THREAD_CS_ENTER(CH3COMM,vc);
 	    mpi_errno = MPIU_CALL(MPIDI_CH3,iStartMsg(vc, put_pkt,
@@ -1454,7 +1454,7 @@ int MPIDI_Win_unlock(int dest, MPID_Win *win_ptr)
         
     single_op_opt = 0;
 
-    MPIDI_Comm_get_vc(comm_ptr, dest, &vc);
+    MPIDI_Comm_get_vc_set_active(comm_ptr, dest, &vc);
    
     if (rma_op->next->next == NULL) {
 	/* Single put, get, or accumulate between the lock and unlock. If it
@@ -1860,7 +1860,7 @@ static int MPIDI_CH3I_Send_lock_put_or_acc(MPID_Win *win_ptr)
     }
 
     MPID_Comm_get_ptr(win_ptr->comm, comm_ptr);
-    MPIDI_Comm_get_vc(comm_ptr, rma_op->target_rank, &vc);
+    MPIDI_Comm_get_vc_set_active(comm_ptr, rma_op->target_rank, &vc);
 
     MPIDI_CH3I_DATATYPE_IS_PREDEFINED(rma_op->origin_datatype, predefined);
     if (!predefined)
@@ -2036,7 +2036,7 @@ static int MPIDI_CH3I_Send_lock_get(MPID_Win *win_ptr)
     lock_get_unlock_pkt->request_handle = rreq->handle;
 
     MPID_Comm_get_ptr(win_ptr->comm, comm_ptr);
-    MPIDI_Comm_get_vc(comm_ptr, rma_op->target_rank, &vc);
+    MPIDI_Comm_get_vc_set_active(comm_ptr, rma_op->target_rank, &vc);
 
     mpi_errno = MPIU_CALL(MPIDI_CH3,iStartMsg(vc, lock_get_unlock_pkt, 
 				      sizeof(*lock_get_unlock_pkt), &sreq));

src/mpid/ch3/src/ch3u_rndv.c

@@ -43,7 +43,7 @@ int MPIDI_CH3_RndvSend( MPID_Request **sreq_p, const void * buf, int count,
     rts_pkt->sender_req_id    = sreq->handle;
     rts_pkt->data_sz	      = data_sz;
 
-    MPIDI_Comm_get_vc(comm, rank, &vc);
+    MPIDI_Comm_get_vc_set_active(comm, rank, &vc);
     MPIDI_VC_FAI_send_seqnum(vc, seqnum);
     MPIDI_Pkt_set_seqnum(rts_pkt, seqnum);
     MPIDI_Request_set_seqnum(sreq, seqnum);