Modify comments about atomicity in GACC/FOP handlers.

Signed-off-by: Min Si <msi@il.is.s.u-tokyo.ac.jp>
Signed-off-by: Antonio J. Pena <apenya@mcs.anl.gov>

src/mpid/ch3/src/ch3u_handle_recv_req.c

@@ -263,7 +263,6 @@ int MPIDI_CH3_ReqHandler_GaccumRecvComplete(MPIDI_VC_t * vc, MPID_Request * rreq
 
     MPID_Datatype_is_contig(rreq->dev.datatype, &is_contig);
 
-    /* Copy data into a temporary buffer */
     resp_req = MPID_Request_create();
     MPIU_ERR_CHKANDJUMP(resp_req == NULL, mpi_errno, MPI_ERR_OTHER, "**nomemreq");
     MPIU_Object_set_ref(resp_req, 1);
@@ -272,9 +271,13 @@ int MPIDI_CH3_ReqHandler_GaccumRecvComplete(MPIDI_VC_t * vc, MPID_Request * rreq
     MPIU_CHKPMEM_MALLOC(resp_req->dev.user_buf, void *, rreq->dev.recv_data_sz,
                         mpi_errno, "GACC resp. buffer");
 
+    /* NOTE: 'copy data + ACC' needs to be atomic */
+
     if (win_ptr->shm_allocated == TRUE)
         MPIDI_CH3I_SHM_MUTEX_LOCK(win_ptr);
 
+    /* Copy data from target window to temporary buffer */
+
     if (is_contig) {
         MPIU_Memcpy(resp_req->dev.user_buf,
                     (void *) ((char *) rreq->dev.real_user_buf + rreq->dev.stream_offset),
@@ -397,6 +400,8 @@ int MPIDI_CH3_ReqHandler_FOPRecvComplete(MPIDI_VC_t * vc, MPID_Request * rreq, i
      * operation are completed when counter reaches zero. */
     win_ptr->at_completion_counter++;
 
+    /* NOTE: 'copy data + ACC' needs to be atomic */
+
     if (win_ptr->shm_allocated == TRUE)
         MPIDI_CH3I_SHM_MUTEX_LOCK(win_ptr);
 
@@ -1193,10 +1198,14 @@ static inline int perform_get_acc_in_lock_queue(MPID_Win * win_ptr,
             get_accum_resp_pkt->flags |= MPIDI_CH3_PKT_FLAG_RMA_FLUSH_ACK;
         get_accum_resp_pkt->target_rank = win_ptr->comm_ptr->rank;
 
-        /* Perform ACCUMULATE OP */
+
+        /* NOTE: copy 'data + ACC' needs to be atomic */
+
         if (win_ptr->shm_allocated == TRUE)
             MPIDI_CH3I_SHM_MUTEX_LOCK(win_ptr);
 
+        /* Copy data from target window to response packet header */
+
         void *src = (void *) (get_accum_pkt->addr), *dest =
             (void *) (get_accum_resp_pkt->info.data);
         mpi_errno = immed_copy(src, dest, len);
@@ -1206,6 +1215,8 @@ static inline int perform_get_acc_in_lock_queue(MPID_Win * win_ptr,
             MPIU_ERR_POP(mpi_errno);
         }
 
+        /* Perform ACCUMULATE OP */
+
         /* All data fits in packet header */
         /* NOTE: here we pass 0 as stream_offset to do_accumulate_op(), because the unit
            that is piggybacked with LOCK flag must be the first stream unit */
@@ -1244,10 +1255,13 @@ static inline int perform_get_acc_in_lock_queue(MPID_Win * win_ptr,
 
     MPID_Datatype_is_contig(get_accum_pkt->datatype, &is_contig);
 
-    /* Perform ACCUMULATE OP */
+    /* NOTE: 'copy data + ACC' needs to be atomic */
+
     if (win_ptr->shm_allocated == TRUE)
         MPIDI_CH3I_SHM_MUTEX_LOCK(win_ptr);
 
+    /* Copy data from target window to temporary buffer */
+
     /* NOTE: here we copy data from stream_offset = 0, because
        the unit that is piggybacked with LOCK flag must be the
        first stream unit. */
@@ -1271,6 +1285,8 @@ static inline int perform_get_acc_in_lock_queue(MPID_Win * win_ptr,
         MPID_Segment_free(seg);
     }
 
+    /* Perform ACCUMULATE OP */
+
     /* NOTE: here we pass 0 as stream_offset to do_accumulate_op(), because the unit
        that is piggybacked with LOCK flag must be the first stream unit */
     mpi_errno = do_accumulate_op(lock_entry->data, get_accum_pkt->count, get_accum_pkt->datatype,
@@ -1379,9 +1395,13 @@ static inline int perform_fop_in_lock_queue(MPID_Win * win_ptr, MPIDI_RMA_Lock_e
         win_ptr->at_completion_counter++;
     }
 
+    /* NOTE: 'copy data + ACC' needs to be atomic */
+
     if (win_ptr->shm_allocated == TRUE)
         MPIDI_CH3I_SHM_MUTEX_LOCK(win_ptr);
 
+    /* Copy data from target window to temporary buffer / response packet header */
+
     if (fop_pkt->type == MPIDI_CH3_PKT_FOP_IMMED) {
         /* copy data to resp pkt header */
         void *src = fop_pkt->addr, *dest = fop_resp_pkt->info.data;

src/mpid/ch3/src/ch3u_rma_pkthandler.c

@@ -836,6 +836,8 @@ int MPIDI_CH3_PktHandler_GetAccumulate(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
             (get_accum_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_UNLOCK))
             get_accum_resp_pkt->flags |= MPIDI_CH3_PKT_FLAG_RMA_FLUSH_ACK;
 
+        /* NOTE: 'copy data + ACC' needs to be atomic */
+
         if (win_ptr->shm_allocated == TRUE)
             MPIDI_CH3I_SHM_MUTEX_LOCK(win_ptr);
 
@@ -1232,6 +1234,8 @@ int MPIDI_CH3_PktHandler_FOP(MPIDI_VC_t * vc, MPIDI_CH3_Pkt_t * pkt,
             (fop_pkt->flags & MPIDI_CH3_PKT_FLAG_RMA_UNLOCK))
             fop_resp_pkt->flags |= MPIDI_CH3_PKT_FLAG_RMA_FLUSH_ACK;
 
+        /* NOTE: 'copy data + ACC' needs to be atomic */
+
         if (win_ptr->shm_allocated == TRUE)
             MPIDI_CH3I_SHM_MUTEX_LOCK(win_ptr);