Modify location of setting next_op_to_issue and sync_flag to NONE

After we issue an op, we set the next_op_to_issue to the next op,
and if next op is NULL, we set sync_flag to NONE. When we receive
the lock ACK saying that lock request is discarded, we set the
next_op_to_issue back to the current op, we reset the sync_flag
from NONE to corresponding flag, since we need to re-transmit the
current op.

Signed-off-by: Pavan Balaji <balaji@anl.gov>

src/mpid/ch3/include/mpidrma.h

@@ -600,6 +600,7 @@ static inline int adjust_op_piggybacked_with_lock(MPID_Win * win_ptr,
         op_flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_EXCLUSIVE) {
         if (flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_GRANTED ||
             flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_QUEUED_DATA_QUEUED) {
+
             if (!op->request) {
                 if (op->ureq) {
                     /* Complete user request and release the ch3 ref */
@@ -660,6 +661,10 @@ static inline int adjust_op_piggybacked_with_lock(MPID_Win * win_ptr,
             MPIDI_CH3_PKT_RMA_ERASE_FLAGS(op->pkt, mpi_errno);
 
             target->next_op_to_issue = op;
+            if (op_flags & MPIDI_CH3_PKT_FLAG_RMA_FLUSH)
+                target->sync.sync_flag = MPIDI_RMA_SYNC_FLUSH;
+            else if (op_flags & MPIDI_RMA_SYNC_UNLOCK)
+                target->sync.sync_flag = MPIDI_RMA_SYNC_UNLOCK;
         }
     }
 

src/mpid/ch3/src/ch3u_rma_oplist.c

@@ -354,13 +354,8 @@ static inline int issue_ops_target(MPID_Win * win_ptr, MPIDI_RMA_Target_t * targ
             else if (target->sync.sync_flag == MPIDI_RMA_SYNC_UNLOCK) {
                 flags |= MPIDI_CH3_PKT_FLAG_RMA_UNLOCK;
             }
-
-            /* We are done with ending sync, unset target's sync_flag. */
-            target->sync.sync_flag = MPIDI_RMA_SYNC_NONE;
         }
 
-        target->next_op_to_issue = curr_op->next;
-
         mpi_errno = issue_rma_op(curr_op, win_ptr, target, flags);
         if (mpi_errno != MPI_SUCCESS)
             MPIU_ERR_POP(mpi_errno);
@@ -378,6 +373,14 @@ static inline int issue_ops_target(MPID_Win * win_ptr, MPIDI_RMA_Target_t * targ
                                          * PUT/ACC operation. */
         }
 
+        target->next_op_to_issue = curr_op->next;
+        if (target->next_op_to_issue == NULL) {
+            if (flags & MPIDI_CH3_PKT_FLAG_RMA_FLUSH || flags & MPIDI_CH3_PKT_FLAG_RMA_UNLOCK) {
+                /* We are done with ending sync, unset target's sync_flag. */
+                target->sync.sync_flag = MPIDI_RMA_SYNC_NONE;
+            }
+        }
+
         if (flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_SHARED ||
             flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_EXCLUSIVE) {
             /* If this operation is piggybacked with LOCK,