ch3u_rma_progress.c 30.5 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
/*
 *  (C) 2001 by Argonne National Laboratory.
 *      See COPYRIGHT in top-level directory.
 */

#include "mpidimpl.h"
#include "mpidrma.h"

/*
=== BEGIN_MPI_T_CVAR_INFO_BLOCK ===

cvars:
14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31
    - name        : MPIR_CVAR_CH3_RMA_ACTIVE_REQ_THRESHOLD
      category    : CH3
      type        : int
      default     : 2097152
      class       : none
      verbosity   : MPI_T_VERBOSITY_USER_BASIC
      scope       : MPI_T_SCOPE_ALL_EQ
      description : >-
         Threshold of number of active requests to trigger
         blocking waiting in operation routines. When the
         value is negative, we never blockingly wait in
         operation routines. When the value is zero, we always
         trigger blocking waiting in operation routines to
         wait until no. of active requests becomes zero. When the
         value is positive, we do blocking waiting in operation
         routines to wait until no. of active requests being
         reduced to this value.

32 33 34
=== END_MPI_T_CVAR_INFO_BLOCK ===
*/

35
static inline int check_and_switch_target_state(MPID_Win * win_ptr, MPIDI_RMA_Target_t * target,
36 37
                                                int *is_able_to_issue, int *made_progress);
static inline int check_and_switch_window_state(MPID_Win * win_ptr, int *is_able_to_issue,
38
                                                int *made_progress);
39 40
static inline int issue_ops_target(MPID_Win * win_ptr, MPIDI_RMA_Target_t * target,
                                   int *made_progress);
41

42
/* check if we can switch window-wide state: FENCE_ISSUED, PSCW_ISSUED, LOCK_ALL_ISSUED */
43
#undef FUNCNAME
44
#define FUNCNAME check_and_switch_window_state
45 46
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
47 48
static inline int check_and_switch_window_state(MPID_Win * win_ptr, int *is_able_to_issue,
                                                int *made_progress)
49
{
50
    MPID_Request *fence_req_ptr = NULL;
51
    int i, mpi_errno = MPI_SUCCESS;
52
    MPIDI_STATE_DECL(MPID_STATE_CHECK_AND_SWITCH_WINDOW_STATE);
53

54
    MPIDI_RMA_FUNC_ENTER(MPID_STATE_CHECK_AND_SWITCH_WINDOW_STATE);
55 56

    (*made_progress) = 0;
57
    (*is_able_to_issue) = 0;
58

59 60
    switch (win_ptr->states.access_state) {
    case MPIDI_RMA_FENCE_ISSUED:
61 62 63
        MPID_Request_get_ptr(win_ptr->fence_sync_req, fence_req_ptr);
        if (MPID_Request_is_complete(fence_req_ptr)) {
            win_ptr->states.access_state = MPIDI_RMA_FENCE_GRANTED;
64 65
            (*is_able_to_issue) = 1;

66 67 68
            MPID_Request_release(fence_req_ptr);
            win_ptr->fence_sync_req = MPI_REQUEST_NULL;

69 70
            MPIDI_CH3I_num_active_issued_win--;
            MPIU_Assert(MPIDI_CH3I_num_active_issued_win >= 0);
71

72 73
            (*made_progress) = 1;
        }
74 75 76
        break;

    case MPIDI_RMA_PSCW_ISSUED:
77 78
        if (win_ptr->start_req == NULL) {
            /* for MPI_MODE_NOCHECK and all targets on SHM,
79
             * we do not create PSCW requests on window. */
80
            win_ptr->states.access_state = MPIDI_RMA_PSCW_GRANTED;
81
            (*is_able_to_issue) = 1;
82

83 84
            MPIDI_CH3I_num_active_issued_win--;
            MPIU_Assert(MPIDI_CH3I_num_active_issued_win >= 0);
85

86 87 88 89 90 91 92 93 94 95 96 97 98
            (*made_progress) = 1;
        }
        else {
            for (i = 0; i < win_ptr->start_grp_size; i++) {
                MPID_Request *start_req_ptr = NULL;
                if (win_ptr->start_req[i] == MPI_REQUEST_NULL)
                    continue;
                MPID_Request_get_ptr(win_ptr->start_req[i], start_req_ptr);
                if (MPID_Request_is_complete(start_req_ptr)) {
                    MPID_Request_release(start_req_ptr);
                    win_ptr->start_req[i] = MPI_REQUEST_NULL;
                }
                else {
99
                    break;
100 101 102
                }
            }

103 104
            if (i == win_ptr->start_grp_size) {
                win_ptr->states.access_state = MPIDI_RMA_PSCW_GRANTED;
105
                (*is_able_to_issue) = 1;
106

107 108
                MPIDI_CH3I_num_active_issued_win--;
                MPIU_Assert(MPIDI_CH3I_num_active_issued_win >= 0);
109

110 111 112 113 114
                (*made_progress) = 1;

                MPIU_Free(win_ptr->start_req);
                win_ptr->start_req = NULL;
            }
115
        }
116 117 118
        break;

    case MPIDI_RMA_LOCK_ALL_ISSUED:
119 120
        if (win_ptr->outstanding_locks == 0) {
            win_ptr->states.access_state = MPIDI_RMA_LOCK_ALL_GRANTED;
121 122
            (*is_able_to_issue) = 1;

123 124
            (*made_progress) = 1;
        }
125 126
        break;

127 128 129 130 131 132 133 134
    case MPIDI_RMA_PER_TARGET:
    case MPIDI_RMA_LOCK_ALL_CALLED:
    case MPIDI_RMA_FENCE_GRANTED:
    case MPIDI_RMA_PSCW_GRANTED:
    case MPIDI_RMA_LOCK_ALL_GRANTED:
        (*is_able_to_issue) = 1;
        break;

135 136
    default:
        break;
137
    }   /* end of switch */
138 139

  fn_exit:
140
    MPIDI_RMA_FUNC_EXIT(MPID_STATE_CHECK_AND_SWITCH_WINDOW_STATE);
141 142 143 144 145 146 147 148 149
    return mpi_errno;
    /* --BEGIN ERROR HANDLING-- */
  fn_fail:
    goto fn_exit;
    /* --END ERROR HANDLING-- */
}


#undef FUNCNAME
150
#define FUNCNAME check_and_switch_target_state
151 152
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
153
static inline int check_and_switch_target_state(MPID_Win * win_ptr, MPIDI_RMA_Target_t * target,
154
                                                int *is_able_to_issue, int *made_progress)
155 156 157 158 159
{
    int rank = win_ptr->comm_ptr->rank;
    int mpi_errno = MPI_SUCCESS;

    (*made_progress) = 0;
160
    (*is_able_to_issue) = 0;
161

162
    if (target == NULL)
163 164
        goto fn_exit;

165 166 167 168 169
    switch (target->access_state) {
    case MPIDI_RMA_LOCK_CALLED:
        if (target->sync.sync_flag == MPIDI_RMA_SYNC_NONE ||
            target->sync.sync_flag == MPIDI_RMA_SYNC_FLUSH_LOCAL ||
            target->sync.sync_flag == MPIDI_RMA_SYNC_FLUSH) {
170 171
            if (target->pending_op_list_head == NULL ||
                !target->pending_op_list_head->piggyback_lock_candidate) {
172 173 174 175
                /* issue lock request */
                target->access_state = MPIDI_RMA_LOCK_ISSUED;
                if (target->target_rank == rank) {
                    mpi_errno = acquire_local_lock(win_ptr, target->lock_type);
176 177
                    if (mpi_errno != MPI_SUCCESS)
                        MPIU_ERR_POP(mpi_errno);
178 179
                }
                else {
180 181 182
                    mpi_errno = send_lock_msg(target->target_rank, target->lock_type, win_ptr);
                    if (mpi_errno != MPI_SUCCESS)
                        MPIU_ERR_POP(mpi_errno);
183 184 185 186 187 188
                }

                (*made_progress) = 1;
            }
        }
        else if (target->sync.sync_flag == MPIDI_RMA_SYNC_UNLOCK) {
189
            if (target->pending_op_list_head == NULL) {
190
                /* No RMA operation has ever been posted to this target,
191 192
                 * finish issuing, no need to acquire the lock. Cleanup
                 * function will clean it up. */
193 194 195 196 197 198 199 200 201
                target->access_state = MPIDI_RMA_LOCK_GRANTED;

                /* We are done with ending synchronization, unset target's sync_flag. */
                target->sync.sync_flag = MPIDI_RMA_SYNC_NONE;

                (*made_progress) = 1;
            }
            else {
                /* if we reach WIN_UNLOCK and there is still operation existing
202 203
                 * in pending list, this operation must be the only operation
                 * and it is prepared to piggyback LOCK and UNLOCK. */
204
                MPIU_Assert(MPIR_CVAR_CH3_RMA_DELAY_ISSUING_FOR_PIGGYBACKING);
205 206
                MPIU_Assert(target->pending_op_list_head->next == NULL);
                MPIU_Assert(target->pending_op_list_head->piggyback_lock_candidate);
207 208 209 210 211 212
            }
        }
        break;

    case MPIDI_RMA_LOCK_GRANTED:
    case MPIDI_RMA_NONE:
213 214 215 216 217 218 219
        if (target->win_complete_flag) {
            if (target->pending_op_list_head == NULL) {
                mpi_errno = send_decr_at_cnt_msg(target->target_rank, win_ptr);
                if (mpi_errno != MPI_SUCCESS)
                    MPIU_ERR_POP(mpi_errno);
            }
        }
220
        if (target->sync.sync_flag == MPIDI_RMA_SYNC_FLUSH) {
221
            if (target->pending_op_list_head == NULL) {
222
                if (target->target_rank != rank) {
223
                    if (target->put_acc_issued) {
224 225 226

                        target->sync.outstanding_acks++;

227
                        mpi_errno = send_flush_msg(target->target_rank, win_ptr);
228 229
                        if (mpi_errno != MPI_SUCCESS)
                            MPIU_ERR_POP(mpi_errno);
230 231
                    }
                }
232 233 234 235 236

                /* We are done with ending synchronization, unset target's sync_flag. */
                target->sync.sync_flag = MPIDI_RMA_SYNC_NONE;

                (*made_progress) = 1;
237
            }
238 239
        }
        else if (target->sync.sync_flag == MPIDI_RMA_SYNC_UNLOCK) {
240
            if (target->pending_op_list_head == NULL) {
241 242
                if (target->target_rank == rank) {
                    mpi_errno = MPIDI_CH3I_Release_lock(win_ptr);
243 244
                    if (mpi_errno != MPI_SUCCESS)
                        MPIU_ERR_POP(mpi_errno);
245
                }
246 247 248 249
                else {
                    MPIDI_CH3_Pkt_flags_t flag = MPIDI_CH3_PKT_FLAG_NONE;
                    if (!target->put_acc_issued) {
                        /* We did not issue PUT/ACC since the last
250 251
                         * synchronization call, therefore here we
                         * don't need ACK back */
252 253 254

                        flag = MPIDI_CH3_PKT_FLAG_RMA_UNLOCK_NO_ACK;
                    }
255 256 257
                    else {
                        target->sync.outstanding_acks++;
                    }
258
                    mpi_errno = send_unlock_msg(target->target_rank, win_ptr, flag);
259 260
                    if (mpi_errno != MPI_SUCCESS)
                        MPIU_ERR_POP(mpi_errno);
261 262 263 264 265
                }

                /* We are done with ending synchronization, unset target's sync_flag. */
                target->sync.sync_flag = MPIDI_RMA_SYNC_NONE;

266 267 268
                (*made_progress) = 1;
            }
        }
269
        break;
270

271 272
    default:
        break;
273
    }   /* end of switch */
274

275
    if (target->access_state != MPIDI_RMA_LOCK_ISSUED) {
276 277 278
        (*is_able_to_issue) = 1;
    }

279
  fn_exit:
280
    return mpi_errno;
281
  fn_fail:
282 283
    goto fn_exit;
}
284 285


286 287 288 289
#undef FUNCNAME
#define FUNCNAME issue_ops_target
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
290
static inline int issue_ops_target(MPID_Win * win_ptr, MPIDI_RMA_Target_t * target,
291 292 293 294
                                   int *made_progress)
{
    MPIDI_RMA_Op_t *curr_op = NULL;
    MPIDI_CH3_Pkt_flags_t flags;
295
    int is_able_to_issue = 0;
296
    int first_op = 1, mpi_errno = MPI_SUCCESS;
297

298
    (*made_progress) = 0;
299

300 301 302 303 304 305 306
    /* check and try to switch target state */
    mpi_errno = check_and_switch_target_state(win_ptr, target, &is_able_to_issue, made_progress);
    if (mpi_errno != MPI_SUCCESS)
        MPIU_ERR_POP(mpi_errno);
    if (!is_able_to_issue)
        goto fn_exit;

307
    if (win_ptr->non_empty_slots == 0 || target == NULL || target->pending_op_list_head == NULL)
308
        goto fn_exit;
309

310 311 312 313
    /* Issue out operations in the list. */
    curr_op = target->next_op_to_issue;
    while (curr_op != NULL) {

314 315
        if (target->access_state == MPIDI_RMA_LOCK_ISSUED) {
            /* It is possible that the previous OP+LOCK changes
316
             * lock state to LOCK_ISSUED. */
317 318
            break;
        }
319

320
        if (MPIR_CVAR_CH3_RMA_DELAY_ISSUING_FOR_PIGGYBACKING && curr_op->next == NULL &&
321
            target->sync.sync_flag == MPIDI_RMA_SYNC_NONE && curr_op->ureq == NULL) {
322 323
            /* If DELAY_ISSUING_FOR_PIGGYBACKING is turned on,
             * skip the last OP if sync_flag is NONE since we
324
             * want to leave it to the ending synchronization
325
             * so that we can piggyback UNLOCK / FLUSH.
326 327 328
             * However, if it is a request-based RMA, do not
             * skip it (otherwise a wait call before unlock
             * will be blocked). */
329
            break;
330 331
        }

332 333
        flags = MPIDI_CH3_PKT_FLAG_NONE;

334 335 336 337
        if (first_op) {
            /* piggyback on first OP. */
            if (target->access_state == MPIDI_RMA_LOCK_CALLED) {
                MPIU_Assert(curr_op->piggyback_lock_candidate);
338 339 340 341 342 343
                if (target->lock_type == MPI_LOCK_SHARED)
                    flags |= MPIDI_CH3_PKT_FLAG_RMA_LOCK_SHARED;
                else {
                    MPIU_Assert(target->lock_type == MPI_LOCK_EXCLUSIVE);
                    flags |= MPIDI_CH3_PKT_FLAG_RMA_LOCK_EXCLUSIVE;
                }
344 345 346 347 348
                target->access_state = MPIDI_RMA_LOCK_ISSUED;
            }
            first_op = 0;
        }

349 350 351 352
        /* piggyback FLUSH on every OP if ordered flush is not guaranteed. */
        if (!MPIDI_CH3U_Win_pkt_orderings.am_flush_ordered)
            flags |= MPIDI_CH3_PKT_FLAG_RMA_FLUSH;

353 354 355 356 357 358 359
        if (curr_op->next == NULL) {
            /* piggyback on last OP. */
            if (target->sync.sync_flag == MPIDI_RMA_SYNC_FLUSH) {
                flags |= MPIDI_CH3_PKT_FLAG_RMA_FLUSH;
            }
            else if (target->sync.sync_flag == MPIDI_RMA_SYNC_UNLOCK) {
                flags |= MPIDI_CH3_PKT_FLAG_RMA_UNLOCK;
360 361 362 363

                /* if piggyback UNLOCK then unset FLUSH (set for every
                 * operation on out-of-order network). */
                flags &= ~MPIDI_CH3_PKT_FLAG_RMA_FLUSH;
364
            }
365 366
            if (target->win_complete_flag)
                flags |= MPIDI_CH3_PKT_FLAG_RMA_DECR_AT_COUNTER;
367 368
        }

369 370 371 372 373 374
        /* only increase ack counter when FLUSH or UNLOCK flag is set,
         * but without LOCK piggyback. */
        if (((flags & MPIDI_CH3_PKT_FLAG_RMA_FLUSH)
             || (flags & MPIDI_CH3_PKT_FLAG_RMA_UNLOCK)))
            target->sync.outstanding_acks++;

375 376 377 378
        mpi_errno = issue_rma_op(curr_op, win_ptr, target, flags);
        if (mpi_errno != MPI_SUCCESS)
            MPIU_ERR_POP(mpi_errno);

379 380
        (*made_progress) = 1;

381
        if (curr_op->pkt.type == MPIDI_CH3_PKT_PUT ||
382 383 384
            curr_op->pkt.type == MPIDI_CH3_PKT_PUT_IMMED ||
            curr_op->pkt.type == MPIDI_CH3_PKT_ACCUMULATE ||
            curr_op->pkt.type == MPIDI_CH3_PKT_ACCUMULATE_IMMED) {
385
            target->put_acc_issued = 1; /* set PUT_ACC_FLAG when sending
386
                                         * PUT/ACC operation. */
387 388
        }

389
        if ((curr_op->pkt.type == MPIDI_CH3_PKT_ACCUMULATE ||
390 391
             curr_op->pkt.type == MPIDI_CH3_PKT_GET_ACCUM) &&
            curr_op->issued_stream_count != ALL_STREAM_UNITS_ISSUED) {
392 393 394 395 396 397 398 399
            /* For ACC-like operations, if not all stream units
             * are issued out, we stick to the current operation,
             * otherwise we move on to the next operation. */
            target->next_op_to_issue = curr_op;
        }
        else
            target->next_op_to_issue = curr_op->next;

400 401 402 403 404 405 406
        if (target->next_op_to_issue == NULL) {
            if (flags & MPIDI_CH3_PKT_FLAG_RMA_FLUSH || flags & MPIDI_CH3_PKT_FLAG_RMA_UNLOCK) {
                /* We are done with ending sync, unset target's sync_flag. */
                target->sync.sync_flag = MPIDI_RMA_SYNC_NONE;
            }
        }

407 408
        if (flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_SHARED ||
            flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_EXCLUSIVE) {
409
            /* If this operation is piggybacked with LOCK,
410 411 412
             * do not move it out of pending list, and do
             * not complete the user request, because we
             * may need to re-transmit it. */
413 414 415
            break;
        }

416 417 418 419 420
        if (curr_op->ureq != NULL) {
            mpi_errno = set_user_req_after_issuing_op(curr_op);
            if (mpi_errno != MPI_SUCCESS)
                MPIU_ERR_POP(mpi_errno);
        }
421

422
        if (curr_op->reqs_size == 0) {
423
            MPIU_Assert(curr_op->single_req == NULL && curr_op->multi_reqs == NULL);
424
            /* Sending is completed immediately. */
425
            MPIDI_CH3I_RMA_Ops_free_elem(win_ptr, &(target->pending_op_list_head),
426 427 428
                                         &(target->pending_op_list_tail), curr_op);
        }
        else {
429 430 431
            MPI_Datatype target_datatype;
            int is_derived = FALSE;

432
            /* Sending is not completed immediately. */
433

434
            MPIDI_CH3I_RMA_Ops_unlink(&(target->pending_op_list_head),
435
                                      &(target->pending_op_list_tail), curr_op);
436 437 438 439 440 441 442 443 444 445 446 447 448

            MPIDI_CH3_PKT_RMA_GET_TARGET_DATATYPE(curr_op->pkt, target_datatype, mpi_errno);

            if ((target_datatype != MPI_DATATYPE_NULL &&
                 !MPIR_DATATYPE_IS_PREDEFINED(target_datatype)) ||
                (curr_op->origin_datatype != MPI_DATATYPE_NULL &&
                 !MPIR_DATATYPE_IS_PREDEFINED(curr_op->origin_datatype)) ||
                (curr_op->result_datatype != MPI_DATATYPE_NULL &&
                 !MPIR_DATATYPE_IS_PREDEFINED(curr_op->result_datatype))) {
                is_derived = TRUE;
            }

            if (is_derived) {
449 450
                MPIDI_CH3I_RMA_Ops_append(&(target->issued_dt_op_list_head),
                                          &(target->issued_dt_op_list_tail), curr_op);
451 452
            }
            else if (curr_op->pkt.type == MPIDI_CH3_PKT_PUT ||
453 454 455
                     curr_op->pkt.type == MPIDI_CH3_PKT_PUT_IMMED ||
                     curr_op->pkt.type == MPIDI_CH3_PKT_ACCUMULATE ||
                     curr_op->pkt.type == MPIDI_CH3_PKT_ACCUMULATE_IMMED) {
456 457
                MPIDI_CH3I_RMA_Ops_append(&(target->issued_write_op_list_head),
                                          &(target->issued_write_op_list_tail), curr_op);
458 459
            }
            else {
460 461
                MPIDI_CH3I_RMA_Ops_append(&(target->issued_read_op_list_head),
                                          &(target->issued_read_op_list_tail), curr_op);
462 463 464 465 466
            }
        }

        curr_op = target->next_op_to_issue;

467
    }   /* end of while loop */
468 469 470 471 472 473 474 475

  fn_exit:
    return mpi_errno;
  fn_fail:
    goto fn_exit;
}


476 477 478 479 480 481 482 483
#undef FUNCNAME
#define FUNCNAME MPIDI_CH3I_RMA_Free_ops_before_completion
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
int MPIDI_CH3I_RMA_Free_ops_before_completion(MPID_Win * win_ptr)
{
    MPIDI_RMA_Op_t *curr_op = NULL;
    MPIDI_RMA_Target_t *curr_target = NULL;
484
    MPIDI_RMA_Op_t **op_list_head = NULL, **op_list_tail = NULL;
485 486 487 488
    int read_flag = 0;
    int i, made_progress = 0;
    int mpi_errno = MPI_SUCCESS;

489 490 491
    /* If we are in an free_ops_before_completion, the window must be holding
     * up resources.  If it isn't, we are in the wrong window and
     * incorrectly entered this function. */
492
    MPIU_ERR_CHKANDJUMP(win_ptr->non_empty_slots == 0, mpi_errno, MPI_ERR_OTHER, "**rmanoop");
493 494

    /* make nonblocking progress once */
495
    mpi_errno = MPIDI_CH3I_RMA_Make_progress_win(win_ptr, &made_progress);
496 497
    if (mpi_errno != MPI_SUCCESS)
        MPIU_ERR_POP(mpi_errno);
498

499
    if (win_ptr->states.access_state == MPIDI_RMA_FENCE_ISSUED ||
500 501
        win_ptr->states.access_state == MPIDI_RMA_PSCW_ISSUED ||
        win_ptr->states.access_state == MPIDI_RMA_LOCK_ALL_ISSUED)
502 503 504 505
        goto fn_exit;

    /* find targets that have operations */
    for (i = 0; i < win_ptr->num_slots; i++) {
506 507
        if (win_ptr->slots[i].target_list_head != NULL) {
            curr_target = win_ptr->slots[i].target_list_head;
508
            while (curr_target != NULL) {
509 510
                if (curr_target->issued_read_op_list_head != NULL ||
                    curr_target->issued_write_op_list_head != NULL) {
511 512 513 514 515 516 517 518 519
                    if (win_ptr->states.access_state == MPIDI_RMA_PER_TARGET ||
                        win_ptr->states.access_state == MPIDI_RMA_LOCK_ALL_CALLED) {
                        if (curr_target->access_state == MPIDI_RMA_LOCK_GRANTED)
                            break;
                    }
                    else {
                        break;
                    }
                }
520
                curr_target = curr_target->next;
521
            }
522 523
            if (curr_target != NULL)
                break;
524 525
        }
    }
526

527 528
    if (curr_target == NULL)
        goto fn_exit;
529

530
    /* After we do this, all following Win_flush_local
531
     * must do a Win_flush instead. */
532
    curr_target->sync.upgrade_flush_local = 1;
533

534 535 536 537 538
    op_list_head = &curr_target->issued_read_op_list_head;
    op_list_tail = &curr_target->issued_read_op_list_tail;
    read_flag = 1;

    curr_op = *op_list_head;
539 540

    /* free all ops in the list since we do not need to maintain them anymore */
541 542
    while (1) {
        if (curr_op != NULL) {
543 544 545 546 547 548 549 550 551
            if (curr_op->reqs_size == 1) {
                MPIU_Assert(curr_op->single_req != NULL);
                MPID_Request_release(curr_op->single_req);
                curr_op->single_req = NULL;
                win_ptr->active_req_cnt--;
                curr_op->reqs_size = 0;
            }
            else if (curr_op->reqs_size > 1) {
                MPIU_Assert(curr_op->multi_reqs != NULL);
552
                for (i = 0; i < curr_op->reqs_size; i++) {
553 554 555
                    if (curr_op->multi_reqs[i] != NULL) {
                        MPID_Request_release(curr_op->multi_reqs[i]);
                        curr_op->multi_reqs[i] = NULL;
556 557
                        win_ptr->active_req_cnt--;
                    }
558 559
                }

560
                /* free req array in this op */
561 562
                MPIU_Free(curr_op->multi_reqs);
                curr_op->multi_reqs = NULL;
563 564
                curr_op->reqs_size = 0;
            }
565
            MPIDI_CH3I_RMA_Ops_free_elem(win_ptr, op_list_head, op_list_tail, curr_op);
566
        }
567
        else {
568
            if (read_flag == 1) {
569
                op_list_head = &curr_target->issued_write_op_list_head;
Xin Zhao's avatar
Xin Zhao committed
570
                op_list_tail = &curr_target->issued_write_op_list_tail;
571 572
                read_flag = 0;
            }
573 574 575 576
            else {
                /* we reach the tail of write_op_list, break out. */
                break;
            }
577
        }
578
        curr_op = *op_list_head;
579
    }
580

581
  fn_exit:
582
    return mpi_errno;
583
  fn_fail:
584 585 586 587
    goto fn_exit;
}


588 589 590 591 592 593
#undef FUNCNAME
#define FUNCNAME MPIDI_CH3I_RMA_Cleanup_ops_aggressive
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
int MPIDI_CH3I_RMA_Cleanup_ops_aggressive(MPID_Win * win_ptr)
{
594
    int i, local_completed = 0, remote_completed ATTRIBUTE((unused)) = 0;
595 596 597 598 599 600 601
    int mpi_errno = MPI_SUCCESS;
    MPIDI_RMA_Target_t *curr_target = NULL;
    int made_progress = 0;

    /* If we are in an aggressive cleanup, the window must be holding
     * up resources.  If it isn't, we are in the wrong window and
     * incorrectly entered this function. */
602
    MPIU_ERR_CHKANDJUMP(win_ptr->non_empty_slots == 0, mpi_errno, MPI_ERR_OTHER, "**rmanoop");
603 604 605

    /* find the first target that has something to issue */
    for (i = 0; i < win_ptr->num_slots; i++) {
606 607 608
        if (win_ptr->slots[i].target_list_head != NULL) {
            curr_target = win_ptr->slots[i].target_list_head;
            while (curr_target != NULL && curr_target->pending_op_list_head == NULL)
609
                curr_target = curr_target->next;
610 611
            if (curr_target != NULL)
                break;
612 613 614
        }
    }

615 616
    if (curr_target == NULL)
        goto fn_exit;
617 618 619 620 621 622 623 624 625 626 627 628

    if (curr_target->sync.sync_flag < MPIDI_RMA_SYNC_FLUSH_LOCAL)
        curr_target->sync.sync_flag = MPIDI_RMA_SYNC_FLUSH_LOCAL;

    /* Issue out all operations. */
    mpi_errno = MPIDI_CH3I_RMA_Make_progress_target(win_ptr, curr_target->target_rank,
                                                    &made_progress);
    if (mpi_errno != MPI_SUCCESS)
        MPIU_ERR_POP(mpi_errno);

    /* Wait for local completion. */
    do {
629
        mpi_errno = MPIDI_CH3I_RMA_Cleanup_ops_target(win_ptr, curr_target);
630 631
        if (mpi_errno != MPI_SUCCESS)
            MPIU_ERR_POP(mpi_errno);
632 633 634

        MPIDI_CH3I_RMA_ops_completion(win_ptr, curr_target, local_completed, remote_completed);

635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654
        if (!local_completed) {
            mpi_errno = wait_progress_engine();
            if (mpi_errno != MPI_SUCCESS)
                MPIU_ERR_POP(mpi_errno);
        }
    } while (!local_completed);

  fn_exit:
    return mpi_errno;
  fn_fail:
    goto fn_exit;
}


#undef FUNCNAME
#define FUNCNAME MPIDI_CH3I_RMA_Cleanup_target_aggressive
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
int MPIDI_CH3I_RMA_Cleanup_target_aggressive(MPID_Win * win_ptr, MPIDI_RMA_Target_t ** target)
{
655
    int i, local_completed ATTRIBUTE((unused)) = 0, remote_completed = 0;
656 657 658 659 660 661 662 663 664
    int made_progress = 0;
    MPIDI_RMA_Target_t *curr_target = NULL;
    int mpi_errno = MPI_SUCCESS;

    (*target) = NULL;

    /* If we are in an aggressive cleanup, the window must be holding
     * up resources.  If it isn't, we are in the wrong window and
     * incorrectly entered this function. */
665
    MPIU_ERR_CHKANDJUMP(win_ptr->non_empty_slots == 0, mpi_errno, MPI_ERR_OTHER, "**rmanotarget");
666 667 668 669 670 671 672 673 674 675 676

    if (win_ptr->states.access_state == MPIDI_RMA_LOCK_ALL_CALLED) {
        /* switch to window-wide protocol */
        MPIDI_VC_t *orig_vc = NULL, *target_vc = NULL;
        MPIDI_Comm_get_vc(win_ptr->comm_ptr, win_ptr->comm_ptr->rank, &orig_vc);
        for (i = 0; i < win_ptr->comm_ptr->local_size; i++) {
            if (i == win_ptr->comm_ptr->rank)
                continue;
            MPIDI_Comm_get_vc(win_ptr->comm_ptr, i, &target_vc);
            if (orig_vc->node_id != target_vc->node_id) {
                mpi_errno = MPIDI_CH3I_Win_find_target(win_ptr, i, &curr_target);
677 678
                if (mpi_errno)
                    MPIU_ERR_POP(mpi_errno);
679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694
                if (curr_target == NULL) {
                    win_ptr->outstanding_locks++;
                    mpi_errno = send_lock_msg(i, MPI_LOCK_SHARED, win_ptr);
                    if (mpi_errno != MPI_SUCCESS)
                        MPIU_ERR_POP(mpi_errno);
                }
            }
        }
        win_ptr->states.access_state = MPIDI_RMA_LOCK_ALL_ISSUED;
    }

    do {
        /* find a non-empty slot and set the FLUSH flag on the first
         * target */
        /* TODO: we should think about better strategies on selecting the target */
        for (i = 0; i < win_ptr->num_slots; i++)
695
            if (win_ptr->slots[i].target_list_head != NULL)
696
                break;
697
        curr_target = win_ptr->slots[i].target_list_head;
698 699 700 701 702 703 704 705 706 707 708 709
        if (curr_target->sync.sync_flag < MPIDI_RMA_SYNC_FLUSH) {
            curr_target->sync.sync_flag = MPIDI_RMA_SYNC_FLUSH;
        }

        /* Issue out all operations. */
        mpi_errno = MPIDI_CH3I_RMA_Make_progress_target(win_ptr, curr_target->target_rank,
                                                        &made_progress);
        if (mpi_errno != MPI_SUCCESS)
            MPIU_ERR_POP(mpi_errno);

        /* Wait for remote completion. */
        do {
710
            mpi_errno = MPIDI_CH3I_RMA_Cleanup_ops_target(win_ptr, curr_target);
711 712
            if (mpi_errno != MPI_SUCCESS)
                MPIU_ERR_POP(mpi_errno);
713 714 715

            MPIDI_CH3I_RMA_ops_completion(win_ptr, curr_target, local_completed, remote_completed);

716 717 718 719 720 721 722 723
            if (!remote_completed) {
                mpi_errno = wait_progress_engine();
                if (mpi_errno != MPI_SUCCESS)
                    MPIU_ERR_POP(mpi_errno);
            }
        } while (!remote_completed);

        /* Cleanup the target. */
Xin Zhao's avatar
Xin Zhao committed
724
        mpi_errno = MPIDI_CH3I_Win_target_dequeue_and_free(win_ptr, curr_target);
725 726
        if (mpi_errno != MPI_SUCCESS)
            MPIU_ERR_POP(mpi_errno);
727 728 729 730 731 732 733 734 735 736 737 738 739

        /* check if we got a target */
        (*target) = MPIDI_CH3I_Win_target_alloc(win_ptr);

    } while ((*target) == NULL);

  fn_exit:
    return mpi_errno;
  fn_fail:
    goto fn_exit;
}


740 741 742 743 744 745
#undef FUNCNAME
#define FUNCNAME MPIDI_CH3I_RMA_Make_progress_target
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
int MPIDI_CH3I_RMA_Make_progress_target(MPID_Win * win_ptr, int target_rank, int *made_progress)
{
746
    int temp_progress = 0;
747
    int is_able_to_issue = 0;
748
    MPIDI_RMA_Target_t *target = NULL;
749 750 751 752
    int mpi_errno = MPI_SUCCESS;

    (*made_progress) = 0;

753
    /* check window state */
754
    mpi_errno = check_and_switch_window_state(win_ptr, &is_able_to_issue, &temp_progress);
755 756 757 758
    if (mpi_errno != MPI_SUCCESS)
        MPIU_ERR_POP(mpi_errno);
    if (temp_progress)
        (*made_progress) = 1;
759 760
    if (!is_able_to_issue)
        goto fn_exit;
761

762 763
    /* find target element */
    mpi_errno = MPIDI_CH3I_Win_find_target(win_ptr, target_rank, &target);
764 765
    if (mpi_errno != MPI_SUCCESS)
        MPIU_ERR_POP(mpi_errno);
766

767 768
    /* issue operations to this target */
    mpi_errno = issue_ops_target(win_ptr, target, &temp_progress);
769 770 771 772
    if (mpi_errno)
        MPIU_ERR_POP(mpi_errno);
    if (temp_progress)
        (*made_progress) = 1;
773 774 775 776 777 778 779 780 781 782 783 784 785 786 787

  fn_exit:
    return mpi_errno;
  fn_fail:
    goto fn_exit;
}


#undef FUNCNAME
#define FUNCNAME MPIDI_CH3I_RMA_Make_progress_win
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
int MPIDI_CH3I_RMA_Make_progress_win(MPID_Win * win_ptr, int *made_progress)
{
    int mpi_errno = MPI_SUCCESS;
Xin Zhao's avatar
Xin Zhao committed
788 789 790
    int start_slot, end_slot, i, idx;
    int is_able_to_issue = 0;
    MPIDI_RMA_Target_t *target = NULL;
791 792 793

    (*made_progress) = 0;

Xin Zhao's avatar
Xin Zhao committed
794 795 796
    /* check and try to switch window state */
    mpi_errno = check_and_switch_window_state(win_ptr, &is_able_to_issue, made_progress);
    if (mpi_errno != MPI_SUCCESS)
797
        MPIU_ERR_POP(mpi_errno);
Xin Zhao's avatar
Xin Zhao committed
798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827
    if (!is_able_to_issue)
        goto fn_exit;

    if (win_ptr->non_empty_slots == 0)
        goto fn_exit;

    /* FIXME: we should optimize the issuing pattern here. */

    start_slot = win_ptr->comm_ptr->rank % win_ptr->num_slots;
    end_slot = start_slot + win_ptr->num_slots;
    for (i = start_slot; i < end_slot; i++) {
        if (i < win_ptr->num_slots)
            idx = i;
        else
            idx = i - win_ptr->num_slots;

        target = win_ptr->slots[idx].target_list_head;
        while (target != NULL) {
            int temp_progress = 0;

            /* issue operations to this target */
            mpi_errno = issue_ops_target(win_ptr, target, &temp_progress);
            if (mpi_errno != MPI_SUCCESS)
                MPIU_ERR_POP(mpi_errno);
            if (temp_progress)
                (*made_progress) = 1;

            target = target->next;
        }
    }
828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846

  fn_exit:
    return mpi_errno;
  fn_fail:
    goto fn_exit;
}


#undef FUNCNAME
#define FUNCNAME MPIDI_CH3I_RMA_Make_progress_global
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
int MPIDI_CH3I_RMA_Make_progress_global(int *made_progress)
{
    MPIDI_RMA_Win_list_t *win_elem = MPIDI_RMA_Win_list;
    int mpi_errno = MPI_SUCCESS;

    (*made_progress) = 0;

847 848 849
    if (MPIDI_CH3I_num_active_issued_win == 0 && MPIDI_CH3I_num_passive_win == 0)
        goto fn_exit;

850
    for (win_elem = MPIDI_RMA_Win_list; win_elem; win_elem = win_elem->next) {
851
        int temp_progress = 0;
852

853 854 855 856
        if (win_elem->win_ptr->states.access_state == MPIDI_RMA_NONE ||
            win_elem->win_ptr->states.access_state == MPIDI_RMA_FENCE_GRANTED ||
            win_elem->win_ptr->states.access_state == MPIDI_RMA_PSCW_GRANTED)
            continue;
857

858
        mpi_errno = MPIDI_CH3I_RMA_Make_progress_win(win_elem->win_ptr, &temp_progress);
859 860 861 862
        if (mpi_errno != MPI_SUCCESS)
            MPIU_ERR_POP(mpi_errno);
        if (temp_progress)
            (*made_progress) = 1;
863 864 865 866 867 868 869
    }

  fn_exit:
    return mpi_errno;
  fn_fail:
    goto fn_exit;
}