ch3u_rma_progress.c 29.6 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
/*
 *  (C) 2001 by Argonne National Laboratory.
 *      See COPYRIGHT in top-level directory.
 */

#include "mpidimpl.h"
#include "mpidrma.h"

/*
=== BEGIN_MPI_T_CVAR_INFO_BLOCK ===

cvars:
14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31
    - name        : MPIR_CVAR_CH3_RMA_ACTIVE_REQ_THRESHOLD
      category    : CH3
      type        : int
      default     : 2097152
      class       : none
      verbosity   : MPI_T_VERBOSITY_USER_BASIC
      scope       : MPI_T_SCOPE_ALL_EQ
      description : >-
         Threshold of number of active requests to trigger
         blocking waiting in operation routines. When the
         value is negative, we never blockingly wait in
         operation routines. When the value is zero, we always
         trigger blocking waiting in operation routines to
         wait until no. of active requests becomes zero. When the
         value is positive, we do blocking waiting in operation
         routines to wait until no. of active requests being
         reduced to this value.

32 33 34
=== END_MPI_T_CVAR_INFO_BLOCK ===
*/

35
static inline int check_and_switch_target_state(MPID_Win * win_ptr, MPIDI_RMA_Target_t * target,
36 37
                                                int *is_able_to_issue, int *made_progress);
static inline int check_and_switch_window_state(MPID_Win * win_ptr, int *is_able_to_issue,
38
                                                int *made_progress);
39 40
static inline int issue_ops_target(MPID_Win * win_ptr, MPIDI_RMA_Target_t * target,
                                   int *made_progress);
41

42
/* check if we can switch window-wide state: FENCE_ISSUED, PSCW_ISSUED, LOCK_ALL_ISSUED */
43
#undef FUNCNAME
44
#define FUNCNAME check_and_switch_window_state
45 46
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
47 48
static inline int check_and_switch_window_state(MPID_Win * win_ptr, int *is_able_to_issue,
                                                int *made_progress)
49
{
50
    MPID_Request *fence_req_ptr = NULL;
51
    int i, mpi_errno = MPI_SUCCESS;
52
    MPIDI_STATE_DECL(MPID_STATE_CHECK_AND_SWITCH_WINDOW_STATE);
53

54
    MPIDI_RMA_FUNC_ENTER(MPID_STATE_CHECK_AND_SWITCH_WINDOW_STATE);
55 56

    (*made_progress) = 0;
57
    (*is_able_to_issue) = 0;
58

59 60
    switch (win_ptr->states.access_state) {
    case MPIDI_RMA_FENCE_ISSUED:
61 62 63
        MPID_Request_get_ptr(win_ptr->fence_sync_req, fence_req_ptr);
        if (MPID_Request_is_complete(fence_req_ptr)) {
            win_ptr->states.access_state = MPIDI_RMA_FENCE_GRANTED;
64 65
            (*is_able_to_issue) = 1;

66 67 68
            MPID_Request_release(fence_req_ptr);
            win_ptr->fence_sync_req = MPI_REQUEST_NULL;

69 70
            MPIDI_CH3I_num_active_issued_win--;
            MPIU_Assert(MPIDI_CH3I_num_active_issued_win >= 0);
71

72 73
            (*made_progress) = 1;
        }
74 75 76
        break;

    case MPIDI_RMA_PSCW_ISSUED:
77 78
        if (win_ptr->start_req == NULL) {
            /* for MPI_MODE_NOCHECK and all targets on SHM,
79
             * we do not create PSCW requests on window. */
80
            win_ptr->states.access_state = MPIDI_RMA_PSCW_GRANTED;
81
            (*is_able_to_issue) = 1;
82

83 84
            MPIDI_CH3I_num_active_issued_win--;
            MPIU_Assert(MPIDI_CH3I_num_active_issued_win >= 0);
85

86 87 88 89 90 91 92 93 94 95 96 97 98
            (*made_progress) = 1;
        }
        else {
            for (i = 0; i < win_ptr->start_grp_size; i++) {
                MPID_Request *start_req_ptr = NULL;
                if (win_ptr->start_req[i] == MPI_REQUEST_NULL)
                    continue;
                MPID_Request_get_ptr(win_ptr->start_req[i], start_req_ptr);
                if (MPID_Request_is_complete(start_req_ptr)) {
                    MPID_Request_release(start_req_ptr);
                    win_ptr->start_req[i] = MPI_REQUEST_NULL;
                }
                else {
99
                    break;
100 101 102
                }
            }

103 104
            if (i == win_ptr->start_grp_size) {
                win_ptr->states.access_state = MPIDI_RMA_PSCW_GRANTED;
105
                (*is_able_to_issue) = 1;
106

107 108
                MPIDI_CH3I_num_active_issued_win--;
                MPIU_Assert(MPIDI_CH3I_num_active_issued_win >= 0);
109

110 111 112 113 114
                (*made_progress) = 1;

                MPIU_Free(win_ptr->start_req);
                win_ptr->start_req = NULL;
            }
115
        }
116 117 118
        break;

    case MPIDI_RMA_LOCK_ALL_ISSUED:
119 120
        if (win_ptr->outstanding_locks == 0) {
            win_ptr->states.access_state = MPIDI_RMA_LOCK_ALL_GRANTED;
121 122
            (*is_able_to_issue) = 1;

123 124
            (*made_progress) = 1;
        }
125 126
        break;

127 128 129 130 131 132 133 134
    case MPIDI_RMA_PER_TARGET:
    case MPIDI_RMA_LOCK_ALL_CALLED:
    case MPIDI_RMA_FENCE_GRANTED:
    case MPIDI_RMA_PSCW_GRANTED:
    case MPIDI_RMA_LOCK_ALL_GRANTED:
        (*is_able_to_issue) = 1;
        break;

135 136
    default:
        break;
137
    }   /* end of switch */
138 139

  fn_exit:
140
    MPIDI_RMA_FUNC_EXIT(MPID_STATE_CHECK_AND_SWITCH_WINDOW_STATE);
141 142 143 144 145 146 147 148 149
    return mpi_errno;
    /* --BEGIN ERROR HANDLING-- */
  fn_fail:
    goto fn_exit;
    /* --END ERROR HANDLING-- */
}


#undef FUNCNAME
150
#define FUNCNAME check_and_switch_target_state
151 152
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
153
static inline int check_and_switch_target_state(MPID_Win * win_ptr, MPIDI_RMA_Target_t * target,
154
                                                int *is_able_to_issue, int *made_progress)
155 156 157 158 159
{
    int rank = win_ptr->comm_ptr->rank;
    int mpi_errno = MPI_SUCCESS;

    (*made_progress) = 0;
160
    (*is_able_to_issue) = 0;
161

162
    if (target == NULL)
163 164
        goto fn_exit;

165 166 167 168 169
    switch (target->access_state) {
    case MPIDI_RMA_LOCK_CALLED:
        if (target->sync.sync_flag == MPIDI_RMA_SYNC_NONE ||
            target->sync.sync_flag == MPIDI_RMA_SYNC_FLUSH_LOCAL ||
            target->sync.sync_flag == MPIDI_RMA_SYNC_FLUSH) {
170 171
            if (target->pending_op_list_head == NULL ||
                !target->pending_op_list_head->piggyback_lock_candidate) {
172 173 174 175
                /* issue lock request */
                target->access_state = MPIDI_RMA_LOCK_ISSUED;
                if (target->target_rank == rank) {
                    mpi_errno = acquire_local_lock(win_ptr, target->lock_type);
176 177
                    if (mpi_errno != MPI_SUCCESS)
                        MPIU_ERR_POP(mpi_errno);
178 179
                }
                else {
180 181 182
                    mpi_errno = send_lock_msg(target->target_rank, target->lock_type, win_ptr);
                    if (mpi_errno != MPI_SUCCESS)
                        MPIU_ERR_POP(mpi_errno);
183 184 185 186 187 188
                }

                (*made_progress) = 1;
            }
        }
        else if (target->sync.sync_flag == MPIDI_RMA_SYNC_UNLOCK) {
189
            if (target->pending_op_list_head == NULL) {
190
                /* No RMA operation has ever been posted to this target,
191 192
                 * finish issuing, no need to acquire the lock. Cleanup
                 * function will clean it up. */
193 194 195 196 197 198 199 200 201
                target->access_state = MPIDI_RMA_LOCK_GRANTED;

                /* We are done with ending synchronization, unset target's sync_flag. */
                target->sync.sync_flag = MPIDI_RMA_SYNC_NONE;

                (*made_progress) = 1;
            }
            else {
                /* if we reach WIN_UNLOCK and there is still operation existing
202 203
                 * in pending list, this operation must be the only operation
                 * and it is prepared to piggyback LOCK and UNLOCK. */
204 205
                MPIU_Assert(target->pending_op_list_head->next == NULL);
                MPIU_Assert(target->pending_op_list_head->piggyback_lock_candidate);
206 207 208 209 210 211 212
            }
        }
        break;

    case MPIDI_RMA_LOCK_GRANTED:
    case MPIDI_RMA_NONE:
        if (target->sync.sync_flag == MPIDI_RMA_SYNC_FLUSH) {
213
            if (target->pending_op_list_head == NULL) {
214
                if (target->target_rank != rank) {
215
                    if (target->put_acc_issued) {
216 217 218

                        target->sync.outstanding_acks++;

219
                        mpi_errno = send_flush_msg(target->target_rank, win_ptr);
220 221
                        if (mpi_errno != MPI_SUCCESS)
                            MPIU_ERR_POP(mpi_errno);
222 223
                    }
                }
224 225 226 227 228

                /* We are done with ending synchronization, unset target's sync_flag. */
                target->sync.sync_flag = MPIDI_RMA_SYNC_NONE;

                (*made_progress) = 1;
229
            }
230 231
        }
        else if (target->sync.sync_flag == MPIDI_RMA_SYNC_UNLOCK) {
232
            if (target->pending_op_list_head == NULL) {
233 234
                if (target->target_rank == rank) {
                    mpi_errno = MPIDI_CH3I_Release_lock(win_ptr);
235 236
                    if (mpi_errno != MPI_SUCCESS)
                        MPIU_ERR_POP(mpi_errno);
237
                }
238 239 240 241
                else {
                    MPIDI_CH3_Pkt_flags_t flag = MPIDI_CH3_PKT_FLAG_NONE;
                    if (!target->put_acc_issued) {
                        /* We did not issue PUT/ACC since the last
242 243
                         * synchronization call, therefore here we
                         * don't need ACK back */
244 245 246

                        flag = MPIDI_CH3_PKT_FLAG_RMA_UNLOCK_NO_ACK;
                    }
247 248 249
                    else {
                        target->sync.outstanding_acks++;
                    }
250
                    mpi_errno = send_unlock_msg(target->target_rank, win_ptr, flag);
251 252
                    if (mpi_errno != MPI_SUCCESS)
                        MPIU_ERR_POP(mpi_errno);
253 254 255 256 257
                }

                /* We are done with ending synchronization, unset target's sync_flag. */
                target->sync.sync_flag = MPIDI_RMA_SYNC_NONE;

258 259 260
                (*made_progress) = 1;
            }
        }
261
        break;
262

263 264
    default:
        break;
265
    }   /* end of switch */
266

267 268 269 270
    if (target->pending_op_list_head != NULL && target->access_state != MPIDI_RMA_LOCK_ISSUED) {
        (*is_able_to_issue) = 1;
    }

271
  fn_exit:
272
    return mpi_errno;
273
  fn_fail:
274 275
    goto fn_exit;
}
276 277


278 279 280 281
#undef FUNCNAME
#define FUNCNAME issue_ops_target
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
282
static inline int issue_ops_target(MPID_Win * win_ptr, MPIDI_RMA_Target_t * target,
283 284 285 286
                                   int *made_progress)
{
    MPIDI_RMA_Op_t *curr_op = NULL;
    MPIDI_CH3_Pkt_flags_t flags;
287
    int is_able_to_issue = 0;
288
    int first_op = 1, mpi_errno = MPI_SUCCESS;
289

290
    (*made_progress) = 0;
291

292 293 294 295 296 297 298
    /* check and try to switch target state */
    mpi_errno = check_and_switch_target_state(win_ptr, target, &is_able_to_issue, made_progress);
    if (mpi_errno != MPI_SUCCESS)
        MPIU_ERR_POP(mpi_errno);
    if (!is_able_to_issue)
        goto fn_exit;

299 300
    if (win_ptr->non_empty_slots == 0 || target == NULL)
        goto fn_exit;
301

302 303 304 305
    /* Issue out operations in the list. */
    curr_op = target->next_op_to_issue;
    while (curr_op != NULL) {

306 307
        if (target->access_state == MPIDI_RMA_LOCK_ISSUED) {
            /* It is possible that the previous OP+LOCK changes
308
             * lock state to LOCK_ISSUED. */
309 310
            break;
        }
311 312

        if (curr_op->next == NULL &&
313
            target->sync.sync_flag == MPIDI_RMA_SYNC_NONE && curr_op->ureq == NULL) {
314
            /* Skip the last OP if sync_flag is NONE since we
315 316 317 318 319
             * want to leave it to the ending synchronization
             * so that we can piggyback LOCK / FLUSH.
             * However, if it is a request-based RMA, do not
             * skip it (otherwise a wait call before unlock
             * will be blocked). */
320
            break;
321 322
        }

323 324
        flags = MPIDI_CH3_PKT_FLAG_NONE;

325 326 327 328
        if (first_op) {
            /* piggyback on first OP. */
            if (target->access_state == MPIDI_RMA_LOCK_CALLED) {
                MPIU_Assert(curr_op->piggyback_lock_candidate);
329 330 331 332 333 334
                if (target->lock_type == MPI_LOCK_SHARED)
                    flags |= MPIDI_CH3_PKT_FLAG_RMA_LOCK_SHARED;
                else {
                    MPIU_Assert(target->lock_type == MPI_LOCK_EXCLUSIVE);
                    flags |= MPIDI_CH3_PKT_FLAG_RMA_LOCK_EXCLUSIVE;
                }
335 336 337 338 339
                target->access_state = MPIDI_RMA_LOCK_ISSUED;
            }
            first_op = 0;
        }

340 341 342 343
        /* piggyback FLUSH on every OP if ordered flush is not guaranteed. */
        if (!MPIDI_CH3U_Win_pkt_orderings.am_flush_ordered)
            flags |= MPIDI_CH3_PKT_FLAG_RMA_FLUSH;

344 345 346 347 348 349 350 351 352
        if (curr_op->next == NULL) {
            /* piggyback on last OP. */
            if (target->sync.sync_flag == MPIDI_RMA_SYNC_FLUSH) {
                flags |= MPIDI_CH3_PKT_FLAG_RMA_FLUSH;
                if (target->win_complete_flag)
                    flags |= MPIDI_CH3_PKT_FLAG_RMA_DECR_AT_COUNTER;
            }
            else if (target->sync.sync_flag == MPIDI_RMA_SYNC_UNLOCK) {
                flags |= MPIDI_CH3_PKT_FLAG_RMA_UNLOCK;
353 354 355 356

                /* if piggyback UNLOCK then unset FLUSH (set for every
                 * operation on out-of-order network). */
                flags &= ~MPIDI_CH3_PKT_FLAG_RMA_FLUSH;
357 358 359
            }
        }

360 361 362 363 364 365
        /* only increase ack counter when FLUSH or UNLOCK flag is set,
         * but without LOCK piggyback. */
        if (((flags & MPIDI_CH3_PKT_FLAG_RMA_FLUSH)
             || (flags & MPIDI_CH3_PKT_FLAG_RMA_UNLOCK)))
            target->sync.outstanding_acks++;

366 367 368 369
        mpi_errno = issue_rma_op(curr_op, win_ptr, target, flags);
        if (mpi_errno != MPI_SUCCESS)
            MPIU_ERR_POP(mpi_errno);

370 371
        (*made_progress) = 1;

372
        if (curr_op->pkt.type == MPIDI_CH3_PKT_PUT ||
373 374 375
            curr_op->pkt.type == MPIDI_CH3_PKT_PUT_IMMED ||
            curr_op->pkt.type == MPIDI_CH3_PKT_ACCUMULATE ||
            curr_op->pkt.type == MPIDI_CH3_PKT_ACCUMULATE_IMMED) {
376
            target->put_acc_issued = 1; /* set PUT_ACC_FLAG when sending
377
                                         * PUT/ACC operation. */
378 379
        }

380 381 382 383 384 385 386 387 388 389
        if ((curr_op->pkt.type == MPIDI_CH3_PKT_ACCUMULATE ||
             curr_op->pkt.type == MPIDI_CH3_PKT_GET_ACCUM) && curr_op->issued_stream_count > 0) {
            /* For ACC-like operations, if not all stream units
             * are issued out, we stick to the current operation,
             * otherwise we move on to the next operation. */
            target->next_op_to_issue = curr_op;
        }
        else
            target->next_op_to_issue = curr_op->next;

390 391 392 393 394 395 396
        if (target->next_op_to_issue == NULL) {
            if (flags & MPIDI_CH3_PKT_FLAG_RMA_FLUSH || flags & MPIDI_CH3_PKT_FLAG_RMA_UNLOCK) {
                /* We are done with ending sync, unset target's sync_flag. */
                target->sync.sync_flag = MPIDI_RMA_SYNC_NONE;
            }
        }

397 398
        if (flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_SHARED ||
            flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_EXCLUSIVE) {
399
            /* If this operation is piggybacked with LOCK,
400 401 402
             * do not move it out of pending list, and do
             * not complete the user request, because we
             * may need to re-transmit it. */
403 404 405
            break;
        }

406 407 408 409 410
        if (curr_op->ureq != NULL) {
            mpi_errno = set_user_req_after_issuing_op(curr_op);
            if (mpi_errno != MPI_SUCCESS)
                MPIU_ERR_POP(mpi_errno);
        }
411

412 413
        if (curr_op->reqs_size == 0) {
            MPIU_Assert(curr_op->reqs == NULL);
414
            /* Sending is completed immediately. */
415
            MPIDI_CH3I_RMA_Ops_free_elem(win_ptr, &(target->pending_op_list_head),
416 417 418
                                         &(target->pending_op_list_tail), curr_op);
        }
        else {
419 420 421
            MPI_Datatype target_datatype;
            int is_derived = FALSE;

422
            /* Sending is not completed immediately. */
423

424
            MPIDI_CH3I_RMA_Ops_unlink(&(target->pending_op_list_head),
425
                                      &(target->pending_op_list_tail), curr_op);
426 427 428 429 430 431 432 433 434 435 436 437 438

            MPIDI_CH3_PKT_RMA_GET_TARGET_DATATYPE(curr_op->pkt, target_datatype, mpi_errno);

            if ((target_datatype != MPI_DATATYPE_NULL &&
                 !MPIR_DATATYPE_IS_PREDEFINED(target_datatype)) ||
                (curr_op->origin_datatype != MPI_DATATYPE_NULL &&
                 !MPIR_DATATYPE_IS_PREDEFINED(curr_op->origin_datatype)) ||
                (curr_op->result_datatype != MPI_DATATYPE_NULL &&
                 !MPIR_DATATYPE_IS_PREDEFINED(curr_op->result_datatype))) {
                is_derived = TRUE;
            }

            if (is_derived) {
439 440
                MPIDI_CH3I_RMA_Ops_append(&(target->issued_dt_op_list_head),
                                          &(target->issued_dt_op_list_tail), curr_op);
441 442
            }
            else if (curr_op->pkt.type == MPIDI_CH3_PKT_PUT ||
443 444 445
                     curr_op->pkt.type == MPIDI_CH3_PKT_PUT_IMMED ||
                     curr_op->pkt.type == MPIDI_CH3_PKT_ACCUMULATE ||
                     curr_op->pkt.type == MPIDI_CH3_PKT_ACCUMULATE_IMMED) {
446 447
                MPIDI_CH3I_RMA_Ops_append(&(target->issued_write_op_list_head),
                                          &(target->issued_write_op_list_tail), curr_op);
448 449
            }
            else {
450 451
                MPIDI_CH3I_RMA_Ops_append(&(target->issued_read_op_list_head),
                                          &(target->issued_read_op_list_tail), curr_op);
452 453 454 455 456
            }
        }

        curr_op = target->next_op_to_issue;

457
    }   /* end of while loop */
458 459 460 461 462 463 464 465

  fn_exit:
    return mpi_errno;
  fn_fail:
    goto fn_exit;
}


466 467 468 469 470 471 472 473
#undef FUNCNAME
#define FUNCNAME MPIDI_CH3I_RMA_Free_ops_before_completion
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
int MPIDI_CH3I_RMA_Free_ops_before_completion(MPID_Win * win_ptr)
{
    MPIDI_RMA_Op_t *curr_op = NULL;
    MPIDI_RMA_Target_t *curr_target = NULL;
474
    MPIDI_RMA_Op_t **op_list_head = NULL, **op_list_tail = NULL;
475 476 477 478
    int read_flag = 0;
    int i, made_progress = 0;
    int mpi_errno = MPI_SUCCESS;

479 480 481
    /* If we are in an free_ops_before_completion, the window must be holding
     * up resources.  If it isn't, we are in the wrong window and
     * incorrectly entered this function. */
482
    MPIU_ERR_CHKANDJUMP(win_ptr->non_empty_slots == 0, mpi_errno, MPI_ERR_OTHER, "**rmanoop");
483 484

    /* make nonblocking progress once */
485
    mpi_errno = MPIDI_CH3I_RMA_Make_progress_win(win_ptr, &made_progress);
486 487
    if (mpi_errno != MPI_SUCCESS)
        MPIU_ERR_POP(mpi_errno);
488

489
    if (win_ptr->states.access_state == MPIDI_RMA_FENCE_ISSUED ||
490 491
        win_ptr->states.access_state == MPIDI_RMA_PSCW_ISSUED ||
        win_ptr->states.access_state == MPIDI_RMA_LOCK_ALL_ISSUED)
492 493 494 495
        goto fn_exit;

    /* find targets that have operations */
    for (i = 0; i < win_ptr->num_slots; i++) {
496 497
        if (win_ptr->slots[i].target_list_head != NULL) {
            curr_target = win_ptr->slots[i].target_list_head;
498
            while (curr_target != NULL) {
499 500
                if (curr_target->issued_read_op_list_head != NULL ||
                    curr_target->issued_write_op_list_head != NULL) {
501 502 503 504 505 506 507 508 509
                    if (win_ptr->states.access_state == MPIDI_RMA_PER_TARGET ||
                        win_ptr->states.access_state == MPIDI_RMA_LOCK_ALL_CALLED) {
                        if (curr_target->access_state == MPIDI_RMA_LOCK_GRANTED)
                            break;
                    }
                    else {
                        break;
                    }
                }
510
                curr_target = curr_target->next;
511
            }
512 513
            if (curr_target != NULL)
                break;
514 515
        }
    }
516

517 518
    if (curr_target == NULL)
        goto fn_exit;
519

520
    /* After we do this, all following Win_flush_local
521
     * must do a Win_flush instead. */
522
    curr_target->sync.upgrade_flush_local = 1;
523

524 525 526 527 528
    op_list_head = &curr_target->issued_read_op_list_head;
    op_list_tail = &curr_target->issued_read_op_list_tail;
    read_flag = 1;

    curr_op = *op_list_head;
529 530

    /* free all ops in the list since we do not need to maintain them anymore */
531 532 533 534 535 536 537 538 539 540
    while (1) {
        if (curr_op != NULL) {
            if (curr_op->reqs_size > 0) {
                MPIU_Assert(curr_op->reqs != NULL);
                for (i = 0; i < curr_op->reqs_size; i++) {
                    if (curr_op->reqs[i] != NULL) {
                        MPID_Request_release(curr_op->reqs[i]);
                        curr_op->reqs[i] = NULL;
                        win_ptr->active_req_cnt--;
                    }
541 542
                }

543 544 545 546 547
                /* free req array in this op */
                MPIU_Free(curr_op->reqs);
                curr_op->reqs = NULL;
                curr_op->reqs_size = 0;
            }
548
            MPIDI_CH3I_RMA_Ops_free_elem(win_ptr, op_list_head, op_list_tail, curr_op);
549
        }
550
        else {
551
            if (read_flag == 1) {
552
                op_list_head = &curr_target->issued_write_op_list_head;
Xin Zhao's avatar
Xin Zhao committed
553
                op_list_tail = &curr_target->issued_write_op_list_tail;
554 555
                read_flag = 0;
            }
556 557 558 559
            else {
                /* we reach the tail of write_op_list, break out. */
                break;
            }
560
        }
561
        curr_op = *op_list_head;
562
    }
563

564
  fn_exit:
565
    return mpi_errno;
566
  fn_fail:
567 568 569 570
    goto fn_exit;
}


571 572 573 574 575 576
#undef FUNCNAME
#define FUNCNAME MPIDI_CH3I_RMA_Cleanup_ops_aggressive
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
int MPIDI_CH3I_RMA_Cleanup_ops_aggressive(MPID_Win * win_ptr)
{
577
    int i, local_completed = 0, remote_completed ATTRIBUTE((unused)) = 0;
578 579 580 581 582 583 584
    int mpi_errno = MPI_SUCCESS;
    MPIDI_RMA_Target_t *curr_target = NULL;
    int made_progress = 0;

    /* If we are in an aggressive cleanup, the window must be holding
     * up resources.  If it isn't, we are in the wrong window and
     * incorrectly entered this function. */
585
    MPIU_ERR_CHKANDJUMP(win_ptr->non_empty_slots == 0, mpi_errno, MPI_ERR_OTHER, "**rmanoop");
586 587 588

    /* find the first target that has something to issue */
    for (i = 0; i < win_ptr->num_slots; i++) {
589 590 591
        if (win_ptr->slots[i].target_list_head != NULL) {
            curr_target = win_ptr->slots[i].target_list_head;
            while (curr_target != NULL && curr_target->pending_op_list_head == NULL)
592
                curr_target = curr_target->next;
593 594
            if (curr_target != NULL)
                break;
595 596 597
        }
    }

598 599
    if (curr_target == NULL)
        goto fn_exit;
600 601 602 603 604 605 606 607 608 609 610 611

    if (curr_target->sync.sync_flag < MPIDI_RMA_SYNC_FLUSH_LOCAL)
        curr_target->sync.sync_flag = MPIDI_RMA_SYNC_FLUSH_LOCAL;

    /* Issue out all operations. */
    mpi_errno = MPIDI_CH3I_RMA_Make_progress_target(win_ptr, curr_target->target_rank,
                                                    &made_progress);
    if (mpi_errno != MPI_SUCCESS)
        MPIU_ERR_POP(mpi_errno);

    /* Wait for local completion. */
    do {
612
        mpi_errno = MPIDI_CH3I_RMA_Cleanup_ops_target(win_ptr, curr_target);
613 614
        if (mpi_errno != MPI_SUCCESS)
            MPIU_ERR_POP(mpi_errno);
615 616 617

        MPIDI_CH3I_RMA_ops_completion(win_ptr, curr_target, local_completed, remote_completed);

618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637
        if (!local_completed) {
            mpi_errno = wait_progress_engine();
            if (mpi_errno != MPI_SUCCESS)
                MPIU_ERR_POP(mpi_errno);
        }
    } while (!local_completed);

  fn_exit:
    return mpi_errno;
  fn_fail:
    goto fn_exit;
}


#undef FUNCNAME
#define FUNCNAME MPIDI_CH3I_RMA_Cleanup_target_aggressive
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
int MPIDI_CH3I_RMA_Cleanup_target_aggressive(MPID_Win * win_ptr, MPIDI_RMA_Target_t ** target)
{
638
    int i, local_completed ATTRIBUTE((unused)) = 0, remote_completed = 0;
639 640 641 642 643 644 645 646 647
    int made_progress = 0;
    MPIDI_RMA_Target_t *curr_target = NULL;
    int mpi_errno = MPI_SUCCESS;

    (*target) = NULL;

    /* If we are in an aggressive cleanup, the window must be holding
     * up resources.  If it isn't, we are in the wrong window and
     * incorrectly entered this function. */
648
    MPIU_ERR_CHKANDJUMP(win_ptr->non_empty_slots == 0, mpi_errno, MPI_ERR_OTHER, "**rmanotarget");
649 650 651 652 653 654 655 656 657 658 659

    if (win_ptr->states.access_state == MPIDI_RMA_LOCK_ALL_CALLED) {
        /* switch to window-wide protocol */
        MPIDI_VC_t *orig_vc = NULL, *target_vc = NULL;
        MPIDI_Comm_get_vc(win_ptr->comm_ptr, win_ptr->comm_ptr->rank, &orig_vc);
        for (i = 0; i < win_ptr->comm_ptr->local_size; i++) {
            if (i == win_ptr->comm_ptr->rank)
                continue;
            MPIDI_Comm_get_vc(win_ptr->comm_ptr, i, &target_vc);
            if (orig_vc->node_id != target_vc->node_id) {
                mpi_errno = MPIDI_CH3I_Win_find_target(win_ptr, i, &curr_target);
660 661
                if (mpi_errno)
                    MPIU_ERR_POP(mpi_errno);
662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677
                if (curr_target == NULL) {
                    win_ptr->outstanding_locks++;
                    mpi_errno = send_lock_msg(i, MPI_LOCK_SHARED, win_ptr);
                    if (mpi_errno != MPI_SUCCESS)
                        MPIU_ERR_POP(mpi_errno);
                }
            }
        }
        win_ptr->states.access_state = MPIDI_RMA_LOCK_ALL_ISSUED;
    }

    do {
        /* find a non-empty slot and set the FLUSH flag on the first
         * target */
        /* TODO: we should think about better strategies on selecting the target */
        for (i = 0; i < win_ptr->num_slots; i++)
678
            if (win_ptr->slots[i].target_list_head != NULL)
679
                break;
680
        curr_target = win_ptr->slots[i].target_list_head;
681 682 683 684 685 686 687 688 689 690 691 692
        if (curr_target->sync.sync_flag < MPIDI_RMA_SYNC_FLUSH) {
            curr_target->sync.sync_flag = MPIDI_RMA_SYNC_FLUSH;
        }

        /* Issue out all operations. */
        mpi_errno = MPIDI_CH3I_RMA_Make_progress_target(win_ptr, curr_target->target_rank,
                                                        &made_progress);
        if (mpi_errno != MPI_SUCCESS)
            MPIU_ERR_POP(mpi_errno);

        /* Wait for remote completion. */
        do {
693
            mpi_errno = MPIDI_CH3I_RMA_Cleanup_ops_target(win_ptr, curr_target);
694 695
            if (mpi_errno != MPI_SUCCESS)
                MPIU_ERR_POP(mpi_errno);
696 697 698

            MPIDI_CH3I_RMA_ops_completion(win_ptr, curr_target, local_completed, remote_completed);

699 700 701 702 703 704 705 706
            if (!remote_completed) {
                mpi_errno = wait_progress_engine();
                if (mpi_errno != MPI_SUCCESS)
                    MPIU_ERR_POP(mpi_errno);
            }
        } while (!remote_completed);

        /* Cleanup the target. */
Xin Zhao's avatar
Xin Zhao committed
707
        mpi_errno = MPIDI_CH3I_Win_target_dequeue_and_free(win_ptr, curr_target);
708 709
        if (mpi_errno != MPI_SUCCESS)
            MPIU_ERR_POP(mpi_errno);
710 711 712 713 714 715 716 717 718 719 720 721 722

        /* check if we got a target */
        (*target) = MPIDI_CH3I_Win_target_alloc(win_ptr);

    } while ((*target) == NULL);

  fn_exit:
    return mpi_errno;
  fn_fail:
    goto fn_exit;
}


723 724 725 726 727 728
#undef FUNCNAME
#define FUNCNAME MPIDI_CH3I_RMA_Make_progress_target
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
int MPIDI_CH3I_RMA_Make_progress_target(MPID_Win * win_ptr, int target_rank, int *made_progress)
{
729
    int temp_progress = 0;
730
    int is_able_to_issue = 0;
731
    MPIDI_RMA_Target_t *target = NULL;
732 733 734 735
    int mpi_errno = MPI_SUCCESS;

    (*made_progress) = 0;

736
    /* check window state */
737
    mpi_errno = check_and_switch_window_state(win_ptr, &is_able_to_issue, &temp_progress);
738 739 740 741
    if (mpi_errno != MPI_SUCCESS)
        MPIU_ERR_POP(mpi_errno);
    if (temp_progress)
        (*made_progress) = 1;
742 743
    if (!is_able_to_issue)
        goto fn_exit;
744

745 746
    /* find target element */
    mpi_errno = MPIDI_CH3I_Win_find_target(win_ptr, target_rank, &target);
747 748
    if (mpi_errno != MPI_SUCCESS)
        MPIU_ERR_POP(mpi_errno);
749

750 751
    /* issue operations to this target */
    mpi_errno = issue_ops_target(win_ptr, target, &temp_progress);
752 753 754 755
    if (mpi_errno)
        MPIU_ERR_POP(mpi_errno);
    if (temp_progress)
        (*made_progress) = 1;
756 757 758 759 760 761 762 763 764 765 766 767 768 769 770

  fn_exit:
    return mpi_errno;
  fn_fail:
    goto fn_exit;
}


#undef FUNCNAME
#define FUNCNAME MPIDI_CH3I_RMA_Make_progress_win
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
int MPIDI_CH3I_RMA_Make_progress_win(MPID_Win * win_ptr, int *made_progress)
{
    int mpi_errno = MPI_SUCCESS;
Xin Zhao's avatar
Xin Zhao committed
771 772 773
    int start_slot, end_slot, i, idx;
    int is_able_to_issue = 0;
    MPIDI_RMA_Target_t *target = NULL;
774 775 776

    (*made_progress) = 0;

Xin Zhao's avatar
Xin Zhao committed
777 778 779
    /* check and try to switch window state */
    mpi_errno = check_and_switch_window_state(win_ptr, &is_able_to_issue, made_progress);
    if (mpi_errno != MPI_SUCCESS)
780
        MPIU_ERR_POP(mpi_errno);
Xin Zhao's avatar
Xin Zhao committed
781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810
    if (!is_able_to_issue)
        goto fn_exit;

    if (win_ptr->non_empty_slots == 0)
        goto fn_exit;

    /* FIXME: we should optimize the issuing pattern here. */

    start_slot = win_ptr->comm_ptr->rank % win_ptr->num_slots;
    end_slot = start_slot + win_ptr->num_slots;
    for (i = start_slot; i < end_slot; i++) {
        if (i < win_ptr->num_slots)
            idx = i;
        else
            idx = i - win_ptr->num_slots;

        target = win_ptr->slots[idx].target_list_head;
        while (target != NULL) {
            int temp_progress = 0;

            /* issue operations to this target */
            mpi_errno = issue_ops_target(win_ptr, target, &temp_progress);
            if (mpi_errno != MPI_SUCCESS)
                MPIU_ERR_POP(mpi_errno);
            if (temp_progress)
                (*made_progress) = 1;

            target = target->next;
        }
    }
811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830

  fn_exit:
    return mpi_errno;
  fn_fail:
    goto fn_exit;
}


#undef FUNCNAME
#define FUNCNAME MPIDI_CH3I_RMA_Make_progress_global
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
int MPIDI_CH3I_RMA_Make_progress_global(int *made_progress)
{
    MPIDI_RMA_Win_list_t *win_elem = MPIDI_RMA_Win_list;
    int mpi_errno = MPI_SUCCESS;

    (*made_progress) = 0;

    for (win_elem = MPIDI_RMA_Win_list; win_elem; win_elem = win_elem->next) {
831
        int temp_progress = 0;
832

833 834 835 836
        if (win_elem->win_ptr->states.access_state == MPIDI_RMA_NONE ||
            win_elem->win_ptr->states.access_state == MPIDI_RMA_FENCE_GRANTED ||
            win_elem->win_ptr->states.access_state == MPIDI_RMA_PSCW_GRANTED)
            continue;
837

838
        mpi_errno = MPIDI_CH3I_RMA_Make_progress_win(win_elem->win_ptr, &temp_progress);
839 840 841 842
        if (mpi_errno != MPI_SUCCESS)
            MPIU_ERR_POP(mpi_errno);
        if (temp_progress)
            (*made_progress) = 1;
843 844 845 846 847 848 849
    }

  fn_exit:
    return mpi_errno;
  fn_fail:
    goto fn_exit;
}