ch3u_rma_progress.c 29.7 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
/*
 *  (C) 2001 by Argonne National Laboratory.
 *      See COPYRIGHT in top-level directory.
 */

#include "mpidimpl.h"
#include "mpidrma.h"

/*
=== BEGIN_MPI_T_CVAR_INFO_BLOCK ===

cvars:
14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31
    - name        : MPIR_CVAR_CH3_RMA_ACTIVE_REQ_THRESHOLD
      category    : CH3
      type        : int
      default     : 2097152
      class       : none
      verbosity   : MPI_T_VERBOSITY_USER_BASIC
      scope       : MPI_T_SCOPE_ALL_EQ
      description : >-
         Threshold of number of active requests to trigger
         blocking waiting in operation routines. When the
         value is negative, we never blockingly wait in
         operation routines. When the value is zero, we always
         trigger blocking waiting in operation routines to
         wait until no. of active requests becomes zero. When the
         value is positive, we do blocking waiting in operation
         routines to wait until no. of active requests being
         reduced to this value.

32 33 34
=== END_MPI_T_CVAR_INFO_BLOCK ===
*/

35
static inline int check_and_switch_target_state(MPID_Win * win_ptr, MPIDI_RMA_Target_t * target,
36 37
                                                int *is_able_to_issue, int *made_progress);
static inline int check_and_switch_window_state(MPID_Win * win_ptr, int *is_able_to_issue,
38
                                                int *made_progress);
39 40
static inline int issue_ops_target(MPID_Win * win_ptr, MPIDI_RMA_Target_t * target,
                                   int *made_progress);
41

42
/* check if we can switch window-wide state: FENCE_ISSUED, PSCW_ISSUED, LOCK_ALL_ISSUED */
43
#undef FUNCNAME
44
#define FUNCNAME check_and_switch_window_state
45 46
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
47 48
static inline int check_and_switch_window_state(MPID_Win * win_ptr, int *is_able_to_issue,
                                                int *made_progress)
49
{
50
    MPID_Request *fence_req_ptr = NULL;
51
    int i, mpi_errno = MPI_SUCCESS;
52
    MPIDI_STATE_DECL(MPID_STATE_CHECK_AND_SWITCH_WINDOW_STATE);
53

54
    MPIDI_RMA_FUNC_ENTER(MPID_STATE_CHECK_AND_SWITCH_WINDOW_STATE);
55 56

    (*made_progress) = 0;
57
    (*is_able_to_issue) = 0;
58

59 60
    switch (win_ptr->states.access_state) {
    case MPIDI_RMA_FENCE_ISSUED:
61 62 63
        MPID_Request_get_ptr(win_ptr->fence_sync_req, fence_req_ptr);
        if (MPID_Request_is_complete(fence_req_ptr)) {
            win_ptr->states.access_state = MPIDI_RMA_FENCE_GRANTED;
64 65
            (*is_able_to_issue) = 1;

66 67 68
            MPID_Request_release(fence_req_ptr);
            win_ptr->fence_sync_req = MPI_REQUEST_NULL;

69 70
            MPIDI_CH3I_num_active_issued_win--;
            MPIU_Assert(MPIDI_CH3I_num_active_issued_win >= 0);
71

72 73
            (*made_progress) = 1;
        }
74 75 76
        break;

    case MPIDI_RMA_PSCW_ISSUED:
77 78
        if (win_ptr->start_req == NULL) {
            /* for MPI_MODE_NOCHECK and all targets on SHM,
79
             * we do not create PSCW requests on window. */
80
            win_ptr->states.access_state = MPIDI_RMA_PSCW_GRANTED;
81
            (*is_able_to_issue) = 1;
82

83 84
            MPIDI_CH3I_num_active_issued_win--;
            MPIU_Assert(MPIDI_CH3I_num_active_issued_win >= 0);
85

86 87 88 89 90 91 92 93 94 95 96 97 98
            (*made_progress) = 1;
        }
        else {
            for (i = 0; i < win_ptr->start_grp_size; i++) {
                MPID_Request *start_req_ptr = NULL;
                if (win_ptr->start_req[i] == MPI_REQUEST_NULL)
                    continue;
                MPID_Request_get_ptr(win_ptr->start_req[i], start_req_ptr);
                if (MPID_Request_is_complete(start_req_ptr)) {
                    MPID_Request_release(start_req_ptr);
                    win_ptr->start_req[i] = MPI_REQUEST_NULL;
                }
                else {
99
                    break;
100 101 102
                }
            }

103 104
            if (i == win_ptr->start_grp_size) {
                win_ptr->states.access_state = MPIDI_RMA_PSCW_GRANTED;
105
                (*is_able_to_issue) = 1;
106

107 108
                MPIDI_CH3I_num_active_issued_win--;
                MPIU_Assert(MPIDI_CH3I_num_active_issued_win >= 0);
109

110 111 112 113 114
                (*made_progress) = 1;

                MPIU_Free(win_ptr->start_req);
                win_ptr->start_req = NULL;
            }
115
        }
116 117 118
        break;

    case MPIDI_RMA_LOCK_ALL_ISSUED:
119 120
        if (win_ptr->outstanding_locks == 0) {
            win_ptr->states.access_state = MPIDI_RMA_LOCK_ALL_GRANTED;
121 122
            (*is_able_to_issue) = 1;

123 124
            (*made_progress) = 1;
        }
125 126
        break;

127 128 129 130 131 132 133 134
    case MPIDI_RMA_PER_TARGET:
    case MPIDI_RMA_LOCK_ALL_CALLED:
    case MPIDI_RMA_FENCE_GRANTED:
    case MPIDI_RMA_PSCW_GRANTED:
    case MPIDI_RMA_LOCK_ALL_GRANTED:
        (*is_able_to_issue) = 1;
        break;

135 136
    default:
        break;
137
    }   /* end of switch */
138 139

  fn_exit:
140
    MPIDI_RMA_FUNC_EXIT(MPID_STATE_CHECK_AND_SWITCH_WINDOW_STATE);
141 142 143 144 145 146 147 148 149
    return mpi_errno;
    /* --BEGIN ERROR HANDLING-- */
  fn_fail:
    goto fn_exit;
    /* --END ERROR HANDLING-- */
}


#undef FUNCNAME
150
#define FUNCNAME check_and_switch_target_state
151 152
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
153
static inline int check_and_switch_target_state(MPID_Win * win_ptr, MPIDI_RMA_Target_t * target,
154
                                                int *is_able_to_issue, int *made_progress)
155 156 157 158 159
{
    int rank = win_ptr->comm_ptr->rank;
    int mpi_errno = MPI_SUCCESS;

    (*made_progress) = 0;
160
    (*is_able_to_issue) = 0;
161

162
    if (target == NULL)
163 164
        goto fn_exit;

165 166 167 168 169
    switch (target->access_state) {
    case MPIDI_RMA_LOCK_CALLED:
        if (target->sync.sync_flag == MPIDI_RMA_SYNC_NONE ||
            target->sync.sync_flag == MPIDI_RMA_SYNC_FLUSH_LOCAL ||
            target->sync.sync_flag == MPIDI_RMA_SYNC_FLUSH) {
170 171
            if (target->pending_op_list_head == NULL ||
                !target->pending_op_list_head->piggyback_lock_candidate) {
172 173 174 175
                /* issue lock request */
                target->access_state = MPIDI_RMA_LOCK_ISSUED;
                if (target->target_rank == rank) {
                    mpi_errno = acquire_local_lock(win_ptr, target->lock_type);
176 177
                    if (mpi_errno != MPI_SUCCESS)
                        MPIU_ERR_POP(mpi_errno);
178 179
                }
                else {
180 181 182
                    mpi_errno = send_lock_msg(target->target_rank, target->lock_type, win_ptr);
                    if (mpi_errno != MPI_SUCCESS)
                        MPIU_ERR_POP(mpi_errno);
183 184 185 186 187 188
                }

                (*made_progress) = 1;
            }
        }
        else if (target->sync.sync_flag == MPIDI_RMA_SYNC_UNLOCK) {
189
            if (target->pending_op_list_head == NULL) {
190
                /* No RMA operation has ever been posted to this target,
191 192
                 * finish issuing, no need to acquire the lock. Cleanup
                 * function will clean it up. */
193 194 195 196 197 198 199 200 201 202 203 204
                target->access_state = MPIDI_RMA_LOCK_GRANTED;

                target->sync.outstanding_acks--;
                MPIU_Assert(target->sync.outstanding_acks >= 0);

                /* We are done with ending synchronization, unset target's sync_flag. */
                target->sync.sync_flag = MPIDI_RMA_SYNC_NONE;

                (*made_progress) = 1;
            }
            else {
                /* if we reach WIN_UNLOCK and there is still operation existing
205 206
                 * in pending list, this operation must be the only operation
                 * and it is prepared to piggyback LOCK and UNLOCK. */
207 208
                MPIU_Assert(target->pending_op_list_head->next == NULL);
                MPIU_Assert(target->pending_op_list_head->piggyback_lock_candidate);
209 210 211 212 213 214 215
            }
        }
        break;

    case MPIDI_RMA_LOCK_GRANTED:
    case MPIDI_RMA_NONE:
        if (target->sync.sync_flag == MPIDI_RMA_SYNC_FLUSH) {
216
            if (target->pending_op_list_head == NULL) {
217 218 219 220 221 222 223
                if (target->target_rank == rank) {
                    target->sync.outstanding_acks--;
                    MPIU_Assert(target->sync.outstanding_acks >= 0);
                }
                else {
                    if (target->put_acc_issued) {
                        mpi_errno = send_flush_msg(target->target_rank, win_ptr);
224 225
                        if (mpi_errno != MPI_SUCCESS)
                            MPIU_ERR_POP(mpi_errno);
226 227
                    }
                    else {
228
                        /* We did not issue PUT/ACC since the last
229 230
                         * synchronization call, therefore here we
                         * don't need ACK back */
231 232
                        target->sync.outstanding_acks--;
                        MPIU_Assert(target->sync.outstanding_acks >= 0);
233 234
                    }
                }
235 236 237 238 239

                /* We are done with ending synchronization, unset target's sync_flag. */
                target->sync.sync_flag = MPIDI_RMA_SYNC_NONE;

                (*made_progress) = 1;
240
            }
241 242
        }
        else if (target->sync.sync_flag == MPIDI_RMA_SYNC_UNLOCK) {
243
            if (target->pending_op_list_head == NULL) {
244
                if (target->target_rank == rank) {
245
                    target->sync.outstanding_acks--;
246
                    MPIU_Assert(target->sync.outstanding_acks >= 0);
247

248
                    mpi_errno = MPIDI_CH3I_Release_lock(win_ptr);
249 250
                    if (mpi_errno != MPI_SUCCESS)
                        MPIU_ERR_POP(mpi_errno);
251
                }
252 253 254 255
                else {
                    MPIDI_CH3_Pkt_flags_t flag = MPIDI_CH3_PKT_FLAG_NONE;
                    if (!target->put_acc_issued) {
                        /* We did not issue PUT/ACC since the last
256 257
                         * synchronization call, therefore here we
                         * don't need ACK back */
258 259 260 261 262 263
                        target->sync.outstanding_acks--;
                        MPIU_Assert(target->sync.outstanding_acks >= 0);

                        flag = MPIDI_CH3_PKT_FLAG_RMA_UNLOCK_NO_ACK;
                    }
                    mpi_errno = send_unlock_msg(target->target_rank, win_ptr, flag);
264 265
                    if (mpi_errno != MPI_SUCCESS)
                        MPIU_ERR_POP(mpi_errno);
266 267 268 269 270
                }

                /* We are done with ending synchronization, unset target's sync_flag. */
                target->sync.sync_flag = MPIDI_RMA_SYNC_NONE;

271 272 273
                (*made_progress) = 1;
            }
        }
274
        break;
275

276 277
    default:
        break;
278
    }   /* end of switch */
279

280 281 282 283
    if (target->pending_op_list_head != NULL && target->access_state != MPIDI_RMA_LOCK_ISSUED) {
        (*is_able_to_issue) = 1;
    }

284
  fn_exit:
285
    return mpi_errno;
286
  fn_fail:
287 288
    goto fn_exit;
}
289 290


291 292 293 294
#undef FUNCNAME
#define FUNCNAME issue_ops_target
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
295
static inline int issue_ops_target(MPID_Win * win_ptr, MPIDI_RMA_Target_t * target,
296 297 298 299
                                   int *made_progress)
{
    MPIDI_RMA_Op_t *curr_op = NULL;
    MPIDI_CH3_Pkt_flags_t flags;
300
    int is_able_to_issue = 0;
301
    int first_op = 1, mpi_errno = MPI_SUCCESS;
302

303
    (*made_progress) = 0;
304

305 306 307 308 309 310 311
    /* check and try to switch target state */
    mpi_errno = check_and_switch_target_state(win_ptr, target, &is_able_to_issue, made_progress);
    if (mpi_errno != MPI_SUCCESS)
        MPIU_ERR_POP(mpi_errno);
    if (!is_able_to_issue)
        goto fn_exit;

312 313
    if (win_ptr->non_empty_slots == 0 || target == NULL)
        goto fn_exit;
314

315 316 317 318
    /* Issue out operations in the list. */
    curr_op = target->next_op_to_issue;
    while (curr_op != NULL) {

319 320
        if (target->access_state == MPIDI_RMA_LOCK_ISSUED) {
            /* It is possible that the previous OP+LOCK changes
321
             * lock state to LOCK_ISSUED. */
322 323
            break;
        }
324 325

        if (curr_op->next == NULL &&
326
            target->sync.sync_flag == MPIDI_RMA_SYNC_NONE && curr_op->ureq == NULL) {
327
            /* Skip the last OP if sync_flag is NONE since we
328 329 330 331 332
             * want to leave it to the ending synchronization
             * so that we can piggyback LOCK / FLUSH.
             * However, if it is a request-based RMA, do not
             * skip it (otherwise a wait call before unlock
             * will be blocked). */
333
            break;
334 335
        }

336 337
        flags = MPIDI_CH3_PKT_FLAG_NONE;

338 339 340 341
        if (first_op) {
            /* piggyback on first OP. */
            if (target->access_state == MPIDI_RMA_LOCK_CALLED) {
                MPIU_Assert(curr_op->piggyback_lock_candidate);
342 343 344 345 346 347
                if (target->lock_type == MPI_LOCK_SHARED)
                    flags |= MPIDI_CH3_PKT_FLAG_RMA_LOCK_SHARED;
                else {
                    MPIU_Assert(target->lock_type == MPI_LOCK_EXCLUSIVE);
                    flags |= MPIDI_CH3_PKT_FLAG_RMA_LOCK_EXCLUSIVE;
                }
348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368
                target->access_state = MPIDI_RMA_LOCK_ISSUED;
            }
            first_op = 0;
        }

        if (curr_op->next == NULL) {
            /* piggyback on last OP. */
            if (target->sync.sync_flag == MPIDI_RMA_SYNC_FLUSH) {
                flags |= MPIDI_CH3_PKT_FLAG_RMA_FLUSH;
                if (target->win_complete_flag)
                    flags |= MPIDI_CH3_PKT_FLAG_RMA_DECR_AT_COUNTER;
            }
            else if (target->sync.sync_flag == MPIDI_RMA_SYNC_UNLOCK) {
                flags |= MPIDI_CH3_PKT_FLAG_RMA_UNLOCK;
            }
        }

        mpi_errno = issue_rma_op(curr_op, win_ptr, target, flags);
        if (mpi_errno != MPI_SUCCESS)
            MPIU_ERR_POP(mpi_errno);

369 370
        (*made_progress) = 1;

371
        if (curr_op->pkt.type == MPIDI_CH3_PKT_PUT ||
372 373 374
            curr_op->pkt.type == MPIDI_CH3_PKT_PUT_IMMED ||
            curr_op->pkt.type == MPIDI_CH3_PKT_ACCUMULATE ||
            curr_op->pkt.type == MPIDI_CH3_PKT_ACCUMULATE_IMMED) {
375
            target->put_acc_issued = 1; /* set PUT_ACC_FLAG when sending
376
                                         * PUT/ACC operation. */
377 378
        }

379 380 381 382 383 384 385 386 387 388
        if ((curr_op->pkt.type == MPIDI_CH3_PKT_ACCUMULATE ||
             curr_op->pkt.type == MPIDI_CH3_PKT_GET_ACCUM) && curr_op->issued_stream_count > 0) {
            /* For ACC-like operations, if not all stream units
             * are issued out, we stick to the current operation,
             * otherwise we move on to the next operation. */
            target->next_op_to_issue = curr_op;
        }
        else
            target->next_op_to_issue = curr_op->next;

389 390 391 392 393 394 395
        if (target->next_op_to_issue == NULL) {
            if (flags & MPIDI_CH3_PKT_FLAG_RMA_FLUSH || flags & MPIDI_CH3_PKT_FLAG_RMA_UNLOCK) {
                /* We are done with ending sync, unset target's sync_flag. */
                target->sync.sync_flag = MPIDI_RMA_SYNC_NONE;
            }
        }

396 397
        if (flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_SHARED ||
            flags & MPIDI_CH3_PKT_FLAG_RMA_LOCK_EXCLUSIVE) {
398
            /* If this operation is piggybacked with LOCK,
399 400 401
             * do not move it out of pending list, and do
             * not complete the user request, because we
             * may need to re-transmit it. */
402 403 404
            break;
        }

405 406 407 408 409
        if (curr_op->ureq != NULL) {
            mpi_errno = set_user_req_after_issuing_op(curr_op);
            if (mpi_errno != MPI_SUCCESS)
                MPIU_ERR_POP(mpi_errno);
        }
410

411 412
        if (curr_op->reqs_size == 0) {
            MPIU_Assert(curr_op->reqs == NULL);
413
            /* Sending is completed immediately. */
414
            MPIDI_CH3I_RMA_Ops_free_elem(win_ptr, &(target->pending_op_list_head),
415 416 417
                                         &(target->pending_op_list_tail), curr_op);
        }
        else {
418 419 420
            MPI_Datatype target_datatype;
            int is_derived = FALSE;

421
            /* Sending is not completed immediately. */
422

423
            MPIDI_CH3I_RMA_Ops_unlink(&(target->pending_op_list_head),
424
                                      &(target->pending_op_list_tail), curr_op);
425 426 427 428 429 430 431 432 433 434 435 436 437

            MPIDI_CH3_PKT_RMA_GET_TARGET_DATATYPE(curr_op->pkt, target_datatype, mpi_errno);

            if ((target_datatype != MPI_DATATYPE_NULL &&
                 !MPIR_DATATYPE_IS_PREDEFINED(target_datatype)) ||
                (curr_op->origin_datatype != MPI_DATATYPE_NULL &&
                 !MPIR_DATATYPE_IS_PREDEFINED(curr_op->origin_datatype)) ||
                (curr_op->result_datatype != MPI_DATATYPE_NULL &&
                 !MPIR_DATATYPE_IS_PREDEFINED(curr_op->result_datatype))) {
                is_derived = TRUE;
            }

            if (is_derived) {
438 439
                MPIDI_CH3I_RMA_Ops_append(&(target->issued_dt_op_list_head),
                                          &(target->issued_dt_op_list_tail), curr_op);
440 441
            }
            else if (curr_op->pkt.type == MPIDI_CH3_PKT_PUT ||
442 443 444
                     curr_op->pkt.type == MPIDI_CH3_PKT_PUT_IMMED ||
                     curr_op->pkt.type == MPIDI_CH3_PKT_ACCUMULATE ||
                     curr_op->pkt.type == MPIDI_CH3_PKT_ACCUMULATE_IMMED) {
445 446
                MPIDI_CH3I_RMA_Ops_append(&(target->issued_write_op_list_head),
                                          &(target->issued_write_op_list_tail), curr_op);
447 448
            }
            else {
449 450
                MPIDI_CH3I_RMA_Ops_append(&(target->issued_read_op_list_head),
                                          &(target->issued_read_op_list_tail), curr_op);
451 452 453 454 455
            }
        }

        curr_op = target->next_op_to_issue;

456
    }   /* end of while loop */
457 458 459 460 461 462 463 464

  fn_exit:
    return mpi_errno;
  fn_fail:
    goto fn_exit;
}


465 466 467 468 469 470 471 472
#undef FUNCNAME
#define FUNCNAME MPIDI_CH3I_RMA_Free_ops_before_completion
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
int MPIDI_CH3I_RMA_Free_ops_before_completion(MPID_Win * win_ptr)
{
    MPIDI_RMA_Op_t *curr_op = NULL;
    MPIDI_RMA_Target_t *curr_target = NULL;
473
    MPIDI_RMA_Op_t **op_list_head = NULL, **op_list_tail = NULL;
474 475 476 477
    int read_flag = 0;
    int i, made_progress = 0;
    int mpi_errno = MPI_SUCCESS;

478 479 480
    /* If we are in an free_ops_before_completion, the window must be holding
     * up resources.  If it isn't, we are in the wrong window and
     * incorrectly entered this function. */
481
    MPIU_ERR_CHKANDJUMP(win_ptr->non_empty_slots == 0, mpi_errno, MPI_ERR_OTHER, "**rmanoop");
482 483

    /* make nonblocking progress once */
484
    mpi_errno = MPIDI_CH3I_RMA_Make_progress_win(win_ptr, &made_progress);
485 486
    if (mpi_errno != MPI_SUCCESS)
        MPIU_ERR_POP(mpi_errno);
487

488
    if (win_ptr->states.access_state == MPIDI_RMA_FENCE_ISSUED ||
489 490
        win_ptr->states.access_state == MPIDI_RMA_PSCW_ISSUED ||
        win_ptr->states.access_state == MPIDI_RMA_LOCK_ALL_ISSUED)
491 492 493 494
        goto fn_exit;

    /* find targets that have operations */
    for (i = 0; i < win_ptr->num_slots; i++) {
495 496
        if (win_ptr->slots[i].target_list_head != NULL) {
            curr_target = win_ptr->slots[i].target_list_head;
497
            while (curr_target != NULL) {
498 499
                if (curr_target->issued_read_op_list_head != NULL ||
                    curr_target->issued_write_op_list_head != NULL) {
500 501 502 503 504 505 506 507 508
                    if (win_ptr->states.access_state == MPIDI_RMA_PER_TARGET ||
                        win_ptr->states.access_state == MPIDI_RMA_LOCK_ALL_CALLED) {
                        if (curr_target->access_state == MPIDI_RMA_LOCK_GRANTED)
                            break;
                    }
                    else {
                        break;
                    }
                }
509
                curr_target = curr_target->next;
510
            }
511 512
            if (curr_target != NULL)
                break;
513 514
        }
    }
515

516 517
    if (curr_target == NULL)
        goto fn_exit;
518

519
    /* After we do this, all following Win_flush_local
520
     * must do a Win_flush instead. */
521
    curr_target->sync.upgrade_flush_local = 1;
522

523 524 525 526 527
    op_list_head = &curr_target->issued_read_op_list_head;
    op_list_tail = &curr_target->issued_read_op_list_tail;
    read_flag = 1;

    curr_op = *op_list_head;
528 529

    /* free all ops in the list since we do not need to maintain them anymore */
530 531 532 533 534 535 536 537 538 539
    while (1) {
        if (curr_op != NULL) {
            if (curr_op->reqs_size > 0) {
                MPIU_Assert(curr_op->reqs != NULL);
                for (i = 0; i < curr_op->reqs_size; i++) {
                    if (curr_op->reqs[i] != NULL) {
                        MPID_Request_release(curr_op->reqs[i]);
                        curr_op->reqs[i] = NULL;
                        win_ptr->active_req_cnt--;
                    }
540 541
                }

542 543 544 545 546 547 548
                /* free req array in this op */
                MPIU_Free(curr_op->reqs);
                curr_op->reqs = NULL;
                curr_op->reqs_size = 0;
            }
            MPL_LL_DELETE(*op_list_head, *op_list_tail, curr_op);
            MPIDI_CH3I_Win_op_free(win_ptr, curr_op);
549
        }
550
        else {
551
            if (read_flag == 1) {
552 553
                op_list_head = &curr_target->issued_write_op_list_head;
                op_list_head = &curr_target->issued_write_op_list_tail;
554 555
                read_flag = 0;
            }
556 557 558 559
            else {
                /* we reach the tail of write_op_list, break out. */
                break;
            }
560
        }
561
        curr_op = *op_list_head;
562
    }
563

564
  fn_exit:
565
    return mpi_errno;
566
  fn_fail:
567 568 569 570
    goto fn_exit;
}


571 572 573 574 575 576
#undef FUNCNAME
#define FUNCNAME MPIDI_CH3I_RMA_Cleanup_ops_aggressive
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
int MPIDI_CH3I_RMA_Cleanup_ops_aggressive(MPID_Win * win_ptr)
{
577
    int i, local_completed = 0, remote_completed ATTRIBUTE((unused)) = 0;
578 579 580 581 582 583 584
    int mpi_errno = MPI_SUCCESS;
    MPIDI_RMA_Target_t *curr_target = NULL;
    int made_progress = 0;

    /* If we are in an aggressive cleanup, the window must be holding
     * up resources.  If it isn't, we are in the wrong window and
     * incorrectly entered this function. */
585
    MPIU_ERR_CHKANDJUMP(win_ptr->non_empty_slots == 0, mpi_errno, MPI_ERR_OTHER, "**rmanoop");
586 587 588

    /* find the first target that has something to issue */
    for (i = 0; i < win_ptr->num_slots; i++) {
589 590 591
        if (win_ptr->slots[i].target_list_head != NULL) {
            curr_target = win_ptr->slots[i].target_list_head;
            while (curr_target != NULL && curr_target->pending_op_list_head == NULL)
592
                curr_target = curr_target->next;
593 594
            if (curr_target != NULL)
                break;
595 596 597
        }
    }

598 599
    if (curr_target == NULL)
        goto fn_exit;
600 601 602 603 604 605 606 607 608 609 610 611

    if (curr_target->sync.sync_flag < MPIDI_RMA_SYNC_FLUSH_LOCAL)
        curr_target->sync.sync_flag = MPIDI_RMA_SYNC_FLUSH_LOCAL;

    /* Issue out all operations. */
    mpi_errno = MPIDI_CH3I_RMA_Make_progress_target(win_ptr, curr_target->target_rank,
                                                    &made_progress);
    if (mpi_errno != MPI_SUCCESS)
        MPIU_ERR_POP(mpi_errno);

    /* Wait for local completion. */
    do {
612
        mpi_errno = MPIDI_CH3I_RMA_Cleanup_ops_target(win_ptr, curr_target);
613 614
        if (mpi_errno != MPI_SUCCESS)
            MPIU_ERR_POP(mpi_errno);
615 616 617

        MPIDI_CH3I_RMA_ops_completion(win_ptr, curr_target, local_completed, remote_completed);

618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637
        if (!local_completed) {
            mpi_errno = wait_progress_engine();
            if (mpi_errno != MPI_SUCCESS)
                MPIU_ERR_POP(mpi_errno);
        }
    } while (!local_completed);

  fn_exit:
    return mpi_errno;
  fn_fail:
    goto fn_exit;
}


#undef FUNCNAME
#define FUNCNAME MPIDI_CH3I_RMA_Cleanup_target_aggressive
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
int MPIDI_CH3I_RMA_Cleanup_target_aggressive(MPID_Win * win_ptr, MPIDI_RMA_Target_t ** target)
{
638
    int i, local_completed ATTRIBUTE((unused)) = 0, remote_completed = 0;
639 640 641 642 643 644 645 646 647
    int made_progress = 0;
    MPIDI_RMA_Target_t *curr_target = NULL;
    int mpi_errno = MPI_SUCCESS;

    (*target) = NULL;

    /* If we are in an aggressive cleanup, the window must be holding
     * up resources.  If it isn't, we are in the wrong window and
     * incorrectly entered this function. */
648
    MPIU_ERR_CHKANDJUMP(win_ptr->non_empty_slots == 0, mpi_errno, MPI_ERR_OTHER, "**rmanotarget");
649 650 651 652 653 654 655 656 657 658 659

    if (win_ptr->states.access_state == MPIDI_RMA_LOCK_ALL_CALLED) {
        /* switch to window-wide protocol */
        MPIDI_VC_t *orig_vc = NULL, *target_vc = NULL;
        MPIDI_Comm_get_vc(win_ptr->comm_ptr, win_ptr->comm_ptr->rank, &orig_vc);
        for (i = 0; i < win_ptr->comm_ptr->local_size; i++) {
            if (i == win_ptr->comm_ptr->rank)
                continue;
            MPIDI_Comm_get_vc(win_ptr->comm_ptr, i, &target_vc);
            if (orig_vc->node_id != target_vc->node_id) {
                mpi_errno = MPIDI_CH3I_Win_find_target(win_ptr, i, &curr_target);
660 661
                if (mpi_errno)
                    MPIU_ERR_POP(mpi_errno);
662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677
                if (curr_target == NULL) {
                    win_ptr->outstanding_locks++;
                    mpi_errno = send_lock_msg(i, MPI_LOCK_SHARED, win_ptr);
                    if (mpi_errno != MPI_SUCCESS)
                        MPIU_ERR_POP(mpi_errno);
                }
            }
        }
        win_ptr->states.access_state = MPIDI_RMA_LOCK_ALL_ISSUED;
    }

    do {
        /* find a non-empty slot and set the FLUSH flag on the first
         * target */
        /* TODO: we should think about better strategies on selecting the target */
        for (i = 0; i < win_ptr->num_slots; i++)
678
            if (win_ptr->slots[i].target_list_head != NULL)
679
                break;
680
        curr_target = win_ptr->slots[i].target_list_head;
681 682 683 684 685 686 687 688 689 690 691 692 693
        if (curr_target->sync.sync_flag < MPIDI_RMA_SYNC_FLUSH) {
            curr_target->sync.sync_flag = MPIDI_RMA_SYNC_FLUSH;
            curr_target->sync.outstanding_acks++;
        }

        /* Issue out all operations. */
        mpi_errno = MPIDI_CH3I_RMA_Make_progress_target(win_ptr, curr_target->target_rank,
                                                        &made_progress);
        if (mpi_errno != MPI_SUCCESS)
            MPIU_ERR_POP(mpi_errno);

        /* Wait for remote completion. */
        do {
694
            mpi_errno = MPIDI_CH3I_RMA_Cleanup_ops_target(win_ptr, curr_target);
695 696
            if (mpi_errno != MPI_SUCCESS)
                MPIU_ERR_POP(mpi_errno);
697 698 699

            MPIDI_CH3I_RMA_ops_completion(win_ptr, curr_target, local_completed, remote_completed);

700 701 702 703 704 705 706 707
            if (!remote_completed) {
                mpi_errno = wait_progress_engine();
                if (mpi_errno != MPI_SUCCESS)
                    MPIU_ERR_POP(mpi_errno);
            }
        } while (!remote_completed);

        /* Cleanup the target. */
Xin Zhao's avatar
Xin Zhao committed
708
        mpi_errno = MPIDI_CH3I_Win_target_dequeue_and_free(win_ptr, curr_target);
709 710
        if (mpi_errno != MPI_SUCCESS)
            MPIU_ERR_POP(mpi_errno);
711 712 713 714 715 716 717 718 719 720 721 722 723

        /* check if we got a target */
        (*target) = MPIDI_CH3I_Win_target_alloc(win_ptr);

    } while ((*target) == NULL);

  fn_exit:
    return mpi_errno;
  fn_fail:
    goto fn_exit;
}


724 725 726 727 728 729
#undef FUNCNAME
#define FUNCNAME MPIDI_CH3I_RMA_Make_progress_target
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
int MPIDI_CH3I_RMA_Make_progress_target(MPID_Win * win_ptr, int target_rank, int *made_progress)
{
730
    int temp_progress = 0;
731
    int is_able_to_issue = 0;
732
    MPIDI_RMA_Target_t *target = NULL;
733 734 735 736
    int mpi_errno = MPI_SUCCESS;

    (*made_progress) = 0;

737
    /* check window state */
738
    mpi_errno = check_and_switch_window_state(win_ptr, &is_able_to_issue, &temp_progress);
739 740 741 742
    if (mpi_errno != MPI_SUCCESS)
        MPIU_ERR_POP(mpi_errno);
    if (temp_progress)
        (*made_progress) = 1;
743 744
    if (!is_able_to_issue)
        goto fn_exit;
745

746 747
    /* find target element */
    mpi_errno = MPIDI_CH3I_Win_find_target(win_ptr, target_rank, &target);
748 749
    if (mpi_errno != MPI_SUCCESS)
        MPIU_ERR_POP(mpi_errno);
750

751 752
    /* issue operations to this target */
    mpi_errno = issue_ops_target(win_ptr, target, &temp_progress);
753 754 755 756
    if (mpi_errno)
        MPIU_ERR_POP(mpi_errno);
    if (temp_progress)
        (*made_progress) = 1;
757 758 759 760 761 762 763 764 765 766 767 768 769 770 771

  fn_exit:
    return mpi_errno;
  fn_fail:
    goto fn_exit;
}


#undef FUNCNAME
#define FUNCNAME MPIDI_CH3I_RMA_Make_progress_win
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
int MPIDI_CH3I_RMA_Make_progress_win(MPID_Win * win_ptr, int *made_progress)
{
    int mpi_errno = MPI_SUCCESS;
Xin Zhao's avatar
Xin Zhao committed
772 773 774
    int start_slot, end_slot, i, idx;
    int is_able_to_issue = 0;
    MPIDI_RMA_Target_t *target = NULL;
775 776 777

    (*made_progress) = 0;

Xin Zhao's avatar
Xin Zhao committed
778 779 780
    /* check and try to switch window state */
    mpi_errno = check_and_switch_window_state(win_ptr, &is_able_to_issue, made_progress);
    if (mpi_errno != MPI_SUCCESS)
781
        MPIU_ERR_POP(mpi_errno);
Xin Zhao's avatar
Xin Zhao committed
782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811
    if (!is_able_to_issue)
        goto fn_exit;

    if (win_ptr->non_empty_slots == 0)
        goto fn_exit;

    /* FIXME: we should optimize the issuing pattern here. */

    start_slot = win_ptr->comm_ptr->rank % win_ptr->num_slots;
    end_slot = start_slot + win_ptr->num_slots;
    for (i = start_slot; i < end_slot; i++) {
        if (i < win_ptr->num_slots)
            idx = i;
        else
            idx = i - win_ptr->num_slots;

        target = win_ptr->slots[idx].target_list_head;
        while (target != NULL) {
            int temp_progress = 0;

            /* issue operations to this target */
            mpi_errno = issue_ops_target(win_ptr, target, &temp_progress);
            if (mpi_errno != MPI_SUCCESS)
                MPIU_ERR_POP(mpi_errno);
            if (temp_progress)
                (*made_progress) = 1;

            target = target->next;
        }
    }
812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831

  fn_exit:
    return mpi_errno;
  fn_fail:
    goto fn_exit;
}


#undef FUNCNAME
#define FUNCNAME MPIDI_CH3I_RMA_Make_progress_global
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
int MPIDI_CH3I_RMA_Make_progress_global(int *made_progress)
{
    MPIDI_RMA_Win_list_t *win_elem = MPIDI_RMA_Win_list;
    int mpi_errno = MPI_SUCCESS;

    (*made_progress) = 0;

    for (win_elem = MPIDI_RMA_Win_list; win_elem; win_elem = win_elem->next) {
832
        int temp_progress = 0;
833

834 835 836 837
        if (win_elem->win_ptr->states.access_state == MPIDI_RMA_NONE ||
            win_elem->win_ptr->states.access_state == MPIDI_RMA_FENCE_GRANTED ||
            win_elem->win_ptr->states.access_state == MPIDI_RMA_PSCW_GRANTED)
            continue;
838

839
        mpi_errno = MPIDI_CH3I_RMA_Make_progress_win(win_elem->win_ptr, &temp_progress);
840 841 842 843
        if (mpi_errno != MPI_SUCCESS)
            MPIU_ERR_POP(mpi_errno);
        if (temp_progress)
            (*made_progress) = 1;
844 845 846 847 848 849 850
    }

  fn_exit:
    return mpi_errno;
  fn_fail:
    goto fn_exit;
}