mpid_rma_oplist.h 21.2 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
/*
 *  (C) 2001 by Argonne National Laboratory.
 *      See COPYRIGHT in top-level directory.
 */

#if !defined(MPID_RMA_OPLIST_H_INCLUDED)
#define MPID_RMA_OPLIST_H_INCLUDED

#include "mpl_utlist.h"
#include "mpid_rma_types.h"

13
int MPIDI_CH3I_RMA_Free_ops_before_completion(MPID_Win * win_ptr);
14
15
int MPIDI_CH3I_RMA_Cleanup_ops_aggressive(MPID_Win * win_ptr);
int MPIDI_CH3I_RMA_Cleanup_target_aggressive(MPID_Win * win_ptr, MPIDI_RMA_Target_t ** target);
16
17
18
int MPIDI_CH3I_RMA_Make_progress_target(MPID_Win * win_ptr, int target_rank, int *made_progress);
int MPIDI_CH3I_RMA_Make_progress_win(MPID_Win * win_ptr, int *made_progress);

19
extern MPIDI_RMA_Op_t *global_rma_op_pool, *global_rma_op_pool_tail, *global_rma_op_pool_start;
20
21
extern MPIDI_RMA_Target_t *global_rma_target_pool, *global_rma_target_pool_tail,
    *global_rma_target_pool_start;
22

Xin Zhao's avatar
Xin Zhao committed
23
24
MPIR_T_PVAR_DOUBLE_TIMER_DECL_EXTERN(RMA, rma_rmaqueue_alloc);

25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
/* MPIDI_CH3I_Win_op_alloc(): get a new op element from op pool and
 * initialize it. If we cannot get one, return NULL. */
#undef FUNCNAME
#define FUNCNAME MPIDI_CH3I_Win_op_alloc
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
static inline MPIDI_RMA_Op_t *MPIDI_CH3I_Win_op_alloc(MPID_Win * win_ptr)
{
    MPIDI_RMA_Op_t *e;

    if (win_ptr->op_pool == NULL) {
        /* local pool is empty, try to find something in the global pool */
        if (global_rma_op_pool == NULL)
            return NULL;
        else {
            e = global_rma_op_pool;
            MPL_LL_DELETE(global_rma_op_pool, global_rma_op_pool_tail, e);
        }
    }
    else {
        e = win_ptr->op_pool;
        MPL_LL_DELETE(win_ptr->op_pool, win_ptr->op_pool_tail, e);
    }

    e->dataloop = NULL;
50
51
    e->reqs = NULL;
    e->reqs_size = 0;
52
    e->ureq = NULL;
53
    e->is_dt = 0;
54
    e->piggyback_lock_candidate = 0;
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77

    return e;
}

/* MPIDI_CH3I_Win_op_free(): put an op element back to the op pool which
 * it belongs to. */
#undef FUNCNAME
#define FUNCNAME MPIDI_CH3I_Win_op_free
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
static inline int MPIDI_CH3I_Win_op_free(MPID_Win * win_ptr, MPIDI_RMA_Op_t * e)
{
    int mpi_errno = MPI_SUCCESS;

    /* Check if we allocated a dataloop for this op (see send/recv_rma_msg) */
    if (e->dataloop != NULL) {
        MPIU_Free(e->dataloop);
    }

    /* We enqueue elements to the right pool, so when they get freed
     * at window free time, they won't conflict with the global pool
     * or other windows */
    /* use PREPEND when return objects back to the pool
78
     * in order to improve cache performance */
79
80
81
82
83
84
85
86
    if (e->pool_type == MPIDI_RMA_POOL_WIN)
        MPL_LL_PREPEND(win_ptr->op_pool, win_ptr->op_pool_tail, e);
    else
        MPL_LL_PREPEND(global_rma_op_pool, global_rma_op_pool_tail, e);

    return mpi_errno;
}

87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
/* MPIDI_CH3I_Win_target_alloc(): get a target element from the target pool.
 * If we cannot get one, return NULL. */
#undef FUNCNAME
#define FUNCNAME MPIDI_CH3I_Win_target_alloc
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
static inline MPIDI_RMA_Target_t *MPIDI_CH3I_Win_target_alloc(MPID_Win * win_ptr)
{
    MPIDI_RMA_Target_t *e;

    if (win_ptr->target_pool == NULL) {
        /* local pool is empty, try to find something in the global pool */
        if (global_rma_target_pool == NULL)
            return NULL;
        else {
            e = global_rma_target_pool;
            MPL_LL_DELETE(global_rma_target_pool, global_rma_target_pool_tail, e);
        }
    }
    else {
        e = win_ptr->target_pool;
        MPL_LL_DELETE(win_ptr->target_pool, win_ptr->target_pool_tail, e);
    }

    e->read_op_list = e->read_op_list_tail = NULL;
    e->write_op_list = e->write_op_list_tail = NULL;
    e->dt_op_list = e->dt_op_list_tail = NULL;
    e->pending_op_list = e->pending_op_list_tail = NULL;
    e->next_op_to_issue = NULL;

    e->target_rank = -1;
118
    e->access_state = MPIDI_RMA_NONE;
119
    e->lock_type = MPID_LOCK_NONE;
120
    e->lock_mode = 0;
121
    e->accumulated_ops_cnt = 0;
122
    e->disable_flush_local = 0;
123
    e->win_complete_flag = 0;
124
    e->put_acc_issued = 0;
125

126
    e->sync.sync_flag = MPIDI_RMA_SYNC_NONE;
127
    e->sync.outstanding_acks = 0;
128
129
130
    e->sync.have_remote_incomplete_ops = 1;     /* When I create a new target, there must be
                                                 * incomplete ops until a FLUSH/UNLOCK packet
                                                 * is sent. */
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
    return e;
}

/* MPIDI_CH3I_Win_target_free(): put a target element back to the target pool
 * it belongs to. */
#undef FUNCNAME
#define FUNCNAME MPIDI_CH3I_Win_target_free
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
static inline int MPIDI_CH3I_Win_target_free(MPID_Win * win_ptr, MPIDI_RMA_Target_t * e)
{
    int mpi_errno = MPI_SUCCESS;

    /* We enqueue elements to the right pool, so when they get freed
     * at window free time, they won't conflict with the global pool
     * or other windows */
    MPIU_Assert(e->read_op_list == NULL);
    MPIU_Assert(e->write_op_list == NULL);
    MPIU_Assert(e->dt_op_list == NULL);
    MPIU_Assert(e->pending_op_list == NULL);

    /* use PREPEND when return objects back to the pool
153
     * in order to improve cache performance */
154
155
156
157
158
159
160
161
    if (e->pool_type == MPIDI_RMA_POOL_WIN)
        MPL_LL_PREPEND(win_ptr->target_pool, win_ptr->target_pool_tail, e);
    else
        MPL_LL_PREPEND(global_rma_target_pool, global_rma_target_pool_tail, e);

    return mpi_errno;
}

Xin Zhao's avatar
Xin Zhao committed
162
163
164
165
166
167
168
/* MPIDI_CH3I_Win_create_target(): given a rank, create
 * corresponding target in RMA slots. */
#undef FUNCNAME
#define FUNCNAME MPIDI_CH3I_Win_create_target
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
static inline int MPIDI_CH3I_Win_create_target(MPID_Win * win_ptr, int target_rank,
169
                                               MPIDI_RMA_Target_t ** e)
Xin Zhao's avatar
Xin Zhao committed
170
171
172
173
174
175
176
177
178
179
180
{
    int mpi_errno = MPI_SUCCESS;
    MPIDI_RMA_Slot_t *slot = NULL;
    MPIDI_RMA_Target_t *t = NULL;

    if (win_ptr->num_slots < win_ptr->comm_ptr->local_size)
        slot = &(win_ptr->slots[target_rank % win_ptr->num_slots]);
    else
        slot = &(win_ptr->slots[target_rank]);

    t = MPIDI_CH3I_Win_target_alloc(win_ptr);
181
182
    if (t == NULL) {
        mpi_errno = MPIDI_CH3I_RMA_Cleanup_target_aggressive(win_ptr, &t);
183
184
        if (mpi_errno != MPI_SUCCESS)
            MPIU_ERR_POP(mpi_errno);
185
    }
Xin Zhao's avatar
Xin Zhao committed
186
187
188

    t->target_rank = target_rank;

189
190
191
    if (slot->target_list == NULL)
        win_ptr->non_empty_slots++;

Xin Zhao's avatar
Xin Zhao committed
192
193
194
195
196
197
198
    /* Enqueue target into target list. */
    MPL_LL_APPEND(slot->target_list, slot->target_list_tail, t);

    assert(t != NULL);

    (*e) = t;

199
  fn_exit:
Xin Zhao's avatar
Xin Zhao committed
200
    return mpi_errno;
201
  fn_fail:
Xin Zhao's avatar
Xin Zhao committed
202
203
204
205
206
207
208
209
210
211
    goto fn_exit;
}

/* MPIDI_CH3I_Win_find_target(): given a rank, find
 * corresponding target in RMA slots. */
#undef FUNCNAME
#define FUNCNAME MPIDI_CH3I_Win_find_target
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
static inline int MPIDI_CH3I_Win_find_target(MPID_Win * win_ptr, int target_rank,
212
                                             MPIDI_RMA_Target_t ** e)
Xin Zhao's avatar
Xin Zhao committed
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
{
    int mpi_errno = MPI_SUCCESS;
    MPIDI_RMA_Slot_t *slot = NULL;
    MPIDI_RMA_Target_t *t = NULL;

    if (win_ptr->num_slots < win_ptr->comm_ptr->local_size)
        slot = &(win_ptr->slots[target_rank % win_ptr->num_slots]);
    else
        slot = &(win_ptr->slots[target_rank]);

    t = slot->target_list;
    while (t != NULL) {
        if (t->target_rank == target_rank)
            break;
    }

    (*e) = t;

231
  fn_exit:
Xin Zhao's avatar
Xin Zhao committed
232
    return mpi_errno;
233
  fn_fail:
Xin Zhao's avatar
Xin Zhao committed
234
235
236
    goto fn_exit;
}

237
238
239
240
241
242
243
/* MPIDI_CH3I_Win_enqueue_op(): given an operation, enqueue it to the
 * corresponding operation lists in corresponding target element. This
 * routines is only called from operation routines. */
#undef FUNCNAME
#define FUNCNAME MPIDI_CH3I_Win_enqueue_op
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
244
static inline int MPIDI_CH3I_Win_enqueue_op(MPID_Win * win_ptr, MPIDI_RMA_Op_t * op)
245
246
247
248
249
{
    int mpi_errno = MPI_SUCCESS;
    MPIDI_RMA_Target_t *target = NULL;

    mpi_errno = MPIDI_CH3I_Win_find_target(win_ptr, op->target_rank, &target);
250
251
    if (mpi_errno != MPI_SUCCESS)
        MPIU_ERR_POP(mpi_errno);
252
253
    if (target == NULL) {
        mpi_errno = MPIDI_CH3I_Win_create_target(win_ptr, op->target_rank, &target);
254
255
        if (mpi_errno != MPI_SUCCESS)
            MPIU_ERR_POP(mpi_errno);
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270

        if (win_ptr->states.access_state == MPIDI_RMA_PER_TARGET ||
            win_ptr->states.access_state == MPIDI_RMA_LOCK_ALL_GRANTED) {
            /* If global state is MPIDI_RMA_PER_TARGET, this must not
             * be the first time to create this target (The first time
             * is in Win_lock). Here we recreated it and set the access
             * state to LOCK_GRANTED because before we free the previous
             * one, the lock should already be granted. */
            /* If global state is MPIDI_RMA_LOCK_ALL_GRANTED, all locks
             * should already be granted. So the access state for this
             * target is also set to MPIDI_RMA_LOCK_GRANTED. */
            target->access_state = MPIDI_RMA_LOCK_GRANTED;
        }
        else if (win_ptr->states.access_state == MPIDI_RMA_LOCK_ALL_CALLED) {
            /* If global state is MPIDI_RMA_LOCK_ALL_CALLED, this must
271
272
             * the first time to create this target, set its access state
             * to MPIDI_RMA_LOCK_CALLED. */
273
274
275
276
277
278
279
280
281
282
            target->access_state = MPIDI_RMA_LOCK_CALLED;
            target->lock_type = MPI_LOCK_SHARED;
        }
    }

    /* Enqueue operation into pending list. */
    MPL_LL_APPEND(target->pending_op_list, target->pending_op_list_tail, op);
    if (target->next_op_to_issue == NULL)
        target->next_op_to_issue = op;

283
284
285
286
    /* Increment the counter for accumulated posted operations */
    target->accumulated_ops_cnt++;
    win_ptr->accumulated_ops_cnt++;

287
  fn_exit:
288
    return mpi_errno;
289
  fn_fail:
290
291
292
    goto fn_exit;
}

Xin Zhao's avatar
Xin Zhao committed
293
294
295
296
297
298
299

/* MPIDI_CH3I_Win_target_dequeue_and_free(): dequeue and free
 * the target in RMA slots. */
#undef FUNCNAME
#define FUNCNAME MPIDI_CH3I_Win_target_dequeue_and_free
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
300
static inline int MPIDI_CH3I_Win_target_dequeue_and_free(MPID_Win * win_ptr, MPIDI_RMA_Target_t * e)
Xin Zhao's avatar
Xin Zhao committed
301
302
303
304
305
306
307
308
309
310
311
312
313
{
    int mpi_errno = MPI_SUCCESS;
    int target_rank = e->target_rank;
    MPIDI_RMA_Slot_t *slot;

    if (win_ptr->num_slots < win_ptr->comm_ptr->local_size)
        slot = &(win_ptr->slots[target_rank % win_ptr->num_slots]);
    else
        slot = &(win_ptr->slots[target_rank]);

    MPL_LL_DELETE(slot->target_list, slot->target_list_tail, e);

    mpi_errno = MPIDI_CH3I_Win_target_free(win_ptr, e);
314
315
    if (mpi_errno != MPI_SUCCESS)
        MPIU_ERR_POP(mpi_errno);
Xin Zhao's avatar
Xin Zhao committed
316

317
318
319
    if (slot->target_list == NULL)
        win_ptr->non_empty_slots--;

320
  fn_exit:
Xin Zhao's avatar
Xin Zhao committed
321
    return mpi_errno;
322
  fn_fail:
Xin Zhao's avatar
Xin Zhao committed
323
324
325
    goto fn_exit;
}

326
327
328
329
330

#undef FUNCNAME
#define FUNCNAME MPIDI_CH3I_RMA_Cleanup_ops_target
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
331
static inline int MPIDI_CH3I_RMA_Cleanup_ops_target(MPID_Win * win_ptr, MPIDI_RMA_Target_t * target,
332
333
334
335
336
337
                                                    int *local_completed, int *remote_completed)
{
    MPIDI_RMA_Op_t *curr_op = NULL;
    MPIDI_RMA_Op_t **op_list = NULL, **op_list_tail = NULL;
    int read_flag = 0, write_flag = 0;
    int mpi_errno = MPI_SUCCESS;
338
    int i;
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355

    (*local_completed) = 0;
    (*remote_completed) = 0;

    if (win_ptr->states.access_state == MPIDI_RMA_FENCE_ISSUED ||
        win_ptr->states.access_state == MPIDI_RMA_PSCW_ISSUED ||
        win_ptr->states.access_state == MPIDI_RMA_LOCK_ALL_ISSUED)
        goto fn_exit;

    if (target == NULL)
        goto fn_exit;

    if (target->access_state == MPIDI_RMA_LOCK_CALLED ||
        target->access_state == MPIDI_RMA_LOCK_ISSUED)
        goto fn_exit;

    if (target->pending_op_list == NULL &&
356
        target->read_op_list == NULL && target->write_op_list == NULL && target->dt_op_list == NULL)
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
        goto cleanup_target;

    if (target->read_op_list != NULL) {
        op_list = &(target->read_op_list);
        op_list_tail = &(target->read_op_list_tail);
        read_flag = 1;
    }
    else if (target->write_op_list != NULL) {
        op_list = &(target->write_op_list);
        op_list_tail = &(target->write_op_list_tail);
        write_flag = 1;
    }
    else if (target->dt_op_list != NULL) {
        op_list = &(target->dt_op_list);
        op_list_tail = &(target->dt_op_list_tail);
    }
    else {
        /* only pending op list is not NULL, nothing we can do here. */
        goto fn_exit;
    }

    curr_op = *op_list;
    while (curr_op != NULL) {
380
381
382
        for (i = 0; i < curr_op->reqs_size; i++) {
            if (curr_op->reqs[i] == NULL)
                continue;
383

384
385
386
387
            if (MPID_Request_is_complete(curr_op->reqs[i])) {
                /* If there's an error, return it */
                mpi_errno = curr_op->reqs[i]->status.MPI_ERROR;
                MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**ch3|rma_msg");
388

389
390
391
392
393
394
395
396
397
398
399
400
                /* No errors, free the request */
                MPID_Request_release(curr_op->reqs[i]);

                curr_op->reqs[i] = NULL;

                win_ptr->active_req_cnt--;
            }
            else
                break;
        }

        if (i == curr_op->reqs_size) {
401
402
403
404
405
406
407
408
409
            /* Release user request */
            if (curr_op->ureq) {
                /* User request must be completed by progress engine */
                MPIU_Assert(MPID_Request_is_complete(curr_op->ureq));

                /* Release the ch3 ref */
                MPID_Request_release(curr_op->ureq);
            }

410
411
412
413
414
            /* free request array in op struct */
            MPIU_Free(curr_op->reqs);
            curr_op->reqs = NULL;
            curr_op->reqs_size = 0;

415
416
417
            /* dequeue the operation and free it */
            MPL_LL_DELETE(*op_list, *op_list_tail, curr_op);
            MPIDI_CH3I_Win_op_free(win_ptr, curr_op);
418

419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
            if (*op_list == NULL) {
                if (read_flag == 1) {
                    read_flag = 0;
                    if (target->write_op_list != NULL) {
                        op_list = &(target->write_op_list);
                        op_list_tail = &(target->write_op_list_tail);
                        write_flag = 1;
                    }
                    else if (target->dt_op_list != NULL) {
                        op_list = &(target->dt_op_list);
                        op_list_tail = &(target->dt_op_list_tail);
                    }
                    else
                        break;
                }
                else if (write_flag == 1) {
                    write_flag = 0;
                    if (target->dt_op_list != NULL) {
                        op_list = &(target->dt_op_list);
                        op_list_tail = &(target->dt_op_list_tail);
                    }
                    else
                        break;
                }
            }
            /* next op */
            curr_op = *op_list;
        }
        else
            break;
    }

  cleanup_target:
    if (target->pending_op_list == NULL &&
        target->read_op_list == NULL && target->write_op_list == NULL &&
        target->dt_op_list == NULL) {

        (*local_completed) = 1;

        /* for the conditions that need to be satisfied before we free the
         * target, see the MPIDI_RMA_Target definition in
         * mpid_rma_types.h */
        if (target->sync.sync_flag == MPIDI_RMA_SYNC_NONE &&
462
            target->sync.outstanding_acks == 0 && target->sync.have_remote_incomplete_ops == 0) {
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
            (*remote_completed) = 1;
        }
    }

  fn_exit:
    return mpi_errno;
  fn_fail:
    goto fn_exit;
}


#undef FUNCNAME
#define FUNCNAME MPIDI_CH3I_RMA_Cleanup_ops_win
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
478
static inline int MPIDI_CH3I_RMA_Cleanup_ops_win(MPID_Win * win_ptr,
479
480
481
482
483
484
485
486
487
488
                                                 int *local_completed, int *remote_completed)
{
    MPIDI_RMA_Target_t *target = NULL;
    int num_targets = 0, local_completed_targets = 0, remote_completed_targets = 0;
    int i, mpi_errno = MPI_SUCCESS;

    (*local_completed) = 0;
    (*remote_completed) = 0;

    for (i = 0; i < win_ptr->num_slots; i++) {
489
        for (target = win_ptr->slots[i].target_list; target;) {
490
491
492
            int local = 0, remote = 0;

            mpi_errno = MPIDI_CH3I_RMA_Cleanup_ops_target(win_ptr, target, &local, &remote);
493
494
            if (mpi_errno != MPI_SUCCESS)
                MPIU_ERR_POP(mpi_errno);
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519

            num_targets++;
            local_completed_targets += local;
            remote_completed_targets += remote;

            target = target->next;
        }
    }

    if (num_targets == local_completed_targets)
        (*local_completed) = 1;
    if (num_targets == remote_completed_targets)
        (*remote_completed) = 1;

  fn_exit:
    return mpi_errno;
  fn_fail:
    goto fn_exit;
}


#undef FUNCNAME
#define FUNCNAME MPIDI_CH3I_RMA_Cleanup_single_target
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
520
521
static inline int MPIDI_CH3I_RMA_Cleanup_single_target(MPID_Win * win_ptr,
                                                       MPIDI_RMA_Target_t * target)
522
523
524
525
526
{
    int mpi_errno = MPI_SUCCESS;

    /* dequeue the target and free it. */
    mpi_errno = MPIDI_CH3I_Win_target_dequeue_and_free(win_ptr, target);
527
528
    if (mpi_errno != MPI_SUCCESS)
        MPIU_ERR_POP(mpi_errno);
529

530
  fn_exit:
531
    return mpi_errno;
532
  fn_fail:
533
534
535
536
537
538
539
540
    goto fn_exit;
}


#undef FUNCNAME
#define FUNCNAME MPIDI_CH3I_RMA_Cleanup_targets_win
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
541
static inline int MPIDI_CH3I_RMA_Cleanup_targets_win(MPID_Win * win_ptr)
542
543
544
545
546
{
    MPIDI_RMA_Target_t *target = NULL, *next_target = NULL;
    int i, mpi_errno = MPI_SUCCESS;

    for (i = 0; i < win_ptr->num_slots; i++) {
547
        for (target = win_ptr->slots[i].target_list; target;) {
548
549
            next_target = target->next;
            mpi_errno = MPIDI_CH3I_RMA_Cleanup_single_target(win_ptr, target);
550
551
            if (mpi_errno != MPI_SUCCESS)
                MPIU_ERR_POP(mpi_errno);
552
553
554
555
556
557
            target = next_target;
        }
    }

    MPIU_Assert(win_ptr->non_empty_slots == 0);

558
  fn_exit:
559
    return mpi_errno;
560
  fn_fail:
561
562
563
    goto fn_exit;
}

564
565
566
567
#undef FUNCNAME
#define FUNCNAME MPIDI_CH3I_Win_get_op
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
568
static inline int MPIDI_CH3I_Win_get_op(MPID_Win * win_ptr, MPIDI_RMA_Op_t ** e)
569
570
571
572
573
574
{
    MPIDI_RMA_Op_t *new_ptr = NULL;
    int local_completed = 0, remote_completed = 0;
    int mpi_errno = MPI_SUCCESS;

    while (1) {
Xin Zhao's avatar
Xin Zhao committed
575
        MPIR_T_PVAR_TIMER_START(RMA, rma_rmaqueue_alloc);
576
        new_ptr = MPIDI_CH3I_Win_op_alloc(win_ptr);
Xin Zhao's avatar
Xin Zhao committed
577
        MPIR_T_PVAR_TIMER_END(RMA, rma_rmaqueue_alloc);
578
579
        if (new_ptr != NULL)
            break;
580

581
582
583
        mpi_errno = MPIDI_CH3I_RMA_Cleanup_ops_win(win_ptr, &local_completed, &remote_completed);
        if (mpi_errno != MPI_SUCCESS)
            MPIU_ERR_POP(mpi_errno);
584

Xin Zhao's avatar
Xin Zhao committed
585
        MPIR_T_PVAR_TIMER_START(RMA, rma_rmaqueue_alloc);
586
        new_ptr = MPIDI_CH3I_Win_op_alloc(win_ptr);
Xin Zhao's avatar
Xin Zhao committed
587
        MPIR_T_PVAR_TIMER_END(RMA, rma_rmaqueue_alloc);
588
589
        if (new_ptr != NULL)
            break;
590
591
592

        if (MPIDI_RMA_Pkt_orderings->flush_remote) {
            mpi_errno = MPIDI_CH3I_RMA_Free_ops_before_completion(win_ptr);
593
594
            if (mpi_errno != MPI_SUCCESS)
                MPIU_ERR_POP(mpi_errno);
595
596
        }

Xin Zhao's avatar
Xin Zhao committed
597
        MPIR_T_PVAR_TIMER_START(RMA, rma_rmaqueue_alloc);
598
        new_ptr = MPIDI_CH3I_Win_op_alloc(win_ptr);
Xin Zhao's avatar
Xin Zhao committed
599
        MPIR_T_PVAR_TIMER_END(RMA, rma_rmaqueue_alloc);
600
601
        if (new_ptr != NULL)
            break;
602
603

        mpi_errno = MPIDI_CH3I_RMA_Cleanup_ops_aggressive(win_ptr);
604
605
        if (mpi_errno != MPI_SUCCESS)
            MPIU_ERR_POP(mpi_errno);
606
607
608
609
    }

    (*e) = new_ptr;

610
  fn_exit:
611
    return mpi_errno;
612
  fn_fail:
613
614
615
616
    goto fn_exit;
}


617
618
619
620
621
622
623
624
625
/* Append an element to the tail of the RMA ops list
 *
 * @param IN    list      Pointer to the RMA ops list
 * @param IN    elem      Pointer to the element to be appended
 */
#undef FUNCNAME
#define FUNCNAME MPIDI_CH3I_RMA_Ops_append
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
626
627
static inline void MPIDI_CH3I_RMA_Ops_append(MPIDI_RMA_Ops_list_t * list,
                                             MPIDI_RMA_Ops_list_t * list_tail,
628
                                             MPIDI_RMA_Op_t * elem)
629
{
630
    MPL_LL_APPEND(*list, *list_tail, elem);
631
632
633
634
635
636
637
638
639
640
641
642
}


/* Unlink an element from the RMA ops list
 *
 * @param IN    list      Pointer to the RMA ops list
 * @param IN    elem      Pointer to the element to be unlinked
 */
#undef FUNCNAME
#define FUNCNAME MPIDI_CH3I_RMA_Ops_unlink
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
643
644
static inline void MPIDI_CH3I_RMA_Ops_unlink(MPIDI_RMA_Ops_list_t * list,
                                             MPIDI_RMA_Ops_list_t * list_tail,
645
                                             MPIDI_RMA_Op_t * elem)
646
{
647
    MPL_LL_DELETE(*list, *list_tail, elem);
648
649
650
651
652
653
654
655
656
657
658
659
}


/* Free an element in the RMA operations list.
 *
 * @param IN    list      Pointer to the RMA ops list
 * @param IN    curr_ptr  Pointer to the element to be freed.
 */
#undef FUNCNAME
#define FUNCNAME MPIDI_CH3I_RMA_Ops_free_elem
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
660
661
static inline void MPIDI_CH3I_RMA_Ops_free_elem(MPID_Win * win_ptr, MPIDI_RMA_Ops_list_t * list,
                                                MPIDI_RMA_Ops_list_t * list_tail,
662
663
664
665
666
667
                                                MPIDI_RMA_Op_t * curr_ptr)
{
    MPIDI_RMA_Op_t *tmp_ptr = curr_ptr;

    MPIU_Assert(curr_ptr != NULL);

668
    MPL_LL_DELETE(*list, *list_tail, curr_ptr);
669

670
    MPIDI_CH3I_Win_op_free(win_ptr, tmp_ptr);
671
672
673
674
}


#endif /* MPID_RMA_OPLIST_H_INCLUDED */