ch3u_rma_ops.c 52.7 KB
Newer Older
1
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
2 3 4 5 6 7 8
/*
 *  (C) 2001 by Argonne National Laboratory.
 *      See COPYRIGHT in top-level directory.
 */

#include "mpidrma.h"

Xin Zhao's avatar
Xin Zhao committed
9 10
MPIR_T_PVAR_DOUBLE_TIMER_DECL_EXTERN(RMA, rma_rmaqueue_set);

11 12 13
#define MPIDI_PASSIVE_TARGET_DONE_TAG  348297
#define MPIDI_PASSIVE_TARGET_RMA_TAG 563924

14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34
/*
=== BEGIN_MPI_T_CVAR_INFO_BLOCK ===

cvars:
    - name        : MPIR_CVAR_CH3_RMA_OP_POKING_PROGRESS
      category    : CH3
      type        : int
      default     : 100
      class       : none
      verbosity   : MPI_T_VERBOSITY_USER_BASIC
      scope       : MPI_T_SCOPE_ALL_EQ
      description : >-
          Specify the threshold of number of posted operations
          when starting poking progress in operation routines.
          When the value is negative, runtime never pokes progress
          engine in operation routines; when the value is zero,
          runtime always pokes progress engine in operation
          routines; when the value is larger than zero, runtime
          starts to poke progress engine when number of posted
          operations reaches that value.

35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53
    - name        : MPIR_CVAR_CH3_RMA_OP_PIGGYBACK_LOCK_DATA_SIZE
      category    : CH3
      type        : int
      default     : 65536
      class       : none
      verbosity   : MPI_T_VERBOSITY_USER_BASIC
      scope       : MPI_T_SCOPE_ALL_EQ
      description : >-
          Specify the threshold of data size of a RMA operation
          which can be piggybacked with a LOCK message. It is
          always a positive value and should not be smaller
          than MPIDI_RMA_IMMED_BYTES.
          If user sets it as a small value, for middle and large
          data size, we will lose performance because of always
          waiting for round-trip of LOCK synchronization; if
          user sets it as a large value, we need to consume
          more memory on target side to buffer this lock request
          when lock is not satisfied.

54 55 56
=== END_MPI_T_CVAR_INFO_BLOCK ===
*/

57
#undef FUNCNAME
58
#define FUNCNAME MPIDI_CH3I_Put
59 60
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
61 62 63 64
int MPIDI_CH3I_Put(const void *origin_addr, int origin_count, MPI_Datatype
                   origin_datatype, int target_rank, MPI_Aint target_disp,
                   int target_count, MPI_Datatype target_datatype, MPID_Win * win_ptr,
                   MPID_Request * ureq)
65 66
{
    int mpi_errno = MPI_SUCCESS;
67
    int dt_contig ATTRIBUTE((unused)), rank;
68
    MPID_Datatype *dtp;
Pavan Balaji's avatar
Pavan Balaji committed
69
    MPI_Aint dt_true_lb ATTRIBUTE((unused));
70
    MPIDI_msg_sz_t data_sz;
71
    MPIDI_VC_t *orig_vc = NULL, *target_vc = NULL;
72
    int made_progress = 0;
73
    MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3I_PUT);
74

75
    MPIDI_RMA_FUNC_ENTER(MPID_STATE_MPIDI_CH3I_PUT);
76

77 78 79
    MPIU_ERR_CHKANDJUMP(win_ptr->states.access_state == MPIDI_RMA_NONE,
                        mpi_errno, MPI_ERR_RMA_SYNC, "**rmasync");

80 81 82 83
    if (target_rank == MPI_PROC_NULL) {
        goto fn_exit;
    }

84 85
    MPIDI_Datatype_get_info(origin_count, origin_datatype, dt_contig, data_sz, dtp, dt_true_lb);

86
    if (data_sz == 0) {
87
        goto fn_exit;
88 89
    }

90
    rank = win_ptr->comm_ptr->rank;
91 92 93

    if (win_ptr->shm_allocated == TRUE && target_rank != rank &&
        win_ptr->create_flavor != MPI_WIN_FLAVOR_SHARED) {
Xin Zhao's avatar
Xin Zhao committed
94
        /* check if target is local and shared memory is allocated on window,
95
         * if so, we directly perform this operation on shared memory region. */
Xin Zhao's avatar
Xin Zhao committed
96 97

        /* FIXME: Here we decide whether to perform SHM operations by checking if origin and target are on
98 99 100 101 102
         * the same node. However, in ch3:sock, even if origin and target are on the same node, they do
         * not within the same SHM region. Here we filter out ch3:sock by checking shm_allocated flag first,
         * which is only set to TRUE when SHM region is allocated in nemesis.
         * In future we need to figure out a way to check if origin and target are in the same "SHM comm".
         */
Xin Zhao's avatar
Xin Zhao committed
103 104 105 106
        MPIDI_Comm_get_vc(win_ptr->comm_ptr, rank, &orig_vc);
        MPIDI_Comm_get_vc(win_ptr->comm_ptr, target_rank, &target_vc);
    }

107
    /* If the put is a local operation, do it here */
Xin Zhao's avatar
Xin Zhao committed
108
    if (target_rank == rank || win_ptr->create_flavor == MPI_WIN_FLAVOR_SHARED ||
109
        (win_ptr->shm_allocated == TRUE && orig_vc->node_id == target_vc->node_id)) {
110 111
        mpi_errno = MPIDI_CH3I_Shm_put_op(origin_addr, origin_count, origin_datatype, target_rank,
                                          target_disp, target_count, target_datatype, win_ptr);
112 113
        if (mpi_errno)
            MPIU_ERR_POP(mpi_errno);
114 115 116 117 118 119

        if (ureq) {
            /* Complete user request and release the ch3 ref */
            MPID_Request_set_completed(ureq);
            MPID_Request_release(ureq);
        }
120
    }
121
    else {
122
        MPIDI_RMA_Op_t *new_ptr = NULL;
123
        MPIDI_CH3_Pkt_put_t *put_pkt = NULL;
124 125 126
        MPI_Aint origin_type_size;
        size_t immed_len, len;
        int use_immed_pkt = FALSE;
127

128
        /* queue it up */
129
        mpi_errno = MPIDI_CH3I_Win_get_op(win_ptr, &new_ptr);
130 131
        if (mpi_errno != MPI_SUCCESS)
            MPIU_ERR_POP(mpi_errno);
132

Xin Zhao's avatar
Xin Zhao committed
133 134
        MPIR_T_PVAR_TIMER_START(RMA, rma_rmaqueue_set);

135
        /******************** Setting operation struct areas ***********************/
136

137 138 139 140 141
        /* FIXME: For contig and very short operations, use a streamlined op */
        new_ptr->origin_addr = (void *) origin_addr;
        new_ptr->origin_count = origin_count;
        new_ptr->origin_datatype = origin_datatype;
        new_ptr->target_rank = target_rank;
142
        new_ptr->ureq = NULL;   /* reset user request */
143 144 145 146 147

        /* Remember user request */
        if (ureq) {
            new_ptr->ureq = ureq;
        }
148 149 150 151 152 153

        /* if source or target datatypes are derived, increment their
         * reference counts */
        if (!MPIR_DATATYPE_IS_PREDEFINED(origin_datatype)) {
            MPID_Datatype_get_ptr(origin_datatype, dtp);
            MPID_Datatype_add_ref(dtp);
154
            new_ptr->is_dt = 1;
155 156 157 158
        }
        if (!MPIR_DATATYPE_IS_PREDEFINED(target_datatype)) {
            MPID_Datatype_get_ptr(target_datatype, dtp);
            MPID_Datatype_add_ref(dtp);
159
            new_ptr->is_dt = 1;
160
        }
161

162 163 164 165 166 167
        MPID_Datatype_get_size_macro(origin_datatype, origin_type_size);
        MPIU_Assign_trunc(len, origin_count * origin_type_size, size_t);

        /* Judge if we can use IMMED data packet */
        if (!new_ptr->is_dt) {
            MPIU_Assign_trunc(immed_len,
168
                              (MPIDI_RMA_IMMED_BYTES / origin_type_size) * origin_type_size,
169 170 171 172 173
                              size_t);
            if (len <= immed_len)
                use_immed_pkt = TRUE;
        }

174
        /* Judge if this operation is an piggyback candidate */
Xin Zhao's avatar
Xin Zhao committed
175
        if (!new_ptr->is_dt) {
176
            if (len <= MPIR_CVAR_CH3_RMA_OP_PIGGYBACK_LOCK_DATA_SIZE)
177
                new_ptr->piggyback_lock_candidate = 1;
Xin Zhao's avatar
Xin Zhao committed
178 179
        }

180 181 182
        /************** Setting packet struct areas in operation ****************/

        put_pkt = &(new_ptr->pkt.put);
183 184 185 186 187 188 189 190

        if (use_immed_pkt) {
            MPIDI_Pkt_init(put_pkt, MPIDI_CH3_PKT_PUT_IMMED);
        }
        else {
            MPIDI_Pkt_init(put_pkt, MPIDI_CH3_PKT_PUT);
        }

191 192 193 194
        put_pkt->addr = (char *) win_ptr->base_addrs[target_rank] +
            win_ptr->disp_units[target_rank] * target_disp;
        put_pkt->count = target_count;
        put_pkt->datatype = target_datatype;
195
        put_pkt->info.dataloop_size = 0;
196 197 198
        put_pkt->target_win_handle = win_ptr->all_win_handles[target_rank];
        put_pkt->source_win_handle = win_ptr->handle;
        put_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE;
199
        if (use_immed_pkt) {
200
            void *src = (void *) origin_addr, *dest = (void *) (put_pkt->info.data);
201
            mpi_errno = immed_copy(src, dest, len);
202 203
            if (mpi_errno != MPI_SUCCESS)
                MPIU_ERR_POP(mpi_errno);
204
        }
205

Xin Zhao's avatar
Xin Zhao committed
206 207
        MPIR_T_PVAR_TIMER_END(RMA, rma_rmaqueue_set);

208 209 210 211
        mpi_errno = MPIDI_CH3I_Win_enqueue_op(win_ptr, new_ptr);
        if (mpi_errno)
            MPIU_ERR_POP(mpi_errno);

212
        mpi_errno = MPIDI_CH3I_RMA_Make_progress_target(win_ptr, target_rank, &made_progress);
213 214
        if (mpi_errno != MPI_SUCCESS)
            MPIU_ERR_POP(mpi_errno);
215 216

        if (MPIR_CVAR_CH3_RMA_OP_POKING_PROGRESS >= 0 &&
217
            win_ptr->accumulated_ops_cnt >= MPIR_CVAR_CH3_RMA_OP_POKING_PROGRESS) {
218 219 220 221
            mpi_errno = poke_progress_engine();
            if (mpi_errno != MPI_SUCCESS)
                MPIU_ERR_POP(mpi_errno);
        }
222 223 224 225 226 227 228 229 230 231 232 233 234 235

        if (MPIR_CVAR_CH3_RMA_ACTIVE_REQ_THRESHOLD >= 0 &&
            win_ptr->active_req_cnt >= MPIR_CVAR_CH3_RMA_ACTIVE_REQ_THRESHOLD) {
            while (win_ptr->active_req_cnt >= MPIR_CVAR_CH3_RMA_ACTIVE_REQ_THRESHOLD) {
                int local_completed = 0, remote_completed = 0;
                mpi_errno = MPIDI_CH3I_RMA_Cleanup_ops_win(win_ptr, &local_completed,
                                                           &remote_completed);
                if (mpi_errno != MPI_SUCCESS)
                    MPIU_ERR_POP(mpi_errno);
                mpi_errno = poke_progress_engine();
                if (mpi_errno != MPI_SUCCESS)
                    MPIU_ERR_POP(mpi_errno);
            }
        }
236 237 238
    }

  fn_exit:
239
    MPIDI_RMA_FUNC_EXIT(MPID_STATE_MPIDI_CH3I_PUT);
240 241 242 243 244 245 246 247 248
    return mpi_errno;

    /* --BEGIN ERROR HANDLING-- */
  fn_fail:
    goto fn_exit;
    /* --END ERROR HANDLING-- */
}

#undef FUNCNAME
249
#define FUNCNAME MPIDI_CH3I_Get
250 251
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
252 253 254 255
int MPIDI_CH3I_Get(void *origin_addr, int origin_count, MPI_Datatype
                   origin_datatype, int target_rank, MPI_Aint target_disp,
                   int target_count, MPI_Datatype target_datatype, MPID_Win * win_ptr,
                   MPID_Request * ureq)
256 257 258
{
    int mpi_errno = MPI_SUCCESS;
    MPIDI_msg_sz_t data_sz;
259
    int dt_contig ATTRIBUTE((unused)), rank;
Pavan Balaji's avatar
Pavan Balaji committed
260
    MPI_Aint dt_true_lb ATTRIBUTE((unused));
261
    MPID_Datatype *dtp;
262
    MPIDI_VC_t *orig_vc = NULL, *target_vc = NULL;
263
    int made_progress = 0;
264
    MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3I_GET);
265

266
    MPIDI_RMA_FUNC_ENTER(MPID_STATE_MPIDI_CH3I_GET);
267

268 269 270
    MPIU_ERR_CHKANDJUMP(win_ptr->states.access_state == MPIDI_RMA_NONE,
                        mpi_errno, MPI_ERR_RMA_SYNC, "**rmasync");

271 272 273 274
    if (target_rank == MPI_PROC_NULL) {
        goto fn_exit;
    }

275
    MPIDI_Datatype_get_info(origin_count, origin_datatype, dt_contig, data_sz, dtp, dt_true_lb);
276

277
    if (data_sz == 0) {
278
        goto fn_exit;
279 280
    }

281
    rank = win_ptr->comm_ptr->rank;
Xin Zhao's avatar
Xin Zhao committed
282

283 284
    if (win_ptr->shm_allocated == TRUE && target_rank != rank &&
        win_ptr->create_flavor != MPI_WIN_FLAVOR_SHARED) {
Xin Zhao's avatar
Xin Zhao committed
285
        /* check if target is local and shared memory is allocated on window,
286
         * if so, we directly perform this operation on shared memory region. */
Xin Zhao's avatar
Xin Zhao committed
287 288

        /* FIXME: Here we decide whether to perform SHM operations by checking if origin and target are on
289 290 291 292 293
         * the same node. However, in ch3:sock, even if origin and target are on the same node, they do
         * not within the same SHM region. Here we filter out ch3:sock by checking shm_allocated flag first,
         * which is only set to TRUE when SHM region is allocated in nemesis.
         * In future we need to figure out a way to check if origin and target are in the same "SHM comm".
         */
Xin Zhao's avatar
Xin Zhao committed
294 295 296
        MPIDI_Comm_get_vc(win_ptr->comm_ptr, rank, &orig_vc);
        MPIDI_Comm_get_vc(win_ptr->comm_ptr, target_rank, &target_vc);
    }
297

298
    /* If the get is a local operation, do it here */
Xin Zhao's avatar
Xin Zhao committed
299
    if (target_rank == rank || win_ptr->create_flavor == MPI_WIN_FLAVOR_SHARED ||
300
        (win_ptr->shm_allocated == TRUE && orig_vc->node_id == target_vc->node_id)) {
301 302
        mpi_errno = MPIDI_CH3I_Shm_get_op(origin_addr, origin_count, origin_datatype, target_rank,
                                          target_disp, target_count, target_datatype, win_ptr);
303 304
        if (mpi_errno)
            MPIU_ERR_POP(mpi_errno);
305 306 307 308 309 310

        if (ureq) {
            /* Complete user request and release the ch3 ref */
            MPID_Request_set_completed(ureq);
            MPID_Request_release(ureq);
        }
311
    }
312
    else {
313
        MPIDI_RMA_Op_t *new_ptr = NULL;
314
        MPIDI_CH3_Pkt_get_t *get_pkt = NULL;
315 316 317
        MPI_Aint target_type_size;
        size_t immed_len, len;
        int use_immed_resp_pkt = FALSE;
318

319
        /* queue it up */
320
        mpi_errno = MPIDI_CH3I_Win_get_op(win_ptr, &new_ptr);
321 322
        if (mpi_errno != MPI_SUCCESS)
            MPIU_ERR_POP(mpi_errno);
323

Xin Zhao's avatar
Xin Zhao committed
324 325
        MPIR_T_PVAR_TIMER_START(RMA, rma_rmaqueue_set);

326
        /******************** Setting operation struct areas ***********************/
327

328 329 330 331 332
        /* FIXME: For contig and very short operations, use a streamlined op */
        new_ptr->origin_addr = origin_addr;
        new_ptr->origin_count = origin_count;
        new_ptr->origin_datatype = origin_datatype;
        new_ptr->target_rank = target_rank;
333
        new_ptr->ureq = NULL;   /* reset user request */
334 335 336 337 338

        /* Remember user request */
        if (ureq) {
            new_ptr->ureq = ureq;
        }
339 340 341 342 343 344

        /* if source or target datatypes are derived, increment their
         * reference counts */
        if (!MPIR_DATATYPE_IS_PREDEFINED(origin_datatype)) {
            MPID_Datatype_get_ptr(origin_datatype, dtp);
            MPID_Datatype_add_ref(dtp);
345
            new_ptr->is_dt = 1;
346 347 348 349
        }
        if (!MPIR_DATATYPE_IS_PREDEFINED(target_datatype)) {
            MPID_Datatype_get_ptr(target_datatype, dtp);
            MPID_Datatype_add_ref(dtp);
350
            new_ptr->is_dt = 1;
351
        }
352

353 354 355 356 357 358
        MPID_Datatype_get_size_macro(target_datatype, target_type_size);
        MPIU_Assign_trunc(len, target_count * target_type_size, size_t);

        /* Judge if we can use IMMED data response packet */
        if (!new_ptr->is_dt) {
            MPIU_Assign_trunc(immed_len,
359
                              (MPIDI_RMA_IMMED_BYTES / target_type_size) * target_type_size,
360 361 362 363 364
                              size_t);
            if (len <= immed_len)
                use_immed_resp_pkt = TRUE;
        }

365
        /* Judge if this operation is an piggyback candidate. */
366 367 368 369
        if (!new_ptr->is_dt) {
            new_ptr->piggyback_lock_candidate = 1;
        }

370 371 372 373 374 375 376 377
        /************** Setting packet struct areas in operation ****************/

        get_pkt = &(new_ptr->pkt.get);
        MPIDI_Pkt_init(get_pkt, MPIDI_CH3_PKT_GET);
        get_pkt->addr = (char *) win_ptr->base_addrs[target_rank] +
            win_ptr->disp_units[target_rank] * target_disp;
        get_pkt->count = target_count;
        get_pkt->datatype = target_datatype;
378
        get_pkt->info.dataloop_size = 0;
379 380
        get_pkt->target_win_handle = win_ptr->all_win_handles[target_rank];
        get_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE;
381 382
        if (use_immed_resp_pkt)
            get_pkt->flags |= MPIDI_CH3_PKT_FLAG_RMA_IMMED_RESP;
383

Xin Zhao's avatar
Xin Zhao committed
384 385
        MPIR_T_PVAR_TIMER_END(RMA, rma_rmaqueue_set);

386 387 388 389
        mpi_errno = MPIDI_CH3I_Win_enqueue_op(win_ptr, new_ptr);
        if (mpi_errno)
            MPIU_ERR_POP(mpi_errno);

390
        mpi_errno = MPIDI_CH3I_RMA_Make_progress_target(win_ptr, target_rank, &made_progress);
391 392
        if (mpi_errno != MPI_SUCCESS)
            MPIU_ERR_POP(mpi_errno);
393 394

        if (MPIR_CVAR_CH3_RMA_OP_POKING_PROGRESS >= 0 &&
395
            win_ptr->accumulated_ops_cnt >= MPIR_CVAR_CH3_RMA_OP_POKING_PROGRESS) {
396 397 398 399
            mpi_errno = poke_progress_engine();
            if (mpi_errno != MPI_SUCCESS)
                MPIU_ERR_POP(mpi_errno);
        }
400 401 402 403 404 405 406 407 408 409 410 411 412 413

        if (MPIR_CVAR_CH3_RMA_ACTIVE_REQ_THRESHOLD >= 0 &&
            win_ptr->active_req_cnt >= MPIR_CVAR_CH3_RMA_ACTIVE_REQ_THRESHOLD) {
            while (win_ptr->active_req_cnt >= MPIR_CVAR_CH3_RMA_ACTIVE_REQ_THRESHOLD) {
                int local_completed = 0, remote_completed = 0;
                mpi_errno = MPIDI_CH3I_RMA_Cleanup_ops_win(win_ptr, &local_completed,
                                                           &remote_completed);
                if (mpi_errno != MPI_SUCCESS)
                    MPIU_ERR_POP(mpi_errno);
                mpi_errno = poke_progress_engine();
                if (mpi_errno != MPI_SUCCESS)
                    MPIU_ERR_POP(mpi_errno);
            }
        }
414 415 416
    }

  fn_exit:
417
    MPIDI_RMA_FUNC_EXIT(MPID_STATE_MPIDI_CH3I_GET);
418 419 420 421 422 423 424 425 426 427
    return mpi_errno;

    /* --BEGIN ERROR HANDLING-- */
  fn_fail:
    goto fn_exit;
    /* --END ERROR HANDLING-- */
}


#undef FUNCNAME
428
#define FUNCNAME MPIDI_CH3I_Accumulate
429 430
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
431 432 433 434
int MPIDI_CH3I_Accumulate(const void *origin_addr, int origin_count, MPI_Datatype
                          origin_datatype, int target_rank, MPI_Aint target_disp,
                          int target_count, MPI_Datatype target_datatype, MPI_Op op,
                          MPID_Win * win_ptr, MPID_Request * ureq)
435
{
436
    int mpi_errno = MPI_SUCCESS;
437
    MPIDI_msg_sz_t data_sz;
438
    int dt_contig ATTRIBUTE((unused)), rank;
Pavan Balaji's avatar
Pavan Balaji committed
439
    MPI_Aint dt_true_lb ATTRIBUTE((unused));
440
    MPID_Datatype *dtp;
441
    MPIDI_VC_t *orig_vc = NULL, *target_vc = NULL;
442
    int made_progress = 0;
443
    MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3I_ACCUMULATE);
444

445
    MPIDI_RMA_FUNC_ENTER(MPID_STATE_MPIDI_CH3I_ACCUMULATE);
446

447 448 449
    MPIU_ERR_CHKANDJUMP(win_ptr->states.access_state == MPIDI_RMA_NONE,
                        mpi_errno, MPI_ERR_RMA_SYNC, "**rmasync");

450 451 452 453
    if (target_rank == MPI_PROC_NULL) {
        goto fn_exit;
    }

454 455
    MPIDI_Datatype_get_info(origin_count, origin_datatype, dt_contig, data_sz, dtp, dt_true_lb);

456
    if (data_sz == 0) {
457
        goto fn_exit;
458
    }
459

460
    rank = win_ptr->comm_ptr->rank;
461 462 463

    if (win_ptr->shm_allocated == TRUE && target_rank != rank &&
        win_ptr->create_flavor != MPI_WIN_FLAVOR_SHARED) {
Xin Zhao's avatar
Xin Zhao committed
464
        /* check if target is local and shared memory is allocated on window,
465
         * if so, we directly perform this operation on shared memory region. */
Xin Zhao's avatar
Xin Zhao committed
466 467

        /* FIXME: Here we decide whether to perform SHM operations by checking if origin and target are on
468 469 470 471 472
         * the same node. However, in ch3:sock, even if origin and target are on the same node, they do
         * not within the same SHM region. Here we filter out ch3:sock by checking shm_allocated flag first,
         * which is only set to TRUE when SHM region is allocated in nemesis.
         * In future we need to figure out a way to check if origin and target are in the same "SHM comm".
         */
Xin Zhao's avatar
Xin Zhao committed
473 474 475 476
        MPIDI_Comm_get_vc(win_ptr->comm_ptr, rank, &orig_vc);
        MPIDI_Comm_get_vc(win_ptr->comm_ptr, target_rank, &target_vc);
    }

477
    /* Do =! rank first (most likely branch?) */
Xin Zhao's avatar
Xin Zhao committed
478
    if (target_rank == rank || win_ptr->create_flavor == MPI_WIN_FLAVOR_SHARED ||
479 480 481 482 483 484
        (win_ptr->shm_allocated == TRUE && orig_vc->node_id == target_vc->node_id)) {
        mpi_errno = MPIDI_CH3I_Shm_acc_op(origin_addr, origin_count, origin_datatype,
                                          target_rank, target_disp, target_count, target_datatype,
                                          op, win_ptr);
        if (mpi_errno)
            MPIU_ERR_POP(mpi_errno);
485 486 487 488 489 490

        if (ureq) {
            /* Complete user request and release the ch3 ref */
            MPID_Request_set_completed(ureq);
            MPID_Request_release(ureq);
        }
491
    }
492
    else {
493
        MPIDI_RMA_Op_t *new_ptr = NULL;
494
        MPIDI_CH3_Pkt_accum_t *accum_pkt = NULL;
495 496 497
        MPI_Aint origin_type_size;
        size_t immed_len, len;
        int use_immed_pkt = FALSE;
498

499
        /* queue it up */
500
        mpi_errno = MPIDI_CH3I_Win_get_op(win_ptr, &new_ptr);
501 502
        if (mpi_errno != MPI_SUCCESS)
            MPIU_ERR_POP(mpi_errno);
503

Xin Zhao's avatar
Xin Zhao committed
504 505
        MPIR_T_PVAR_TIMER_START(RMA, rma_rmaqueue_set);

506
        /******************** Setting operation struct areas ***********************/
507

508 509 510 511
        new_ptr->origin_addr = (void *) origin_addr;
        new_ptr->origin_count = origin_count;
        new_ptr->origin_datatype = origin_datatype;
        new_ptr->target_rank = target_rank;
512
        new_ptr->ureq = NULL;   /* reset user request */
513 514 515 516 517

        /* Remember user request */
        if (ureq) {
            new_ptr->ureq = ureq;
        }
518 519 520 521 522 523

        /* if source or target datatypes are derived, increment their
         * reference counts */
        if (!MPIR_DATATYPE_IS_PREDEFINED(origin_datatype)) {
            MPID_Datatype_get_ptr(origin_datatype, dtp);
            MPID_Datatype_add_ref(dtp);
524
            new_ptr->is_dt = 1;
525 526 527 528
        }
        if (!MPIR_DATATYPE_IS_PREDEFINED(target_datatype)) {
            MPID_Datatype_get_ptr(target_datatype, dtp);
            MPID_Datatype_add_ref(dtp);
529
            new_ptr->is_dt = 1;
530
        }
531

532 533 534 535 536 537
        MPID_Datatype_get_size_macro(origin_datatype, origin_type_size);
        MPIU_Assign_trunc(len, origin_count * origin_type_size, size_t);

        /* Judge if we can use IMMED data packet */
        if (!new_ptr->is_dt) {
            MPIU_Assign_trunc(immed_len,
538
                              (MPIDI_RMA_IMMED_BYTES / origin_type_size) * origin_type_size,
539 540 541 542 543
                              size_t);
            if (len <= immed_len)
                use_immed_pkt = TRUE;
        }

544
        /* Judge if this operation is an piggyback candidate. */
Xin Zhao's avatar
Xin Zhao committed
545
        if (!new_ptr->is_dt) {
546
            if (len <= MPIR_CVAR_CH3_RMA_OP_PIGGYBACK_LOCK_DATA_SIZE)
547
                new_ptr->piggyback_lock_candidate = 1;
Xin Zhao's avatar
Xin Zhao committed
548 549
        }

550 551 552 553
        /************** Setting packet struct areas in operation ****************/

        accum_pkt = &(new_ptr->pkt.accum);

554 555 556 557 558 559 560
        if (use_immed_pkt) {
            MPIDI_Pkt_init(accum_pkt, MPIDI_CH3_PKT_ACCUMULATE_IMMED);
        }
        else {
            MPIDI_Pkt_init(accum_pkt, MPIDI_CH3_PKT_ACCUMULATE);
        }

561 562 563 564
        accum_pkt->addr = (char *) win_ptr->base_addrs[target_rank] +
            win_ptr->disp_units[target_rank] * target_disp;
        accum_pkt->count = target_count;
        accum_pkt->datatype = target_datatype;
565
        accum_pkt->info.dataloop_size = 0;
566 567 568 569
        accum_pkt->op = op;
        accum_pkt->target_win_handle = win_ptr->all_win_handles[target_rank];
        accum_pkt->source_win_handle = win_ptr->handle;
        accum_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE;
570
        if (use_immed_pkt) {
571
            void *src = (void *) origin_addr, *dest = (void *) (accum_pkt->info.data);
572
            mpi_errno = immed_copy(src, dest, len);
573 574
            if (mpi_errno != MPI_SUCCESS)
                MPIU_ERR_POP(mpi_errno);
575
        }
576

Xin Zhao's avatar
Xin Zhao committed
577 578
        MPIR_T_PVAR_TIMER_END(RMA, rma_rmaqueue_set);

579 580 581 582
        mpi_errno = MPIDI_CH3I_Win_enqueue_op(win_ptr, new_ptr);
        if (mpi_errno)
            MPIU_ERR_POP(mpi_errno);

583
        mpi_errno = MPIDI_CH3I_RMA_Make_progress_target(win_ptr, target_rank, &made_progress);
584 585
        if (mpi_errno != MPI_SUCCESS)
            MPIU_ERR_POP(mpi_errno);
586 587

        if (MPIR_CVAR_CH3_RMA_OP_POKING_PROGRESS >= 0 &&
588
            win_ptr->accumulated_ops_cnt >= MPIR_CVAR_CH3_RMA_OP_POKING_PROGRESS) {
589 590 591 592
            mpi_errno = poke_progress_engine();
            if (mpi_errno != MPI_SUCCESS)
                MPIU_ERR_POP(mpi_errno);
        }
593 594 595 596 597 598 599 600 601 602 603 604 605 606

        if (MPIR_CVAR_CH3_RMA_ACTIVE_REQ_THRESHOLD >= 0 &&
            win_ptr->active_req_cnt >= MPIR_CVAR_CH3_RMA_ACTIVE_REQ_THRESHOLD) {
            while (win_ptr->active_req_cnt >= MPIR_CVAR_CH3_RMA_ACTIVE_REQ_THRESHOLD) {
                int local_completed = 0, remote_completed = 0;
                mpi_errno = MPIDI_CH3I_RMA_Cleanup_ops_win(win_ptr, &local_completed,
                                                           &remote_completed);
                if (mpi_errno != MPI_SUCCESS)
                    MPIU_ERR_POP(mpi_errno);
                mpi_errno = poke_progress_engine();
                if (mpi_errno != MPI_SUCCESS)
                    MPIU_ERR_POP(mpi_errno);
            }
        }
607 608
    }

609
  fn_exit:
610
    MPIDI_RMA_FUNC_EXIT(MPID_STATE_MPIDI_CH3I_ACCUMULATE);
611 612 613 614 615 616 617 618 619 620
    return mpi_errno;

    /* --BEGIN ERROR HANDLING-- */
  fn_fail:
    goto fn_exit;
    /* --END ERROR HANDLING-- */
}


#undef FUNCNAME
621
#define FUNCNAME MPIDI_CH3I_Get_accumulate
622 623
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
624 625 626 627 628
int MPIDI_CH3I_Get_accumulate(const void *origin_addr, int origin_count,
                              MPI_Datatype origin_datatype, void *result_addr, int result_count,
                              MPI_Datatype result_datatype, int target_rank, MPI_Aint target_disp,
                              int target_count, MPI_Datatype target_datatype, MPI_Op op,
                              MPID_Win * win_ptr, MPID_Request * ureq)
629 630 631 632 633 634 635 636
{
    int mpi_errno = MPI_SUCCESS;
    MPIDI_msg_sz_t data_sz;
    int rank;
    int dt_contig ATTRIBUTE((unused));
    MPI_Aint dt_true_lb ATTRIBUTE((unused));
    MPID_Datatype *dtp;
    MPIDI_VC_t *orig_vc = NULL, *target_vc = NULL;
637
    int made_progress = 0;
638
    MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3I_GET_ACCUMULATE);
639

640
    MPIDI_RMA_FUNC_ENTER(MPID_STATE_MPIDI_CH3I_GET_ACCUMULATE);
641

642 643 644
    MPIU_ERR_CHKANDJUMP(win_ptr->states.access_state == MPIDI_RMA_NONE,
                        mpi_errno, MPI_ERR_RMA_SYNC, "**rmasync");

645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680
    if (target_rank == MPI_PROC_NULL) {
        goto fn_exit;
    }

    MPIDI_Datatype_get_info(target_count, target_datatype, dt_contig, data_sz, dtp, dt_true_lb);

    if (data_sz == 0) {
        goto fn_exit;
    }

    rank = win_ptr->comm_ptr->rank;

    if (win_ptr->shm_allocated == TRUE && target_rank != rank &&
        win_ptr->create_flavor != MPI_WIN_FLAVOR_SHARED) {
        /* check if target is local and shared memory is allocated on window,
         * if so, we directly perform this operation on shared memory region. */

        /* FIXME: Here we decide whether to perform SHM operations by checking if origin and target are on
         * the same node. However, in ch3:sock, even if origin and target are on the same node, they do
         * not within the same SHM region. Here we filter out ch3:sock by checking shm_allocated flag first,
         * which is only set to TRUE when SHM region is allocated in nemesis.
         * In future we need to figure out a way to check if origin and target are in the same "SHM comm".
         */
        MPIDI_Comm_get_vc(win_ptr->comm_ptr, rank, &orig_vc);
        MPIDI_Comm_get_vc(win_ptr->comm_ptr, target_rank, &target_vc);
    }

    /* Do =! rank first (most likely branch?) */
    if (target_rank == rank || win_ptr->create_flavor == MPI_WIN_FLAVOR_SHARED ||
        (win_ptr->shm_allocated == TRUE && orig_vc->node_id == target_vc->node_id)) {
        mpi_errno = MPIDI_CH3I_Shm_get_acc_op(origin_addr, origin_count, origin_datatype,
                                              result_addr, result_count, result_datatype,
                                              target_rank, target_disp, target_count,
                                              target_datatype, op, win_ptr);
        if (mpi_errno)
            MPIU_ERR_POP(mpi_errno);
681 682 683 684 685 686

        if (ureq) {
            /* Complete user request and release the ch3 ref */
            MPID_Request_set_completed(ureq);
            MPID_Request_release(ureq);
        }
687 688 689 690 691
    }
    else {
        MPIDI_RMA_Op_t *new_ptr = NULL;

        /* Append the operation to the window's RMA ops queue */
692
        mpi_errno = MPIDI_CH3I_Win_get_op(win_ptr, &new_ptr);
693 694
        if (mpi_errno != MPI_SUCCESS)
            MPIU_ERR_POP(mpi_errno);
695 696 697

        /* TODO: Can we use the MPIDI_RMA_ACC_CONTIG optimization? */

Xin Zhao's avatar
Xin Zhao committed
698 699
        MPIR_T_PVAR_TIMER_START(RMA, rma_rmaqueue_set);

700 701
        if (op == MPI_NO_OP) {
            /* Convert GAcc to a Get */
702
            MPIDI_CH3_Pkt_get_t *get_pkt;
703 704 705
            MPI_Aint target_type_size;
            size_t len, immed_len;
            int use_immed_resp_pkt = FALSE;
706 707

            /******************** Setting operation struct areas ***********************/
708 709 710 711 712

            new_ptr->origin_addr = result_addr;
            new_ptr->origin_count = result_count;
            new_ptr->origin_datatype = result_datatype;
            new_ptr->target_rank = target_rank;
713
            new_ptr->ureq = NULL;       /* reset user request */
714 715 716 717 718

            /* Remember user request */
            if (ureq) {
                new_ptr->ureq = ureq;
            }
719 720 721 722 723 724 725 726 727 728 729

            if (!MPIR_DATATYPE_IS_PREDEFINED(result_datatype)) {
                MPID_Datatype_get_ptr(result_datatype, dtp);
                MPID_Datatype_add_ref(dtp);
                new_ptr->is_dt = 1;
            }
            if (!MPIR_DATATYPE_IS_PREDEFINED(target_datatype)) {
                MPID_Datatype_get_ptr(target_datatype, dtp);
                MPID_Datatype_add_ref(dtp);
                new_ptr->is_dt = 1;
            }
730

731 732 733 734 735 736
            MPID_Datatype_get_size_macro(target_datatype, target_type_size);
            MPIU_Assign_trunc(len, target_count * target_type_size, size_t);

            /* Judge if we can use IMMED data response packet */
            if (!new_ptr->is_dt) {
                MPIU_Assign_trunc(immed_len,
737
                                  (MPIDI_RMA_IMMED_BYTES / target_type_size) * target_type_size,
738 739 740 741 742
                                  size_t);
                if (len <= immed_len)
                    use_immed_resp_pkt = TRUE;
            }

743
            /* Judge if this operation is a piggyback candidate */
744 745 746
            if (!new_ptr->is_dt) {
                new_ptr->piggyback_lock_candidate = 1;
            }
747 748 749 750 751 752 753 754 755

            /************** Setting packet struct areas in operation ****************/

            get_pkt = &(new_ptr->pkt.get);
            MPIDI_Pkt_init(get_pkt, MPIDI_CH3_PKT_GET);
            get_pkt->addr = (char *) win_ptr->base_addrs[target_rank] +
                win_ptr->disp_units[target_rank] * target_disp;
            get_pkt->count = target_count;
            get_pkt->datatype = target_datatype;
756
            get_pkt->info.dataloop_size = 0;
757 758
            get_pkt->target_win_handle = win_ptr->all_win_handles[target_rank];
            get_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE;
759 760
            if (use_immed_resp_pkt == TRUE)
                get_pkt->flags |= MPIDI_CH3_PKT_FLAG_RMA_IMMED_RESP;
761 762 763
        }

        else {
764
            MPIDI_CH3_Pkt_get_accum_t *get_accum_pkt;
765 766 767
            MPI_Aint origin_type_size, target_type_size;
            size_t immed_len, orig_len, tar_len;
            int use_immed_pkt = FALSE, use_immed_resp_pkt = FALSE;
768 769

            /******************** Setting operation struct areas ***********************/
770 771 772 773 774 775 776 777

            new_ptr->origin_addr = (void *) origin_addr;
            new_ptr->origin_count = origin_count;
            new_ptr->origin_datatype = origin_datatype;
            new_ptr->result_addr = result_addr;
            new_ptr->result_count = result_count;
            new_ptr->result_datatype = result_datatype;
            new_ptr->target_rank = target_rank;
778
            new_ptr->ureq = NULL;       /* reset user request */
779 780 781 782 783

            /* Remember user request */
            if (ureq) {
                new_ptr->ureq = ureq;
            }
784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801

            /* if source or target datatypes are derived, increment their
             * reference counts */
            if (!MPIR_DATATYPE_IS_PREDEFINED(origin_datatype)) {
                MPID_Datatype_get_ptr(origin_datatype, dtp);
                MPID_Datatype_add_ref(dtp);
                new_ptr->is_dt = 1;
            }
            if (!MPIR_DATATYPE_IS_PREDEFINED(result_datatype)) {
                MPID_Datatype_get_ptr(result_datatype, dtp);
                MPID_Datatype_add_ref(dtp);
                new_ptr->is_dt = 1;
            }
            if (!MPIR_DATATYPE_IS_PREDEFINED(target_datatype)) {
                MPID_Datatype_get_ptr(target_datatype, dtp);
                MPID_Datatype_add_ref(dtp);
                new_ptr->is_dt = 1;
            }
Xin Zhao's avatar
Xin Zhao committed
802

803 804 805 806 807 808 809 810 811
            MPID_Datatype_get_size_macro(origin_datatype, origin_type_size);
            MPIU_Assign_trunc(orig_len, origin_count * origin_type_size, size_t);

            MPID_Datatype_get_size_macro(target_datatype, target_type_size);
            MPIU_Assign_trunc(tar_len, target_count * target_type_size, size_t);

            /* Judge if we can use IMMED data packet */
            if (!new_ptr->is_dt) {
                MPIU_Assign_trunc(immed_len,
812
                                  (MPIDI_RMA_IMMED_BYTES / origin_type_size) * origin_type_size,
813 814 815 816 817
                                  size_t);
                if (orig_len <= immed_len)
                    use_immed_pkt = TRUE;

                MPIU_Assign_trunc(immed_len,
818
                                  (MPIDI_RMA_IMMED_BYTES / target_type_size) * target_type_size,
819 820 821 822 823
                                  size_t);
                if (tar_len <= immed_len)
                    use_immed_resp_pkt = TRUE;
            }

824
            /* Judge if this operation is a piggyback candidate */
Xin Zhao's avatar
Xin Zhao committed
825
            if (!new_ptr->is_dt) {
826
                if (orig_len <= MPIR_CVAR_CH3_RMA_OP_PIGGYBACK_LOCK_DATA_SIZE)
827
                    new_ptr->piggyback_lock_candidate = 1;
Xin Zhao's avatar
Xin Zhao committed
828
            }
829

830
            /************** Setting packet struct areas in operation ****************/
831

832
            get_accum_pkt = &(new_ptr->pkt.get_accum);
833 834 835 836 837 838 839 840

            if (use_immed_pkt) {
                MPIDI_Pkt_init(get_accum_pkt, MPIDI_CH3_PKT_GET_ACCUM_IMMED);
            }
            else {
                MPIDI_Pkt_init(get_accum_pkt, MPIDI_CH3_PKT_GET_ACCUM);
            }

841 842 843 844
            get_accum_pkt->addr = (char *) win_ptr->base_addrs[target_rank] +
                win_ptr->disp_units[target_rank] * target_disp;
            get_accum_pkt->count = target_count;
            get_accum_pkt->datatype = target_datatype;
845
            get_accum_pkt->info.dataloop_size = 0;
846 847 848
            get_accum_pkt->op = op;
            get_accum_pkt->target_win_handle = win_ptr->all_win_handles[target_rank];
            get_accum_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE;
849
            if (use_immed_pkt) {
850
                void *src = (void *) origin_addr, *dest = (void *) (get_accum_pkt->info.data);
851
                mpi_errno = immed_copy(src, dest, orig_len);
852 853
                if (mpi_errno != MPI_SUCCESS)
                    MPIU_ERR_POP(mpi_errno);
854 855 856
            }
            if (use_immed_resp_pkt)
                get_accum_pkt->flags |= MPIDI_CH3_PKT_FLAG_RMA_IMMED_RESP;
857 858
        }

Xin Zhao's avatar
Xin Zhao committed
859 860
        MPIR_T_PVAR_TIMER_END(RMA, rma_rmaqueue_set);

861 862 863 864
        mpi_errno = MPIDI_CH3I_Win_enqueue_op(win_ptr, new_ptr);
        if (mpi_errno)
            MPIU_ERR_POP(mpi_errno);

865
        mpi_errno = MPIDI_CH3I_RMA_Make_progress_target(win_ptr, target_rank, &made_progress);
866 867
        if (mpi_errno != MPI_SUCCESS)
            MPIU_ERR_POP(mpi_errno);
868 869

        if (MPIR_CVAR_CH3_RMA_OP_POKING_PROGRESS >= 0 &&
870
            win_ptr->accumulated_ops_cnt >= MPIR_CVAR_CH3_RMA_OP_POKING_PROGRESS) {
871 872 873 874
            mpi_errno = poke_progress_engine();
            if (mpi_errno != MPI_SUCCESS)
                MPIU_ERR_POP(mpi_errno);
        }
875 876 877 878 879 880 881 882 883 884 885 886 887 888

        if (MPIR_CVAR_CH3_RMA_ACTIVE_REQ_THRESHOLD >= 0 &&
            win_ptr->active_req_cnt >= MPIR_CVAR_CH3_RMA_ACTIVE_REQ_THRESHOLD) {
            while (win_ptr->active_req_cnt >= MPIR_CVAR_CH3_RMA_ACTIVE_REQ_THRESHOLD) {
                int local_completed = 0, remote_completed = 0;
                mpi_errno = MPIDI_CH3I_RMA_Cleanup_ops_win(win_ptr, &local_completed,
                                                           &remote_completed);
                if (mpi_errno != MPI_SUCCESS)
                    MPIU_ERR_POP(mpi_errno);
                mpi_errno = poke_progress_engine();
                if (mpi_errno != MPI_SUCCESS)
                    MPIU_ERR_POP(mpi_errno);
            }
        }
889 890
    }

891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002
  fn_exit:
    MPIDI_RMA_FUNC_EXIT(MPID_STATE_MPIDI_CH3I_GET_ACCUMULATE);
    return mpi_errno;

    /* --BEGIN ERROR HANDLING-- */
  fn_fail:
    goto fn_exit;
    /* --END ERROR HANDLING-- */
}


#undef FUNCNAME
#define FUNCNAME MPIDI_Put
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
int MPIDI_Put(const void *origin_addr, int origin_count, MPI_Datatype
              origin_datatype, int target_rank, MPI_Aint target_disp,
              int target_count, MPI_Datatype target_datatype, MPID_Win * win_ptr)
{
    int mpi_errno = MPI_SUCCESS;

    MPIDI_STATE_DECL(MPID_STATE_MPIDI_PUT);
    MPIDI_RMA_FUNC_ENTER(MPID_STATE_MPIDI_PUT);

    mpi_errno = MPIDI_CH3I_Put(origin_addr, origin_count, origin_datatype,
                               target_rank, target_disp, target_count, target_datatype,
                               win_ptr, NULL);

  fn_exit:
    MPIDI_RMA_FUNC_EXIT(MPID_STATE_MPIDI_PUT);
    return mpi_errno;

    /* --BEGIN ERROR HANDLING-- */
  fn_fail:
    goto fn_exit;
    /* --END ERROR HANDLING-- */
}

#undef FUNCNAME
#define FUNCNAME MPIDI_Get
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
int MPIDI_Get(void *origin_addr, int origin_count, MPI_Datatype
              origin_datatype, int target_rank, MPI_Aint target_disp,
              int target_count, MPI_Datatype target_datatype, MPID_Win * win_ptr)
{
    int mpi_errno = MPI_SUCCESS;

    MPIDI_STATE_DECL(MPID_STATE_MPIDI_GET);
    MPIDI_RMA_FUNC_ENTER(MPID_STATE_MPIDI_GET);

    mpi_errno = MPIDI_CH3I_Get(origin_addr, origin_count, origin_datatype,
                               target_rank, target_disp, target_count, target_datatype,
                               win_ptr, NULL);

  fn_exit:
    MPIDI_RMA_FUNC_EXIT(MPID_STATE_MPIDI_GET);
    return mpi_errno;

    /* --BEGIN ERROR HANDLING-- */
  fn_fail:
    goto fn_exit;
    /* --END ERROR HANDLING-- */
}

#undef FUNCNAME
#define FUNCNAME MPIDI_Accumulate
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
int MPIDI_Accumulate(const void *origin_addr, int origin_count, MPI_Datatype
                     origin_datatype, int target_rank, MPI_Aint target_disp,
                     int target_count, MPI_Datatype target_datatype, MPI_Op op, MPID_Win * win_ptr)
{
    int mpi_errno = MPI_SUCCESS;

    MPIDI_STATE_DECL(MPID_STATE_MPIDI_ACCUMULATE);
    MPIDI_RMA_FUNC_ENTER(MPID_STATE_MPIDI_ACCUMULATE);

    mpi_errno = MPIDI_CH3I_Accumulate(origin_addr, origin_count, origin_datatype,
                                      target_rank, target_disp, target_count, target_datatype,
                                      op, win_ptr, NULL);

  fn_exit:
    MPIDI_RMA_FUNC_EXIT(MPID_STATE_MPIDI_ACCUMULATE);
    return mpi_errno;

    /* --BEGIN ERROR HANDLING-- */
  fn_fail:
    goto fn_exit;
    /* --END ERROR HANDLING-- */
}

#undef FUNCNAME
#define FUNCNAME MPIDI_Get_accumulate
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
int MPIDI_Get_accumulate(const void *origin_addr, int origin_count,
                         MPI_Datatype origin_datatype, void *result_addr, int result_count,
                         MPI_Datatype result_datatype, int target_rank, MPI_Aint target_disp,
                         int target_count, MPI_Datatype target_datatype, MPI_Op op,
                         MPID_Win * win_ptr)
{
    int mpi_errno = MPI_SUCCESS;

    MPIDI_STATE_DECL(MPID_STATE_MPIDI_GET_ACCUMULATE);
    MPIDI_RMA_FUNC_ENTER(MPID_STATE_MPIDI_GET_ACCUMULATE);

    mpi_errno = MPIDI_CH3I_Get_accumulate(origin_addr, origin_count, origin_datatype,
                                          result_addr, result_count, result_datatype,
                                          target_rank, target_disp, target_count,
                                          target_datatype, op, win_ptr, NULL);

1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015
  fn_exit:
    MPIDI_RMA_FUNC_EXIT(MPID_STATE_MPIDI_GET_ACCUMULATE);
    return mpi_errno;

    /* --BEGIN ERROR HANDLING-- */
  fn_fail:
    goto fn_exit;
    /* --END ERROR HANDLING-- */
}


#undef FUNCNAME
#define FUNCNAME MPIDI_Compare_and_swap
1016 1017
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
1018 1019 1020
int MPIDI_Compare_and_swap(const void *origin_addr, const void *compare_addr,
                           void *result_addr, MPI_Datatype datatype, int target_rank,
                           MPI_Aint target_disp, MPID_Win * win_ptr)
1021
{
1022 1023 1024
    int mpi_errno = MPI_SUCCESS;
    int rank;
    MPIDI_VC_t *orig_vc = NULL, *target_vc = NULL;
1025
    int made_progress = 0;
1026 1027

    MPIDI_STATE_DECL(MPID_STATE_MPIDI_COMPARE_AND_SWAP);
1028

1029 1030
    MPIDI_RMA_FUNC_ENTER(MPID_STATE_MPIDI_COMPARE_AND_SWAP);

1031 1032 1033
    MPIU_ERR_CHKANDJUMP(win_ptr->states.access_state == MPIDI_RMA_NONE