ch3u_rma_ops.c 53.4 KB
Newer Older
1
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
2 3 4 5 6 7 8
/*
 *  (C) 2001 by Argonne National Laboratory.
 *      See COPYRIGHT in top-level directory.
 */

#include "mpidrma.h"

Xin Zhao's avatar
Xin Zhao committed
9 10
MPIR_T_PVAR_DOUBLE_TIMER_DECL_EXTERN(RMA, rma_rmaqueue_set);

11 12 13
#define MPIDI_PASSIVE_TARGET_DONE_TAG  348297
#define MPIDI_PASSIVE_TARGET_RMA_TAG 563924

14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34
/*
=== BEGIN_MPI_T_CVAR_INFO_BLOCK ===

cvars:
    - name        : MPIR_CVAR_CH3_RMA_OP_POKING_PROGRESS
      category    : CH3
      type        : int
      default     : 100
      class       : none
      verbosity   : MPI_T_VERBOSITY_USER_BASIC
      scope       : MPI_T_SCOPE_ALL_EQ
      description : >-
          Specify the threshold of number of posted operations
          when starting poking progress in operation routines.
          When the value is negative, runtime never pokes progress
          engine in operation routines; when the value is zero,
          runtime always pokes progress engine in operation
          routines; when the value is larger than zero, runtime
          starts to poke progress engine when number of posted
          operations reaches that value.

35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53
    - name        : MPIR_CVAR_CH3_RMA_OP_PIGGYBACK_LOCK_DATA_SIZE
      category    : CH3
      type        : int
      default     : 65536
      class       : none
      verbosity   : MPI_T_VERBOSITY_USER_BASIC
      scope       : MPI_T_SCOPE_ALL_EQ
      description : >-
          Specify the threshold of data size of a RMA operation
          which can be piggybacked with a LOCK message. It is
          always a positive value and should not be smaller
          than MPIDI_RMA_IMMED_BYTES.
          If user sets it as a small value, for middle and large
          data size, we will lose performance because of always
          waiting for round-trip of LOCK synchronization; if
          user sets it as a large value, we need to consume
          more memory on target side to buffer this lock request
          when lock is not satisfied.

54 55 56
=== END_MPI_T_CVAR_INFO_BLOCK ===
*/

57
#undef FUNCNAME
58
#define FUNCNAME MPIDI_CH3I_Put
59 60
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
61 62 63 64
int MPIDI_CH3I_Put(const void *origin_addr, int origin_count, MPI_Datatype
                   origin_datatype, int target_rank, MPI_Aint target_disp,
                   int target_count, MPI_Datatype target_datatype, MPID_Win * win_ptr,
                   MPID_Request * ureq)
65 66
{
    int mpi_errno = MPI_SUCCESS;
67
    int dt_contig ATTRIBUTE((unused)), rank;
68
    MPID_Datatype *dtp;
Pavan Balaji's avatar
Pavan Balaji committed
69
    MPI_Aint dt_true_lb ATTRIBUTE((unused));
70
    MPIDI_msg_sz_t data_sz;
71
    MPIDI_VC_t *orig_vc = NULL, *target_vc = NULL;
72
    int made_progress = 0;
73
    MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3I_PUT);
74

75
    MPIDI_RMA_FUNC_ENTER(MPID_STATE_MPIDI_CH3I_PUT);
76

77 78 79
    MPIU_ERR_CHKANDJUMP(win_ptr->states.access_state == MPIDI_RMA_NONE,
                        mpi_errno, MPI_ERR_RMA_SYNC, "**rmasync");

80 81 82 83
    if (target_rank == MPI_PROC_NULL) {
        goto fn_exit;
    }

84 85
    MPIDI_Datatype_get_info(origin_count, origin_datatype, dt_contig, data_sz, dtp, dt_true_lb);

86
    if (data_sz == 0) {
87
        goto fn_exit;
88 89
    }

90
    rank = win_ptr->comm_ptr->rank;
91 92 93

    if (win_ptr->shm_allocated == TRUE && target_rank != rank &&
        win_ptr->create_flavor != MPI_WIN_FLAVOR_SHARED) {
Xin Zhao's avatar
Xin Zhao committed
94
        /* check if target is local and shared memory is allocated on window,
95
         * if so, we directly perform this operation on shared memory region. */
Xin Zhao's avatar
Xin Zhao committed
96 97

        /* FIXME: Here we decide whether to perform SHM operations by checking if origin and target are on
98 99 100 101 102
         * the same node. However, in ch3:sock, even if origin and target are on the same node, they do
         * not within the same SHM region. Here we filter out ch3:sock by checking shm_allocated flag first,
         * which is only set to TRUE when SHM region is allocated in nemesis.
         * In future we need to figure out a way to check if origin and target are in the same "SHM comm".
         */
Xin Zhao's avatar
Xin Zhao committed
103 104 105 106
        MPIDI_Comm_get_vc(win_ptr->comm_ptr, rank, &orig_vc);
        MPIDI_Comm_get_vc(win_ptr->comm_ptr, target_rank, &target_vc);
    }

107
    /* If the put is a local operation, do it here */
Xin Zhao's avatar
Xin Zhao committed
108
    if (target_rank == rank || win_ptr->create_flavor == MPI_WIN_FLAVOR_SHARED ||
109
        (win_ptr->shm_allocated == TRUE && orig_vc->node_id == target_vc->node_id)) {
110 111
        mpi_errno = MPIDI_CH3I_Shm_put_op(origin_addr, origin_count, origin_datatype, target_rank,
                                          target_disp, target_count, target_datatype, win_ptr);
112 113
        if (mpi_errno)
            MPIU_ERR_POP(mpi_errno);
114 115 116 117 118 119

        if (ureq) {
            /* Complete user request and release the ch3 ref */
            MPID_Request_set_completed(ureq);
            MPID_Request_release(ureq);
        }
120
    }
121
    else {
122
        MPIDI_RMA_Op_t *new_ptr = NULL;
123
        MPIDI_CH3_Pkt_put_t *put_pkt = NULL;
124 125 126
        MPI_Aint origin_type_size;
        size_t immed_len, len;
        int use_immed_pkt = FALSE;
127

128
        /* queue it up */
129
        mpi_errno = MPIDI_CH3I_Win_get_op(win_ptr, &new_ptr);
130 131
        if (mpi_errno != MPI_SUCCESS)
            MPIU_ERR_POP(mpi_errno);
132

Xin Zhao's avatar
Xin Zhao committed
133 134
        MPIR_T_PVAR_TIMER_START(RMA, rma_rmaqueue_set);

135
        /******************** Setting operation struct areas ***********************/
136

137 138 139 140 141
        /* FIXME: For contig and very short operations, use a streamlined op */
        new_ptr->origin_addr = (void *) origin_addr;
        new_ptr->origin_count = origin_count;
        new_ptr->origin_datatype = origin_datatype;
        new_ptr->target_rank = target_rank;
142
        new_ptr->ureq = NULL;   /* reset user request */
143 144 145 146 147

        /* Remember user request */
        if (ureq) {
            new_ptr->ureq = ureq;
        }
148 149 150 151 152 153

        /* if source or target datatypes are derived, increment their
         * reference counts */
        if (!MPIR_DATATYPE_IS_PREDEFINED(origin_datatype)) {
            MPID_Datatype_get_ptr(origin_datatype, dtp);
            MPID_Datatype_add_ref(dtp);
154
            new_ptr->is_dt = 1;
155 156 157 158
        }
        if (!MPIR_DATATYPE_IS_PREDEFINED(target_datatype)) {
            MPID_Datatype_get_ptr(target_datatype, dtp);
            MPID_Datatype_add_ref(dtp);
159
            new_ptr->is_dt = 1;
160
        }
161

162 163 164 165 166 167
        MPID_Datatype_get_size_macro(origin_datatype, origin_type_size);
        MPIU_Assign_trunc(len, origin_count * origin_type_size, size_t);

        /* Judge if we can use IMMED data packet */
        if (!new_ptr->is_dt) {
            MPIU_Assign_trunc(immed_len,
168
                              (MPIDI_RMA_IMMED_BYTES / origin_type_size) * origin_type_size,
169 170 171 172 173
                              size_t);
            if (len <= immed_len)
                use_immed_pkt = TRUE;
        }

174
        /* Judge if this operation is an piggyback candidate */
Xin Zhao's avatar
Xin Zhao committed
175
        if (!new_ptr->is_dt) {
176 177 178
            /* FIXME: currently we only piggyback LOCK flag with op using predefined datatypes
             * for both origin and target data. We should extend this optimization to derived
             * datatypes as well. */
179
            if (len <= MPIR_CVAR_CH3_RMA_OP_PIGGYBACK_LOCK_DATA_SIZE)
180
                new_ptr->piggyback_lock_candidate = 1;
Xin Zhao's avatar
Xin Zhao committed
181 182
        }

183 184 185
        /************** Setting packet struct areas in operation ****************/

        put_pkt = &(new_ptr->pkt.put);
186 187 188 189 190 191 192 193

        if (use_immed_pkt) {
            MPIDI_Pkt_init(put_pkt, MPIDI_CH3_PKT_PUT_IMMED);
        }
        else {
            MPIDI_Pkt_init(put_pkt, MPIDI_CH3_PKT_PUT);
        }

194 195
        put_pkt->addr = (char *) win_ptr->basic_info_table[target_rank].base_addr +
            win_ptr->basic_info_table[target_rank].disp_unit * target_disp;
196 197
        put_pkt->count = target_count;
        put_pkt->datatype = target_datatype;
198
        put_pkt->info.dataloop_size = 0;
199
        put_pkt->target_win_handle = win_ptr->basic_info_table[target_rank].win_handle;
200 201
        put_pkt->source_win_handle = win_ptr->handle;
        put_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE;
202
        if (use_immed_pkt) {
203
            void *src = (void *) origin_addr, *dest = (void *) (put_pkt->info.data);
204
            mpi_errno = immed_copy(src, dest, len);
205 206
            if (mpi_errno != MPI_SUCCESS)
                MPIU_ERR_POP(mpi_errno);
207
        }
208

Xin Zhao's avatar
Xin Zhao committed
209 210
        MPIR_T_PVAR_TIMER_END(RMA, rma_rmaqueue_set);

211 212 213 214
        mpi_errno = MPIDI_CH3I_Win_enqueue_op(win_ptr, new_ptr);
        if (mpi_errno)
            MPIU_ERR_POP(mpi_errno);

215
        mpi_errno = MPIDI_CH3I_RMA_Make_progress_target(win_ptr, target_rank, &made_progress);
216 217
        if (mpi_errno != MPI_SUCCESS)
            MPIU_ERR_POP(mpi_errno);
218 219

        if (MPIR_CVAR_CH3_RMA_OP_POKING_PROGRESS >= 0 &&
220
            win_ptr->accumulated_ops_cnt >= MPIR_CVAR_CH3_RMA_OP_POKING_PROGRESS) {
221 222 223 224
            mpi_errno = poke_progress_engine();
            if (mpi_errno != MPI_SUCCESS)
                MPIU_ERR_POP(mpi_errno);
        }
225 226 227 228 229 230 231 232 233 234 235 236 237 238

        if (MPIR_CVAR_CH3_RMA_ACTIVE_REQ_THRESHOLD >= 0 &&
            win_ptr->active_req_cnt >= MPIR_CVAR_CH3_RMA_ACTIVE_REQ_THRESHOLD) {
            while (win_ptr->active_req_cnt >= MPIR_CVAR_CH3_RMA_ACTIVE_REQ_THRESHOLD) {
                int local_completed = 0, remote_completed = 0;
                mpi_errno = MPIDI_CH3I_RMA_Cleanup_ops_win(win_ptr, &local_completed,
                                                           &remote_completed);
                if (mpi_errno != MPI_SUCCESS)
                    MPIU_ERR_POP(mpi_errno);
                mpi_errno = poke_progress_engine();
                if (mpi_errno != MPI_SUCCESS)
                    MPIU_ERR_POP(mpi_errno);
            }
        }
239 240 241
    }

  fn_exit:
242
    MPIDI_RMA_FUNC_EXIT(MPID_STATE_MPIDI_CH3I_PUT);
243 244 245 246 247 248 249 250 251
    return mpi_errno;

    /* --BEGIN ERROR HANDLING-- */
  fn_fail:
    goto fn_exit;
    /* --END ERROR HANDLING-- */
}

#undef FUNCNAME
252
#define FUNCNAME MPIDI_CH3I_Get
253 254
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
255 256 257 258
int MPIDI_CH3I_Get(void *origin_addr, int origin_count, MPI_Datatype
                   origin_datatype, int target_rank, MPI_Aint target_disp,
                   int target_count, MPI_Datatype target_datatype, MPID_Win * win_ptr,
                   MPID_Request * ureq)
259 260 261
{
    int mpi_errno = MPI_SUCCESS;
    MPIDI_msg_sz_t data_sz;
262
    int dt_contig ATTRIBUTE((unused)), rank;
Pavan Balaji's avatar
Pavan Balaji committed
263
    MPI_Aint dt_true_lb ATTRIBUTE((unused));
264
    MPID_Datatype *dtp;
265
    MPIDI_VC_t *orig_vc = NULL, *target_vc = NULL;
266
    int made_progress = 0;
267
    MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3I_GET);
268

269
    MPIDI_RMA_FUNC_ENTER(MPID_STATE_MPIDI_CH3I_GET);
270

271 272 273
    MPIU_ERR_CHKANDJUMP(win_ptr->states.access_state == MPIDI_RMA_NONE,
                        mpi_errno, MPI_ERR_RMA_SYNC, "**rmasync");

274 275 276 277
    if (target_rank == MPI_PROC_NULL) {
        goto fn_exit;
    }

278
    MPIDI_Datatype_get_info(origin_count, origin_datatype, dt_contig, data_sz, dtp, dt_true_lb);
279

280
    if (data_sz == 0) {
281
        goto fn_exit;
282 283
    }

284
    rank = win_ptr->comm_ptr->rank;
Xin Zhao's avatar
Xin Zhao committed
285

286 287
    if (win_ptr->shm_allocated == TRUE && target_rank != rank &&
        win_ptr->create_flavor != MPI_WIN_FLAVOR_SHARED) {
Xin Zhao's avatar
Xin Zhao committed
288
        /* check if target is local and shared memory is allocated on window,
289
         * if so, we directly perform this operation on shared memory region. */
Xin Zhao's avatar
Xin Zhao committed
290 291

        /* FIXME: Here we decide whether to perform SHM operations by checking if origin and target are on
292 293 294 295 296
         * the same node. However, in ch3:sock, even if origin and target are on the same node, they do
         * not within the same SHM region. Here we filter out ch3:sock by checking shm_allocated flag first,
         * which is only set to TRUE when SHM region is allocated in nemesis.
         * In future we need to figure out a way to check if origin and target are in the same "SHM comm".
         */
Xin Zhao's avatar
Xin Zhao committed
297 298 299
        MPIDI_Comm_get_vc(win_ptr->comm_ptr, rank, &orig_vc);
        MPIDI_Comm_get_vc(win_ptr->comm_ptr, target_rank, &target_vc);
    }
300

301
    /* If the get is a local operation, do it here */
Xin Zhao's avatar
Xin Zhao committed
302
    if (target_rank == rank || win_ptr->create_flavor == MPI_WIN_FLAVOR_SHARED ||
303
        (win_ptr->shm_allocated == TRUE && orig_vc->node_id == target_vc->node_id)) {
304 305
        mpi_errno = MPIDI_CH3I_Shm_get_op(origin_addr, origin_count, origin_datatype, target_rank,
                                          target_disp, target_count, target_datatype, win_ptr);
306 307
        if (mpi_errno)
            MPIU_ERR_POP(mpi_errno);
308 309 310 311 312 313

        if (ureq) {
            /* Complete user request and release the ch3 ref */
            MPID_Request_set_completed(ureq);
            MPID_Request_release(ureq);
        }
314
    }
315
    else {
316
        MPIDI_RMA_Op_t *new_ptr = NULL;
317
        MPIDI_CH3_Pkt_get_t *get_pkt = NULL;
318 319 320
        MPI_Aint target_type_size;
        size_t immed_len, len;
        int use_immed_resp_pkt = FALSE;
321

322
        /* queue it up */
323
        mpi_errno = MPIDI_CH3I_Win_get_op(win_ptr, &new_ptr);
324 325
        if (mpi_errno != MPI_SUCCESS)
            MPIU_ERR_POP(mpi_errno);
326

Xin Zhao's avatar
Xin Zhao committed
327 328
        MPIR_T_PVAR_TIMER_START(RMA, rma_rmaqueue_set);

329
        /******************** Setting operation struct areas ***********************/
330

331 332 333 334 335
        /* FIXME: For contig and very short operations, use a streamlined op */
        new_ptr->origin_addr = origin_addr;
        new_ptr->origin_count = origin_count;
        new_ptr->origin_datatype = origin_datatype;
        new_ptr->target_rank = target_rank;
336
        new_ptr->ureq = NULL;   /* reset user request */
337 338 339 340 341

        /* Remember user request */
        if (ureq) {
            new_ptr->ureq = ureq;
        }
342 343 344 345 346 347

        /* if source or target datatypes are derived, increment their
         * reference counts */
        if (!MPIR_DATATYPE_IS_PREDEFINED(origin_datatype)) {
            MPID_Datatype_get_ptr(origin_datatype, dtp);
            MPID_Datatype_add_ref(dtp);
348
            new_ptr->is_dt = 1;
349 350 351 352
        }
        if (!MPIR_DATATYPE_IS_PREDEFINED(target_datatype)) {
            MPID_Datatype_get_ptr(target_datatype, dtp);
            MPID_Datatype_add_ref(dtp);
353
            new_ptr->is_dt = 1;
354
        }
355

356 357 358 359 360 361
        MPID_Datatype_get_size_macro(target_datatype, target_type_size);
        MPIU_Assign_trunc(len, target_count * target_type_size, size_t);

        /* Judge if we can use IMMED data response packet */
        if (!new_ptr->is_dt) {
            MPIU_Assign_trunc(immed_len,
362
                              (MPIDI_RMA_IMMED_BYTES / target_type_size) * target_type_size,
363 364 365 366 367
                              size_t);
            if (len <= immed_len)
                use_immed_resp_pkt = TRUE;
        }

368
        /* Judge if this operation is an piggyback candidate. */
369
        if (!new_ptr->is_dt) {
370 371 372
            /* FIXME: currently we only piggyback LOCK flag with op using predefined datatypes
             * for both origin and target data. We should extend this optimization to derived
             * datatypes as well. */
373 374 375
            new_ptr->piggyback_lock_candidate = 1;
        }

376 377 378 379
        /************** Setting packet struct areas in operation ****************/

        get_pkt = &(new_ptr->pkt.get);
        MPIDI_Pkt_init(get_pkt, MPIDI_CH3_PKT_GET);
380 381
        get_pkt->addr = (char *) win_ptr->basic_info_table[target_rank].base_addr +
            win_ptr->basic_info_table[target_rank].disp_unit * target_disp;
382 383
        get_pkt->count = target_count;
        get_pkt->datatype = target_datatype;
384
        get_pkt->info.dataloop_size = 0;
385
        get_pkt->target_win_handle = win_ptr->basic_info_table[target_rank].win_handle;
386
        get_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE;
387 388
        if (use_immed_resp_pkt)
            get_pkt->flags |= MPIDI_CH3_PKT_FLAG_RMA_IMMED_RESP;
389

Xin Zhao's avatar
Xin Zhao committed
390 391
        MPIR_T_PVAR_TIMER_END(RMA, rma_rmaqueue_set);

392 393 394 395
        mpi_errno = MPIDI_CH3I_Win_enqueue_op(win_ptr, new_ptr);
        if (mpi_errno)
            MPIU_ERR_POP(mpi_errno);

396
        mpi_errno = MPIDI_CH3I_RMA_Make_progress_target(win_ptr, target_rank, &made_progress);
397 398
        if (mpi_errno != MPI_SUCCESS)
            MPIU_ERR_POP(mpi_errno);
399 400

        if (MPIR_CVAR_CH3_RMA_OP_POKING_PROGRESS >= 0 &&
401
            win_ptr->accumulated_ops_cnt >= MPIR_CVAR_CH3_RMA_OP_POKING_PROGRESS) {
402 403 404 405
            mpi_errno = poke_progress_engine();
            if (mpi_errno != MPI_SUCCESS)
                MPIU_ERR_POP(mpi_errno);
        }
406 407 408 409 410 411 412 413 414 415 416 417 418 419

        if (MPIR_CVAR_CH3_RMA_ACTIVE_REQ_THRESHOLD >= 0 &&
            win_ptr->active_req_cnt >= MPIR_CVAR_CH3_RMA_ACTIVE_REQ_THRESHOLD) {
            while (win_ptr->active_req_cnt >= MPIR_CVAR_CH3_RMA_ACTIVE_REQ_THRESHOLD) {
                int local_completed = 0, remote_completed = 0;
                mpi_errno = MPIDI_CH3I_RMA_Cleanup_ops_win(win_ptr, &local_completed,
                                                           &remote_completed);
                if (mpi_errno != MPI_SUCCESS)
                    MPIU_ERR_POP(mpi_errno);
                mpi_errno = poke_progress_engine();
                if (mpi_errno != MPI_SUCCESS)
                    MPIU_ERR_POP(mpi_errno);
            }
        }
420 421 422
    }

  fn_exit:
423
    MPIDI_RMA_FUNC_EXIT(MPID_STATE_MPIDI_CH3I_GET);
424 425 426 427 428 429 430 431 432 433
    return mpi_errno;

    /* --BEGIN ERROR HANDLING-- */
  fn_fail:
    goto fn_exit;
    /* --END ERROR HANDLING-- */
}


#undef FUNCNAME
434
#define FUNCNAME MPIDI_CH3I_Accumulate
435 436
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
437 438 439 440
int MPIDI_CH3I_Accumulate(const void *origin_addr, int origin_count, MPI_Datatype
                          origin_datatype, int target_rank, MPI_Aint target_disp,
                          int target_count, MPI_Datatype target_datatype, MPI_Op op,
                          MPID_Win * win_ptr, MPID_Request * ureq)
441
{
442
    int mpi_errno = MPI_SUCCESS;
443
    MPIDI_msg_sz_t data_sz;
444
    int dt_contig ATTRIBUTE((unused)), rank;
Pavan Balaji's avatar
Pavan Balaji committed
445
    MPI_Aint dt_true_lb ATTRIBUTE((unused));
446
    MPID_Datatype *dtp;
447
    MPIDI_VC_t *orig_vc = NULL, *target_vc = NULL;
448
    int made_progress = 0;
449
    MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3I_ACCUMULATE);
450

451
    MPIDI_RMA_FUNC_ENTER(MPID_STATE_MPIDI_CH3I_ACCUMULATE);
452

453 454 455
    MPIU_ERR_CHKANDJUMP(win_ptr->states.access_state == MPIDI_RMA_NONE,
                        mpi_errno, MPI_ERR_RMA_SYNC, "**rmasync");

456 457 458 459
    if (target_rank == MPI_PROC_NULL) {
        goto fn_exit;
    }

460 461
    MPIDI_Datatype_get_info(origin_count, origin_datatype, dt_contig, data_sz, dtp, dt_true_lb);

462
    if (data_sz == 0) {
463
        goto fn_exit;
464
    }
465

466
    rank = win_ptr->comm_ptr->rank;
467 468 469

    if (win_ptr->shm_allocated == TRUE && target_rank != rank &&
        win_ptr->create_flavor != MPI_WIN_FLAVOR_SHARED) {
Xin Zhao's avatar
Xin Zhao committed
470
        /* check if target is local and shared memory is allocated on window,
471
         * if so, we directly perform this operation on shared memory region. */
Xin Zhao's avatar
Xin Zhao committed
472 473

        /* FIXME: Here we decide whether to perform SHM operations by checking if origin and target are on
474 475 476 477 478
         * the same node. However, in ch3:sock, even if origin and target are on the same node, they do
         * not within the same SHM region. Here we filter out ch3:sock by checking shm_allocated flag first,
         * which is only set to TRUE when SHM region is allocated in nemesis.
         * In future we need to figure out a way to check if origin and target are in the same "SHM comm".
         */
Xin Zhao's avatar
Xin Zhao committed
479 480 481 482
        MPIDI_Comm_get_vc(win_ptr->comm_ptr, rank, &orig_vc);
        MPIDI_Comm_get_vc(win_ptr->comm_ptr, target_rank, &target_vc);
    }

483
    /* Do =! rank first (most likely branch?) */
Xin Zhao's avatar
Xin Zhao committed
484
    if (target_rank == rank || win_ptr->create_flavor == MPI_WIN_FLAVOR_SHARED ||
485 486 487 488 489 490
        (win_ptr->shm_allocated == TRUE && orig_vc->node_id == target_vc->node_id)) {
        mpi_errno = MPIDI_CH3I_Shm_acc_op(origin_addr, origin_count, origin_datatype,
                                          target_rank, target_disp, target_count, target_datatype,
                                          op, win_ptr);
        if (mpi_errno)
            MPIU_ERR_POP(mpi_errno);
491 492 493 494 495 496

        if (ureq) {
            /* Complete user request and release the ch3 ref */
            MPID_Request_set_completed(ureq);
            MPID_Request_release(ureq);
        }
497
    }
498
    else {
499
        MPIDI_RMA_Op_t *new_ptr = NULL;
500
        MPIDI_CH3_Pkt_accum_t *accum_pkt = NULL;
501 502 503
        MPI_Aint origin_type_size;
        size_t immed_len, len;
        int use_immed_pkt = FALSE;
504

505
        /* queue it up */
506
        mpi_errno = MPIDI_CH3I_Win_get_op(win_ptr, &new_ptr);
507 508
        if (mpi_errno != MPI_SUCCESS)
            MPIU_ERR_POP(mpi_errno);
509

Xin Zhao's avatar
Xin Zhao committed
510 511
        MPIR_T_PVAR_TIMER_START(RMA, rma_rmaqueue_set);

512
        /******************** Setting operation struct areas ***********************/
513

514 515 516 517
        new_ptr->origin_addr = (void *) origin_addr;
        new_ptr->origin_count = origin_count;
        new_ptr->origin_datatype = origin_datatype;
        new_ptr->target_rank = target_rank;
518
        new_ptr->ureq = NULL;   /* reset user request */
519 520 521 522 523

        /* Remember user request */
        if (ureq) {
            new_ptr->ureq = ureq;
        }
524 525 526 527 528 529

        /* if source or target datatypes are derived, increment their
         * reference counts */
        if (!MPIR_DATATYPE_IS_PREDEFINED(origin_datatype)) {
            MPID_Datatype_get_ptr(origin_datatype, dtp);
            MPID_Datatype_add_ref(dtp);
530
            new_ptr->is_dt = 1;
531 532 533 534
        }
        if (!MPIR_DATATYPE_IS_PREDEFINED(target_datatype)) {
            MPID_Datatype_get_ptr(target_datatype, dtp);
            MPID_Datatype_add_ref(dtp);
535
            new_ptr->is_dt = 1;
536
        }
537

538 539 540 541 542 543
        MPID_Datatype_get_size_macro(origin_datatype, origin_type_size);
        MPIU_Assign_trunc(len, origin_count * origin_type_size, size_t);

        /* Judge if we can use IMMED data packet */
        if (!new_ptr->is_dt) {
            MPIU_Assign_trunc(immed_len,
544
                              (MPIDI_RMA_IMMED_BYTES / origin_type_size) * origin_type_size,
545 546 547 548 549
                              size_t);
            if (len <= immed_len)
                use_immed_pkt = TRUE;
        }

550
        /* Judge if this operation is an piggyback candidate. */
Xin Zhao's avatar
Xin Zhao committed
551
        if (!new_ptr->is_dt) {
552 553 554
            /* FIXME: currently we only piggyback LOCK flag with op using predefined datatypes
             * for both origin and target data. We should extend this optimization to derived
             * datatypes as well. */
555
            if (len <= MPIR_CVAR_CH3_RMA_OP_PIGGYBACK_LOCK_DATA_SIZE)
556
                new_ptr->piggyback_lock_candidate = 1;
Xin Zhao's avatar
Xin Zhao committed
557 558
        }

559 560 561 562
        /************** Setting packet struct areas in operation ****************/

        accum_pkt = &(new_ptr->pkt.accum);

563 564 565 566 567 568 569
        if (use_immed_pkt) {
            MPIDI_Pkt_init(accum_pkt, MPIDI_CH3_PKT_ACCUMULATE_IMMED);
        }
        else {
            MPIDI_Pkt_init(accum_pkt, MPIDI_CH3_PKT_ACCUMULATE);
        }

570 571
        accum_pkt->addr = (char *) win_ptr->basic_info_table[target_rank].base_addr +
            win_ptr->basic_info_table[target_rank].disp_unit * target_disp;
572 573
        accum_pkt->count = target_count;
        accum_pkt->datatype = target_datatype;
574
        accum_pkt->info.dataloop_size = 0;
575
        accum_pkt->op = op;
576
        accum_pkt->target_win_handle = win_ptr->basic_info_table[target_rank].win_handle;
577 578
        accum_pkt->source_win_handle = win_ptr->handle;
        accum_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE;
579
        if (use_immed_pkt) {
580
            void *src = (void *) origin_addr, *dest = (void *) (accum_pkt->info.data);
581
            mpi_errno = immed_copy(src, dest, len);
582 583
            if (mpi_errno != MPI_SUCCESS)
                MPIU_ERR_POP(mpi_errno);
584
        }
585

Xin Zhao's avatar
Xin Zhao committed
586 587
        MPIR_T_PVAR_TIMER_END(RMA, rma_rmaqueue_set);

588 589 590 591
        mpi_errno = MPIDI_CH3I_Win_enqueue_op(win_ptr, new_ptr);
        if (mpi_errno)
            MPIU_ERR_POP(mpi_errno);

592
        mpi_errno = MPIDI_CH3I_RMA_Make_progress_target(win_ptr, target_rank, &made_progress);
593 594
        if (mpi_errno != MPI_SUCCESS)
            MPIU_ERR_POP(mpi_errno);
595 596

        if (MPIR_CVAR_CH3_RMA_OP_POKING_PROGRESS >= 0 &&
597
            win_ptr->accumulated_ops_cnt >= MPIR_CVAR_CH3_RMA_OP_POKING_PROGRESS) {
598 599 600 601
            mpi_errno = poke_progress_engine();
            if (mpi_errno != MPI_SUCCESS)
                MPIU_ERR_POP(mpi_errno);
        }
602 603 604 605 606 607 608 609 610 611 612 613 614 615

        if (MPIR_CVAR_CH3_RMA_ACTIVE_REQ_THRESHOLD >= 0 &&
            win_ptr->active_req_cnt >= MPIR_CVAR_CH3_RMA_ACTIVE_REQ_THRESHOLD) {
            while (win_ptr->active_req_cnt >= MPIR_CVAR_CH3_RMA_ACTIVE_REQ_THRESHOLD) {
                int local_completed = 0, remote_completed = 0;
                mpi_errno = MPIDI_CH3I_RMA_Cleanup_ops_win(win_ptr, &local_completed,
                                                           &remote_completed);
                if (mpi_errno != MPI_SUCCESS)
                    MPIU_ERR_POP(mpi_errno);
                mpi_errno = poke_progress_engine();
                if (mpi_errno != MPI_SUCCESS)
                    MPIU_ERR_POP(mpi_errno);
            }
        }
616 617
    }

618
  fn_exit:
619
    MPIDI_RMA_FUNC_EXIT(MPID_STATE_MPIDI_CH3I_ACCUMULATE);
620 621 622 623 624 625 626 627 628 629
    return mpi_errno;

    /* --BEGIN ERROR HANDLING-- */
  fn_fail:
    goto fn_exit;
    /* --END ERROR HANDLING-- */
}


#undef FUNCNAME
630
#define FUNCNAME MPIDI_CH3I_Get_accumulate
631 632
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
633 634 635 636 637
int MPIDI_CH3I_Get_accumulate(const void *origin_addr, int origin_count,
                              MPI_Datatype origin_datatype, void *result_addr, int result_count,
                              MPI_Datatype result_datatype, int target_rank, MPI_Aint target_disp,
                              int target_count, MPI_Datatype target_datatype, MPI_Op op,
                              MPID_Win * win_ptr, MPID_Request * ureq)
638 639 640 641 642 643 644 645
{
    int mpi_errno = MPI_SUCCESS;
    MPIDI_msg_sz_t data_sz;
    int rank;
    int dt_contig ATTRIBUTE((unused));
    MPI_Aint dt_true_lb ATTRIBUTE((unused));
    MPID_Datatype *dtp;
    MPIDI_VC_t *orig_vc = NULL, *target_vc = NULL;
646
    int made_progress = 0;
647
    MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3I_GET_ACCUMULATE);
648

649
    MPIDI_RMA_FUNC_ENTER(MPID_STATE_MPIDI_CH3I_GET_ACCUMULATE);
650

651 652 653
    MPIU_ERR_CHKANDJUMP(win_ptr->states.access_state == MPIDI_RMA_NONE,
                        mpi_errno, MPI_ERR_RMA_SYNC, "**rmasync");

654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689
    if (target_rank == MPI_PROC_NULL) {
        goto fn_exit;
    }

    MPIDI_Datatype_get_info(target_count, target_datatype, dt_contig, data_sz, dtp, dt_true_lb);

    if (data_sz == 0) {
        goto fn_exit;
    }

    rank = win_ptr->comm_ptr->rank;

    if (win_ptr->shm_allocated == TRUE && target_rank != rank &&
        win_ptr->create_flavor != MPI_WIN_FLAVOR_SHARED) {
        /* check if target is local and shared memory is allocated on window,
         * if so, we directly perform this operation on shared memory region. */

        /* FIXME: Here we decide whether to perform SHM operations by checking if origin and target are on
         * the same node. However, in ch3:sock, even if origin and target are on the same node, they do
         * not within the same SHM region. Here we filter out ch3:sock by checking shm_allocated flag first,
         * which is only set to TRUE when SHM region is allocated in nemesis.
         * In future we need to figure out a way to check if origin and target are in the same "SHM comm".
         */
        MPIDI_Comm_get_vc(win_ptr->comm_ptr, rank, &orig_vc);
        MPIDI_Comm_get_vc(win_ptr->comm_ptr, target_rank, &target_vc);
    }

    /* Do =! rank first (most likely branch?) */
    if (target_rank == rank || win_ptr->create_flavor == MPI_WIN_FLAVOR_SHARED ||
        (win_ptr->shm_allocated == TRUE && orig_vc->node_id == target_vc->node_id)) {
        mpi_errno = MPIDI_CH3I_Shm_get_acc_op(origin_addr, origin_count, origin_datatype,
                                              result_addr, result_count, result_datatype,
                                              target_rank, target_disp, target_count,
                                              target_datatype, op, win_ptr);
        if (mpi_errno)
            MPIU_ERR_POP(mpi_errno);
690 691 692 693 694 695

        if (ureq) {
            /* Complete user request and release the ch3 ref */
            MPID_Request_set_completed(ureq);
            MPID_Request_release(ureq);
        }
696 697 698 699 700
    }
    else {
        MPIDI_RMA_Op_t *new_ptr = NULL;

        /* Append the operation to the window's RMA ops queue */
701
        mpi_errno = MPIDI_CH3I_Win_get_op(win_ptr, &new_ptr);
702 703
        if (mpi_errno != MPI_SUCCESS)
            MPIU_ERR_POP(mpi_errno);
704 705 706

        /* TODO: Can we use the MPIDI_RMA_ACC_CONTIG optimization? */

Xin Zhao's avatar
Xin Zhao committed
707 708
        MPIR_T_PVAR_TIMER_START(RMA, rma_rmaqueue_set);

709 710
        if (op == MPI_NO_OP) {
            /* Convert GAcc to a Get */
711
            MPIDI_CH3_Pkt_get_t *get_pkt;
712 713 714
            MPI_Aint target_type_size;
            size_t len, immed_len;
            int use_immed_resp_pkt = FALSE;
715 716

            /******************** Setting operation struct areas ***********************/
717 718 719 720 721

            new_ptr->origin_addr = result_addr;
            new_ptr->origin_count = result_count;
            new_ptr->origin_datatype = result_datatype;
            new_ptr->target_rank = target_rank;
722
            new_ptr->ureq = NULL;       /* reset user request */
723 724 725 726 727

            /* Remember user request */
            if (ureq) {
                new_ptr->ureq = ureq;
            }
728 729 730 731 732 733 734 735 736 737 738

            if (!MPIR_DATATYPE_IS_PREDEFINED(result_datatype)) {
                MPID_Datatype_get_ptr(result_datatype, dtp);
                MPID_Datatype_add_ref(dtp);
                new_ptr->is_dt = 1;
            }
            if (!MPIR_DATATYPE_IS_PREDEFINED(target_datatype)) {
                MPID_Datatype_get_ptr(target_datatype, dtp);
                MPID_Datatype_add_ref(dtp);
                new_ptr->is_dt = 1;
            }
739

740 741 742 743 744 745
            MPID_Datatype_get_size_macro(target_datatype, target_type_size);
            MPIU_Assign_trunc(len, target_count * target_type_size, size_t);

            /* Judge if we can use IMMED data response packet */
            if (!new_ptr->is_dt) {
                MPIU_Assign_trunc(immed_len,
746
                                  (MPIDI_RMA_IMMED_BYTES / target_type_size) * target_type_size,
747 748 749 750 751
                                  size_t);
                if (len <= immed_len)
                    use_immed_resp_pkt = TRUE;
            }

752
            /* Judge if this operation is a piggyback candidate */
753
            if (!new_ptr->is_dt) {
754 755 756
                /* FIXME: currently we only piggyback LOCK flag with op using predefined datatypes
                 * for both origin and target data. We should extend this optimization to derived
                 * datatypes as well. */
757 758
                new_ptr->piggyback_lock_candidate = 1;
            }
759 760 761 762 763

            /************** Setting packet struct areas in operation ****************/

            get_pkt = &(new_ptr->pkt.get);
            MPIDI_Pkt_init(get_pkt, MPIDI_CH3_PKT_GET);
764 765
            get_pkt->addr = (char *) win_ptr->basic_info_table[target_rank].base_addr +
                win_ptr->basic_info_table[target_rank].disp_unit * target_disp;
766 767
            get_pkt->count = target_count;
            get_pkt->datatype = target_datatype;
768
            get_pkt->info.dataloop_size = 0;
769
            get_pkt->target_win_handle = win_ptr->basic_info_table[target_rank].win_handle;
770
            get_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE;
771 772
            if (use_immed_resp_pkt == TRUE)
                get_pkt->flags |= MPIDI_CH3_PKT_FLAG_RMA_IMMED_RESP;
773 774 775
        }

        else {
776
            MPIDI_CH3_Pkt_get_accum_t *get_accum_pkt;
777 778 779
            MPI_Aint origin_type_size;
            size_t immed_len, orig_len;
            int use_immed_pkt = FALSE;
780 781

            /******************** Setting operation struct areas ***********************/
782 783 784 785 786 787 788 789

            new_ptr->origin_addr = (void *) origin_addr;
            new_ptr->origin_count = origin_count;
            new_ptr->origin_datatype = origin_datatype;
            new_ptr->result_addr = result_addr;
            new_ptr->result_count = result_count;
            new_ptr->result_datatype = result_datatype;
            new_ptr->target_rank = target_rank;
790
            new_ptr->ureq = NULL;       /* reset user request */
791 792 793 794 795

            /* Remember user request */
            if (ureq) {
                new_ptr->ureq = ureq;
            }
796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813

            /* if source or target datatypes are derived, increment their
             * reference counts */
            if (!MPIR_DATATYPE_IS_PREDEFINED(origin_datatype)) {
                MPID_Datatype_get_ptr(origin_datatype, dtp);
                MPID_Datatype_add_ref(dtp);
                new_ptr->is_dt = 1;
            }
            if (!MPIR_DATATYPE_IS_PREDEFINED(result_datatype)) {
                MPID_Datatype_get_ptr(result_datatype, dtp);
                MPID_Datatype_add_ref(dtp);
                new_ptr->is_dt = 1;
            }
            if (!MPIR_DATATYPE_IS_PREDEFINED(target_datatype)) {
                MPID_Datatype_get_ptr(target_datatype, dtp);
                MPID_Datatype_add_ref(dtp);
                new_ptr->is_dt = 1;
            }
Xin Zhao's avatar
Xin Zhao committed
814

815 816 817 818 819 820
            MPID_Datatype_get_size_macro(origin_datatype, origin_type_size);
            MPIU_Assign_trunc(orig_len, origin_count * origin_type_size, size_t);

            /* Judge if we can use IMMED data packet */
            if (!new_ptr->is_dt) {
                MPIU_Assign_trunc(immed_len,
821
                                  (MPIDI_RMA_IMMED_BYTES / origin_type_size) * origin_type_size,
822 823 824 825 826
                                  size_t);
                if (orig_len <= immed_len)
                    use_immed_pkt = TRUE;
            }

827
            /* Judge if this operation is a piggyback candidate */
Xin Zhao's avatar
Xin Zhao committed
828
            if (!new_ptr->is_dt) {
829 830 831
                /* FIXME: currently we only piggyback LOCK flag with op using predefined datatypes
                 * for origin, target and result data. We should extend this optimization to derived
                 * datatypes as well. */
832
                if (orig_len <= MPIR_CVAR_CH3_RMA_OP_PIGGYBACK_LOCK_DATA_SIZE)
833
                    new_ptr->piggyback_lock_candidate = 1;
Xin Zhao's avatar
Xin Zhao committed
834
            }
835

836
            /************** Setting packet struct areas in operation ****************/
837

838
            get_accum_pkt = &(new_ptr->pkt.get_accum);
839 840 841 842 843 844 845 846

            if (use_immed_pkt) {
                MPIDI_Pkt_init(get_accum_pkt, MPIDI_CH3_PKT_GET_ACCUM_IMMED);
            }
            else {
                MPIDI_Pkt_init(get_accum_pkt, MPIDI_CH3_PKT_GET_ACCUM);
            }

847 848
            get_accum_pkt->addr = (char *) win_ptr->basic_info_table[target_rank].base_addr +
                win_ptr->basic_info_table[target_rank].disp_unit * target_disp;
849 850
            get_accum_pkt->count = target_count;
            get_accum_pkt->datatype = target_datatype;
851
            get_accum_pkt->info.dataloop_size = 0;
852
            get_accum_pkt->op = op;
853
            get_accum_pkt->target_win_handle = win_ptr->basic_info_table[target_rank].win_handle;
854
            get_accum_pkt->flags = MPIDI_CH3_PKT_FLAG_NONE;
855
            if (use_immed_pkt) {
856
                void *src = (void *) origin_addr, *dest = (void *) (get_accum_pkt->info.data);
857
                mpi_errno = immed_copy(src, dest, orig_len);
858 859
                if (mpi_errno != MPI_SUCCESS)
                    MPIU_ERR_POP(mpi_errno);
860
            }
861 862
        }

Xin Zhao's avatar
Xin Zhao committed
863 864
        MPIR_T_PVAR_TIMER_END(RMA, rma_rmaqueue_set);

865 866 867 868
        mpi_errno = MPIDI_CH3I_Win_enqueue_op(win_ptr, new_ptr);
        if (mpi_errno)
            MPIU_ERR_POP(mpi_errno);

869
        mpi_errno = MPIDI_CH3I_RMA_Make_progress_target(win_ptr, target_rank, &made_progress);
870 871
        if (mpi_errno != MPI_SUCCESS)
            MPIU_ERR_POP(mpi_errno);
872 873

        if (MPIR_CVAR_CH3_RMA_OP_POKING_PROGRESS >= 0 &&
874
            win_ptr->accumulated_ops_cnt >= MPIR_CVAR_CH3_RMA_OP_POKING_PROGRESS) {
875 876 877 878
            mpi_errno = poke_progress_engine();
            if (mpi_errno != MPI_SUCCESS)
                MPIU_ERR_POP(mpi_errno);
        }
879 880 881 882 883 884 885 886 887 888 889 890 891 892

        if (MPIR_CVAR_CH3_RMA_ACTIVE_REQ_THRESHOLD >= 0 &&
            win_ptr->active_req_cnt >= MPIR_CVAR_CH3_RMA_ACTIVE_REQ_THRESHOLD) {
            while (win_ptr->active_req_cnt >= MPIR_CVAR_CH3_RMA_ACTIVE_REQ_THRESHOLD) {
                int local_completed = 0, remote_completed = 0;
                mpi_errno = MPIDI_CH3I_RMA_Cleanup_ops_win(win_ptr, &local_completed,
                                                           &remote_completed);
                if (mpi_errno != MPI_SUCCESS)
                    MPIU_ERR_POP(mpi_errno);
                mpi_errno = poke_progress_engine();
                if (mpi_errno != MPI_SUCCESS)
                    MPIU_ERR_POP(mpi_errno);
            }
        }
893 894
    }

895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006
  fn_exit:
    MPIDI_RMA_FUNC_EXIT(MPID_STATE_MPIDI_CH3I_GET_ACCUMULATE);
    return mpi_errno;

    /* --BEGIN ERROR HANDLING-- */
  fn_fail:
    goto fn_exit;
    /* --END ERROR HANDLING-- */
}


#undef FUNCNAME
#define FUNCNAME MPIDI_Put
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
int MPIDI_Put(const void *origin_addr, int origin_count, MPI_Datatype
              origin_datatype, int target_rank, MPI_Aint target_disp,
              int target_count, MPI_Datatype target_datatype, MPID_Win * win_ptr)
{
    int mpi_errno = MPI_SUCCESS;

    MPIDI_STATE_DECL(MPID_STATE_MPIDI_PUT);
    MPIDI_RMA_FUNC_ENTER(MPID_STATE_MPIDI_PUT);

    mpi_errno = MPIDI_CH3I_Put(origin_addr, origin_count, origin_datatype,
                               target_rank, target_disp, target_count, target_datatype,
                               win_ptr, NULL);

  fn_exit:
    MPIDI_RMA_FUNC_EXIT(MPID_STATE_MPIDI_PUT);
    return mpi_errno;

    /* --BEGIN ERROR HANDLING-- */
  fn_fail:
    goto fn_exit;
    /* --END ERROR HANDLING-- */
}

#undef FUNCNAME
#define FUNCNAME MPIDI_Get
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
int MPIDI_Get(void *origin_addr, int origin_count, MPI_Datatype
              origin_datatype, int target_rank, MPI_Aint target_disp,
              int target_count, MPI_Datatype target_datatype, MPID_Win * win_ptr)
{
    int mpi_errno = MPI_SUCCESS;

    MPIDI_STATE_DECL(MPID_STATE_MPIDI_GET);
    MPIDI_RMA_FUNC_ENTER(MPID_STATE_MPIDI_GET);

    mpi_errno = MPIDI_CH3I_Get(origin_addr, origin_count, origin_datatype,
                               target_rank, target_disp, target_count, target_datatype,
                               win_ptr, NULL);

  fn_exit:
    MPIDI_RMA_FUNC_EXIT(MPID_STATE_MPIDI_GET);
    return mpi_errno;

    /* --BEGIN ERROR HANDLING-- */
  fn_fail:
    goto fn_exit;
    /* --END ERROR HANDLING-- */
}

#undef FUNCNAME
#define FUNCNAME MPIDI_Accumulate
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
int MPIDI_Accumulate(const void *origin_addr, int origin_count, MPI_Datatype
                     origin_datatype, int target_rank, MPI_Aint target_disp,
                     int target_count, MPI_Datatype target_datatype, MPI_Op op, MPID_Win * win_ptr)
{
    int mpi_errno = MPI_SUCCESS;

    MPIDI_STATE_DECL(MPID_STATE_MPIDI_ACCUMULATE);
    MPIDI_RMA_FUNC_ENTER(MPID_STATE_MPIDI_ACCUMULATE);

    mpi_errno = MPIDI_CH3I_Accumulate(origin_addr, origin_count, origin_datatype,
                                      target_rank, target_disp, target_count, target_datatype,
                                      op, win_ptr, NULL);

  fn_exit:
    MPIDI_RMA_FUNC_EXIT(MPID_STATE_MPIDI_ACCUMULATE);
    return mpi_errno;

    /* --BEGIN ERROR HANDLING-- */
  fn_fail:
    goto fn_exit;
    /* --END ERROR HANDLING-- */
}

#undef FUNCNAME
#define FUNCNAME MPIDI_Get_accumulate
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
int MPIDI_Get_accumulate(const void *origin_addr, int origin_count,