ch3u_rma_acc_ops.c 12.8 KB
Newer Older
1
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
2
3
4
5
6
7
8
/*
 *  (C) 2001 by Argonne National Laboratory.
 *      See COPYRIGHT in top-level directory.
 */

#include "mpidrma.h"

9
10
MPIR_T_PVAR_DOUBLE_TIMER_DECL_EXTERN(RMA, rma_rmaqueue_alloc);
MPIR_T_PVAR_DOUBLE_TIMER_DECL_EXTERN(RMA, rma_rmaqueue_set);
11
12
13
14
15
16
17
18

#undef FUNCNAME
#define FUNCNAME MPIDI_Get_accumulate
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
int MPIDI_Get_accumulate(const void *origin_addr, int origin_count,
                         MPI_Datatype origin_datatype, void *result_addr, int result_count,
                         MPI_Datatype result_datatype, int target_rank, MPI_Aint target_disp,
19
20
                         int target_count, MPI_Datatype target_datatype, MPI_Op op,
                         MPID_Win * win_ptr)
21
{
22
23
    int mpi_errno = MPI_SUCCESS;
    MPIDI_msg_sz_t data_sz;
24
    int rank;
25
26
27
    int dt_contig ATTRIBUTE((unused));
    MPI_Aint dt_true_lb ATTRIBUTE((unused));
    MPID_Datatype *dtp;
28
    MPIDI_VC_t *orig_vc = NULL, *target_vc = NULL;
29
    MPIDI_STATE_DECL(MPID_STATE_MPIDI_GET_ACCUMULATE);
30

31
32
    MPIDI_RMA_FUNC_ENTER(MPID_STATE_MPIDI_GET_ACCUMULATE);

James Dinan's avatar
James Dinan committed
33
34
35
36
    if (target_rank == MPI_PROC_NULL) {
        goto fn_exit;
    }

37
    if (win_ptr->epoch_state == MPIDI_EPOCH_NONE && win_ptr->fence_issued) {
James Dinan's avatar
James Dinan committed
38
39
40
41
42
43
        win_ptr->epoch_state = MPIDI_EPOCH_FENCE;
    }

    MPIU_ERR_CHKANDJUMP(win_ptr->epoch_state == MPIDI_EPOCH_NONE,
                        mpi_errno, MPI_ERR_RMA_SYNC, "**rmasync");

44
    MPIDI_Datatype_get_info(target_count, target_datatype, dt_contig, data_sz, dtp, dt_true_lb);
45

James Dinan's avatar
James Dinan committed
46
    if (data_sz == 0) {
47
48
49
        goto fn_exit;
    }

50
    rank = win_ptr->comm_ptr->rank;
51

52
53
    if (win_ptr->shm_allocated == TRUE && target_rank != rank &&
        win_ptr->create_flavor != MPI_WIN_FLAVOR_SHARED) {
Xin Zhao's avatar
Xin Zhao committed
54
        /* check if target is local and shared memory is allocated on window,
55
         * if so, we directly perform this operation on shared memory region. */
Xin Zhao's avatar
Xin Zhao committed
56
57

        /* FIXME: Here we decide whether to perform SHM operations by checking if origin and target are on
58
59
60
61
62
         * the same node. However, in ch3:sock, even if origin and target are on the same node, they do
         * not within the same SHM region. Here we filter out ch3:sock by checking shm_allocated flag first,
         * which is only set to TRUE when SHM region is allocated in nemesis.
         * In future we need to figure out a way to check if origin and target are in the same "SHM comm".
         */
Xin Zhao's avatar
Xin Zhao committed
63
64
65
66
        MPIDI_Comm_get_vc(win_ptr->comm_ptr, rank, &orig_vc);
        MPIDI_Comm_get_vc(win_ptr->comm_ptr, target_rank, &target_vc);
    }

67
    /* Do =! rank first (most likely branch?) */
Xin Zhao's avatar
Xin Zhao committed
68
    if (target_rank == rank || win_ptr->create_flavor == MPI_WIN_FLAVOR_SHARED ||
69
        (win_ptr->shm_allocated == TRUE && orig_vc->node_id == target_vc->node_id)) {
70
71
        mpi_errno = MPIDI_CH3I_Shm_get_acc_op(origin_addr, origin_count, origin_datatype,
                                              result_addr, result_count, result_datatype,
72
73
74
75
                                              target_rank, target_disp, target_count,
                                              target_datatype, op, win_ptr);
        if (mpi_errno)
            MPIU_ERR_POP(mpi_errno);
76
77
    }
    else {
78
        MPIDI_RMA_Ops_list_t *ops_list = MPIDI_CH3I_RMA_Get_ops_list(win_ptr, target_rank);
79
        MPIDI_RMA_Op_t *new_ptr = NULL;
80

81
        /* Append the operation to the window's RMA ops queue */
82
        MPIR_T_PVAR_TIMER_START(RMA, rma_rmaqueue_alloc);
83
        mpi_errno = MPIDI_CH3I_RMA_Ops_alloc_tail(ops_list, &new_ptr);
84
        MPIR_T_PVAR_TIMER_END(RMA, rma_rmaqueue_alloc);
85
86
87
        if (mpi_errno) {
            MPIU_ERR_POP(mpi_errno);
        }
88
89
90

        /* TODO: Can we use the MPIDI_RMA_ACC_CONTIG optimization? */

91
        MPIR_T_PVAR_TIMER_START(RMA, rma_rmaqueue_set);
92
93
94
95
96
97
98
99
100
101
102
103
104
105
        new_ptr->type = MPIDI_RMA_GET_ACCUMULATE;
        /* Cast away const'ness for origin_address as MPIDI_RMA_Op_t
         * contain both PUT and GET like ops */
        new_ptr->origin_addr = (void *) origin_addr;
        new_ptr->origin_count = origin_count;
        new_ptr->origin_datatype = origin_datatype;
        new_ptr->result_addr = result_addr;
        new_ptr->result_count = result_count;
        new_ptr->result_datatype = result_datatype;
        new_ptr->target_rank = target_rank;
        new_ptr->target_disp = target_disp;
        new_ptr->target_count = target_count;
        new_ptr->target_datatype = target_datatype;
        new_ptr->op = op;
106
        MPIR_T_PVAR_TIMER_END(RMA, rma_rmaqueue_set);
107

108
        /* if source or target datatypes are derived, increment their
109
         * reference counts */
110
        if (op != MPI_NO_OP && !MPIR_DATATYPE_IS_PREDEFINED(origin_datatype)) {
111
112
113
            MPID_Datatype_get_ptr(origin_datatype, dtp);
            MPID_Datatype_add_ref(dtp);
        }
114
        if (!MPIR_DATATYPE_IS_PREDEFINED(result_datatype)) {
115
116
117
            MPID_Datatype_get_ptr(result_datatype, dtp);
            MPID_Datatype_add_ref(dtp);
        }
118
        if (!MPIR_DATATYPE_IS_PREDEFINED(target_datatype)) {
119
120
            MPID_Datatype_get_ptr(target_datatype, dtp);
            MPID_Datatype_add_ref(dtp);
121
        }
122
123
    }

124
  fn_exit:
125
126
127
128
129
130
131
132
    MPIDI_RMA_FUNC_EXIT(MPID_STATE_MPIDI_GET_ACCUMULATE);
    return mpi_errno;

    /* --BEGIN ERROR HANDLING-- */
  fn_fail:
    goto fn_exit;
    /* --END ERROR HANDLING-- */
}
133
134
135
136
137
138
139


#undef FUNCNAME
#define FUNCNAME MPIDI_Compare_and_swap
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
int MPIDI_Compare_and_swap(const void *origin_addr, const void *compare_addr,
140
141
                           void *result_addr, MPI_Datatype datatype, int target_rank,
                           MPI_Aint target_disp, MPID_Win * win_ptr)
142
143
144
{
    int mpi_errno = MPI_SUCCESS;
    int rank;
145
    MPIDI_VC_t *orig_vc = NULL, *target_vc = NULL;
146
147
148
149
150
151
152
153

    MPIDI_STATE_DECL(MPID_STATE_MPIDI_COMPARE_AND_SWAP);
    MPIDI_RMA_FUNC_ENTER(MPID_STATE_MPIDI_COMPARE_AND_SWAP);

    if (target_rank == MPI_PROC_NULL) {
        goto fn_exit;
    }

154
    if (win_ptr->epoch_state == MPIDI_EPOCH_NONE && win_ptr->fence_issued) {
James Dinan's avatar
James Dinan committed
155
156
157
158
159
160
        win_ptr->epoch_state = MPIDI_EPOCH_FENCE;
    }

    MPIU_ERR_CHKANDJUMP(win_ptr->epoch_state == MPIDI_EPOCH_NONE,
                        mpi_errno, MPI_ERR_RMA_SYNC, "**rmasync");

161
    rank = win_ptr->comm_ptr->rank;
162

163
164
    if (win_ptr->shm_allocated == TRUE && target_rank != rank &&
        win_ptr->create_flavor != MPI_WIN_FLAVOR_SHARED) {
Xin Zhao's avatar
Xin Zhao committed
165
        /* check if target is local and shared memory is allocated on window,
166
         * if so, we directly perform this operation on shared memory region. */
Xin Zhao's avatar
Xin Zhao committed
167
168

        /* FIXME: Here we decide whether to perform SHM operations by checking if origin and target are on
169
170
171
172
173
         * the same node. However, in ch3:sock, even if origin and target are on the same node, they do
         * not within the same SHM region. Here we filter out ch3:sock by checking shm_allocated flag first,
         * which is only set to TRUE when SHM region is allocated in nemesis.
         * In future we need to figure out a way to check if origin and target are in the same "SHM comm".
         */
Xin Zhao's avatar
Xin Zhao committed
174
175
176
177
        MPIDI_Comm_get_vc(win_ptr->comm_ptr, rank, &orig_vc);
        MPIDI_Comm_get_vc(win_ptr->comm_ptr, target_rank, &target_vc);
    }

178
179
180
181
    /* The datatype must be predefined, and one of: C integer, Fortran integer,
     * Logical, Multi-language types, or Byte.  This is checked above the ADI,
     * so there's no need to check it again here. */

182
183
    /* FIXME: For shared memory windows, we should provide an implementation
     * that uses a processor atomic operation. */
Xin Zhao's avatar
Xin Zhao committed
184
    if (target_rank == rank || win_ptr->create_flavor == MPI_WIN_FLAVOR_SHARED ||
185
        (win_ptr->shm_allocated == TRUE && orig_vc->node_id == target_vc->node_id)) {
186
187
        mpi_errno = MPIDI_CH3I_Shm_cas_op(origin_addr, compare_addr, result_addr,
                                          datatype, target_rank, target_disp, win_ptr);
188
189
        if (mpi_errno)
            MPIU_ERR_POP(mpi_errno);
190
191
    }
    else {
192
        MPIDI_RMA_Ops_list_t *ops_list = MPIDI_CH3I_RMA_Get_ops_list(win_ptr, target_rank);
193
        MPIDI_RMA_Op_t *new_ptr = NULL;
194

195
        /* Append this operation to the RMA ops queue */
196
        MPIR_T_PVAR_TIMER_START(RMA, rma_rmaqueue_alloc);
197
        mpi_errno = MPIDI_CH3I_RMA_Ops_alloc_tail(ops_list, &new_ptr);
198
        MPIR_T_PVAR_TIMER_END(RMA, rma_rmaqueue_alloc);
199
200
201
        if (mpi_errno) {
            MPIU_ERR_POP(mpi_errno);
        }
202

203
        MPIR_T_PVAR_TIMER_START(RMA, rma_rmaqueue_set);
204
        new_ptr->type = MPIDI_RMA_COMPARE_AND_SWAP;
205
206
207
        new_ptr->origin_addr = (void *) origin_addr;
        new_ptr->origin_count = 1;
        new_ptr->origin_datatype = datatype;
208
209
210
211
        new_ptr->target_rank = target_rank;
        new_ptr->target_disp = target_disp;
        new_ptr->target_count = 1;
        new_ptr->target_datatype = datatype;
212
        new_ptr->result_addr = result_addr;
213
        new_ptr->result_count = 1;
214
215
        new_ptr->result_datatype = datatype;
        new_ptr->compare_addr = (void *) compare_addr;
216
        new_ptr->compare_count = 1;
217
        new_ptr->compare_datatype = datatype;
218
        MPIR_T_PVAR_TIMER_END(RMA, rma_rmaqueue_set);
219
220
    }

221
  fn_exit:
222
223
224
    MPIDI_RMA_FUNC_EXIT(MPID_STATE_MPIDI_COMPARE_AND_SWAP);
    return mpi_errno;
    /* --BEGIN ERROR HANDLING-- */
225
  fn_fail:
226
227
228
229
230
    goto fn_exit;
    /* --END ERROR HANDLING-- */
}


231
232
233
234
235
236
#undef FUNCNAME
#define FUNCNAME MPIDI_Fetch_and_op
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
int MPIDI_Fetch_and_op(const void *origin_addr, void *result_addr,
                       MPI_Datatype datatype, int target_rank,
237
                       MPI_Aint target_disp, MPI_Op op, MPID_Win * win_ptr)
238
239
240
{
    int mpi_errno = MPI_SUCCESS;
    int rank;
241
    MPIDI_VC_t *orig_vc = NULL, *target_vc = NULL;
242

243
244
    MPIDI_STATE_DECL(MPID_STATE_MPIDI_FETCH_AND_OP);
    MPIDI_RMA_FUNC_ENTER(MPID_STATE_MPIDI_FETCH_AND_OP);
245
246
247
248
249

    if (target_rank == MPI_PROC_NULL) {
        goto fn_exit;
    }

250
    if (win_ptr->epoch_state == MPIDI_EPOCH_NONE && win_ptr->fence_issued) {
James Dinan's avatar
James Dinan committed
251
252
253
254
255
256
        win_ptr->epoch_state = MPIDI_EPOCH_FENCE;
    }

    MPIU_ERR_CHKANDJUMP(win_ptr->epoch_state == MPIDI_EPOCH_NONE,
                        mpi_errno, MPI_ERR_RMA_SYNC, "**rmasync");

257
    rank = win_ptr->comm_ptr->rank;
258

259
260
    if (win_ptr->shm_allocated == TRUE && target_rank != rank &&
        win_ptr->create_flavor != MPI_WIN_FLAVOR_SHARED) {
Xin Zhao's avatar
Xin Zhao committed
261
        /* check if target is local and shared memory is allocated on window,
262
         * if so, we directly perform this operation on shared memory region. */
Xin Zhao's avatar
Xin Zhao committed
263
264

        /* FIXME: Here we decide whether to perform SHM operations by checking if origin and target are on
265
266
267
268
269
         * the same node. However, in ch3:sock, even if origin and target are on the same node, they do
         * not within the same SHM region. Here we filter out ch3:sock by checking shm_allocated flag first,
         * which is only set to TRUE when SHM region is allocated in nemesis.
         * In future we need to figure out a way to check if origin and target are in the same "SHM comm".
         */
Xin Zhao's avatar
Xin Zhao committed
270
271
272
273
        MPIDI_Comm_get_vc(win_ptr->comm_ptr, rank, &orig_vc);
        MPIDI_Comm_get_vc(win_ptr->comm_ptr, target_rank, &target_vc);
    }

274
275
276
    /* The datatype and op must be predefined.  This is checked above the ADI,
     * so there's no need to check it again here. */

277
278
    /* FIXME: For shared memory windows, we should provide an implementation
     * that uses a processor atomic operation. */
Xin Zhao's avatar
Xin Zhao committed
279
    if (target_rank == rank || win_ptr->create_flavor == MPI_WIN_FLAVOR_SHARED ||
280
        (win_ptr->shm_allocated == TRUE && orig_vc->node_id == target_vc->node_id)) {
281
282
        mpi_errno = MPIDI_CH3I_Shm_fop_op(origin_addr, result_addr, datatype,
                                          target_rank, target_disp, op, win_ptr);
283
284
        if (mpi_errno)
            MPIU_ERR_POP(mpi_errno);
285
286
    }
    else {
287
        MPIDI_RMA_Ops_list_t *ops_list = MPIDI_CH3I_RMA_Get_ops_list(win_ptr, target_rank);
288
        MPIDI_RMA_Op_t *new_ptr = NULL;
289

290
        /* Append this operation to the RMA ops queue */
291
        MPIR_T_PVAR_TIMER_START(RMA, rma_rmaqueue_alloc);
292
        mpi_errno = MPIDI_CH3I_RMA_Ops_alloc_tail(ops_list, &new_ptr);
293
        MPIR_T_PVAR_TIMER_END(RMA, rma_rmaqueue_alloc);
294
295
296
        if (mpi_errno) {
            MPIU_ERR_POP(mpi_errno);
        }
297

298
        MPIR_T_PVAR_TIMER_START(RMA, rma_rmaqueue_set);
299
        new_ptr->type = MPIDI_RMA_FETCH_AND_OP;
300
301
302
        new_ptr->origin_addr = (void *) origin_addr;
        new_ptr->origin_count = 1;
        new_ptr->origin_datatype = datatype;
303
304
305
306
        new_ptr->target_rank = target_rank;
        new_ptr->target_disp = target_disp;
        new_ptr->target_count = 1;
        new_ptr->target_datatype = datatype;
307
        new_ptr->result_addr = result_addr;
308
        new_ptr->result_count = 1;
309
        new_ptr->result_datatype = datatype;
310
        new_ptr->op = op;
311
        MPIR_T_PVAR_TIMER_END(RMA, rma_rmaqueue_set);
312
313
    }

314
  fn_exit:
315
    MPIDI_RMA_FUNC_EXIT(MPID_STATE_MPIDI_FETCH_AND_OP);
316
317
    return mpi_errno;
    /* --BEGIN ERROR HANDLING-- */
318
  fn_fail:
319
320
321
    goto fn_exit;
    /* --END ERROR HANDLING-- */
}