ch3u_rma_acc_ops.c 12.1 KB
Newer Older
1
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
2
3
4
5
6
7
8
9
10
11
12
13
14
15
/*
 *  (C) 2001 by Argonne National Laboratory.
 *      See COPYRIGHT in top-level directory.
 */

#include "mpidrma.h"

#undef FUNCNAME
#define FUNCNAME MPIDI_Get_accumulate
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
int MPIDI_Get_accumulate(const void *origin_addr, int origin_count,
                         MPI_Datatype origin_datatype, void *result_addr, int result_count,
                         MPI_Datatype result_datatype, int target_rank, MPI_Aint target_disp,
16
17
                         int target_count, MPI_Datatype target_datatype, MPI_Op op,
                         MPID_Win * win_ptr)
18
{
19
20
    int mpi_errno = MPI_SUCCESS;
    MPIDI_msg_sz_t data_sz;
21
    int rank;
22
23
24
    int dt_contig ATTRIBUTE((unused));
    MPI_Aint dt_true_lb ATTRIBUTE((unused));
    MPID_Datatype *dtp;
25
    MPIDI_VC_t *orig_vc = NULL, *target_vc = NULL;
26
    MPIDI_STATE_DECL(MPID_STATE_MPIDI_GET_ACCUMULATE);
27

28
29
    MPIDI_RMA_FUNC_ENTER(MPID_STATE_MPIDI_GET_ACCUMULATE);

James Dinan's avatar
James Dinan committed
30
31
32
33
    if (target_rank == MPI_PROC_NULL) {
        goto fn_exit;
    }

34
    if (win_ptr->epoch_state == MPIDI_EPOCH_NONE && win_ptr->fence_issued) {
James Dinan's avatar
James Dinan committed
35
36
37
38
39
40
        win_ptr->epoch_state = MPIDI_EPOCH_FENCE;
    }

    MPIU_ERR_CHKANDJUMP(win_ptr->epoch_state == MPIDI_EPOCH_NONE,
                        mpi_errno, MPI_ERR_RMA_SYNC, "**rmasync");

41
    MPIDI_Datatype_get_info(target_count, target_datatype, dt_contig, data_sz, dtp, dt_true_lb);
42

James Dinan's avatar
James Dinan committed
43
    if (data_sz == 0) {
44
45
46
        goto fn_exit;
    }

47
    rank = win_ptr->comm_ptr->rank;
48

49
50
    if (win_ptr->shm_allocated == TRUE && target_rank != rank &&
        win_ptr->create_flavor != MPI_WIN_FLAVOR_SHARED) {
Xin Zhao's avatar
Xin Zhao committed
51
        /* check if target is local and shared memory is allocated on window,
52
         * if so, we directly perform this operation on shared memory region. */
Xin Zhao's avatar
Xin Zhao committed
53
54

        /* FIXME: Here we decide whether to perform SHM operations by checking if origin and target are on
55
56
57
58
59
         * the same node. However, in ch3:sock, even if origin and target are on the same node, they do
         * not within the same SHM region. Here we filter out ch3:sock by checking shm_allocated flag first,
         * which is only set to TRUE when SHM region is allocated in nemesis.
         * In future we need to figure out a way to check if origin and target are in the same "SHM comm".
         */
Xin Zhao's avatar
Xin Zhao committed
60
61
62
63
        MPIDI_Comm_get_vc(win_ptr->comm_ptr, rank, &orig_vc);
        MPIDI_Comm_get_vc(win_ptr->comm_ptr, target_rank, &target_vc);
    }

64
    /* Do =! rank first (most likely branch?) */
Xin Zhao's avatar
Xin Zhao committed
65
    if (target_rank == rank || win_ptr->create_flavor == MPI_WIN_FLAVOR_SHARED ||
66
        (win_ptr->shm_allocated == TRUE && orig_vc->node_id == target_vc->node_id)) {
67
68
        mpi_errno = MPIDI_CH3I_Shm_get_acc_op(origin_addr, origin_count, origin_datatype,
                                              result_addr, result_count, result_datatype,
69
70
71
72
                                              target_rank, target_disp, target_count,
                                              target_datatype, op, win_ptr);
        if (mpi_errno)
            MPIU_ERR_POP(mpi_errno);
73
74
    }
    else {
75
        MPIDI_RMA_Ops_list_t *ops_list = MPIDI_CH3I_RMA_Get_ops_list(win_ptr, target_rank);
76
        MPIDI_RMA_Op_t *new_ptr = NULL;
77

78
        /* Append the operation to the window's RMA ops queue */
79
        mpi_errno = MPIDI_CH3I_RMA_Ops_alloc_tail(ops_list, &new_ptr);
80
81
82
        if (mpi_errno) {
            MPIU_ERR_POP(mpi_errno);
        }
83
84
85

        /* TODO: Can we use the MPIDI_RMA_ACC_CONTIG optimization? */

86
87
88
89
90
91
92
93
94
95
96
97
98
99
        new_ptr->type = MPIDI_RMA_GET_ACCUMULATE;
        /* Cast away const'ness for origin_address as MPIDI_RMA_Op_t
         * contain both PUT and GET like ops */
        new_ptr->origin_addr = (void *) origin_addr;
        new_ptr->origin_count = origin_count;
        new_ptr->origin_datatype = origin_datatype;
        new_ptr->result_addr = result_addr;
        new_ptr->result_count = result_count;
        new_ptr->result_datatype = result_datatype;
        new_ptr->target_rank = target_rank;
        new_ptr->target_disp = target_disp;
        new_ptr->target_count = target_count;
        new_ptr->target_datatype = target_datatype;
        new_ptr->op = op;
100

101
        /* if source or target datatypes are derived, increment their
102
         * reference counts */
103
        if (op != MPI_NO_OP && !MPIR_DATATYPE_IS_PREDEFINED(origin_datatype)) {
104
105
106
            MPID_Datatype_get_ptr(origin_datatype, dtp);
            MPID_Datatype_add_ref(dtp);
        }
107
        if (!MPIR_DATATYPE_IS_PREDEFINED(result_datatype)) {
108
109
110
            MPID_Datatype_get_ptr(result_datatype, dtp);
            MPID_Datatype_add_ref(dtp);
        }
111
        if (!MPIR_DATATYPE_IS_PREDEFINED(target_datatype)) {
112
113
            MPID_Datatype_get_ptr(target_datatype, dtp);
            MPID_Datatype_add_ref(dtp);
114
        }
115
116
    }

117
  fn_exit:
118
119
120
121
122
123
124
125
    MPIDI_RMA_FUNC_EXIT(MPID_STATE_MPIDI_GET_ACCUMULATE);
    return mpi_errno;

    /* --BEGIN ERROR HANDLING-- */
  fn_fail:
    goto fn_exit;
    /* --END ERROR HANDLING-- */
}
126
127
128
129
130
131
132


#undef FUNCNAME
#define FUNCNAME MPIDI_Compare_and_swap
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
int MPIDI_Compare_and_swap(const void *origin_addr, const void *compare_addr,
133
134
                           void *result_addr, MPI_Datatype datatype, int target_rank,
                           MPI_Aint target_disp, MPID_Win * win_ptr)
135
136
137
{
    int mpi_errno = MPI_SUCCESS;
    int rank;
138
    MPIDI_VC_t *orig_vc = NULL, *target_vc = NULL;
139
140
141
142
143
144
145
146

    MPIDI_STATE_DECL(MPID_STATE_MPIDI_COMPARE_AND_SWAP);
    MPIDI_RMA_FUNC_ENTER(MPID_STATE_MPIDI_COMPARE_AND_SWAP);

    if (target_rank == MPI_PROC_NULL) {
        goto fn_exit;
    }

147
    if (win_ptr->epoch_state == MPIDI_EPOCH_NONE && win_ptr->fence_issued) {
James Dinan's avatar
James Dinan committed
148
149
150
151
152
153
        win_ptr->epoch_state = MPIDI_EPOCH_FENCE;
    }

    MPIU_ERR_CHKANDJUMP(win_ptr->epoch_state == MPIDI_EPOCH_NONE,
                        mpi_errno, MPI_ERR_RMA_SYNC, "**rmasync");

154
    rank = win_ptr->comm_ptr->rank;
155

156
157
    if (win_ptr->shm_allocated == TRUE && target_rank != rank &&
        win_ptr->create_flavor != MPI_WIN_FLAVOR_SHARED) {
Xin Zhao's avatar
Xin Zhao committed
158
        /* check if target is local and shared memory is allocated on window,
159
         * if so, we directly perform this operation on shared memory region. */
Xin Zhao's avatar
Xin Zhao committed
160
161

        /* FIXME: Here we decide whether to perform SHM operations by checking if origin and target are on
162
163
164
165
166
         * the same node. However, in ch3:sock, even if origin and target are on the same node, they do
         * not within the same SHM region. Here we filter out ch3:sock by checking shm_allocated flag first,
         * which is only set to TRUE when SHM region is allocated in nemesis.
         * In future we need to figure out a way to check if origin and target are in the same "SHM comm".
         */
Xin Zhao's avatar
Xin Zhao committed
167
168
169
170
        MPIDI_Comm_get_vc(win_ptr->comm_ptr, rank, &orig_vc);
        MPIDI_Comm_get_vc(win_ptr->comm_ptr, target_rank, &target_vc);
    }

171
172
173
174
    /* The datatype must be predefined, and one of: C integer, Fortran integer,
     * Logical, Multi-language types, or Byte.  This is checked above the ADI,
     * so there's no need to check it again here. */

175
176
    /* FIXME: For shared memory windows, we should provide an implementation
     * that uses a processor atomic operation. */
Xin Zhao's avatar
Xin Zhao committed
177
    if (target_rank == rank || win_ptr->create_flavor == MPI_WIN_FLAVOR_SHARED ||
178
        (win_ptr->shm_allocated == TRUE && orig_vc->node_id == target_vc->node_id)) {
179
180
        mpi_errno = MPIDI_CH3I_Shm_cas_op(origin_addr, compare_addr, result_addr,
                                          datatype, target_rank, target_disp, win_ptr);
181
182
        if (mpi_errno)
            MPIU_ERR_POP(mpi_errno);
183
184
    }
    else {
185
        MPIDI_RMA_Ops_list_t *ops_list = MPIDI_CH3I_RMA_Get_ops_list(win_ptr, target_rank);
186
        MPIDI_RMA_Op_t *new_ptr = NULL;
187

188
        /* Append this operation to the RMA ops queue */
189
        mpi_errno = MPIDI_CH3I_RMA_Ops_alloc_tail(ops_list, &new_ptr);
190
191
192
        if (mpi_errno) {
            MPIU_ERR_POP(mpi_errno);
        }
193

194
        new_ptr->type = MPIDI_RMA_COMPARE_AND_SWAP;
195
196
197
        new_ptr->origin_addr = (void *) origin_addr;
        new_ptr->origin_count = 1;
        new_ptr->origin_datatype = datatype;
198
199
200
201
        new_ptr->target_rank = target_rank;
        new_ptr->target_disp = target_disp;
        new_ptr->target_count = 1;
        new_ptr->target_datatype = datatype;
202
        new_ptr->result_addr = result_addr;
203
        new_ptr->result_count = 1;
204
205
        new_ptr->result_datatype = datatype;
        new_ptr->compare_addr = (void *) compare_addr;
206
        new_ptr->compare_count = 1;
207
208
209
        new_ptr->compare_datatype = datatype;
    }

210
  fn_exit:
211
212
213
    MPIDI_RMA_FUNC_EXIT(MPID_STATE_MPIDI_COMPARE_AND_SWAP);
    return mpi_errno;
    /* --BEGIN ERROR HANDLING-- */
214
  fn_fail:
215
216
217
218
219
    goto fn_exit;
    /* --END ERROR HANDLING-- */
}


220
221
222
223
224
225
#undef FUNCNAME
#define FUNCNAME MPIDI_Fetch_and_op
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
int MPIDI_Fetch_and_op(const void *origin_addr, void *result_addr,
                       MPI_Datatype datatype, int target_rank,
226
                       MPI_Aint target_disp, MPI_Op op, MPID_Win * win_ptr)
227
228
229
{
    int mpi_errno = MPI_SUCCESS;
    int rank;
230
    MPIDI_VC_t *orig_vc = NULL, *target_vc = NULL;
231

232
233
    MPIDI_STATE_DECL(MPID_STATE_MPIDI_FETCH_AND_OP);
    MPIDI_RMA_FUNC_ENTER(MPID_STATE_MPIDI_FETCH_AND_OP);
234
235
236
237
238

    if (target_rank == MPI_PROC_NULL) {
        goto fn_exit;
    }

239
    if (win_ptr->epoch_state == MPIDI_EPOCH_NONE && win_ptr->fence_issued) {
James Dinan's avatar
James Dinan committed
240
241
242
243
244
245
        win_ptr->epoch_state = MPIDI_EPOCH_FENCE;
    }

    MPIU_ERR_CHKANDJUMP(win_ptr->epoch_state == MPIDI_EPOCH_NONE,
                        mpi_errno, MPI_ERR_RMA_SYNC, "**rmasync");

246
    rank = win_ptr->comm_ptr->rank;
247

248
249
    if (win_ptr->shm_allocated == TRUE && target_rank != rank &&
        win_ptr->create_flavor != MPI_WIN_FLAVOR_SHARED) {
Xin Zhao's avatar
Xin Zhao committed
250
        /* check if target is local and shared memory is allocated on window,
251
         * if so, we directly perform this operation on shared memory region. */
Xin Zhao's avatar
Xin Zhao committed
252
253

        /* FIXME: Here we decide whether to perform SHM operations by checking if origin and target are on
254
255
256
257
258
         * the same node. However, in ch3:sock, even if origin and target are on the same node, they do
         * not within the same SHM region. Here we filter out ch3:sock by checking shm_allocated flag first,
         * which is only set to TRUE when SHM region is allocated in nemesis.
         * In future we need to figure out a way to check if origin and target are in the same "SHM comm".
         */
Xin Zhao's avatar
Xin Zhao committed
259
260
261
262
        MPIDI_Comm_get_vc(win_ptr->comm_ptr, rank, &orig_vc);
        MPIDI_Comm_get_vc(win_ptr->comm_ptr, target_rank, &target_vc);
    }

263
264
265
    /* The datatype and op must be predefined.  This is checked above the ADI,
     * so there's no need to check it again here. */

266
267
    /* FIXME: For shared memory windows, we should provide an implementation
     * that uses a processor atomic operation. */
Xin Zhao's avatar
Xin Zhao committed
268
    if (target_rank == rank || win_ptr->create_flavor == MPI_WIN_FLAVOR_SHARED ||
269
        (win_ptr->shm_allocated == TRUE && orig_vc->node_id == target_vc->node_id)) {
270
271
        mpi_errno = MPIDI_CH3I_Shm_fop_op(origin_addr, result_addr, datatype,
                                          target_rank, target_disp, op, win_ptr);
272
273
        if (mpi_errno)
            MPIU_ERR_POP(mpi_errno);
274
275
    }
    else {
276
        MPIDI_RMA_Ops_list_t *ops_list = MPIDI_CH3I_RMA_Get_ops_list(win_ptr, target_rank);
277
        MPIDI_RMA_Op_t *new_ptr = NULL;
278

279
        /* Append this operation to the RMA ops queue */
280
        mpi_errno = MPIDI_CH3I_RMA_Ops_alloc_tail(ops_list, &new_ptr);
281
282
283
        if (mpi_errno) {
            MPIU_ERR_POP(mpi_errno);
        }
284

285
        new_ptr->type = MPIDI_RMA_FETCH_AND_OP;
286
287
288
        new_ptr->origin_addr = (void *) origin_addr;
        new_ptr->origin_count = 1;
        new_ptr->origin_datatype = datatype;
289
290
291
292
        new_ptr->target_rank = target_rank;
        new_ptr->target_disp = target_disp;
        new_ptr->target_count = 1;
        new_ptr->target_datatype = datatype;
293
        new_ptr->result_addr = result_addr;
294
        new_ptr->result_count = 1;
295
        new_ptr->result_datatype = datatype;
296
        new_ptr->op = op;
297
298
    }

299
  fn_exit:
300
    MPIDI_RMA_FUNC_EXIT(MPID_STATE_MPIDI_FETCH_AND_OP);
301
302
    return mpi_errno;
    /* --BEGIN ERROR HANDLING-- */
303
  fn_fail:
304
305
306
    goto fn_exit;
    /* --END ERROR HANDLING-- */
}