ch3u_rma_reqops.c 17.5 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32

/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
/*
 *  (C) 2012 by Argonne National Laboratory.
 *      See COPYRIGHT in top-level directory.
 */

#include "mpidi_ch3_impl.h"
#include "mpidrma.h"

/* Request-based RMA operations are implemented using generalized requests.
 * Below are the generalized request state and handlers for these operations.
 */

typedef struct {
    MPID_Request *request;
    MPID_Win     *win_ptr;
    int           target_rank;
} MPIDI_CH3I_Rma_req_state_t;


#undef FUNCNAME
#define FUNCNAME MPIDI_CH3I_Rma_req_poll
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
static int MPIDI_CH3I_Rma_req_poll(void *state, MPI_Status *status)
{
    int mpi_errno = MPI_SUCCESS;
    MPIDI_CH3I_Rma_req_state_t *req_state = (MPIDI_CH3I_Rma_req_state_t*) state;

    MPIU_UNREFERENCED_ARG(status);

33
34
35
    /* Call flush to complete the operation.  Check that a passive target epoch
     * is still active first; the user could complete the request after calling
     * unlock. */
36
    /* FIXME: We need per-operation completion to make this more efficient. */
37
38
39
40
41
42
    if (req_state->win_ptr->targets[req_state->target_rank].remote_lock_state
        != MPIDI_CH3_WIN_LOCK_NONE)
    {
        mpi_errno = req_state->win_ptr->RMAFns.Win_flush(req_state->target_rank,
                                                         req_state->win_ptr);
    }
43
44

    if (mpi_errno != MPI_SUCCESS) { MPIU_ERR_POP(mpi_errno); }
45
46
47
48
49
50
51
52
53
54

    MPIR_Grequest_complete_impl(req_state->request);

 fn_exit:
    return mpi_errno;
 fn_fail:
    goto fn_exit;
}


55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
#undef FUNCNAME
#define FUNCNAME MPIDI_CH3I_Rma_req_wait
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
static int MPIDI_CH3I_Rma_req_wait(int count, void **states, double timeout,
                                   MPI_Status *status)
{
    int mpi_errno = MPI_SUCCESS;
    int i;

    for (i = 0; i < count; i++) {
        /* Call poll to complete the operation */
        mpi_errno = MPIDI_CH3I_Rma_req_poll(states[i], status);
        if (mpi_errno != MPI_SUCCESS) { MPIU_ERR_POP(mpi_errno); }
    }

 fn_exit:
    return mpi_errno;
 fn_fail:
    goto fn_exit;
}


78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
#undef FUNCNAME
#define FUNCNAME MPIDI_CH3I_Rma_req_query
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
static int MPIDI_CH3I_Rma_req_query(void *state, MPI_Status *status)
{
    int mpi_errno = MPI_SUCCESS;

    MPIU_UNREFERENCED_ARG(state);

    /* All status fields, except the error code, are undefined */
    status->count = 0;
    status->cancelled = FALSE;
    status->MPI_SOURCE = MPI_UNDEFINED;
    status->MPI_TAG = MPI_UNDEFINED;

 fn_exit:
    status->MPI_ERROR = mpi_errno;
    return mpi_errno;
 fn_fail:
    goto fn_exit;
}


#undef FUNCNAME
#define FUNCNAME MPIDI_CH3I_Rma_req_free
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
static int MPIDI_CH3I_Rma_req_free(void *state)
{
    MPIU_Free(state);

    return MPI_SUCCESS;
}


#undef FUNCNAME
#define FUNCNAME MPIDI_CH3I_Rma_req_cancel
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
static int MPIDI_CH3I_Rma_req_cancel(void *state, int complete)
{
    int mpi_errno = MPI_SUCCESS;

    MPIU_UNREFERENCED_ARG(state);

    /* This operation can't be cancelled */
    if (!complete) {
        MPIU_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**rmareqcancel");
    }

 fn_exit:
    return mpi_errno;
 fn_fail:
    goto fn_exit;
}


#undef FUNCNAME
#define FUNCNAME MPIDI_Rput
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
int MPIDI_Rput(const void *origin_addr, int origin_count,
               MPI_Datatype origin_datatype, int target_rank, MPI_Aint target_disp,
               int target_count, MPI_Datatype target_datatype, MPID_Win *win_ptr,
               MPID_Request **request)
{
    int mpi_errno = MPI_SUCCESS;
146
147
148
149
    int dt_contig ATTRIBUTE((unused));
    MPID_Datatype *dtp;
    MPI_Aint dt_true_lb ATTRIBUTE((unused));
    MPIDI_msg_sz_t data_sz;
150
    MPIDI_CH3I_Rma_req_state_t *req_state;
151
    MPIDI_VC_t *orig_vc, *target_vc;
152
    MPIU_CHKPMEM_DECL(1);
153
    MPIDI_STATE_DECL(MPID_STATE_MPIDI_RPUT);
154
155
156

    MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_RPUT);

157
    MPIU_ERR_CHKANDJUMP(win_ptr->epoch_state != MPIDI_EPOCH_LOCK &&
158
159
                        win_ptr->epoch_state != MPIDI_EPOCH_LOCK_ALL &&
                        target_rank != MPI_PROC_NULL,
160
161
                        mpi_errno, MPI_ERR_RMA_SYNC, "**rmasync");

162
163
164
165
166
167
168
    MPIU_CHKPMEM_MALLOC(req_state, MPIDI_CH3I_Rma_req_state_t*,
                        sizeof(MPIDI_CH3I_Rma_req_state_t), mpi_errno,
                        "req-based RMA state");

    req_state->win_ptr = win_ptr;
    req_state->target_rank = target_rank;

169
170
171
    MPIDI_Datatype_get_info(origin_count, origin_datatype,
                            dt_contig, data_sz, dtp, dt_true_lb);

172
    /* Enqueue or perform the RMA operation */
173
    if (target_rank != MPI_PROC_NULL && data_sz != 0) {
174
175
176
177
        mpi_errno = win_ptr->RMAFns.Put(origin_addr, origin_count,
                                        origin_datatype, target_rank,
                                        target_disp, target_count,
                                        target_datatype, win_ptr);
178

179
180
        if (mpi_errno != MPI_SUCCESS) { MPIU_ERR_POP(mpi_errno); }
    }
181

182
183
184
    MPIDI_Comm_get_vc(win_ptr->comm_ptr, win_ptr->comm_ptr->rank, &orig_vc);
    MPIDI_Comm_get_vc(win_ptr->comm_ptr, target_rank, &target_vc);

185
186
187
    /* If the operation is already complete, return a completed request.
     * Otherwise, generate a grequest. */
    /* FIXME: We still may need to flush or sync for shared memory windows */
188
    if (target_rank == MPI_PROC_NULL || target_rank == win_ptr->comm_ptr->rank ||
189
        (win_ptr->shm_allocated == TRUE && orig_vc->node_id == target_vc->node_id) || data_sz == 0)
190
    {
191
192
193
194
195
        mpi_errno = MPIR_Grequest_start_impl(MPIDI_CH3I_Rma_req_query,
                                             MPIDI_CH3I_Rma_req_free,
                                             MPIDI_CH3I_Rma_req_cancel,
                                             req_state, &req_state->request);
        if (mpi_errno != MPI_SUCCESS) { MPIU_ERR_POP(mpi_errno); }
196

197
198
199
200
201
202
203
204
205
206
207
208
        MPIR_Grequest_complete_impl(req_state->request);
    }
    else {
        mpi_errno = MPIX_Grequest_start_impl(MPIDI_CH3I_Rma_req_query,
                                             MPIDI_CH3I_Rma_req_free,
                                             MPIDI_CH3I_Rma_req_cancel,
                                             MPIDI_CH3I_Rma_req_poll,
                                             MPIDI_CH3I_Rma_req_wait,
                                             req_state, &req_state->request);

        if (mpi_errno != MPI_SUCCESS) { MPIU_ERR_POP(mpi_errno); }
    }
209
210
211
212

    *request = req_state->request;

 fn_exit:
213
    MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_RPUT);
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
    return mpi_errno;
 fn_fail:
    MPIU_CHKPMEM_REAP();
    goto fn_exit;
}


#undef FUNCNAME
#define FUNCNAME MPIDI_Rget
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
int MPIDI_Rget(void *origin_addr, int origin_count,
               MPI_Datatype origin_datatype, int target_rank, MPI_Aint target_disp,
               int target_count, MPI_Datatype target_datatype, MPID_Win *win_ptr,
               MPID_Request **request)
{
    int mpi_errno = MPI_SUCCESS;
231
232
233
234
    int dt_contig ATTRIBUTE((unused));
    MPID_Datatype *dtp;
    MPI_Aint dt_true_lb ATTRIBUTE((unused));
    MPIDI_msg_sz_t data_sz;
235
    MPIDI_CH3I_Rma_req_state_t *req_state;
236
    MPIDI_VC_t *orig_vc, *target_vc;
237
    MPIU_CHKPMEM_DECL(1);
238
    MPIDI_STATE_DECL(MPID_STATE_MPIDI_RGET);
239
240
241

    MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_RGET);

242
    MPIU_ERR_CHKANDJUMP(win_ptr->epoch_state != MPIDI_EPOCH_LOCK &&
243
244
                        win_ptr->epoch_state != MPIDI_EPOCH_LOCK_ALL &&
                        target_rank != MPI_PROC_NULL,
245
246
                        mpi_errno, MPI_ERR_RMA_SYNC, "**rmasync");

247
248
249
250
251
252
253
    MPIU_CHKPMEM_MALLOC(req_state, MPIDI_CH3I_Rma_req_state_t*,
                        sizeof(MPIDI_CH3I_Rma_req_state_t), mpi_errno,
                        "req-based RMA state");

    req_state->win_ptr = win_ptr;
    req_state->target_rank = target_rank;

254
255
256
    MPIDI_Datatype_get_info(origin_count, origin_datatype,
                            dt_contig, data_sz, dtp, dt_true_lb);

257
    /* Enqueue or perform the RMA operation */
258
    if (target_rank != MPI_PROC_NULL && data_sz != 0) {
259
260
261
262
        mpi_errno = win_ptr->RMAFns.Get(origin_addr, origin_count,
                                        origin_datatype, target_rank,
                                        target_disp, target_count,
                                        target_datatype, win_ptr);
263

264
265
        if (mpi_errno != MPI_SUCCESS) { MPIU_ERR_POP(mpi_errno); }
    }
266

267
268
269
    MPIDI_Comm_get_vc(win_ptr->comm_ptr, win_ptr->comm_ptr->rank, &orig_vc);
    MPIDI_Comm_get_vc(win_ptr->comm_ptr, target_rank, &target_vc);

270
271
272
    /* If the operation is already complete, return a completed request.
     * Otherwise, generate a grequest. */
    /* FIXME: We still may need to flush or sync for shared memory windows */
273
    if (target_rank == MPI_PROC_NULL || target_rank == win_ptr->comm_ptr->rank ||
274
        (win_ptr->shm_allocated == TRUE && orig_vc->node_id == target_vc->node_id) || data_sz == 0)
275
    {
276
277
278
279
280
        mpi_errno = MPIR_Grequest_start_impl(MPIDI_CH3I_Rma_req_query,
                                             MPIDI_CH3I_Rma_req_free,
                                             MPIDI_CH3I_Rma_req_cancel,
                                             req_state, &req_state->request);
        if (mpi_errno != MPI_SUCCESS) { MPIU_ERR_POP(mpi_errno); }
281

282
283
284
285
286
287
288
289
290
291
292
293
        MPIR_Grequest_complete_impl(req_state->request);
    }
    else {
        mpi_errno = MPIX_Grequest_start_impl(MPIDI_CH3I_Rma_req_query,
                                             MPIDI_CH3I_Rma_req_free,
                                             MPIDI_CH3I_Rma_req_cancel,
                                             MPIDI_CH3I_Rma_req_poll,
                                             MPIDI_CH3I_Rma_req_wait,
                                             req_state, &req_state->request);

        if (mpi_errno != MPI_SUCCESS) { MPIU_ERR_POP(mpi_errno); }
    }
294
295
296
297

    *request = req_state->request;

 fn_exit:
298
    MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_RGET);
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
    return mpi_errno;
 fn_fail:
    MPIU_CHKPMEM_REAP();
    goto fn_exit;
}


#undef FUNCNAME
#define FUNCNAME MPIDI_Raccumulate
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
int MPIDI_Raccumulate(const void *origin_addr, int origin_count,
                      MPI_Datatype origin_datatype, int target_rank, MPI_Aint target_disp,
                      int target_count, MPI_Datatype target_datatype, MPI_Op op, MPID_Win *win_ptr,
                      MPID_Request **request)
{
    int mpi_errno = MPI_SUCCESS;
316
317
318
319
    int dt_contig ATTRIBUTE((unused));
    MPID_Datatype *dtp;
    MPI_Aint dt_true_lb ATTRIBUTE((unused));
    MPIDI_msg_sz_t data_sz;
320
    MPIDI_CH3I_Rma_req_state_t *req_state;
321
    MPIDI_VC_t *orig_vc, *target_vc;
322
    MPIU_CHKPMEM_DECL(1);
323
    MPIDI_STATE_DECL(MPID_STATE_MPIDI_RACCUMULATE);
324
325
326

    MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_RACCUMULATE);

327
    MPIU_ERR_CHKANDJUMP(win_ptr->epoch_state != MPIDI_EPOCH_LOCK &&
328
329
                        win_ptr->epoch_state != MPIDI_EPOCH_LOCK_ALL &&
                        target_rank != MPI_PROC_NULL,
330
331
                        mpi_errno, MPI_ERR_RMA_SYNC, "**rmasync");

332
333
334
335
336
337
338
    MPIU_CHKPMEM_MALLOC(req_state, MPIDI_CH3I_Rma_req_state_t*,
                        sizeof(MPIDI_CH3I_Rma_req_state_t), mpi_errno,
                        "req-based RMA state");

    req_state->win_ptr = win_ptr;
    req_state->target_rank = target_rank;

339
340
341
    MPIDI_Datatype_get_info(origin_count, origin_datatype,
                            dt_contig, data_sz, dtp, dt_true_lb);

342
    /* Enqueue or perform the RMA operation */
343
    if (target_rank != MPI_PROC_NULL && data_sz != 0) {
344
345
346
347
348
349
        mpi_errno = win_ptr->RMAFns.Accumulate(origin_addr, origin_count,
                                               origin_datatype, target_rank,
                                               target_disp, target_count,
                                               target_datatype, op, win_ptr);
        if (mpi_errno != MPI_SUCCESS) { MPIU_ERR_POP(mpi_errno); }
    }
350

351
352
353
    MPIDI_Comm_get_vc(win_ptr->comm_ptr, win_ptr->comm_ptr->rank, &orig_vc);
    MPIDI_Comm_get_vc(win_ptr->comm_ptr, target_rank, &target_vc);

354
355
356
    /* If the operation is already complete, return a completed request.
     * Otherwise, generate a grequest. */
    /* FIXME: We still may need to flush or sync for shared memory windows */
357
    if (target_rank == MPI_PROC_NULL || target_rank == win_ptr->comm_ptr->rank ||
358
        (win_ptr->shm_allocated == TRUE && orig_vc->node_id == target_vc->node_id) || data_sz == 0)
359
    {
360
361
362
363
364
        mpi_errno = MPIR_Grequest_start_impl(MPIDI_CH3I_Rma_req_query,
                                             MPIDI_CH3I_Rma_req_free,
                                             MPIDI_CH3I_Rma_req_cancel,
                                             req_state, &req_state->request);
        if (mpi_errno != MPI_SUCCESS) { MPIU_ERR_POP(mpi_errno); }
365

366
367
368
369
370
371
372
373
374
        MPIR_Grequest_complete_impl(req_state->request);
    }
    else {
        mpi_errno = MPIX_Grequest_start_impl(MPIDI_CH3I_Rma_req_query,
                                             MPIDI_CH3I_Rma_req_free,
                                             MPIDI_CH3I_Rma_req_cancel,
                                             MPIDI_CH3I_Rma_req_poll,
                                             MPIDI_CH3I_Rma_req_wait,
                                             req_state, &req_state->request);
375

376
377
        if (mpi_errno != MPI_SUCCESS) { MPIU_ERR_POP(mpi_errno); }
    }
378
379
380
381

    *request = req_state->request;

 fn_exit:
382
    MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_RACCUMULATE);
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
    return mpi_errno;
 fn_fail:
    MPIU_CHKPMEM_REAP();
    goto fn_exit;
}


#undef FUNCNAME
#define FUNCNAME MPIDI_Rget_accumulate
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
int MPIDI_Rget_accumulate(const void *origin_addr, int origin_count,
                          MPI_Datatype origin_datatype, void *result_addr, int result_count,
                          MPI_Datatype result_datatype, int target_rank, MPI_Aint target_disp,
                          int target_count, MPI_Datatype target_datatype, MPI_Op op, MPID_Win *win_ptr,
                          MPID_Request **request)
{
    int mpi_errno = MPI_SUCCESS;
401
402
403
404
    int dt_contig ATTRIBUTE((unused));
    MPID_Datatype *dtp;
    MPI_Aint dt_true_lb ATTRIBUTE((unused));
    MPIDI_msg_sz_t data_sz, trg_data_sz;
405
    MPIDI_CH3I_Rma_req_state_t *req_state;
406
    MPIDI_VC_t *orig_vc, *target_vc;
407
    MPIU_CHKPMEM_DECL(1);
408
    MPIDI_STATE_DECL(MPID_STATE_MPIDI_RGET_ACCUMULATE);
409
410
411

    MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_RGET_ACCUMULATE);

412
    MPIU_ERR_CHKANDJUMP(win_ptr->epoch_state != MPIDI_EPOCH_LOCK &&
413
414
                        win_ptr->epoch_state != MPIDI_EPOCH_LOCK_ALL &&
                        target_rank != MPI_PROC_NULL,
415
416
                        mpi_errno, MPI_ERR_RMA_SYNC, "**rmasync");

417
418
419
420
421
422
423
    MPIU_CHKPMEM_MALLOC(req_state, MPIDI_CH3I_Rma_req_state_t*,
                        sizeof(MPIDI_CH3I_Rma_req_state_t), mpi_errno,
                        "req-based RMA state");

    req_state->win_ptr = win_ptr;
    req_state->target_rank = target_rank;

424
425
426
427
428
429
    /* Note that GACC is only a no-op if no data goes in both directions */
    MPIDI_Datatype_get_info(origin_count, origin_datatype,
                            dt_contig, data_sz, dtp, dt_true_lb);
    MPIDI_Datatype_get_info(origin_count, origin_datatype,
                            dt_contig, trg_data_sz, dtp, dt_true_lb);

430
    /* Enqueue or perform the RMA operation */
431
    if (target_rank != MPI_PROC_NULL && (data_sz != 0 || trg_data_sz != 0)) {
432
433
434
435
436
437
438
439
        mpi_errno = win_ptr->RMAFns.Get_accumulate(origin_addr, origin_count,
                                                   origin_datatype, result_addr,
                                                   result_count, result_datatype,
                                                   target_rank, target_disp,
                                                   target_count, target_datatype,
                                                   op, win_ptr);
        if (mpi_errno != MPI_SUCCESS) { MPIU_ERR_POP(mpi_errno); }
    }
440

441
442
443
    MPIDI_Comm_get_vc(win_ptr->comm_ptr, win_ptr->comm_ptr->rank, &orig_vc);
    MPIDI_Comm_get_vc(win_ptr->comm_ptr, target_rank, &target_vc);

444
445
446
    /* If the operation is already complete, return a completed request.
     * Otherwise, generate a grequest. */
    /* FIXME: We still may need to flush or sync for shared memory windows */
447
    if (target_rank == MPI_PROC_NULL || target_rank == win_ptr->comm_ptr->rank ||
448
        (win_ptr->shm_allocated == TRUE && orig_vc->node_id == target_vc->node_id) ||
449
        (data_sz == 0 && trg_data_sz == 0))
450
    {
451
452
453
454
455
        mpi_errno = MPIR_Grequest_start_impl(MPIDI_CH3I_Rma_req_query,
                                             MPIDI_CH3I_Rma_req_free,
                                             MPIDI_CH3I_Rma_req_cancel,
                                             req_state, &req_state->request);
        if (mpi_errno != MPI_SUCCESS) { MPIU_ERR_POP(mpi_errno); }
456

457
458
459
460
461
462
463
464
465
        MPIR_Grequest_complete_impl(req_state->request);
    }
    else {
        mpi_errno = MPIX_Grequest_start_impl(MPIDI_CH3I_Rma_req_query,
                                             MPIDI_CH3I_Rma_req_free,
                                             MPIDI_CH3I_Rma_req_cancel,
                                             MPIDI_CH3I_Rma_req_poll,
                                             MPIDI_CH3I_Rma_req_wait,
                                             req_state, &req_state->request);
466

467
468
        if (mpi_errno != MPI_SUCCESS) { MPIU_ERR_POP(mpi_errno); }
    }
469
470
471
472

    *request = req_state->request;

 fn_exit:
473
    MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_RGET_ACCUMULATE);
474
475
476
477
478
479
    return mpi_errno;
 fn_fail:
    MPIU_CHKPMEM_REAP();
    goto fn_exit;
}