Commit 3530af43 authored by James Dinan's avatar James Dinan
Browse files

[svn-r10142] Shared mem window: added disp_unit, fixed size=0.

Added the missing disp_unit argument (was added in a later revision of the MPI
3.0 spec) and fixed a bug in base pointer calculations when processes pass a
size of 0.  Added a test case to test MPI-2 ops on shared memory windows.

Reviewer: buntinas
parent aa77aac2
......@@ -997,7 +997,7 @@ int MPI_Win_wait(MPI_Win);
/* MPI-3 One-Sided Communication Routines */
int MPIX_Win_allocate(MPI_Aint size, int disp_unit, MPI_Info info,
MPI_Comm comm, void *baseptr, MPI_Win *win);
int MPIX_Win_allocate_shared(MPI_Aint size, MPI_Info info, MPI_Comm comm,
int MPIX_Win_allocate_shared(MPI_Aint size, int disp_unit, MPI_Info info, MPI_Comm comm,
void *baseptr, MPI_Win *win);
int MPIX_Win_shared_query(MPI_Win win, int rank, MPI_Aint *size, int *disp_unit, void *baseptr);
int MPIX_Win_create_dynamic(MPI_Info info, MPI_Comm comm, MPI_Win *win);
......@@ -1444,7 +1444,7 @@ int PMPI_Win_wait(MPI_Win);
/* MPI-3 One-Sided Communication Routines */
int PMPIX_Win_allocate(MPI_Aint size, int disp_unit, MPI_Info info,
MPI_Comm comm, void *baseptr, MPI_Win *win);
int PMPIX_Win_allocate_shared(MPI_Aint size, MPI_Info info, MPI_Comm comm,
int PMPIX_Win_allocate_shared(MPI_Aint size, int disp_unit, MPI_Info info, MPI_Comm comm,
void *baseptr, MPI_Win *win);
int PMPIX_Win_shared_query(MPI_Win win, int rank, MPI_Aint *size, int *disp_unit, void *baseptr);
int PMPIX_Win_create_dynamic(MPI_Info info, MPI_Comm comm, MPI_Win *win);
......
......@@ -3220,7 +3220,7 @@ int MPID_Win_unlock(int dest, MPID_Win *win_ptr);
int MPID_Win_allocate(MPI_Aint size, int disp_unit, MPID_Info *info,
MPID_Comm *comm, void *baseptr, MPID_Win **win);
int MPID_Win_allocate_shared(MPI_Aint size, MPID_Info *info_ptr, MPID_Comm *comm_ptr,
int MPID_Win_allocate_shared(MPI_Aint size, int disp_unit, MPID_Info *info_ptr, MPID_Comm *comm_ptr,
void **base_ptr, MPID_Win **win_ptr);
int MPID_Win_shared_query(MPI_Win win, int rank, MPI_Aint *size,
void *baseptr);
......
......@@ -55,7 +55,7 @@
.N MPI_ERR_OTHER
.N MPI_ERR_SIZE
@*/
int MPIX_Win_allocate_shared(MPI_Aint size, MPI_Info info, MPI_Comm comm,
int MPIX_Win_allocate_shared(MPI_Aint size, int disp_unit, MPI_Info info, MPI_Comm comm,
void *baseptr, MPI_Win *win)
{
int mpi_errno = MPI_SUCCESS;
......@@ -65,7 +65,7 @@ int MPIX_Win_allocate_shared(MPI_Aint size, MPI_Info info, MPI_Comm comm,
MPID_MPI_STATE_DECL(MPID_STATE_MPIX_WIN_ALLOCATE_SHARED);
MPIR_ERRTEST_INITIALIZED_ORDIE();
MPIU_THREAD_CS_ENTER(ALLFUNC,);
MPID_MPI_RMA_FUNC_ENTER(MPID_STATE_MPIX_WIN_ALLOCATE_SHARED);
......@@ -95,21 +95,17 @@ int MPIX_Win_allocate_shared(MPI_Aint size, MPI_Info info, MPI_Comm comm,
/* Validate pointers */
MPID_Comm_valid_ptr( comm_ptr, mpi_errno );
if (mpi_errno != MPI_SUCCESS) goto fn_fail;
if (size < 0)
mpi_errno = MPIR_Err_create_code( MPI_SUCCESS,
MPIR_ERR_RECOVERABLE,
FCNAME, __LINE__,
MPI_ERR_SIZE,
"**rmasize",
"**rmasize %d", size);
if (size > 0 && baseptr == NULL)
mpi_errno = MPIR_Err_create_code( MPI_SUCCESS,
MPIR_ERR_RECOVERABLE,
FCNAME, __LINE__,
MPI_ERR_ARG,
"**nullptr",
"**nullptr %s",
"NULL base pointer is invalid when size is nonzero");
MPIU_ERR_CHKANDJUMP1(disp_unit != 0, mpi_errno, MPI_ERR_ARG,
"**arg", "**arg %s", "disp_unit must be positive");
MPIU_ERR_CHKANDJUMP1(size < 0, mpi_errno, MPI_ERR_SIZE,
"**rmasize", "**rmasize %d", size);
MPIU_ERR_CHKANDJUMP1(size > 0 && baseptr == NULL, mpi_errno, MPI_ERR_ARG,
"**nullptr", "**nullptr %s",
"NULL base pointer is invalid when size is nonzero");
if (mpi_errno != MPI_SUCCESS) goto fn_fail;
}
MPID_END_ERROR_CHECKS;
......@@ -117,8 +113,8 @@ int MPIX_Win_allocate_shared(MPI_Aint size, MPI_Info info, MPI_Comm comm,
# endif /* HAVE_ERROR_CHECKING */
/* ... body of routine ... */
mpi_errno = MPID_Win_allocate_shared(size, info_ptr, comm_ptr, baseptr, &win_ptr);
mpi_errno = MPID_Win_allocate_shared(size, disp_unit, info_ptr, comm_ptr, baseptr, &win_ptr);
if (mpi_errno != MPI_SUCCESS) goto fn_fail;
/* Initialize a few fields that have specific defaults */
......
......@@ -17,7 +17,7 @@
MPIU_INSTR_DURATION_EXTERN_DECL(wincreate_allgather);
#endif
static int MPIDI_CH3I_Win_allocate_shared(MPI_Aint size, MPID_Info *info, MPID_Comm *comm_ptr,
static int MPIDI_CH3I_Win_allocate_shared(MPI_Aint size, int disp_unit, MPID_Info *info, MPID_Comm *comm_ptr,
void **base_ptr, MPID_Win **win_ptr);
#undef FUNCNAME
......@@ -43,8 +43,8 @@ int MPIDI_CH3_Win_fns_init(MPIDI_CH3U_Win_fns_t *win_fns)
#define FUNCNAME MPIDI_CH3I_Win_allocate_shared
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
static int MPIDI_CH3I_Win_allocate_shared(MPI_Aint size, MPID_Info *info, MPID_Comm *comm_ptr,
void **base_ptr, MPID_Win **win_ptr)
static int MPIDI_CH3I_Win_allocate_shared(MPI_Aint size, int disp_unit, MPID_Info *info,
MPID_Comm *comm_ptr, void **base_ptr, MPID_Win **win_ptr)
{
int mpi_errno = MPI_SUCCESS;
int i, k, comm_size, rank;
......@@ -83,7 +83,6 @@ static int MPIDI_CH3I_Win_allocate_shared(MPI_Aint size, MPID_Info *info, MPID_C
MPIU_CHKPMEM_MALLOC((*win_ptr)->disp_units, int *, comm_size*sizeof(int),
mpi_errno, "(*win_ptr)->disp_units");
for (i=0; i<comm_size; i++) (*win_ptr)->disp_units[i] = 1;
MPIU_CHKPMEM_MALLOC((*win_ptr)->all_win_handles, MPI_Win *,
comm_size*sizeof(MPI_Win),
......@@ -96,14 +95,15 @@ static int MPIDI_CH3I_Win_allocate_shared(MPI_Aint size, MPID_Info *info, MPID_C
/* get the sizes of the windows and window objectsof
all processes. allocate temp. buffer for communication */
MPIU_CHKLMEM_MALLOC(tmp_buf, MPI_Aint *, 2*comm_size*sizeof(MPI_Aint), mpi_errno, "tmp_buf");
MPIU_CHKLMEM_MALLOC(tmp_buf, MPI_Aint *, 3*comm_size*sizeof(MPI_Aint), mpi_errno, "tmp_buf");
/* FIXME: This needs to be fixed for heterogeneous systems */
tmp_buf[2*rank] = (MPI_Aint) size;
tmp_buf[2*rank+1] = (MPI_Aint) (*win_ptr)->handle;
tmp_buf[3*rank] = (MPI_Aint) size;
tmp_buf[3*rank+1] = (MPI_Aint) disp_unit;
tmp_buf[3*rank+2] = (MPI_Aint) (*win_ptr)->handle;
mpi_errno = MPIR_Allgather_impl(MPI_IN_PLACE, 0, MPI_DATATYPE_NULL,
tmp_buf, 2 * sizeof(MPI_Aint), MPI_BYTE,
tmp_buf, 3 * sizeof(MPI_Aint), MPI_BYTE,
(*win_ptr)->comm_ptr, &errflag);
MPIU_INSTR_DURATION_END(wincreate_allgather);
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
......@@ -113,6 +113,7 @@ static int MPIDI_CH3I_Win_allocate_shared(MPI_Aint size, MPID_Info *info, MPID_C
k = 0;
for (i = 0; i < comm_size; ++i) {
(*win_ptr)->sizes[i] = tmp_buf[k++];
(*win_ptr)->disp_units[i] = (int) tmp_buf[k++];
(*win_ptr)->all_win_handles[i] = (MPI_Win) tmp_buf[k++];
if (noncontig)
......@@ -172,16 +173,20 @@ static int MPIDI_CH3I_Win_allocate_shared(MPI_Aint size, MPID_Info *info, MPID_C
}
/* compute the base addresses of each process within the shared memory segment */
(*win_ptr)->shm_base_addrs[0] = (*win_ptr)->shm_base_addr;
for (i = 1; i < comm_size; ++i) {
if ((*win_ptr)->sizes[i]) {
if (noncontig)
(*win_ptr)->shm_base_addrs[i] = (char *)(*win_ptr)->shm_base_addrs[i-1] +
MPIDI_CH3_ROUND_UP_PAGESIZE((*win_ptr)->sizes[i-1]);
else
(*win_ptr)->shm_base_addrs[i] = (char *)(*win_ptr)->shm_base_addrs[i-1] + (*win_ptr)->sizes[i-1];
} else {
(*win_ptr)->shm_base_addrs[i] = NULL; /* FIXME: Is this right? */
{
char *cur_base = (*win_ptr)->shm_base_addr;
(*win_ptr)->shm_base_addrs[0] = (*win_ptr)->shm_base_addr;
for (i = 1; i < comm_size; ++i) {
if ((*win_ptr)->sizes[i]) {
if (noncontig) {
(*win_ptr)->shm_base_addrs[i] = cur_base + MPIDI_CH3_ROUND_UP_PAGESIZE((*win_ptr)->sizes[i-1]);
} else {
(*win_ptr)->shm_base_addrs[i] = cur_base + (*win_ptr)->sizes[i-1];
}
cur_base = (*win_ptr)->shm_base_addrs[i];
} else {
(*win_ptr)->shm_base_addrs[i] = NULL; /* FIXME: Is this right? */
}
}
}
......@@ -189,18 +194,16 @@ static int MPIDI_CH3I_Win_allocate_shared(MPI_Aint size, MPID_Info *info, MPID_C
/* get the base addresses of the windows. Note we reuse tmp_buf from above
since it's at least as large as we need it for this allgather. */
/* FIXME: This needs to be fixed for heterogeneous systems */
tmp_buf[rank] = (MPI_Aint) (*win_ptr)->base;
tmp_buf[rank] = MPIU_PtrToAint((*win_ptr)->base);
mpi_errno = MPIR_Allgather_impl(MPI_IN_PLACE, 0, MPI_DATATYPE_NULL,
tmp_buf, sizeof(MPI_Aint), MPI_BYTE,
tmp_buf, 1, MPI_AINT,
(*win_ptr)->comm_ptr, &errflag);
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
MPIU_ERR_CHKANDJUMP(errflag, mpi_errno, MPI_ERR_OTHER, "**coll_fail");
for (i = 0; i < comm_size; ++i)
(*win_ptr)->base_addrs[i] = (void *)(tmp_buf[i]);
(*win_ptr)->base_addrs[i] = MPIU_AintToPtr(tmp_buf[i]);
*base_ptr = (*win_ptr)->base;
......
......@@ -1120,7 +1120,7 @@ int MPIDI_CH3U_Comm_FinishPending( MPID_Comm * );
typedef struct MPIDI_CH3U_Win_fns_s {
int (*create)(void *, MPI_Aint, int, MPID_Info *, MPID_Comm *, MPID_Win **);
int (*allocate)(MPI_Aint, int, MPID_Info *, MPID_Comm *, void *, MPID_Win **);
int (*allocate_shared)(MPI_Aint, MPID_Info *, MPID_Comm *, void **, MPID_Win **);
int (*allocate_shared)(MPI_Aint, int, MPID_Info *, MPID_Comm *, void **, MPID_Win **);
int (*create_dynamic)(MPID_Info *, MPID_Comm *, MPID_Win **);
} MPIDI_CH3U_Win_fns_t;
......@@ -1135,8 +1135,8 @@ int MPIDI_CH3U_Win_create(void *, MPI_Aint, int, MPID_Info *, MPID_Comm *,
MPID_Win **);
int MPIDI_CH3U_Win_allocate(MPI_Aint size, int disp_unit, MPID_Info *info,
MPID_Comm *comm, void *baseptr, MPID_Win **win);
int MPIDI_CH3U_Win_allocate_shared(MPI_Aint size, MPID_Info *info_ptr, MPID_Comm *comm_ptr,
void **baseptr, MPID_Win **win_ptr);
int MPIDI_CH3U_Win_allocate_shared(MPI_Aint size, int disp_unit, MPID_Info *info_ptr,
MPID_Comm *comm_ptr, void **baseptr, MPID_Win **win_ptr);
int MPIDI_CH3U_Win_create_dynamic(MPID_Info *info, MPID_Comm *comm, MPID_Win **win);
......
......@@ -88,7 +88,7 @@ int MPIDI_CH3U_Win_create_gather( void *base, MPI_Aint size, int disp_unit,
tmp_buf[4*rank+3] = (MPI_Aint) (*win_ptr)->handle;
mpi_errno = MPIR_Allgather_impl(MPI_IN_PLACE, 0, MPI_DATATYPE_NULL,
tmp_buf, 4 * sizeof(MPI_Aint), MPI_BYTE,
tmp_buf, 4, MPI_AINT,
(*win_ptr)->comm_ptr, &errflag);
MPIU_INSTR_DURATION_END(wincreate_allgather);
if (mpi_errno) { MPIU_ERR_POP(mpi_errno); }
......@@ -167,7 +167,7 @@ fn_fail:
#define FUNCNAME MPIDI_CH3U_Win_allocate_shared
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
int MPIDI_CH3U_Win_allocate_shared(MPI_Aint size, MPID_Info *info, MPID_Comm *comm_ptr,
int MPIDI_CH3U_Win_allocate_shared(MPI_Aint size, int disp_unit, MPID_Info *info, MPID_Comm *comm_ptr,
void **base_ptr, MPID_Win **win_ptr)
{
int mpi_errno = MPI_SUCCESS;
......
......@@ -186,7 +186,7 @@ int MPID_Free_mem( void *ptr )
#define FUNCNAME MPID_Win_allocate_shared
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
int MPID_Win_allocate_shared(MPI_Aint size, MPID_Info *info, MPID_Comm *comm_ptr,
int MPID_Win_allocate_shared(MPI_Aint size, int disp_unit, MPID_Info *info, MPID_Comm *comm_ptr,
void **base_ptr, MPID_Win **win_ptr)
{
int mpi_errno=MPI_SUCCESS;
......@@ -195,10 +195,10 @@ int MPID_Win_allocate_shared(MPI_Aint size, MPID_Info *info, MPID_Comm *comm_ptr
MPIDI_RMA_FUNC_ENTER(MPID_STATE_MPID_WIN_ALLOCATE_SHARED);
mpi_errno = win_init(size, 1, MPIX_WIN_FLAVOR_SHARED, MPIX_WIN_UNIFIED, info, comm_ptr, win_ptr);
mpi_errno = win_init(size, disp_unit, MPIX_WIN_FLAVOR_SHARED, MPIX_WIN_UNIFIED, info, comm_ptr, win_ptr);
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
mpi_errno = MPIDI_CH3U_Win_fns.allocate_shared(size, info, comm_ptr, base_ptr, win_ptr);
mpi_errno = MPIDI_CH3U_Win_fns.allocate_shared(size, disp_unit, info, comm_ptr, base_ptr, win_ptr);
if (mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
fn_fail:
......
......@@ -78,6 +78,7 @@ noinst_PROGRAMS = \
win_flavors \
win_shared \
win_shared_noncontig \
win_shared_noncontig_put \
get_acc_local
strided_acc_indexed_LDADD = $(LDADD) -lm
......
......@@ -63,4 +63,5 @@ win_flavors 4 strict=false
manyrma2 2 timeLimit=800
win_shared 4 strict=false
win_shared_noncontig 4 strict=false
win_shared_noncontig_put 4 strict=false
get_acc_local 1 strict=false
......@@ -45,7 +45,7 @@ int main(int argc, char **argv) {
MPI_Comm_size(shm_comm, &shm_nproc);
/* Allocate ELEM_PER_PROC integers for each process */
MPIX_Win_allocate_shared(sizeof(int)*ELEM_PER_PROC, MPI_INFO_NULL,
MPIX_Win_allocate_shared(sizeof(int)*ELEM_PER_PROC, sizeof(int), MPI_INFO_NULL,
shm_comm, &my_base, &shm_win);
/* Locate absolute base */
......
......@@ -48,7 +48,7 @@ int main(int argc, char **argv) {
MPI_Comm_size(shm_comm, &shm_nproc);
/* Allocate ELEM_PER_PROC integers for each process */
MPIX_Win_allocate_shared(sizeof(int)*ELEM_PER_PROC, alloc_shared_info,
MPIX_Win_allocate_shared(sizeof(int)*ELEM_PER_PROC, sizeof(int), alloc_shared_info,
shm_comm, &my_base, &shm_win);
MPIX_Win_lock_all(MPI_MODE_NOCHECK, shm_win);
......
/* -*- Mode: C; c-basic-offset:4 ; -*- */
/*
*
* (C) 2012 by Argonne National Laboratory.
* See COPYRIGHT in top-level directory.
*/
#include <stdio.h>
#include <stdlib.h>
#include <assert.h>
#include <mpi.h>
#include "mpitest.h"
/* MPI-3 is not yet standardized -- allow MPI-3 routines to be switched off.
*/
#if !defined(USE_STRICT_MPI) && defined(MPICH2)
# define TEST_MPI3_ROUTINES 1
#endif
#define ELEM_PER_PROC 10000
const int verbose = 0;
int main(int argc, char **argv) {
int i, j, rank, nproc;
int shm_rank, shm_nproc;
MPI_Info alloc_shared_info;
int errors = 0, all_errors = 0;
int disp_unit;
int *my_base, my_size;
MPI_Win shm_win;
MPI_Comm shm_comm;
MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &nproc);
MPI_Info_create(&alloc_shared_info);
MPI_Info_set(alloc_shared_info, "alloc_shared_noncontig", "true");
#ifdef TEST_MPI3_ROUTINES
MPIX_Comm_split_type(MPI_COMM_WORLD, MPIX_COMM_TYPE_SHARED, rank, MPI_INFO_NULL, &shm_comm);
MPI_Comm_rank(shm_comm, &shm_rank);
MPI_Comm_size(shm_comm, &shm_nproc);
/* Allocate ELEM_PER_PROC integers on each even rank process */
my_size = (shm_rank % 2 == 0) ? sizeof(int)*ELEM_PER_PROC : 0;
MPIX_Win_allocate_shared(my_size, sizeof(int), alloc_shared_info,
shm_comm, &my_base, &shm_win);
for (i = 0; i < ELEM_PER_PROC; i++) {
MPI_Win_fence(MPI_MODE_NOPRECEDE, shm_win);
if (shm_rank % 2 == 0) {
MPI_Put(&i, 1, MPI_INT,
(shm_rank + 2 > shm_nproc) ? 0 : (shm_rank+2) % shm_nproc,
i, 1, MPI_INT, shm_win);
}
MPI_Win_fence(MPI_MODE_NOSUCCEED, shm_win);
}
MPI_Barrier(shm_comm);
/* Read and verify everyone's data */
for (i = 0; i < shm_nproc; i++) {
int *base;
MPI_Aint size;
MPIX_Win_shared_query(shm_win, i, &size, &disp_unit, &base);
if (i % 2 == 0) {
assert(size == ELEM_PER_PROC * sizeof(int));
for (j = 0; j < ELEM_PER_PROC; j++) {
if ( base[j] != j ) {
errors++;
printf("%d -- Got %d at rank %d index %d, expected %d\n", shm_rank,
base[j], i, j, j);
}
}
} else {
assert(size == 0);
assert(base == NULL);
}
}
MPI_Win_free(&shm_win);
MPI_Comm_free(&shm_comm);
#endif /* TEST_MPI3_ROUTINES */
MPI_Info_free(&alloc_shared_info);
MPI_Reduce(&errors, &all_errors, 1, MPI_INT, MPI_SUM, 0, MPI_COMM_WORLD);
if (rank == 0 && all_errors == 0)
printf(" No Errors\n");
MPI_Finalize();
return 0;
}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment