Commit c26c6627 authored by Sangmin Seo's avatar Sangmin Seo Committed by Rob Latham
Browse files

Add nonblocking collective I/O functions.

This patch implemented four functions for nonblocking collective I/O,
which will be added to MPI 3.1 standard. Details for these functions
can be found in the MPI-Forum ticket,
https://svn.mpi-forum.org/trac/mpi-forum-web/ticket/273

.
Currently, they are implemented as MPIX functions.
Signed-off-by: Rob Latham's avatarRob Latham <robl@mcs.anl.gov>
parent f0f2c00a
......@@ -145,7 +145,11 @@ my %bufpos = (
"MPI_Unpack_external" => [1, -1, -1, 4, 5, 6],
"MPI_Win_attach" => [1, -1, -1],
"MPI_Win_create" => [0, -1, -1],
"MPI_Win_detach" => [1, -1, -1]
"MPI_Win_detach" => [1, -1, -1],
"MPIX_File_iread_all" => [1, 2, 3],
"MPIX_File_iread_at_all" => [2, 3, 4],
"MPIX_File_iwrite_all" => [1, 2, 3],
"MPIX_File_iwrite_at_all" => [2, 3, 4]
);
# Choice buffers in some functions can be passed in MPI_IN_PLACE. We store such
......
......@@ -56,6 +56,8 @@ struct ADIOI_Fns_struct ADIO_GPFS_operations = {
#elif PEPLATFORM
"GPFS+PE: IBM GPFS for PE",
#else
"GPFS: IBM GPFS"
"GPFS: IBM GPFS",
#endif
ADIOI_GEN_IreadStridedColl, /* IreadStridedColl */
ADIOI_GEN_IwriteStridedColl /* IwriteStridedColl */
};
......@@ -34,4 +34,6 @@ struct ADIOI_Fns_struct ADIO_GRIDFTP_operations = {
ADIOI_GRIDFTP_Resize, /* Resize */
ADIOI_GRIDFTP_Delete, /* Delete */
ADIOI_GRIDFTP_Feature, /* Features */
ADIOI_GEN_IreadStridedColl, /* IreadStridedColl */
ADIOI_GEN_IwriteStridedColl /* IwriteStridedColl */
};
......@@ -33,4 +33,6 @@ struct ADIOI_Fns_struct ADIO_HFS_operations = {
ADIOI_GEN_Flush, /* Flush */
ADIOI_HFS_Resize, /* Resize */
ADIOI_GEN_Delete, /* Delete */
ADIOI_GEN_IreadStridedColl, /* IreadStridedColl */
ADIOI_GEN_IwriteStridedColl /* IwriteStridedColl */
};
......@@ -41,4 +41,6 @@ struct ADIOI_Fns_struct ADIO_LUSTRE_operations = {
ADIOI_GEN_Delete, /* Delete */
ADIOI_GEN_Feature, /* Features */
"LUSTRE:",
ADIOI_GEN_IreadStridedColl, /* IreadStridedColl */
ADIOI_GEN_IwriteStridedColl /* IwriteStridedColl */
};
......@@ -37,5 +37,7 @@ struct ADIOI_Fns_struct ADIO_NFS_operations = {
ADIOI_NFS_Resize, /* Resize */
ADIOI_GEN_Delete, /* Delete */
ADIOI_NFS_Feature, /* Features */
"NFS:" /* fsname: just a string */
"NFS:", /* fsname: just a string */
ADIOI_GEN_IreadStridedColl, /* IreadStridedColl */
ADIOI_GEN_IwriteStridedColl /* IwriteStridedColl */
};
......@@ -34,5 +34,7 @@ struct ADIOI_Fns_struct ADIO_NTFS_operations = {
ADIOI_NTFS_Flush, /* Flush */
ADIOI_NTFS_Resize, /* Resize */
ADIOI_GEN_Delete, /* Delete */
ADIOI_NTFS_Feature /* Features */
ADIOI_NTFS_Feature, /* Features */
ADIOI_GEN_IreadStridedColl, /* IreadStridedColl */
ADIOI_GEN_IwriteStridedColl /* IwriteStridedColl */
};
......@@ -41,5 +41,7 @@ struct ADIOI_Fns_struct ADIO_PANFS_operations = {
ADIOI_PANFS_Resize, /* Resize */
ADIOI_GEN_Delete, /* Delete */
ADIOI_GEN_Feature,
"PANFS: Panasas PanFS"
"PANFS: Panasas PanFS",
ADIOI_GEN_IreadStridedColl, /* IreadStridedColl */
ADIOI_GEN_IwriteStridedColl /* IwriteStridedColl */
};
......@@ -33,4 +33,6 @@ struct ADIOI_Fns_struct ADIO_PFS_operations = {
ADIOI_PFS_Flush, /* Flush */
ADIOI_GEN_Resize, /* Resize */
ADIOI_GEN_Delete, /* Delete */
ADIOI_GEN_IreadStridedColl, /* IreadStridedColl */
ADIOI_GEN_IwriteStridedColl /* IwriteStridedColl */
};
......@@ -34,4 +34,6 @@ struct ADIOI_Fns_struct ADIO_PIOFS_operations = {
ADIOI_GEN_Resize, /* Resize */
ADIOI_GEN_Delete, /* Delete */
ADIOI_PIOFS_Feature,
ADIOI_GEN_IreadStridedColl, /* IreadStridedColl */
ADIOI_GEN_IwriteStridedColl /* IwriteStridedColl */
};
......@@ -34,4 +34,6 @@ struct ADIOI_Fns_struct ADIO_PVFS_operations = {
ADIOI_PVFS_Resize, /* Resize */
ADIOI_PVFS_Delete, /* Delete */
ADIOI_PVFS_Feature, /* Features */
ADIOI_GEN_IreadStridedColl, /* IreadStridedColl */
ADIOI_GEN_IwriteStridedColl /* IwriteStridedColl */
};
......@@ -39,7 +39,9 @@ struct ADIOI_Fns_struct ADIO_PVFS2_operations = {
ADIOI_PVFS2_Resize, /* Resize */
ADIOI_PVFS2_Delete, /* Delete */
ADIOI_PVFS2_Feature,
"PVFS2: the PVFS v2 or OrangeFS file systems"
"PVFS2: the PVFS v2 or OrangeFS file systems",
ADIOI_GEN_IreadStridedColl, /* IreadStridedColl */
ADIOI_GEN_IwriteStridedColl /* IwriteStridedColl */
};
/*
......
......@@ -33,4 +33,6 @@ struct ADIOI_Fns_struct ADIO_SFS_operations = {
ADIOI_SFS_Flush, /* Flush */
ADIOI_GEN_Resize, /* Resize */
ADIOI_GEN_Delete, /* Delete */
ADIOI_GEN_IreadStridedColl, /* IreadStridedColl */
ADIOI_GEN_IwriteStridedColl /* IwriteStridedColl */
};
......@@ -35,5 +35,7 @@ struct ADIOI_Fns_struct ADIO_TESTFS_operations = {
ADIOI_TESTFS_Resize, /* Resize */
ADIOI_TESTFS_Delete, /* Delete */
ADIOI_GEN_Feature, /* Features */
"TESTFS: the logging-only file system"
"TESTFS: the logging-only file system",
ADIOI_GEN_IreadStridedColl, /* IreadStridedColl */
ADIOI_GEN_IwriteStridedColl /* IwriteStridedColl */
};
......@@ -41,4 +41,6 @@ struct ADIOI_Fns_struct ADIO_UFS_operations = {
ADIOI_GEN_Delete, /* Delete */
ADIOI_GEN_Feature, /* Features */
"UFS: Generic ROMIO driver for all UNIX-like file systems",
ADIOI_GEN_IreadStridedColl, /* IreadStridedColl */
ADIOI_GEN_IwriteStridedColl /* IwriteStridedColl */
};
......@@ -40,5 +40,7 @@ struct ADIOI_Fns_struct ADIO_XFS_operations = {
ADIOI_XFS_Resize, /* Resize */
ADIOI_GEN_Delete, /* Delete */
ADIOI_GEN_Feature, /* Features */
"XFS: SGI XFS"
"XFS: SGI XFS",
ADIOI_GEN_IreadStridedColl, /* IreadStridedColl */
ADIOI_GEN_IwriteStridedColl /* IwriteStridedColl */
};
......@@ -35,6 +35,8 @@ struct ADIOI_Fns_struct ADIO_ZOIDFS_operations = {
ADIOI_ZOIDFS_Resize, /* Resize */
ADIOI_ZOIDFS_Delete, /* Delete */
ADIOI_ZOIDFS_Feature,
ADIOI_GEN_IreadStridedColl, /* IreadStridedColl */
ADIOI_GEN_IwriteStridedColl /* IwriteStridedColl */
};
/*
......
......@@ -26,8 +26,10 @@ romio_other_sources += \
adio/common/ad_io_coll.c \
adio/common/ad_iopen.c \
adio/common/ad_iread.c \
adio/common/ad_iread_coll.c \
adio/common/ad_iread_fake.c \
adio/common/ad_iwrite.c \
adio/common/ad_iwrite_coll.c \
adio/common/ad_iwrite_fake.c \
adio/common/ad_open.c \
adio/common/ad_opencoll.c \
......
......@@ -513,3 +513,141 @@ void ADIOI_Calc_others_req(ADIO_File fd, int count_my_req_procs,
MPE_Log_event (5027, 0, NULL);
#endif
}
/* Nonblocking version of ADIOI_Calc_others_req().
It consists of three functions - ADIOI_Icalc_others_req(),
ADIOI_Icalc_others_req_main(), and ADIOI_Icalc_others_req_fini(). */
void ADIOI_Icalc_others_req(ADIOI_NBC_Request *nbc_req, int *error_code)
{
ADIOI_Icalc_others_req_vars *vars = nbc_req->cor_vars;
/* count_others_req_per_proc[i] indicates how many separate contiguous
requests of proc. i lie in this process's file domain. */
/* first find out how much to send/recv and from/to whom */
#ifdef AGGREGATION_PROFILE
MPE_Log_event(5026, 0, NULL);
#endif
vars->count_others_req_per_proc =
(int *)ADIOI_Malloc(vars->nprocs * sizeof(int));
*error_code = MPI_Ialltoall(vars->count_my_req_per_proc, 1, MPI_INT,
vars->count_others_req_per_proc, 1, MPI_INT, vars->fd->comm,
&vars->req1);
if (nbc_req->rdwr == ADIOI_READ) {
nbc_req->data.rd.state = ADIOI_IRC_STATE_ICALC_OTHERS_REQ;
} else {
ADIOI_Assert(nbc_req->rdwr == ADIOI_WRITE);
nbc_req->data.wr.state = ADIOI_IWC_STATE_ICALC_OTHERS_REQ;
}
}
void ADIOI_Icalc_others_req_main(ADIOI_NBC_Request *nbc_req, int *error_code)
{
ADIOI_Icalc_others_req_vars *vars = nbc_req->cor_vars;
ADIO_File fd = vars->fd;
int count_my_req_procs = vars->count_my_req_procs;
ADIOI_Access *my_req = vars->my_req;
int nprocs = vars->nprocs;
int myrank = vars->myrank;
ADIOI_Access **others_req_ptr = vars->others_req_ptr;
/* determine what requests of other processes lie in this process's
file domain */
/* count_others_req_procs = number of processes whose requests lie in
this process's file domain (including this process itself)
count_others_req_per_proc[i] indicates how many separate contiguous
requests of proc. i lie in this process's file domain. */
int *count_others_req_per_proc = vars->count_others_req_per_proc;
int count_others_req_procs;
int i, j;
ADIOI_Access *others_req;
*others_req_ptr = (ADIOI_Access *)ADIOI_Malloc(nprocs*sizeof(ADIOI_Access));
others_req = *others_req_ptr;
count_others_req_procs = 0;
for (i = 0; i < nprocs; i++) {
if (count_others_req_per_proc[i]) {
others_req[i].count = count_others_req_per_proc[i];
others_req[i].offsets = (ADIO_Offset *)
ADIOI_Malloc(count_others_req_per_proc[i]*sizeof(ADIO_Offset));
others_req[i].lens =
ADIOI_Malloc(count_others_req_per_proc[i]*sizeof(ADIO_Offset));
others_req[i].mem_ptrs = (MPI_Aint *)
ADIOI_Malloc(count_others_req_per_proc[i]*sizeof(MPI_Aint));
count_others_req_procs++;
}
else others_req[i].count = 0;
}
vars->count_others_req_procs = count_others_req_procs;
/* now send the calculated offsets and lengths to respective processes */
vars->req2 = (MPI_Request *)
ADIOI_Malloc(1+2*(count_my_req_procs+count_others_req_procs)
*sizeof(MPI_Request));
/* +1 to avoid a 0-size malloc */
j = 0;
for (i = 0; i < nprocs; i++) {
if (others_req[i].count) {
MPI_Irecv(others_req[i].offsets, others_req[i].count,
ADIO_OFFSET, i, i+myrank, fd->comm, &vars->req2[j]);
j++;
MPI_Irecv(others_req[i].lens, others_req[i].count,
ADIO_OFFSET, i, i+myrank+1, fd->comm, &vars->req2[j]);
j++;
}
}
for (i=0; i < nprocs; i++) {
if (my_req[i].count) {
MPI_Isend(my_req[i].offsets, my_req[i].count,
ADIO_OFFSET, i, i+myrank, fd->comm, &vars->req2[j]);
j++;
MPI_Isend(my_req[i].lens, my_req[i].count,
ADIO_OFFSET, i, i+myrank+1, fd->comm, &vars->req2[j]);
j++;
}
}
/* keep the number of requests */
vars->num_req2 = j;
if (nbc_req->rdwr == ADIOI_READ) {
nbc_req->data.rd.state = ADIOI_IRC_STATE_ICALC_OTHERS_REQ_MAIN;
} else {
ADIOI_Assert(nbc_req->rdwr == ADIOI_WRITE);
nbc_req->data.wr.state = ADIOI_IWC_STATE_ICALC_OTHERS_REQ_MAIN;
}
}
void ADIOI_Icalc_others_req_fini(ADIOI_NBC_Request *nbc_req, int *error_code)
{
ADIOI_Icalc_others_req_vars *vars = nbc_req->cor_vars;
void (*next_fn)(ADIOI_NBC_Request *, int *);
ADIOI_Free(vars->req2);
ADIOI_Free(vars->count_others_req_per_proc);
*vars->count_others_req_procs_ptr = vars->count_others_req_procs;
#ifdef AGGREGATION_PROFILE
MPE_Log_event(5027, 0, NULL);
#endif
/* end of the calculation */
next_fn = vars->next_fn;
/* free the struct for parameters and variables */
ADIOI_Free(vars);
nbc_req->cor_vars = NULL;
/* move to the next function */
next_fn(nbc_req, error_code);
}
This diff is collapsed.
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment