Commit c26c6627 authored by Sangmin Seo's avatar Sangmin Seo Committed by Rob Latham
Browse files

Add nonblocking collective I/O functions.

This patch implemented four functions for nonblocking collective I/O,
which will be added to MPI 3.1 standard. Details for these functions
can be found in the MPI-Forum ticket,
https://svn.mpi-forum.org/trac/mpi-forum-web/ticket/273

.
Currently, they are implemented as MPIX functions.
Signed-off-by: Rob Latham's avatarRob Latham <robl@mcs.anl.gov>
parent f0f2c00a
......@@ -145,7 +145,11 @@ my %bufpos = (
"MPI_Unpack_external" => [1, -1, -1, 4, 5, 6],
"MPI_Win_attach" => [1, -1, -1],
"MPI_Win_create" => [0, -1, -1],
"MPI_Win_detach" => [1, -1, -1]
"MPI_Win_detach" => [1, -1, -1],
"MPIX_File_iread_all" => [1, 2, 3],
"MPIX_File_iread_at_all" => [2, 3, 4],
"MPIX_File_iwrite_all" => [1, 2, 3],
"MPIX_File_iwrite_at_all" => [2, 3, 4]
);
# Choice buffers in some functions can be passed in MPI_IN_PLACE. We store such
......
......@@ -56,6 +56,8 @@ struct ADIOI_Fns_struct ADIO_GPFS_operations = {
#elif PEPLATFORM
"GPFS+PE: IBM GPFS for PE",
#else
"GPFS: IBM GPFS"
"GPFS: IBM GPFS",
#endif
ADIOI_GEN_IreadStridedColl, /* IreadStridedColl */
ADIOI_GEN_IwriteStridedColl /* IwriteStridedColl */
};
......@@ -34,4 +34,6 @@ struct ADIOI_Fns_struct ADIO_GRIDFTP_operations = {
ADIOI_GRIDFTP_Resize, /* Resize */
ADIOI_GRIDFTP_Delete, /* Delete */
ADIOI_GRIDFTP_Feature, /* Features */
ADIOI_GEN_IreadStridedColl, /* IreadStridedColl */
ADIOI_GEN_IwriteStridedColl /* IwriteStridedColl */
};
......@@ -33,4 +33,6 @@ struct ADIOI_Fns_struct ADIO_HFS_operations = {
ADIOI_GEN_Flush, /* Flush */
ADIOI_HFS_Resize, /* Resize */
ADIOI_GEN_Delete, /* Delete */
ADIOI_GEN_IreadStridedColl, /* IreadStridedColl */
ADIOI_GEN_IwriteStridedColl /* IwriteStridedColl */
};
......@@ -41,4 +41,6 @@ struct ADIOI_Fns_struct ADIO_LUSTRE_operations = {
ADIOI_GEN_Delete, /* Delete */
ADIOI_GEN_Feature, /* Features */
"LUSTRE:",
ADIOI_GEN_IreadStridedColl, /* IreadStridedColl */
ADIOI_GEN_IwriteStridedColl /* IwriteStridedColl */
};
......@@ -37,5 +37,7 @@ struct ADIOI_Fns_struct ADIO_NFS_operations = {
ADIOI_NFS_Resize, /* Resize */
ADIOI_GEN_Delete, /* Delete */
ADIOI_NFS_Feature, /* Features */
"NFS:" /* fsname: just a string */
"NFS:", /* fsname: just a string */
ADIOI_GEN_IreadStridedColl, /* IreadStridedColl */
ADIOI_GEN_IwriteStridedColl /* IwriteStridedColl */
};
......@@ -34,5 +34,7 @@ struct ADIOI_Fns_struct ADIO_NTFS_operations = {
ADIOI_NTFS_Flush, /* Flush */
ADIOI_NTFS_Resize, /* Resize */
ADIOI_GEN_Delete, /* Delete */
ADIOI_NTFS_Feature /* Features */
ADIOI_NTFS_Feature, /* Features */
ADIOI_GEN_IreadStridedColl, /* IreadStridedColl */
ADIOI_GEN_IwriteStridedColl /* IwriteStridedColl */
};
......@@ -41,5 +41,7 @@ struct ADIOI_Fns_struct ADIO_PANFS_operations = {
ADIOI_PANFS_Resize, /* Resize */
ADIOI_GEN_Delete, /* Delete */
ADIOI_GEN_Feature,
"PANFS: Panasas PanFS"
"PANFS: Panasas PanFS",
ADIOI_GEN_IreadStridedColl, /* IreadStridedColl */
ADIOI_GEN_IwriteStridedColl /* IwriteStridedColl */
};
......@@ -33,4 +33,6 @@ struct ADIOI_Fns_struct ADIO_PFS_operations = {
ADIOI_PFS_Flush, /* Flush */
ADIOI_GEN_Resize, /* Resize */
ADIOI_GEN_Delete, /* Delete */
ADIOI_GEN_IreadStridedColl, /* IreadStridedColl */
ADIOI_GEN_IwriteStridedColl /* IwriteStridedColl */
};
......@@ -34,4 +34,6 @@ struct ADIOI_Fns_struct ADIO_PIOFS_operations = {
ADIOI_GEN_Resize, /* Resize */
ADIOI_GEN_Delete, /* Delete */
ADIOI_PIOFS_Feature,
ADIOI_GEN_IreadStridedColl, /* IreadStridedColl */
ADIOI_GEN_IwriteStridedColl /* IwriteStridedColl */
};
......@@ -34,4 +34,6 @@ struct ADIOI_Fns_struct ADIO_PVFS_operations = {
ADIOI_PVFS_Resize, /* Resize */
ADIOI_PVFS_Delete, /* Delete */
ADIOI_PVFS_Feature, /* Features */
ADIOI_GEN_IreadStridedColl, /* IreadStridedColl */
ADIOI_GEN_IwriteStridedColl /* IwriteStridedColl */
};
......@@ -39,7 +39,9 @@ struct ADIOI_Fns_struct ADIO_PVFS2_operations = {
ADIOI_PVFS2_Resize, /* Resize */
ADIOI_PVFS2_Delete, /* Delete */
ADIOI_PVFS2_Feature,
"PVFS2: the PVFS v2 or OrangeFS file systems"
"PVFS2: the PVFS v2 or OrangeFS file systems",
ADIOI_GEN_IreadStridedColl, /* IreadStridedColl */
ADIOI_GEN_IwriteStridedColl /* IwriteStridedColl */
};
/*
......
......@@ -33,4 +33,6 @@ struct ADIOI_Fns_struct ADIO_SFS_operations = {
ADIOI_SFS_Flush, /* Flush */
ADIOI_GEN_Resize, /* Resize */
ADIOI_GEN_Delete, /* Delete */
ADIOI_GEN_IreadStridedColl, /* IreadStridedColl */
ADIOI_GEN_IwriteStridedColl /* IwriteStridedColl */
};
......@@ -35,5 +35,7 @@ struct ADIOI_Fns_struct ADIO_TESTFS_operations = {
ADIOI_TESTFS_Resize, /* Resize */
ADIOI_TESTFS_Delete, /* Delete */
ADIOI_GEN_Feature, /* Features */
"TESTFS: the logging-only file system"
"TESTFS: the logging-only file system",
ADIOI_GEN_IreadStridedColl, /* IreadStridedColl */
ADIOI_GEN_IwriteStridedColl /* IwriteStridedColl */
};
......@@ -41,4 +41,6 @@ struct ADIOI_Fns_struct ADIO_UFS_operations = {
ADIOI_GEN_Delete, /* Delete */
ADIOI_GEN_Feature, /* Features */
"UFS: Generic ROMIO driver for all UNIX-like file systems",
ADIOI_GEN_IreadStridedColl, /* IreadStridedColl */
ADIOI_GEN_IwriteStridedColl /* IwriteStridedColl */
};
......@@ -40,5 +40,7 @@ struct ADIOI_Fns_struct ADIO_XFS_operations = {
ADIOI_XFS_Resize, /* Resize */
ADIOI_GEN_Delete, /* Delete */
ADIOI_GEN_Feature, /* Features */
"XFS: SGI XFS"
"XFS: SGI XFS",
ADIOI_GEN_IreadStridedColl, /* IreadStridedColl */
ADIOI_GEN_IwriteStridedColl /* IwriteStridedColl */
};
......@@ -35,6 +35,8 @@ struct ADIOI_Fns_struct ADIO_ZOIDFS_operations = {
ADIOI_ZOIDFS_Resize, /* Resize */
ADIOI_ZOIDFS_Delete, /* Delete */
ADIOI_ZOIDFS_Feature,
ADIOI_GEN_IreadStridedColl, /* IreadStridedColl */
ADIOI_GEN_IwriteStridedColl /* IwriteStridedColl */
};
/*
......
......@@ -26,8 +26,10 @@ romio_other_sources += \
adio/common/ad_io_coll.c \
adio/common/ad_iopen.c \
adio/common/ad_iread.c \
adio/common/ad_iread_coll.c \
adio/common/ad_iread_fake.c \
adio/common/ad_iwrite.c \
adio/common/ad_iwrite_coll.c \
adio/common/ad_iwrite_fake.c \
adio/common/ad_open.c \
adio/common/ad_opencoll.c \
......
......@@ -513,3 +513,141 @@ void ADIOI_Calc_others_req(ADIO_File fd, int count_my_req_procs,
MPE_Log_event (5027, 0, NULL);
#endif
}
/* Nonblocking version of ADIOI_Calc_others_req().
It consists of three functions - ADIOI_Icalc_others_req(),
ADIOI_Icalc_others_req_main(), and ADIOI_Icalc_others_req_fini(). */
void ADIOI_Icalc_others_req(ADIOI_NBC_Request *nbc_req, int *error_code)
{
ADIOI_Icalc_others_req_vars *vars = nbc_req->cor_vars;
/* count_others_req_per_proc[i] indicates how many separate contiguous
requests of proc. i lie in this process's file domain. */
/* first find out how much to send/recv and from/to whom */
#ifdef AGGREGATION_PROFILE
MPE_Log_event(5026, 0, NULL);
#endif
vars->count_others_req_per_proc =
(int *)ADIOI_Malloc(vars->nprocs * sizeof(int));
*error_code = MPI_Ialltoall(vars->count_my_req_per_proc, 1, MPI_INT,
vars->count_others_req_per_proc, 1, MPI_INT, vars->fd->comm,
&vars->req1);
if (nbc_req->rdwr == ADIOI_READ) {
nbc_req->data.rd.state = ADIOI_IRC_STATE_ICALC_OTHERS_REQ;
} else {
ADIOI_Assert(nbc_req->rdwr == ADIOI_WRITE);
nbc_req->data.wr.state = ADIOI_IWC_STATE_ICALC_OTHERS_REQ;
}
}
void ADIOI_Icalc_others_req_main(ADIOI_NBC_Request *nbc_req, int *error_code)
{
ADIOI_Icalc_others_req_vars *vars = nbc_req->cor_vars;
ADIO_File fd = vars->fd;
int count_my_req_procs = vars->count_my_req_procs;
ADIOI_Access *my_req = vars->my_req;
int nprocs = vars->nprocs;
int myrank = vars->myrank;
ADIOI_Access **others_req_ptr = vars->others_req_ptr;
/* determine what requests of other processes lie in this process's
file domain */
/* count_others_req_procs = number of processes whose requests lie in
this process's file domain (including this process itself)
count_others_req_per_proc[i] indicates how many separate contiguous
requests of proc. i lie in this process's file domain. */
int *count_others_req_per_proc = vars->count_others_req_per_proc;
int count_others_req_procs;
int i, j;
ADIOI_Access *others_req;
*others_req_ptr = (ADIOI_Access *)ADIOI_Malloc(nprocs*sizeof(ADIOI_Access));
others_req = *others_req_ptr;
count_others_req_procs = 0;
for (i = 0; i < nprocs; i++) {
if (count_others_req_per_proc[i]) {
others_req[i].count = count_others_req_per_proc[i];
others_req[i].offsets = (ADIO_Offset *)
ADIOI_Malloc(count_others_req_per_proc[i]*sizeof(ADIO_Offset));
others_req[i].lens =
ADIOI_Malloc(count_others_req_per_proc[i]*sizeof(ADIO_Offset));
others_req[i].mem_ptrs = (MPI_Aint *)
ADIOI_Malloc(count_others_req_per_proc[i]*sizeof(MPI_Aint));
count_others_req_procs++;
}
else others_req[i].count = 0;
}
vars->count_others_req_procs = count_others_req_procs;
/* now send the calculated offsets and lengths to respective processes */
vars->req2 = (MPI_Request *)
ADIOI_Malloc(1+2*(count_my_req_procs+count_others_req_procs)
*sizeof(MPI_Request));
/* +1 to avoid a 0-size malloc */
j = 0;
for (i = 0; i < nprocs; i++) {
if (others_req[i].count) {
MPI_Irecv(others_req[i].offsets, others_req[i].count,
ADIO_OFFSET, i, i+myrank, fd->comm, &vars->req2[j]);
j++;
MPI_Irecv(others_req[i].lens, others_req[i].count,
ADIO_OFFSET, i, i+myrank+1, fd->comm, &vars->req2[j]);
j++;
}
}
for (i=0; i < nprocs; i++) {
if (my_req[i].count) {
MPI_Isend(my_req[i].offsets, my_req[i].count,
ADIO_OFFSET, i, i+myrank, fd->comm, &vars->req2[j]);
j++;
MPI_Isend(my_req[i].lens, my_req[i].count,
ADIO_OFFSET, i, i+myrank+1, fd->comm, &vars->req2[j]);
j++;
}
}
/* keep the number of requests */
vars->num_req2 = j;
if (nbc_req->rdwr == ADIOI_READ) {
nbc_req->data.rd.state = ADIOI_IRC_STATE_ICALC_OTHERS_REQ_MAIN;
} else {
ADIOI_Assert(nbc_req->rdwr == ADIOI_WRITE);
nbc_req->data.wr.state = ADIOI_IWC_STATE_ICALC_OTHERS_REQ_MAIN;
}
}
void ADIOI_Icalc_others_req_fini(ADIOI_NBC_Request *nbc_req, int *error_code)
{
ADIOI_Icalc_others_req_vars *vars = nbc_req->cor_vars;
void (*next_fn)(ADIOI_NBC_Request *, int *);
ADIOI_Free(vars->req2);
ADIOI_Free(vars->count_others_req_per_proc);
*vars->count_others_req_procs_ptr = vars->count_others_req_procs;
#ifdef AGGREGATION_PROFILE
MPE_Log_event(5027, 0, NULL);
#endif
/* end of the calculation */
next_fn = vars->next_fn;
/* free the struct for parameters and variables */
ADIOI_Free(vars);
nbc_req->cor_vars = NULL;
/* move to the next function */
next_fn(nbc_req, error_code);
}
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
/*
* (C) 2014 by Argonne National Laboratory.
* See COPYRIGHT in top-level directory.
*/
#include "adio.h"
#include "adio_extern.h"
#include "mpiu_greq.h"
#ifdef USE_DBG_LOGGING
#define RDCOLL_DEBUG 1
#endif
#ifdef AGGREGATION_PROFILE
#include "mpe.h"
#endif
/* ADIOI_GEN_IreadStridedColl */
struct ADIOI_GEN_IreadStridedColl_vars {
/* requests */
MPI_Request req_offset[2]; /* ADIOI_IRC_STATE_GEN_IREADSTRIDEDCOLL */
MPI_Request req_ind_io; /* ADIOI_IRC_STATE_GEN_IREADSTRIDEDCOLL_INDIO */
/* parameters */
ADIO_File fd;
void *buf;
int count;
MPI_Datatype datatype;
int file_ptr_type;
ADIO_Offset offset;
/* stack variables */
ADIOI_Access *my_req;
/* array of nprocs structures, one for each other process in
whose file domain this process's request lies */
ADIOI_Access *others_req;
/* array of nprocs structures, one for each other process
whose request lies in this process's file domain. */
int nprocs;
int nprocs_for_coll;
int myrank;
int contig_access_count;
int interleave_count;
int buftype_is_contig;
int *count_my_req_per_proc;
int count_my_req_procs;
int count_others_req_procs;
ADIO_Offset orig_fp;
ADIO_Offset fd_size;
ADIO_Offset min_st_offset;
ADIO_Offset *offset_list;
ADIO_Offset *st_offsets;
ADIO_Offset *fd_start;
ADIO_Offset *fd_end;
ADIO_Offset *end_offsets;
ADIO_Offset *len_list;
int *buf_idx;
};
/* ADIOI_Iread_and_exch */
struct ADIOI_Iread_and_exch_vars {
/* requests */
MPI_Request req1; /* ADIOI_IRC_STATE_IREAD_AND_EXCH */
MPI_Request req2; /* ADIOI_IRC_STATE_IREAD_AND_EXCH_L1_BEGIN */
/* parameters */
ADIO_File fd;
void *buf;
MPI_Datatype datatype;
int nprocs;
int myrank;
ADIOI_Access *others_req;
ADIO_Offset *offset_list;
ADIO_Offset *len_list;
int contig_access_count;
ADIO_Offset min_st_offset;
ADIO_Offset fd_size;
ADIO_Offset *fd_start;
ADIO_Offset *fd_end;
int *buf_idx;
/* stack variables */
int m;
int ntimes;
int max_ntimes;
int buftype_is_contig;
ADIO_Offset st_loc;
ADIO_Offset end_loc;
ADIO_Offset off;
ADIO_Offset done;
char *read_buf;
int *curr_offlen_ptr;
int *count;
int *send_size;
int *recv_size;
int *partial_send;
int *recd_from_proc;
int *start_pos;
/* Not convinced end_loc-st_loc couldn't be > int, so make these offsets*/
ADIO_Offset size;
ADIO_Offset real_size;
ADIO_Offset for_curr_iter;
ADIO_Offset for_next_iter;
ADIOI_Flatlist_node *flat_buf;
MPI_Aint buftype_extent;
int coll_bufsize;
/* next function to be called */
void (*next_fn)(ADIOI_NBC_Request *, int *);
};
/* ADIOI_R_Iexchange_data */
struct ADIOI_R_Iexchange_data_vars {
/* requests */
MPI_Request req1; /* ADIOI_IRC_STATE_R_IEXCHANGE_DATA */
MPI_Request *req2; /* ADIOI_IRC_STATE_R_IEXCHANGE_DATA_RECV & FILL */
/* parameters */
ADIO_File fd;
void *buf;
ADIOI_Flatlist_node *flat_buf;
ADIO_Offset *offset_list;
ADIO_Offset *len_list;
int *send_size;
int *recv_size;
int *count;
int *start_pos;
int *partial_send;
int *recd_from_proc;
int nprocs;
int myrank;
int buftype_is_contig;
int contig_access_count;
ADIO_Offset min_st_offset;
ADIO_Offset fd_size;
ADIO_Offset *fd_start;
ADIO_Offset *fd_end;
ADIOI_Access *others_req;
int iter;
MPI_Aint buftype_extent;
int *buf_idx;
/* stack variables */
int nprocs_recv;
int nprocs_send;
char **recv_buf;
/* next function to be called */
void (*next_fn)(ADIOI_NBC_Request *, int *);
};
void ADIOI_Fill_user_buffer(ADIO_File fd, void *buf, ADIOI_Flatlist_node
*flat_buf, char **recv_buf, ADIO_Offset
*offset_list, ADIO_Offset *len_list,
unsigned *recv_size,
MPI_Request *requests, MPI_Status *statuses,
int *recd_from_proc, int nprocs,
int contig_access_count,
ADIO_Offset min_st_offset,
ADIO_Offset fd_size, ADIO_Offset *fd_start,
ADIO_Offset *fd_end,
MPI_Aint buftype_extent);
/* prototypes of functions used for nonblocking collective reads only. */
static void ADIOI_GEN_IreadStridedColl_inter(ADIOI_NBC_Request *, int *);
static void ADIOI_GEN_IreadStridedColl_indio(ADIOI_NBC_Request *, int *);
static void ADIOI_GEN_IreadStridedColl_read(ADIOI_NBC_Request *, int *);
static void ADIOI_GEN_IreadStridedColl_free(ADIOI_NBC_Request *, int *);
static void ADIOI_GEN_IreadStridedColl_fini(ADIOI_NBC_Request *, int *);
static void ADIOI_Iread_and_exch(ADIOI_NBC_Request *, int *);
static void ADIOI_Iread_and_exch_l1_begin(ADIOI_NBC_Request *, int *);
static void ADIOI_Iread_and_exch_l1_end(ADIOI_NBC_Request *, int *);
static void ADIOI_Iread_and_exch_reset(ADIOI_NBC_Request *, int *);
static void ADIOI_Iread_and_exch_l2_begin(ADIOI_NBC_Request *, int *);
static void ADIOI_Iread_and_exch_l2_end(ADIOI_NBC_Request *, int *);
static void ADIOI_Iread_and_exch_fini(ADIOI_NBC_Request *, int *);
static void ADIOI_R_Iexchange_data(ADIOI_NBC_Request *, int *);
static void ADIOI_R_Iexchange_data_recv(ADIOI_NBC_Request *, int *);
static void ADIOI_R_Iexchange_data_fill(ADIOI_NBC_Request *, int *);
static void ADIOI_R_Iexchange_data_fini(ADIOI_NBC_Request *, int *);
static MPIX_Grequest_class ADIOI_GEN_greq_class = 0;
static int ADIOI_GEN_irc_query_fn(void *extra_state, MPI_Status *status);
static int ADIOI_GEN_irc_free_fn(void *extra_state);
static int ADIOI_GEN_irc_poll_fn(void *extra_state, MPI_Status *status);
static int ADIOI_GEN_irc_wait_fn(int count, void **array_of_states,
double timeout, MPI_Status *status);
/* Nonblocking version of ADIOI_GEN_ReadStridedColl() */
void ADIOI_GEN_IreadStridedColl(ADIO_File fd, void *buf, int count,
MPI_Datatype datatype, int file_ptr_type,
ADIO_Offset offset, MPI_Request *request,
int *error_code)
{
/* Uses a generalized version of the extended two-phase method described
in "An Extended Two-Phase Method for Accessing Sections of
Out-of-Core Arrays", Rajeev Thakur and Alok Choudhary,
Scientific Programming, (5)4:301--317, Winter 1996.
http://www.mcs.anl.gov/home/thakur/ext2ph.ps */
ADIOI_NBC_Request *nbc_req = NULL;
ADIOI_GEN_IreadStridedColl_vars *vars = NULL;
int nprocs, myrank;
#ifdef RDCOLL_DEBUG
int i;
#endif
ADIO_Offset start_offset, end_offset;
/* FIXME: need an implementation of ADIOI_IOIstridedColl
if (fd->hints->cb_pfr != ADIOI_HINT_DISABLE) {
ADIOI_IOIstridedColl(fd, buf, count, ADIOI_READ, datatype,
file_ptr_type, offset, request, error_code);
return;
}
*/
/* top-level struct keeping the status of function progress */
nbc_req = (ADIOI_NBC_Request *)ADIOI_Calloc(1, sizeof(ADIOI_NBC_Request));
nbc_req->rdwr = ADIOI_READ;
/* create a generalized request */
if (ADIOI_GEN_greq_class == 0) {
MPIX_Grequest_class_create(ADIOI_GEN_irc_query_fn,
ADIOI_GEN_irc_free_fn, MPIU_Greq_cancel_fn,
ADIOI_GEN_irc_poll_fn, ADIOI_GEN_irc_wait_fn,
&ADIOI_GEN_greq_class);
}
MPIX_Grequest_class_allocate(ADIOI_GEN_greq_class, nbc_req, request);
memcpy(&nbc_req->req, request, sizeof(MPI_Request));
/* create a struct for parameters and variables */
vars = (ADIOI_GEN_IreadStridedColl_vars *)ADIOI_Calloc(
1, sizeof(ADIOI_GEN_IreadStridedColl_vars));
nbc_req->data.rd.rsc_vars = vars;
/* save the parameters */
vars->fd = fd;
vars->buf = buf;
vars->count = count;
vars->datatype = datatype;
vars->file_ptr_type = file_ptr_type;
vars->offset = offset;
MPI_Comm_size(fd->comm, &nprocs);
MPI_Comm_rank(fd->comm, &myrank);
vars->nprocs = nprocs;
vars->myrank = myrank;
/* number of aggregators, cb_nodes, is stored in the hints */
vars->nprocs_for_coll = fd->hints->cb_nodes;
vars->orig_fp = fd->fp_ind;
/* only check for interleaving if cb_read isn't disabled */
if (fd->hints->cb_read != ADIOI_HINT_DISABLE) {
/* For this process's request, calculate the list of offsets and
lengths in the file and determine the start and end offsets. */
/* Note: end_offset points to the last byte-offset that will be accessed.
e.g., if start_offset=0 and 100 bytes to be read, end_offset=99*/
ADIOI_Calc_my_off_len(fd, count, datatype, file_ptr_type, offset,
&vars->offset_list, &vars->len_list,
&start_offset, &end_offset,
&vars->contig_access_count);
#ifdef RDCOLL_DEBUG
for (i = 0; i < vars->contig_access_count; i++) {
DBG_FPRINTF(stderr, "rank %d off %lld len %lld\n",
myrank, vars->offset_list[i], vars->len_list[i]);
}
#endif
/* each process communicates its start and end offsets to other
processes. The result is an array each of start and end offsets
stored in order of process rank. */
vars->st_offsets = (ADIO_Offset *)ADIOI_Malloc(nprocs*sizeof(ADIO_Offset));
vars->end_offsets = (ADIO_Offset *)ADIOI_Malloc(nprocs*sizeof(ADIO_Offset));
*error_code = MPI_Iallgather(&start_offset, 1, ADIO_OFFSET,
vars->st_offsets, 1, ADIO_OFFSET,