Commit 0ddd8086 authored by Rob Latham's avatar Rob Latham
Browse files

scalable system-hints processing



instead of every process reading and processing the hint config file,
one process will read and broadcast, then everyone will process.  Yay!
fewer file system calls!
Signed-off-by: default avatarPaul Coffman <pkcoff@us.ibm.com>
parent a4868b87
......@@ -78,11 +78,6 @@ void ADIO_Init(int *argc, char ***argv, int *error_code)
else ADIOI_Direct_write = 0;
#endif
/* Assume system-wide hints won't change between runs: move hint processing
* from ADIO_Open to here */
/* FIXME should be checking error code from MPI_Info_create here */
MPI_Info_create(&ADIOI_syshints);
ADIOI_process_system_hints(ADIOI_syshints);
#ifdef ADIOI_MPE_LOGGING
{
......
......@@ -88,6 +88,24 @@ MPI_File ADIO_Open(MPI_Comm orig_comm,
fd->hints->initialized = 0;
fd->info = MPI_INFO_NULL;
/* move system-wide hint processing *back* into open, but this time the
* hintfile reader will do a scalable read-and-broadcast. The global
* ADIOI_syshints will get initialized at first open. subsequent open
* calls will just use result from first open.
*
* We have two goals here:
* 1: avoid processing the hintfile multiple times
* 2: have all processes participate in hintfile processing (so we can read-and-broadcast)
*
* a code might do an "initialize from 0", so we can only skip hint
* processing once everyone has particpiated in hint processing */
int dummy_info;
MPI_Allreduce(&ADIOI_syshints, &dummy_info, 1, MPI_INT, MPI_MIN, fd->comm);
if ((MPI_Info)dummy_info== MPI_INFO_NULL) {
MPI_Info_create(&ADIOI_syshints);
ADIOI_process_system_hints(fd, ADIOI_syshints);
}
ADIOI_incorporate_system_hints(info, ADIOI_syshints, &dupinfo);
ADIO_SetInfo(fd, dupinfo, &err);
if (dupinfo != MPI_INFO_NULL) {
......
......@@ -85,8 +85,11 @@ static int find_file(void)
* alone on the assumption that the caller knows best.
*
* because MPI-IO hints are optional, we can get away with limited error
* reporting. */
static int file_to_info(int fd, MPI_Info info)
* reporting.
*
* for better scalability, the config file will be read on one processor and
* broadcast to all others */
static int file_to_info_all(int fd, MPI_Info info, int rank, MPI_Comm comm)
{
char *buffer, *token, *key, *val, *garbage;
char *pos1=NULL, *pos2=NULL;
......@@ -97,11 +100,19 @@ static int file_to_info(int fd, MPI_Info info)
/* assumption: config files will be small */
#define HINTFILE_MAX_SIZE 1024*4
buffer = (char *)ADIOI_Calloc(HINTFILE_MAX_SIZE, sizeof (char));
if (buffer == NULL) return -1;
ret = read(fd, buffer, HINTFILE_MAX_SIZE);
if (ret < 0) return -1;
if (rank == 0) {
ret = read(fd, buffer, HINTFILE_MAX_SIZE);
/* any error: bad/nonexistent fd, no perms, anything: set up a null
* buffer and the subsequent string parsing will quit immediately */
if (ret == -1)
buffer[0] = '\0';
}
MPI_Bcast(buffer, HINTFILE_MAX_SIZE, MPI_BYTE, 0, comm);
token = strtok_r(buffer, "\n", &pos1);
if (token == NULL)
goto fn_exit;
do {
if ( (key = strtok_r(token, " \t", &pos2)) == NULL)
/* malformed line: found no items */
......@@ -125,23 +136,25 @@ static int file_to_info(int fd, MPI_Info info)
if (flag == 1) continue;
ADIOI_Info_set(info, key, val);
} while ((token = strtok_r(NULL, "\n", &pos1)) != NULL);
fn_exit:
ADIOI_Free(buffer);
return 0;
}
void ADIOI_process_system_hints(MPI_Info info)
void ADIOI_process_system_hints(ADIO_File fd, MPI_Info info)
{
int hintfd;
int hintfd=-1, rank;
hintfd = find_file();
if (hintfd < 0) {
#ifdef SYSHINT_DEBUG
perror("ADIOI_process_system_hints");
#endif
return;
MPI_Comm_rank(fd->comm, &rank);
if (rank == 0) {
hintfd = find_file();
}
file_to_info(hintfd, info);
close(hintfd);
/* hintfd only significant on rank 0. -1 (on rank 0) means no hintfile found */
file_to_info_all(hintfd, info, rank, fd->comm);
if (hintfd != -1)
close(hintfd);
}
/* given 'info', incorporate any hints in 'sysinfo' that are not already set
......
......@@ -340,7 +340,7 @@ void ADIOI_Get_position(ADIO_File fd, ADIO_Offset *offset);
void ADIOI_Get_eof_offset(ADIO_File fd, ADIO_Offset *eof_offset);
void ADIOI_Get_byte_offset(ADIO_File fd, ADIO_Offset offset,
ADIO_Offset *disp);
void ADIOI_process_system_hints(MPI_Info info);
void ADIOI_process_system_hints(ADIO_File fd, MPI_Info info);
void ADIOI_incorporate_system_hints(MPI_Info info, MPI_Info sysinfo,
MPI_Info *new_info);
void ADIOI_Info_print_keyvals(MPI_Info info);
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment